chongming/chongming-utils-cache
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify: 清除diskcache依赖

Browse Source
This commit is contained in:
chakcy
2026-09-04 14:44:06 +08:00
parent 2b5a7b09eb
commit 08657fa166
11 changed files with 424 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/chongming_cache/__init__.py
+6 -2
View File
@@ -1,14 +1,18 @@
from chongming_cache.backends import CacheBackend, DiskBackend, RedisBackend
from chongming_cache.backends import CacheBackend, RedisBackend, SqliteBackend
from chongming_cache.cache import Cache
from chongming_cache.lock import DistributedLock
# 兼容旧名：原 diskcache 后端已由 SQLite 后端 (SqliteBackend) 直接替换
DiskBackend = SqliteBackend
__version__ = "0.2.0"
__all__ = [
"Cache",
"CacheBackend",
"DiskBackend",
"SqliteBackend",
"RedisBackend",
"DistributedLock",
"__version__",
]
]
src/chongming_cache/backends/__init__.py
+2 -2
View File
@@ -1,5 +1,5 @@
from chongming_cache.backends.base import CacheBackend
from chongming_cache.backends.disk import DiskBackend
from chongming_cache.backends.redis_ import RedisBackend
from chongming_cache.backends.sqlite_ import SqliteBackend
__all__ = ["CacheBackend", "DiskBackend", "RedisBackend"]
__all__ = ["CacheBackend", "SqliteBackend", "RedisBackend"]
src/chongming_cache/backends/base.py
+75
View File
@@ -1,8 +1,83 @@
"""缓存后端抽象接口：dict-like 映射 + TTL。"""
import base64
import pickle
from abc import ABC, abstractmethod
from typing import Any, Iterator
try:
import orjson
def _dumps(value: Any) -> bytes:
# default：bytes 无法原生 JSON 化，包成 {__b64__: base64}，读取时还原为 bytes
def _default(obj: Any) -> Any:
if isinstance(obj, bytes):
return {_B64_MARKER: base64.b64encode(obj).decode("ascii")}
raise TypeError(f"无法序列化为 JSON: {type(obj)!r}")
return orjson.dumps(value, default=_default)
def _loads(raw) -> Any:
return orjson.loads(raw)
except ImportError: # pragma: no cover
import json
def _dumps(value: Any) -> bytes:
def _default(obj: Any) -> Any:
if isinstance(obj, bytes):
return {_B64_MARKER: base64.b64encode(obj).decode("ascii")}
raise TypeError(f"无法序列化为 JSON: {type(obj)!r}")
return json.dumps(value, ensure_ascii=False, default=_default).encode("utf-8")
def _loads(raw) -> Any:
return json.loads(raw.decode("utf-8"))
# bytes 的 base64 包裹标记：让 bytes（如 RPC 缓存回复）在 JSON 下也能无损往返。
_B64_MARKER = "__b64__"
# 所有后端统一以 JSON 持久化：跨后端（redis/本地 diskcache）语义一致，
# 且与 Rust 进程插件（serde_json / json）的共享协议互通。
def _restore_bytes(obj: Any) -> Any:
"""递归还原被包裹的 bytes。"""
if isinstance(obj, dict):
if len(obj) == 1 and _B64_MARKER in obj:
try:
return base64.b64decode(obj[_B64_MARKER])
except Exception:
pass
return {k: _restore_bytes(v) for k, v in obj.items()}
if isinstance(obj, list):
return [_restore_bytes(v) for v in obj]
return obj
def encode_value(value: Any) -> bytes:
"""统一序列化：JSON（orjson）。bytes 以 base64 包裹；datetime/set/自定义对象不支持。"""
return _dumps(value)
def decode_value(raw: Any) -> Any:
"""统一反序列化。
- 字节值优先按 JSON 解析（orjson）；bytes 从 base64 包裹还原；
- 解析失败回退 pickle（兼容历史 pickled 数据，跨后端切换窗口）；
- 若后端已把对象透明还原（旧磁盘直存路径），非字节则原样返回。
"""
if not isinstance(raw, (bytes, bytearray)):
return raw
try:
return _restore_bytes(_loads(raw))
except Exception:
pass
try:
return pickle.loads(raw)
except Exception:
return raw
class CacheBackend(ABC):
"""缓存后端统一接口。
src/chongming_cache/backends/disk.py
-64
View File
@@ -1,64 +0,0 @@
"""diskcache 后端：基于 SQLite 的本地持久化缓存，多进程安全。"""
from pathlib import Path
from typing import Any, Iterator
from diskcache import Cache as DiskCache
from chongming_cache.backends.base import CacheBackend
class DiskBackend(CacheBackend):
"""包装 diskcache.Cache，提供统一接口。
Args:
path: 缓存目录（SQLite 数据文件所在目录），父目录不存在时自动创建。
"""
def __init__(self, path: str | Path):
self._cache = DiskCache(path)
@property
def client(self) -> DiskCache:
"""底层 diskcache.Cache 实例（逃生口）。"""
return self._cache
def __contains__(self, key) -> bool:
return key in self._cache
def __getitem__(self, key) -> Any:
return self._cache[key]
def __setitem__(self, key, value) -> None:
self._cache[key] = value
def __delitem__(self, key) -> None:
del self._cache[key]
def __len__(self) -> int:
return len(self._cache)
def __iter__(self) -> Iterator:
return iter(self._cache)
def get(self, key, default=None) -> Any:
return self._cache.get(key, default)
def set(self, key, value, expire: float | None = None) -> None:
self._cache.set(key, value, expire=expire)
def set_nx(self, key, value, expire: float | None = None) -> bool:
# diskcache.add 仅在 key 不存在时写入，返回是否成功（SET NX 语义）且进程安全
return bool(self._cache.add(key, value, expire=expire))
def touch(self, key, expire: float | None = None) -> bool:
return self._cache.touch(key, expire=expire)
def delete(self, key) -> bool:
return self._cache.delete(key)
def clear(self) -> None:
self._cache.clear()
def close(self) -> None:
self._cache.close()
src/chongming_cache/backends/redis_.py
+30 -11
View File
@@ -2,12 +2,16 @@
redis-py 采用懒加载：仅在选择 redis 后端时才 import，
使用 diskcache 后端的打包产物不会引入 redis 依赖。
值统一使用 JSON（orjson）序列化，与本地 diskcache 后端及 Rust 进程插件
（serde_json）共享同一套跨语言协议；``prefix`` 作为 key 命名空间前缀。
"""
import pickle
import datetime
from typing import Any, Iterator
from chongming_cache.backends.base import CacheBackend
from chongming_cache.backends.base import CacheBackend, decode_value, encode_value
# SCAN 批大小与 clear() 管道批大小
_SCAN_COUNT = 1000
@@ -16,8 +20,8 @@ _SCAN_COUNT = 1000
class RedisBackend(CacheBackend):
"""包装 redis-py 客户端，提供统一接口。
值使用 pickle 序列化（与 diskcache 默认序列化语义对齐，支持任意 Python 对象）；
``prefix`` 作为 key 命名空间前缀，便于多实例共享同一 Redis 时互不干扰。
值统一使用 JSON（orjson）序列化；``prefix`` 作为 key 命名空间前缀，
便于多实例共享同一 Redis 时互不干扰。
Args:
host: Redis 地址
@@ -66,6 +70,19 @@ class RedisBackend(CacheBackend):
"""去掉前缀并解码为 str。"""
return raw[len(self._prefix) :].decode()
@staticmethod
def _expire_secs(expire):
"""把秒（可能是 float）规范为 redis-py 可接受的取值。
redis-py 的 ``SET EX`` / ``EXPIRE`` 只接受 ``int`` 或 ``datetime.timedelta``，
传入 float（如 DistributedLock 默认 ttl=30.0）会抛 ``DataError``。
"""
if isinstance(expire, datetime.timedelta):
return expire
if expire is not None:
return int(expire)
return None
def __contains__(self, key) -> bool:
return bool(self._redis.exists(self._key(key)))
@@ -73,10 +90,10 @@ class RedisBackend(CacheBackend):
raw = self._redis.get(self._key(key))
if raw is None:
raise KeyError(key)
return pickle.loads(raw)
return decode_value(raw)
def __setitem__(self, key, value) -> None:
self._redis.set(self._key(key), pickle.dumps(value))
self._redis.set(self._key(key), encode_value(value))
def __delitem__(self, key) -> None:
if not self._redis.delete(self._key(key)):
@@ -95,25 +112,27 @@ class RedisBackend(CacheBackend):
raw = self._redis.get(self._key(key))
if raw is None:
return default
return pickle.loads(raw)
return decode_value(raw)
def set(self, key, value, expire: float | None = None) -> None:
self._redis.set(self._key(key), pickle.dumps(value), ex=expire)
self._redis.set(
self._key(key), encode_value(value), ex=self._expire_secs(expire)
)
def set_nx(self, key, value, expire: float | None = None) -> bool:
# redis SET key val NX EX ttl：原子地仅在不存在时写入，用于分布式锁
return bool(
self._redis.set(
self._key(key),
pickle.dumps(value),
ex=expire,
encode_value(value),
ex=self._expire_secs(expire),
nx=True,
)
)
def touch(self, key, expire: float | None = None) -> bool:
# expire=None 时 redis-py 发 PERSIST（永久），与 diskcache.touch 语义一致
return bool(self._redis.expire(self._key(key), expire))
return bool(self._redis.expire(self._key(key), self._expire_secs(expire)))
def delete(self, key) -> bool:
return bool(self._redis.delete(self._key(key)))
src/chongming_cache/backends/sqlite_.py
+178
View File
@@ -0,0 +1,178 @@
"""sqlite 后端：基于标准库 SQLite 的本地共享键值缓存（替代原 diskcache 后端）。
值以 JSON(orjson) 存入 TEXT 列，序列化协议与 redis 后端 / Rust 进程插件（serde_json）
共享同一套 schema，因此 Rust 侧（sqlx）可直接读写同一份 .db 文件，实现真正的跨语言
本地共享（无需 Redis）。
- 本地、多进程安全：WAL + busy_timeout + SQLite 行锁（等价于原 diskcache 的多进程语义）；
- TTL：``expire`` 列存到期 epoch 秒，读时惰性淘汰（对齐 diskcache/redis 的 get 语义）。
共享 schema（Ruby/Python 与 Rust 必须一致）：
CREATE TABLE IF NOT EXISTS cache (
k TEXT PRIMARY KEY,
v TEXT NOT NULL, -- JSON 文本
expire REAL -- 到期 epoch 秒；NULL 表示永不过期
);
"""
import sqlite3
import threading
import time
from pathlib import Path
from typing import Any, Iterator
from chongming_cache.backends.base import CacheBackend, decode_value, encode_value
_SCHEMA = """
CREATE TABLE IF NOT EXISTS cache (
k TEXT PRIMARY KEY,
v TEXT NOT NULL,
expire REAL
);
"""
class SqliteBackend(CacheBackend):
"""包装 sqlite3，提供统一接口（与 Rust 侧共享同一份 schema）。
Args:
path: SQLite 数据库文件路径（父目录不存在时自动创建）。
"""
def __init__(self, path: str | Path):
p = Path(path)
if p.parent and str(p.parent) != ".":
p.parent.mkdir(parents=True, exist_ok=True)
self._path = str(p)
self._lock = threading.RLock()
self._conn = sqlite3.connect(self._path, check_same_thread=False)
self._conn.execute("PRAGMA journal_mode=WAL")
self._conn.execute("PRAGMA busy_timeout=5000")
self._conn.execute(_SCHEMA)
self._conn.commit()
@property
def client(self) -> sqlite3.Connection:
"""底层 sqlite3.Connection 实例（逃生口）。"""
return self._conn
@staticmethod
def _expire_secs(expire) -> float | None:
"""秒级 TTL -> 到期 epoch 时间戳；None 表示永不过期。"""
if expire is None:
return None
return time.time() + float(expire)
def _live_value(self, k: str) -> str | None:
"""取未过期的原始 JSON 文本；过期或缺失返回 None（过期时顺手删除）。"""
row = self._conn.execute(
"SELECT v, expire FROM cache WHERE k = ?", (k,)
).fetchone()
if row is None:
return None
v, expire = row
if expire is not None and expire <= time.time():
self._conn.execute("DELETE FROM cache WHERE k = ?", (k,))
self._conn.commit()
return None
return v
def __contains__(self, key) -> bool:
with self._lock:
return self._live_value(str(key)) is not None
def __getitem__(self, key) -> Any:
with self._lock:
raw = self._live_value(str(key))
if raw is None:
raise KeyError(key)
return decode_value(raw.encode("utf-8"))
def __setitem__(self, key, value) -> None:
with self._lock:
self._set(str(key), value, None)
def __delitem__(self, key) -> None:
with self._lock:
if not self.delete(key):
raise KeyError(key)
def __len__(self) -> int:
with self._lock:
now = time.time()
self._conn.execute(
"DELETE FROM cache WHERE expire IS NOT NULL AND expire <= ?", (now,)
)
self._conn.commit()
return self._conn.execute("SELECT COUNT(*) FROM cache").fetchone()[0]
def __iter__(self) -> Iterator:
with self._lock:
now = time.time()
self._conn.execute(
"DELETE FROM cache WHERE expire IS NOT NULL AND expire <= ?", (now,)
)
self._conn.commit()
rows = self._conn.execute("SELECT k FROM cache").fetchall()
return iter(row[0] for row in rows)
def get(self, key, default=None) -> Any:
with self._lock:
raw = self._live_value(str(key))
if raw is None:
return default
return decode_value(raw.encode("utf-8"))
def _set(self, k: str, value: Any, expire) -> None:
v = encode_value(value).decode("utf-8")
self._conn.execute(
"INSERT INTO cache(k, v, expire) VALUES(?, ?, ?) "
"ON CONFLICT(k) DO UPDATE SET v=excluded.v, expire=excluded.expire",
(k, v, expire),
)
self._conn.commit()
def set(self, key, value, expire: float | None = None) -> None:
with self._lock:
self._set(str(key), value, self._expire_secs(expire))
def set_nx(self, key, value, expire: float | None = None) -> bool:
with self._lock:
k = str(key)
v = encode_value(value).decode("utf-8")
cur = self._conn.execute(
"INSERT OR IGNORE INTO cache(k, v, expire) VALUES(?, ?, ?)",
(k, v, self._expire_secs(expire)),
)
self._conn.commit()
return cur.rowcount == 1
def touch(self, key, expire: float | None = None) -> bool:
with self._lock:
k = str(key)
if self._live_value(k) is None:
return False
self._conn.execute(
"UPDATE cache SET expire=? WHERE k=?", (self._expire_secs(expire), k)
)
self._conn.commit()
return True
def delete(self, key) -> bool:
with self._lock:
cur = self._conn.execute("DELETE FROM cache WHERE k = ?", (str(key),))
self._conn.commit()
return cur.rowcount > 0
def clear(self) -> None:
with self._lock:
self._conn.execute("DELETE FROM cache")
self._conn.commit()
def close(self) -> None:
with self._lock:
try:
self._conn.close()
except Exception:
pass
src/chongming_cache/cache.py
+23 -14
View File
@@ -1,10 +1,14 @@
"""统一缓存门面：dict-like 接口，后端可切换 diskcache / redis。"""
"""统一缓存门面：dict-like 接口，后端可切换 sqlite(本地) / redis(共享)。"""
from pathlib import Path
from typing import Any, Iterator, Mapping
from chongming_cache.backends import CacheBackend, DiskBackend, RedisBackend
from chongming_cache.backends import CacheBackend, RedisBackend, SqliteBackend
_KNOWN_BACKENDS = {"diskcache", "redis"}
_KNOWN_BACKENDS = {"sqlite", "redis"}
# 旧后端名 diskcache → sqlite（新本地后端直接替换了原 diskcache 实现）
_BACKEND_ALIASES = {"diskcache": "sqlite"}
_DEFAULT_PATH = "./cache_data/cache.db"
class Cache:
@@ -24,11 +28,11 @@ class Cache:
value = cache["k"]
"""
def __init__(self, backend: str = "diskcache", **params):
def __init__(self, backend: str = "sqlite", **params):
"""创建缓存实例。
Args:
backend: "diskcache" 或 "redis"
backend: "sqlite" 或 "redis"（旧名 "diskcache" 兼容映射到 sqlite）
**params: 后端参数，见 ``from_config`` 说明
"""
self._backend = self._create_backend(backend, params)
@@ -37,24 +41,29 @@ class Cache:
def from_config(cls, config: Mapping) -> "Cache":
"""从配置字典创建缓存实例，读取形如主项目 config.toml 的 ``[cache]`` 段：
- ``backend``: "diskcache"（默认）或 "redis"
- ``cache_store_path``: diskcache 缓存目录（默认 "./cache_data"）
- ``backend``: "sqlite"（默认，本地共享）或 "redis"（跨主机）
- ``path`` / ``cache_store_path``: sqlite 数据库文件路径（默认 "./cache_data/cache.db"）
- ``redis_host`` / ``redis_port`` / ``redis_db`` / ``redis_password`` / ``redis_prefix``:
redis 后端连接参数（仅 backend = "redis" 时生效）
所有键均可缺省，旧配置（只有 cache_store_path）直接兼容。
"""
cfg = dict(config)
return cls(cfg.pop("backend", "diskcache"), **cfg)
return cls(cfg.pop("backend", "sqlite"), **cfg)
@staticmethod
def _create_backend(backend: str, params: Mapping) -> CacheBackend:
name = (backend or "diskcache").strip().lower()
name = _BACKEND_ALIASES.get((backend or "sqlite").strip().lower(), (backend or "sqlite").strip().lower())
if name not in _KNOWN_BACKENDS:
raise ValueError(f"不支持的缓存后端: {backend!r}（可选 diskcache / redis）")
raise ValueError(f"不支持的缓存后端: {backend!r}（可选 sqlite / redis）")
if name == "diskcache":
return DiskBackend(params.get("cache_store_path", "./cache_data"))
if name == "sqlite":
path = params.get("path") or params.get("cache_store_path") or _DEFAULT_PATH
p = Path(path)
# 兼容旧 diskcache 配置：路径若指向目录则追加默认 db 文件名
if path.endswith(("/", "\\")) or p.is_dir():
p = p / "cache.db"
return SqliteBackend(p)
return RedisBackend(
host=params.get("redis_host", "127.0.0.1"),
@@ -66,12 +75,12 @@ class Cache:
@property
def backend(self) -> CacheBackend:
"""后端封装实例（DiskBackend / RedisBackend）。"""
"""后端封装实例（SqliteBackend / RedisBackend）。"""
return self._backend
@property
def client(self):
"""底层客户端（diskcache.Cache / redis.Redis），逃生口。"""
"""底层客户端（sqlite3.Connection / redis.Redis），逃生口。"""
return self._backend.client
# ── 映射协议 ──────────────────────────────────────────────