fix+feat: 修复打包/解包主链路,并把存储层重建为内容寻址 + 快照

起始状态:主链路 100% 不可用,而且失败被静默吞掉——打包后库里 0 个文件、
界面仍打印"文件数量: N";解包导出 0 个文件并在磁盘上生成几百层 sub/sub/sub 垃圾目录。

缺陷修复(4 个 P0 + 12 个 P1,逐条见 BUG_REPORT.md):
- 建表列名 compresse_algo 与 INSERT/read_file 用的 compress_algo 不一致
- list_directory 按 path 而非 parent_path 过滤,导致导出无限自递归
- CLI --ignore-timestamps 把 export_filesystem 赋成引用自身的 lambda
- add_file 缺少 commit(直接调用 API 时静默丢数据)
- LIKE 前缀误伤兄弟目录(删 doc 连带删 docs);mv_dir 二次改名
- 失败路径一律返回 exit code 0;--exclude 是子串匹配而非 glob
- FolderPacker.add_file 反向调用 export_file_safe,会用库内内容覆盖源文件

存储层(schema v3,格式变更,提供 vfsmigrate 迁移):
- 内容移入 blobs(hash PRIMARY KEY):相同内容只存一份,跨路径/跨快照共享
- files 的唯一约束改为 UNIQUE(snapshot_id, path):一个库存多个版本
- 完整性:每文件 sha256 + 快照级摘要 + HMAC-SHA256 签名,只读连接即可验证
- 增量打包:按 mtime+size 或 sha256 只写变化项,未变文件不重写不重压
- get_file_info 不再返回 content(元数据查询不该搬 BLOB)

新增能力:
- 可选 FastAPI 静态文件插件(extras,核心仍零第三方依赖)
- 内容寻址发布到对象存储 + 逐对象校验的取回客户端
- 压缩默认改回 zlib:实测比自研哈夫曼体积小 1/3~1/170、快数百倍

其它:
- P2-4:时间戳显式转字符串,清掉 1240 条 DeprecationWarning(盘上格式不变)
- read_file 负数 offset/size 改为报错(原先会走切片语义从末尾取)
- requires-python 从 >=3.7 改为按实际 API 推导的 >=3.9

测试:0 → 343 个用例,纯标准库 unittest,有/无 FastAPI 两种环境都跑通。
证据见 docs/evidence-*.txt。
This commit is contained in:
2026-09-14 00:11:42 +08:00
parent 161ac9129f
commit ecdfacc620
33 changed files with 10063 additions and 632 deletions
+443
View File
@@ -0,0 +1,443 @@
"""CLI 行为测试(以子进程方式运行,覆盖退出码与各选项语义)。"""
import json
import os
import sqlite3
import unittest
import helpers
from sqlite_vfs.core import SQLiteVFS
PACKER = "sqlite_vfs.cli.packer"
UNPACKER = "sqlite_vfs.cli.unpacker"
VERIFY = "sqlite_vfs.cli.verify"
MIGRATE = "sqlite_vfs.cli.migrate"
SMALL_SPEC = {
"top.txt": b"top\n",
"sub/inner.txt": b"inner\n",
"sub/deep/leaf.bin": b"leaf\n",
"empty_dir": None,
}
class CliFixture(helpers.VFSTestCase):
def seed_db(self, name="seeded.svfs"):
"""用库接口直接造一个库,让 CLI 测试不被打包链路的缺陷连带影响。"""
db = self.path(name)
vfs = SQLiteVFS(db)
try:
helpers.seed_rows(
vfs,
[
{"path": "doc", "is_directory": True},
{"path": "doc/a.txt", "content": b"a"},
{"path": "doc/sub", "is_directory": True},
{"path": "doc/sub/b.txt", "content": b"b"},
{"path": "docs", "is_directory": True},
{"path": "top.txt", "content": b"t"},
],
)
finally:
vfs.close()
return db
class CliFailureTest(CliFixture):
"""P1-8:失败路径必须返回非 0 退出码,且给出干净的错误信息而不是 traceback。"""
def test_pack_missing_source_returns_nonzero(self):
result = helpers.run_cli(PACKER, self.path("nope"), self.path("out.svfs"))
self.assertNotEqual(0, result.returncode, "源目录不存在却返回了成功")
self.assertNotIn("Traceback", helpers.cli_output(result))
def test_unpack_missing_db_returns_nonzero(self):
result = helpers.run_cli(UNPACKER, self.path("nope.svfs"), self.path("out"))
self.assertNotEqual(0, result.returncode, "数据库不存在却返回了成功")
self.assertNotIn("Traceback", helpers.cli_output(result))
def test_pack_to_missing_output_directory_reports_clean_error(self):
src = helpers.make_tree(self.path("src"), {"a.txt": b"a"})
result = helpers.run_cli(PACKER, src, self.path("missing_dir", "out.svfs"))
self.assertNotEqual(0, result.returncode)
self.assertNotIn("Traceback", helpers.cli_output(result))
self.assertIn("错误", helpers.cli_output(result))
def test_unpack_non_database_file_reports_clean_error(self):
junk = helpers.write_file(self.path("junk.svfs"), b"definitely not sqlite")
result = helpers.run_cli(UNPACKER, junk, self.path("out"))
self.assertNotEqual(0, result.returncode)
self.assertNotIn("Traceback", helpers.cli_output(result))
def test_unpack_missing_path_returns_nonzero(self):
db = self.seed_db()
result = helpers.run_cli(UNPACKER, db, self.path("out"), "--path", "nope")
self.assertNotEqual(0, result.returncode, "导出不存在的路径却返回了成功")
class CliPackTest(CliFixture):
def test_pack_stores_every_file(self):
src = helpers.make_tree(self.path("src"), SMALL_SPEC)
db = self.path("packed.svfs")
result = helpers.run_cli(PACKER, src, db, "--compress", "--name", "demo")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
vfs = SQLiteVFS(db)
self.addCleanup(vfs.close)
expected = {rel for rel, content in SMALL_SPEC.items() if content is not None}
self.assertEqual(expected, set(helpers.stored_files(vfs)))
self.assertEqual("demo", vfs.get_stats()["name"])
def test_exclude_supports_glob_patterns(self):
"""P1-9: README 承诺 glob 语义,`*.log` 必须真的排除 .log 文件,
同时不能把 catalog.txt 这类"看起来像"的文件误伤。"""
spec = {
"keep.txt": b"k",
"skip.log": b"s",
"catalog.txt": b"c",
"sub/skip.tmp": b"t",
"sub/keep2.txt": b"k2",
}
src = helpers.make_tree(self.path("src"), spec)
db = self.path("packed.svfs")
result = helpers.run_cli(PACKER, src, db, "--exclude", "*.log", "*.tmp")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
vfs = SQLiteVFS(db)
self.addCleanup(vfs.close)
self.assertEqual(
{"keep.txt", "catalog.txt", "sub/keep2.txt"}, set(helpers.stored_files(vfs))
)
def test_exclude_prunes_directory_subtree(self):
spec = {
"keep.txt": b"k",
"vendor/lib.txt": b"v",
"vendor/deep/more.txt": b"m",
}
src = helpers.make_tree(self.path("src"), spec)
db = self.path("packed.svfs")
result = helpers.run_cli(PACKER, src, db, "--exclude", "vendor")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
vfs = SQLiteVFS(db)
self.addCleanup(vfs.close)
self.assertEqual({"keep.txt"}, set(helpers.stored_files(vfs)))
self.assertNotIn("vendor", helpers.all_paths(vfs))
def test_pack_reports_file_count_matching_database(self):
src = helpers.make_tree(self.path("src"), SMALL_SPEC)
db = self.path("packed.svfs")
result = helpers.run_cli(PACKER, src, db)
self.assertEqual(0, result.returncode, helpers.cli_output(result))
vfs = SQLiteVFS(db)
self.addCleanup(vfs.close)
reported = vfs.get_stats()["total_files"]
expected = len([1 for content in SMALL_SPEC.values() if content is not None])
self.assertEqual(expected, reported)
def test_default_compression_algorithm_is_zlib(self):
spec = {
"app.js": b"console.log('hello');\n" * 40, # ~920 字节,高于阈值
"tiny.txt": b"hi", # 低于阈值,应当原样存储
}
src = helpers.make_tree(self.path("src"), spec)
db = self.path("packed.svfs")
result = helpers.run_cli(PACKER, src, db, "--compress")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
vfs = SQLiteVFS(db)
self.addCleanup(vfs.close)
algorithms = {
row["path"]: row["compress_algo"]
for row in vfs.conn.execute(
"""
SELECT f.path AS path, b.compress_algo AS compress_algo
FROM files f JOIN blobs b ON b.hash = f.content_hash
WHERE f.is_directory = 0
"""
)
}
self.assertEqual("zlib", algorithms["app.js"])
self.assertEqual("none", algorithms["tiny.txt"])
self.assertEqual(spec["app.js"], vfs.read_file("app.js"))
def test_compress_algo_huffman_is_still_selectable(self):
"""早期算法仍然可选,且打出来的包能被正常读回。"""
src = helpers.make_tree(self.path("src"), SMALL_SPEC)
db = self.path("packed.svfs")
result = helpers.run_cli(
PACKER, src, db, "--compress", "--compress-algo", "huffman"
)
self.assertEqual(0, result.returncode, helpers.cli_output(result))
vfs = SQLiteVFS(db)
self.addCleanup(vfs.close)
self.assertEqual(b"top\n", vfs.read_file("top.txt"))
out = self.path("out")
unpack_result = helpers.run_cli(UNPACKER, db, out)
self.assertEqual(0, unpack_result.returncode, helpers.cli_output(unpack_result))
helpers.assert_tree_equal(self, helpers.read_tree(src), helpers.read_tree(out))
class CliUnpackTest(CliFixture):
def test_full_roundtrip_via_cli(self):
src = helpers.make_tree(self.path("src"), SMALL_SPEC)
db = self.path("packed.svfs")
out = self.path("out")
pack_result = helpers.run_cli(PACKER, src, db, "--compress")
self.assertEqual(0, pack_result.returncode, helpers.cli_output(pack_result))
unpack_result = helpers.run_cli(UNPACKER, db, out)
self.assertEqual(0, unpack_result.returncode, helpers.cli_output(unpack_result))
# 以磁盘上的源目录为基准(make_tree 会补出 sub、sub/deep 这些中间目录)
helpers.assert_tree_equal(self, helpers.read_tree(src), helpers.read_tree(out))
def test_list_root_only_shows_top_level(self):
"""P0-2: 根目录列表不能把整张表铺平。"""
db = self.seed_db()
result = helpers.run_cli(UNPACKER, db, self.path("out"), "--list-root")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
lines = [
line
for line in result.stdout.splitlines()
if line.strip().startswith(("DIR:", "FILE:"))
]
self.assertEqual(3, len(lines), f"根目录应当只有 3 个条目,实际输出:\n{result.stdout}")
for line in lines:
listed = line.split("路径: ")[-1]
self.assertNotIn("/", listed, "根目录列表里出现了子目录条目")
def test_path_exports_subdirectory_with_contents(self):
db = self.seed_db()
out = self.path("out")
result = helpers.run_cli(UNPACKER, db, out, "--path", "doc")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
expected = {"a.txt": b"a", "sub": None, "sub/b.txt": b"b"}
helpers.assert_tree_equal(self, expected, helpers.read_tree(os.path.join(out, "doc")))
def test_path_exports_single_file(self):
db = self.seed_db()
out = self.path("out")
result = helpers.run_cli(UNPACKER, db, out, "--path", "top.txt")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
with open(os.path.join(out, "top.txt"), "rb") as f:
self.assertEqual(b"t", f.read())
def test_ignore_timestamps_full_export_does_not_recurse(self):
"""P0-3: --ignore-timestamps 曾把 export_filesystem 赋成引用自身的 lambda。"""
db = self.seed_db()
out = self.path("out")
result = helpers.run_cli(UNPACKER, db, out, "--ignore-timestamps")
output = helpers.cli_output(result)
self.assertNotIn("RecursionError", output)
self.assertEqual(0, result.returncode, output)
expected = {
"doc": None,
"doc/a.txt": b"a",
"doc/sub": None,
"doc/sub/b.txt": b"b",
"docs": None,
"top.txt": b"t",
}
helpers.assert_tree_equal(self, expected, helpers.read_tree(out))
def test_ignore_timestamps_with_path_subdirectory(self):
db = self.seed_db()
out = self.path("out")
result = helpers.run_cli(
UNPACKER, db, out, "--path", "doc", "--ignore-timestamps"
)
self.assertEqual(0, result.returncode, helpers.cli_output(result))
expected = {"a.txt": b"a", "sub": None, "sub/b.txt": b"b"}
helpers.assert_tree_equal(self, expected, helpers.read_tree(os.path.join(out, "doc")))
def test_rejects_old_schema_database_cleanly(self):
"""P1-7: 旧格式库要给出明确提示,退出码非 0,且不能是 traceback。"""
db = helpers.make_old_schema_db(self.path("legacy.svfs"))
result = helpers.run_cli(UNPACKER, db, self.path("out"))
output = helpers.cli_output(result)
self.assertNotEqual(0, result.returncode)
self.assertNotIn("Traceback", output)
class CliVerifyTest(CliFixture):
def test_verify_reports_ok(self):
db = self.seed_db()
result = helpers.run_cli(VERIFY, db)
self.assertEqual(0, result.returncode, helpers.cli_output(result))
self.assertIn("完整性校验通过", result.stdout)
self.assertIn("未做内容哈希校验", result.stdout)
def test_verify_with_content_check(self):
db = self.seed_db()
result = helpers.run_cli(VERIFY, db, "--check-content")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
self.assertIn("做了内容哈希校验", result.stdout)
def test_verify_exits_nonzero_on_metadata_tampering(self):
db = self.seed_db()
conn = sqlite3.connect(db)
# 夹具里的文件都是 1 字节,所以要用一个明显不同的值来制造篡改
conn.execute("UPDATE files SET file_size = 999 WHERE is_directory = 0")
conn.commit()
conn.close()
result = helpers.run_cli(VERIFY, db)
self.assertNotEqual(0, result.returncode, "校验失败必须返回非 0")
self.assertIn("校验失败", result.stdout)
self.assertIn("摘要不一致", result.stdout)
def test_content_tampering_needs_check_content(self):
db = self.seed_db()
conn = sqlite3.connect(db)
conn.execute("UPDATE blobs SET content = X'00'")
conn.commit()
conn.close()
self.assertEqual(0, helpers.run_cli(VERIFY, db).returncode, "元数据级校验看不出来")
self.assertNotEqual(
0, helpers.run_cli(VERIFY, db, "--check-content").returncode
)
def test_list_snapshots(self):
db = self.seed_db()
result = helpers.run_cli(VERIFY, db, "--list-snapshots")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
self.assertIn("*", result.stdout, "默认快照应当被标出来")
def test_sign_then_verify(self):
db = self.seed_db()
signed = helpers.run_cli(VERIFY, db, "--sign", "--key", "sec", "--key-id", "k1")
self.assertEqual(0, signed.returncode, helpers.cli_output(signed))
self.assertIn("已签名", signed.stdout)
passed = helpers.run_cli(VERIFY, db, "--key", "sec", "--key-id", "k1")
self.assertEqual(0, passed.returncode, helpers.cli_output(passed))
self.assertIn("签名: OK", passed.stdout)
failed = helpers.run_cli(VERIFY, db, "--key", "wrong")
self.assertNotEqual(0, failed.returncode)
self.assertIn("签名: 失败", failed.stdout)
def test_json_output(self):
db = self.seed_db()
result = helpers.run_cli(VERIFY, db, "--json")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
payload = json.loads(result.stdout)
self.assertTrue(payload["ok"])
self.assertEqual(64, len(payload["actual_digest"]))
def test_snapshot_selection(self):
db = self.seed_db()
result = helpers.run_cli(VERIFY, db, "--snapshot", "1")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
self.assertIn("快照: 1", result.stdout)
def test_missing_db_exits_nonzero(self):
result = helpers.run_cli(VERIFY, self.path("nope.svfs"))
self.assertNotEqual(0, result.returncode)
self.assertNotIn("Traceback", helpers.cli_output(result))
def test_verify_does_not_modify_the_pack(self):
db = self.seed_db()
before = helpers.md5_of(db)
helpers.run_cli(VERIFY, db, "--check-content")
self.assertEqual(before, helpers.md5_of(db), "校验不该改动资源包")
class CliUnpackSnapshotTest(CliFixture):
def test_list_snapshots_option(self):
db = self.seed_db()
result = helpers.run_cli(UNPACKER, db, self.path("out"), "--list-snapshots")
self.assertEqual(0, result.returncode, helpers.cli_output(result))
self.assertIn("ID", result.stdout)
def test_unknown_snapshot_is_rejected_cleanly(self):
db = self.seed_db()
result = helpers.run_cli(UNPACKER, db, self.path("out"), "--snapshot", "999")
self.assertNotEqual(0, result.returncode)
self.assertNotIn("Traceback", helpers.cli_output(result))
class CliMigrateTest(helpers.VFSTestCase):
def _v2_db(self, name="v2.svfs"):
return helpers.make_v2_schema_db(
self.path(name),
rows=[
{"path": "a.txt", "content": b"aaa"},
{"path": "b.txt", "content": b"bbb"},
],
)
def test_migrates_in_place_with_backup(self):
db = self._v2_db()
result = helpers.run_cli(MIGRATE, db)
self.assertEqual(0, result.returncode, helpers.cli_output(result))
self.assertIn("迁移完成", result.stdout)
self.assertIn("备份保留", result.stdout)
vfs = SQLiteVFS(db)
self.addCleanup(vfs.close)
self.assertEqual(b"aaa", vfs.read_file("a.txt"))
self.assertTrue(vfs.verify()["ok"])
def test_reports_when_nothing_to_do(self):
db = self.path("v3.svfs")
SQLiteVFS(db).close()
result = helpers.run_cli(MIGRATE, db)
self.assertEqual(0, result.returncode, helpers.cli_output(result))
self.assertIn("无需迁移", result.stdout)
def test_output_mode_leaves_the_original_alone(self):
db = self._v2_db()
before = helpers.md5_of(db)
target = self.path("new.svfs")
result = helpers.run_cli(MIGRATE, db, "--output", target)
self.assertEqual(0, result.returncode, helpers.cli_output(result))
self.assertEqual(before, helpers.md5_of(db))
self.assertTrue(os.path.exists(target))
def test_non_vfs_file_is_rejected(self):
junk = helpers.write_file(self.path("junk.svfs"), b"not a database")
result = helpers.run_cli(MIGRATE, junk)
self.assertNotEqual(0, result.returncode)
self.assertNotIn("Traceback", helpers.cli_output(result))
class DemoScriptTest(helpers.VFSTestCase):
"""main.py 是给新人的冒烟示例——没人跑它就会烂(上一版引用的路径早已不存在)。"""
def test_demo_runs_and_reports_success(self):
import subprocess
import sys
env = dict(os.environ)
env["PYTHONPATH"] = helpers.SRC_DIR + os.pathsep + env.get("PYTHONPATH", "")
env["PYTHONIOENCODING"] = "utf-8"
result = subprocess.run(
[sys.executable, os.path.join(helpers.REPO_ROOT, "main.py")],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
env=env,
cwd=helpers.REPO_ROOT,
)
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertIn("一致", result.stdout)
self.assertNotIn("不一致", result.stdout)
self.assertNotIn("Traceback", result.stderr)
if __name__ == "__main__":
unittest.main()