chakcy
ecdfacc620
起始状态：主链路 100% 不可用，而且失败被静默吞掉——打包后库里 0 个文件、 界面仍打印"文件数量: N"；解包导出 0 个文件并在磁盘上生成几百层 sub/sub/sub 垃圾目录。 缺陷修复（4 个 P0 + 12 个 P1，逐条见 BUG_REPORT.md）： - 建表列名 compresse_algo 与 INSERT/read_file 用的 compress_algo 不一致 - list_directory 按 path 而非 parent_path 过滤，导致导出无限自递归 - CLI --ignore-timestamps 把 export_filesystem 赋成引用自身的 lambda - add_file 缺少 commit（直接调用 API 时静默丢数据） - LIKE 前缀误伤兄弟目录（删 doc 连带删 docs）；mv_dir 二次改名 - 失败路径一律返回 exit code 0；--exclude 是子串匹配而非 glob - FolderPacker.add_file 反向调用 export_file_safe，会用库内内容覆盖源文件 存储层（schema v3，格式变更，提供 vfsmigrate 迁移）： - 内容移入 blobs(hash PRIMARY KEY)：相同内容只存一份，跨路径/跨快照共享 - files 的唯一约束改为 UNIQUE(snapshot_id, path)：一个库存多个版本 - 完整性：每文件 sha256 + 快照级摘要 + HMAC-SHA256 签名，只读连接即可验证 - 增量打包：按 mtime+size 或 sha256 只写变化项，未变文件不重写不重压 - get_file_info 不再返回 content（元数据查询不该搬 BLOB） 新增能力： - 可选 FastAPI 静态文件插件（extras，核心仍零第三方依赖） - 内容寻址发布到对象存储 + 逐对象校验的取回客户端 - 压缩默认改回 zlib：实测比自研哈夫曼体积小 1/3~1/170、快数百倍 其它： - P2-4：时间戳显式转字符串，清掉 1240 条 DeprecationWarning（盘上格式不变） - read_file 负数 offset/size 改为报错（原先会走切片语义从末尾取） - requires-python 从 >=3.7 改为按实际 API 推导的 >=3.9 测试：0 → 343 个用例，纯标准库 unittest，有/无 FastAPI 两种环境都跑通。 证据见 docs/evidence-*.txt。
338 lines
13 KiB
Python
338 lines
13 KiB
Python
"""增量打包。
|
|
|
|
约定：
|
|
* 默认按 ``mtime + size`` 判定变化（不读文件内容，快）；
|
|
* ``rehash=True`` 时按 sha256 逐文件比对（可靠，但要把文件读一遍）；
|
|
* 磁盘上消失的路径会被删掉，对应的 blob 也会被回收；
|
|
* 内容没变的文件不会被重新压缩或重复存储（去重的附带收益）。
|
|
"""
|
|
|
|
import hashlib
|
|
import os
|
|
import time
|
|
import unittest
|
|
|
|
import helpers
|
|
|
|
from sqlite_vfs.core import SQLiteVFS
|
|
from sqlite_vfs.folder_packer import FolderPacker
|
|
|
|
|
|
class IncrementalTestBase(helpers.VFSTestCase):
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.src = helpers.make_tree(
|
|
self.path("src"),
|
|
{"a.txt": b"aaa", "b.txt": b"bbb", "sub/c.txt": b"ccc"},
|
|
)
|
|
self.db = self.path("inc.svfs")
|
|
self.pack(incremental=False)
|
|
|
|
def pack_with(self, vfs, **kwargs):
|
|
"""用给定的 vfs 打包，并把统计与输出留下来。"""
|
|
packer = FolderPacker(vfs, fs_name="inc")
|
|
with helpers.quiet() as output:
|
|
ok = packer.pack_folder(self.src, **kwargs)
|
|
self.stats = packer.last_stats
|
|
self.output = output.getvalue()
|
|
return ok
|
|
|
|
def pack(self, **kwargs):
|
|
vfs = SQLiteVFS(self.db, compress=True)
|
|
try:
|
|
return self.pack_with(vfs, **kwargs)
|
|
finally:
|
|
vfs.close()
|
|
|
|
def read(self, path):
|
|
vfs = SQLiteVFS(self.db)
|
|
try:
|
|
return vfs.read_file(path)
|
|
finally:
|
|
vfs.close()
|
|
|
|
def blob_hashes(self):
|
|
vfs = SQLiteVFS(self.db)
|
|
try:
|
|
return {row["hash"] for row in vfs.conn.execute("SELECT hash FROM blobs")}
|
|
finally:
|
|
vfs.close()
|
|
|
|
def hash_for(self, path):
|
|
vfs = SQLiteVFS(self.db)
|
|
try:
|
|
return vfs.get_blob_info(path)["hash"]
|
|
finally:
|
|
vfs.close()
|
|
|
|
def touch_later(self, path, data):
|
|
"""改文件内容并把 mtime 推到 1 秒之后，避免同秒同大小的歧义。"""
|
|
helpers.write_file(path, data)
|
|
future = time.time() + 2
|
|
os.utime(path, (future, future))
|
|
|
|
|
|
class ChangeDetectionTest(IncrementalTestBase):
|
|
def test_unchanged_tree_writes_nothing(self):
|
|
self.pack(incremental=True)
|
|
self.assertEqual(
|
|
{"added": 0, "updated": 0, "unchanged": 3, "removed": 0},
|
|
{k: self.stats[k] for k in ("added", "updated", "unchanged", "removed")},
|
|
)
|
|
self.assertIn("未变 3", self.output)
|
|
|
|
def test_modified_file_is_updated(self):
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-changed")
|
|
self.pack(incremental=True)
|
|
self.assertEqual(1, self.stats["updated"])
|
|
self.assertEqual(2, self.stats["unchanged"])
|
|
self.assertEqual(b"aaa-changed", self.read("a.txt"))
|
|
|
|
def test_new_file_is_added(self):
|
|
helpers.write_file(os.path.join(self.src, "new.txt"), b"new")
|
|
self.pack(incremental=True)
|
|
self.assertEqual(1, self.stats["added"])
|
|
self.assertEqual(b"new", self.read("new.txt"))
|
|
|
|
def test_deleted_file_is_removed(self):
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
|
self.pack(incremental=True)
|
|
self.assertEqual(1, self.stats["removed"])
|
|
|
|
vfs = SQLiteVFS(self.db)
|
|
self.addCleanup(vfs.close)
|
|
self.assertFalse(vfs.exists("b.txt"))
|
|
self.assertEqual(2, helpers.count_files(vfs))
|
|
|
|
def test_new_directory_is_created(self):
|
|
helpers.make_tree(self.src, {"fresh/deep/x.txt": b"x"})
|
|
self.pack(incremental=True)
|
|
vfs = SQLiteVFS(self.db)
|
|
self.addCleanup(vfs.close)
|
|
self.assertTrue(vfs.is_dir("fresh"))
|
|
self.assertTrue(vfs.is_dir("fresh/deep"))
|
|
self.assertEqual(b"x", vfs.read_file("fresh/deep/x.txt"))
|
|
|
|
def test_deleted_directory_is_removed(self):
|
|
import shutil
|
|
|
|
shutil.rmtree(os.path.join(self.src, "sub"))
|
|
self.pack(incremental=True)
|
|
vfs = SQLiteVFS(self.db)
|
|
self.addCleanup(vfs.close)
|
|
self.assertFalse(vfs.exists("sub"))
|
|
self.assertFalse(vfs.exists("sub/c.txt"))
|
|
|
|
def test_file_replaced_by_directory(self):
|
|
"""同一路径从文件变成目录时必须换掉类型，而不是留着旧条目。"""
|
|
os.remove(os.path.join(self.src, "a.txt"))
|
|
helpers.make_tree(self.src, {"a.txt": None, "a.txt/inner.txt": b"inner"})
|
|
self.pack(incremental=True)
|
|
vfs = SQLiteVFS(self.db)
|
|
self.addCleanup(vfs.close)
|
|
self.assertTrue(vfs.is_dir("a.txt"))
|
|
self.assertEqual(b"inner", vfs.read_file("a.txt/inner.txt"))
|
|
|
|
def test_mixed_changes_in_one_run(self):
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2")
|
|
helpers.write_file(os.path.join(self.src, "new.txt"), b"new")
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
|
self.pack(incremental=True)
|
|
self.assertEqual(1, self.stats["added"])
|
|
self.assertEqual(1, self.stats["updated"])
|
|
self.assertEqual(1, self.stats["unchanged"])
|
|
self.assertEqual(1, self.stats["removed"])
|
|
|
|
def test_result_still_verifies(self):
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2")
|
|
helpers.write_file(os.path.join(self.src, "new.txt"), b"new")
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
|
self.pack(incremental=True)
|
|
|
|
vfs = SQLiteVFS(self.db)
|
|
self.addCleanup(vfs.close)
|
|
report = vfs.verify(check_content=True)
|
|
self.assertTrue(report["ok"], report["errors"])
|
|
self.assertEqual(3, report["files_checked"])
|
|
|
|
|
|
class NoRewriteTest(IncrementalTestBase):
|
|
"""未变的文件不能被重写、重压或重新存储。"""
|
|
|
|
def test_unchanged_file_keeps_its_blob_hash(self):
|
|
before = self.hash_for("sub/c.txt")
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2")
|
|
self.pack(incremental=True)
|
|
self.assertEqual(before, self.hash_for("sub/c.txt"))
|
|
|
|
def test_unchanged_files_do_not_add_blobs(self):
|
|
before = self.blob_hashes()
|
|
self.pack(incremental=True)
|
|
self.assertEqual(before, self.blob_hashes())
|
|
|
|
def test_dedup_reuses_blob_for_identical_new_file(self):
|
|
"""新路径但内容与已有文件相同 → 直接复用，不新增 blob。"""
|
|
before = self.blob_hashes()
|
|
helpers.write_file(os.path.join(self.src, "copy_of_c.txt"), b"ccc")
|
|
self.pack(incremental=True)
|
|
self.assertEqual(1, self.stats["added"])
|
|
self.assertEqual(before, self.blob_hashes(), "内容相同，不该新增 blob")
|
|
|
|
def test_replacing_content_frees_the_old_blob(self):
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"completely different")
|
|
self.pack(incremental=True)
|
|
vfs = SQLiteVFS(self.db)
|
|
self.addCleanup(vfs.close)
|
|
self.assertEqual(3, helpers.blob_count(vfs), "旧的 aaa 已无人引用，应当回收")
|
|
self.assertEqual(b"completely different", vfs.read_file("a.txt"))
|
|
|
|
def test_rehash_skips_files_whose_content_is_identical(self):
|
|
"""mtime 变了但内容没变：rehash 模式应当判为未变，不做任何写入。"""
|
|
path = os.path.join(self.src, "a.txt")
|
|
with open(path, "rb") as f:
|
|
original = f.read()
|
|
self.touch_later(path, original) # 只改 mtime
|
|
|
|
before = self.hash_for("a.txt")
|
|
self.pack(incremental=True, rehash=True)
|
|
self.assertEqual(3, self.stats["unchanged"])
|
|
self.assertEqual(0, self.stats["updated"])
|
|
self.assertEqual(before, self.hash_for("a.txt"))
|
|
|
|
|
|
class RehashDetectionTest(IncrementalTestBase):
|
|
def test_mtime_only_change_is_detected_without_rehash(self):
|
|
"""同大小 + 同秒修改：默认模式可能漏掉，这正是 rehash 存在的理由。"""
|
|
path = os.path.join(self.src, "a.txt")
|
|
helpers.write_file(path, b"AAAA") # 与原始 aaa 大小不同，先制造同大小场景
|
|
self.pack(incremental=True)
|
|
|
|
# 现在 disk 与库里都是 AAAA；把内容换成同样长度的别的字节，
|
|
# 并把 mtime 强制设回库里记录的那个值
|
|
vfs = SQLiteVFS(self.db)
|
|
recorded = vfs.get_file_info("a.txt")["modified_time"].timestamp()
|
|
vfs.close()
|
|
helpers.write_file(path, b"BBBB")
|
|
os.utime(path, (recorded, recorded))
|
|
|
|
self.pack(incremental=True)
|
|
self.assertEqual(3, self.stats["unchanged"], "mtime+size 相同，默认模式看不出来")
|
|
self.assertEqual(b"AAAA", self.read("a.txt"), "库里还是旧内容（预期的漏检）")
|
|
|
|
def test_rehash_catches_same_size_same_mtime_edit(self):
|
|
path = os.path.join(self.src, "a.txt")
|
|
helpers.write_file(path, b"AAAA")
|
|
self.pack(incremental=True)
|
|
|
|
vfs = SQLiteVFS(self.db)
|
|
recorded = vfs.get_file_info("a.txt")["modified_time"].timestamp()
|
|
vfs.close()
|
|
helpers.write_file(path, b"BBBB")
|
|
os.utime(path, (recorded, recorded))
|
|
|
|
self.pack(incremental=True, rehash=True)
|
|
self.assertEqual(1, self.stats["updated"], "rehash 必须抓到这次修改")
|
|
self.assertEqual(b"BBBB", self.read("a.txt"))
|
|
|
|
|
|
class IncrementalSnapshotTest(IncrementalTestBase):
|
|
def test_first_incremental_on_an_empty_database(self):
|
|
db = self.path("fresh.svfs")
|
|
vfs = SQLiteVFS(db, compress=True)
|
|
try:
|
|
packer = FolderPacker(vfs, fs_name="fresh")
|
|
with helpers.quiet():
|
|
self.assertTrue(packer.pack_folder(self.src, incremental=True))
|
|
finally:
|
|
vfs.close()
|
|
|
|
vfs = SQLiteVFS(db)
|
|
self.addCleanup(vfs.close)
|
|
self.assertEqual(3, helpers.count_files(vfs))
|
|
self.assertEqual("fresh", vfs.list_snapshots()[0]["name"])
|
|
|
|
def test_incremental_into_a_new_snapshot_keeps_history(self):
|
|
"""派生新快照后再增量：旧版本原样保留，新版本只有变化的部分被重写。"""
|
|
vfs = SQLiteVFS(self.db, compress=True)
|
|
try:
|
|
parent = vfs._active_snapshot()
|
|
vfs.create_snapshot("v2", parent_id=parent, copy_entries=True)
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2")
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
|
self.assertTrue(self.pack_with(vfs, incremental=True))
|
|
finally:
|
|
vfs.close()
|
|
self.assertEqual(1, self.stats["updated"])
|
|
self.assertEqual(1, self.stats["removed"])
|
|
|
|
vfs = SQLiteVFS(self.db)
|
|
self.addCleanup(vfs.close)
|
|
self.assertEqual(2, len(vfs.list_snapshots()))
|
|
self.assertEqual(b"aaa-v2", vfs.read_file("a.txt"))
|
|
self.assertFalse(vfs.exists("b.txt"))
|
|
|
|
vfs.use_snapshot(1)
|
|
self.assertEqual(b"aaa", vfs.read_file("a.txt"), "旧版本必须原样保留")
|
|
self.assertEqual(b"bbb", vfs.read_file("b.txt"))
|
|
|
|
def test_incremental_does_not_touch_other_snapshots(self):
|
|
vfs = SQLiteVFS(self.db, compress=True)
|
|
try:
|
|
parent = vfs._active_snapshot()
|
|
vfs.create_snapshot("v2", parent_id=parent, copy_entries=True)
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
|
self.pack_with(vfs, incremental=True)
|
|
finally:
|
|
vfs.close()
|
|
|
|
vfs = SQLiteVFS(self.db, snapshot_id=1)
|
|
self.addCleanup(vfs.close)
|
|
self.assertTrue(vfs.exists("b.txt"), "v1 不该被 v2 的增量影响")
|
|
|
|
|
|
class IncrementalCliTest(helpers.VFSTestCase):
|
|
def test_cli_incremental(self):
|
|
src = helpers.make_tree(self.path("src"), {"a.txt": b"aaa", "b.txt": b"bbb"})
|
|
db = self.path("cli.svfs")
|
|
first = helpers.run_cli("sqlite_vfs.cli.packer", src, db, "--compress")
|
|
self.assertEqual(0, first.returncode, helpers.cli_output(first))
|
|
|
|
helpers.write_file(os.path.join(src, "new.txt"), b"new")
|
|
os.remove(os.path.join(src, "b.txt"))
|
|
second = helpers.run_cli("sqlite_vfs.cli.packer", src, db, "--incremental")
|
|
self.assertEqual(0, second.returncode, helpers.cli_output(second))
|
|
self.assertIn("增量结果", second.stdout)
|
|
self.assertIn("新增 1", second.stdout)
|
|
self.assertIn("删除 1", second.stdout)
|
|
|
|
vfs = SQLiteVFS(db)
|
|
self.addCleanup(vfs.close)
|
|
self.assertEqual(b"new", vfs.read_file("new.txt"))
|
|
self.assertFalse(vfs.exists("b.txt"))
|
|
|
|
def test_cli_snapshot_creates_a_new_version(self):
|
|
src = helpers.make_tree(self.path("src"), {"a.txt": b"aaa"})
|
|
db = self.path("cli2.svfs")
|
|
helpers.run_cli("sqlite_vfs.cli.packer", src, db, "--compress")
|
|
|
|
helpers.write_file(os.path.join(src, "a.txt"), b"aaa-v2")
|
|
result = helpers.run_cli(
|
|
"sqlite_vfs.cli.packer", src, db, "--snapshot", "v2", "--incremental"
|
|
)
|
|
self.assertEqual(0, result.returncode, helpers.cli_output(result))
|
|
|
|
vfs = SQLiteVFS(db)
|
|
self.addCleanup(vfs.close)
|
|
self.assertEqual([1, 2], [s["id"] for s in vfs.list_snapshots()])
|
|
self.assertEqual(b"aaa-v2", vfs.read_file("a.txt"))
|
|
vfs.use_snapshot(1)
|
|
self.assertEqual(b"aaa", vfs.read_file("a.txt"))
|
|
|
|
listing = helpers.run_cli("sqlite_vfs.cli.unpacker", db, self.path("out"), "--list-snapshots")
|
|
self.assertEqual(0, listing.returncode, helpers.cli_output(listing))
|
|
self.assertIn("v2", listing.stdout)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|