"""增量打包。 约定: * 默认按 ``mtime + size`` 判定变化(不读文件内容,快); * ``rehash=True`` 时按 sha256 逐文件比对(可靠,但要把文件读一遍); * 磁盘上消失的路径会被删掉,对应的 blob 也会被回收; * 内容没变的文件不会被重新压缩或重复存储(去重的附带收益)。 """ import hashlib import os import time import unittest import helpers from sqlite_vfs.core import SQLiteVFS from sqlite_vfs.folder_packer import FolderPacker class IncrementalTestBase(helpers.VFSTestCase): def setUp(self): super().setUp() self.src = helpers.make_tree( self.path("src"), {"a.txt": b"aaa", "b.txt": b"bbb", "sub/c.txt": b"ccc"}, ) self.db = self.path("inc.svfs") self.pack(incremental=False) def pack_with(self, vfs, **kwargs): """用给定的 vfs 打包,并把统计与输出留下来。""" packer = FolderPacker(vfs, fs_name="inc") with helpers.quiet() as output: ok = packer.pack_folder(self.src, **kwargs) self.stats = packer.last_stats self.output = output.getvalue() return ok def pack(self, **kwargs): vfs = SQLiteVFS(self.db, compress=True) try: return self.pack_with(vfs, **kwargs) finally: vfs.close() def read(self, path): vfs = SQLiteVFS(self.db) try: return vfs.read_file(path) finally: vfs.close() def blob_hashes(self): vfs = SQLiteVFS(self.db) try: return {row["hash"] for row in vfs.conn.execute("SELECT hash FROM blobs")} finally: vfs.close() def hash_for(self, path): vfs = SQLiteVFS(self.db) try: return vfs.get_blob_info(path)["hash"] finally: vfs.close() def touch_later(self, path, data): """改文件内容并把 mtime 推到 1 秒之后,避免同秒同大小的歧义。""" helpers.write_file(path, data) future = time.time() + 2 os.utime(path, (future, future)) class ChangeDetectionTest(IncrementalTestBase): def test_unchanged_tree_writes_nothing(self): self.pack(incremental=True) self.assertEqual( {"added": 0, "updated": 0, "unchanged": 3, "removed": 0}, {k: self.stats[k] for k in ("added", "updated", "unchanged", "removed")}, ) self.assertIn("未变 3", self.output) def test_modified_file_is_updated(self): self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-changed") self.pack(incremental=True) self.assertEqual(1, self.stats["updated"]) self.assertEqual(2, self.stats["unchanged"]) self.assertEqual(b"aaa-changed", self.read("a.txt")) def test_new_file_is_added(self): helpers.write_file(os.path.join(self.src, "new.txt"), b"new") self.pack(incremental=True) self.assertEqual(1, self.stats["added"]) self.assertEqual(b"new", self.read("new.txt")) def test_deleted_file_is_removed(self): os.remove(os.path.join(self.src, "b.txt")) self.pack(incremental=True) self.assertEqual(1, self.stats["removed"]) vfs = SQLiteVFS(self.db) self.addCleanup(vfs.close) self.assertFalse(vfs.exists("b.txt")) self.assertEqual(2, helpers.count_files(vfs)) def test_new_directory_is_created(self): helpers.make_tree(self.src, {"fresh/deep/x.txt": b"x"}) self.pack(incremental=True) vfs = SQLiteVFS(self.db) self.addCleanup(vfs.close) self.assertTrue(vfs.is_dir("fresh")) self.assertTrue(vfs.is_dir("fresh/deep")) self.assertEqual(b"x", vfs.read_file("fresh/deep/x.txt")) def test_deleted_directory_is_removed(self): import shutil shutil.rmtree(os.path.join(self.src, "sub")) self.pack(incremental=True) vfs = SQLiteVFS(self.db) self.addCleanup(vfs.close) self.assertFalse(vfs.exists("sub")) self.assertFalse(vfs.exists("sub/c.txt")) def test_file_replaced_by_directory(self): """同一路径从文件变成目录时必须换掉类型,而不是留着旧条目。""" os.remove(os.path.join(self.src, "a.txt")) helpers.make_tree(self.src, {"a.txt": None, "a.txt/inner.txt": b"inner"}) self.pack(incremental=True) vfs = SQLiteVFS(self.db) self.addCleanup(vfs.close) self.assertTrue(vfs.is_dir("a.txt")) self.assertEqual(b"inner", vfs.read_file("a.txt/inner.txt")) def test_mixed_changes_in_one_run(self): self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2") helpers.write_file(os.path.join(self.src, "new.txt"), b"new") os.remove(os.path.join(self.src, "b.txt")) self.pack(incremental=True) self.assertEqual(1, self.stats["added"]) self.assertEqual(1, self.stats["updated"]) self.assertEqual(1, self.stats["unchanged"]) self.assertEqual(1, self.stats["removed"]) def test_result_still_verifies(self): self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2") helpers.write_file(os.path.join(self.src, "new.txt"), b"new") os.remove(os.path.join(self.src, "b.txt")) self.pack(incremental=True) vfs = SQLiteVFS(self.db) self.addCleanup(vfs.close) report = vfs.verify(check_content=True) self.assertTrue(report["ok"], report["errors"]) self.assertEqual(3, report["files_checked"]) class NoRewriteTest(IncrementalTestBase): """未变的文件不能被重写、重压或重新存储。""" def test_unchanged_file_keeps_its_blob_hash(self): before = self.hash_for("sub/c.txt") self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2") self.pack(incremental=True) self.assertEqual(before, self.hash_for("sub/c.txt")) def test_unchanged_files_do_not_add_blobs(self): before = self.blob_hashes() self.pack(incremental=True) self.assertEqual(before, self.blob_hashes()) def test_dedup_reuses_blob_for_identical_new_file(self): """新路径但内容与已有文件相同 → 直接复用,不新增 blob。""" before = self.blob_hashes() helpers.write_file(os.path.join(self.src, "copy_of_c.txt"), b"ccc") self.pack(incremental=True) self.assertEqual(1, self.stats["added"]) self.assertEqual(before, self.blob_hashes(), "内容相同,不该新增 blob") def test_replacing_content_frees_the_old_blob(self): self.touch_later(os.path.join(self.src, "a.txt"), b"completely different") self.pack(incremental=True) vfs = SQLiteVFS(self.db) self.addCleanup(vfs.close) self.assertEqual(3, helpers.blob_count(vfs), "旧的 aaa 已无人引用,应当回收") self.assertEqual(b"completely different", vfs.read_file("a.txt")) def test_rehash_skips_files_whose_content_is_identical(self): """mtime 变了但内容没变:rehash 模式应当判为未变,不做任何写入。""" path = os.path.join(self.src, "a.txt") with open(path, "rb") as f: original = f.read() self.touch_later(path, original) # 只改 mtime before = self.hash_for("a.txt") self.pack(incremental=True, rehash=True) self.assertEqual(3, self.stats["unchanged"]) self.assertEqual(0, self.stats["updated"]) self.assertEqual(before, self.hash_for("a.txt")) class RehashDetectionTest(IncrementalTestBase): def test_mtime_only_change_is_detected_without_rehash(self): """同大小 + 同秒修改:默认模式可能漏掉,这正是 rehash 存在的理由。""" path = os.path.join(self.src, "a.txt") helpers.write_file(path, b"AAAA") # 与原始 aaa 大小不同,先制造同大小场景 self.pack(incremental=True) # 现在 disk 与库里都是 AAAA;把内容换成同样长度的别的字节, # 并把 mtime 强制设回库里记录的那个值 vfs = SQLiteVFS(self.db) recorded = vfs.get_file_info("a.txt")["modified_time"].timestamp() vfs.close() helpers.write_file(path, b"BBBB") os.utime(path, (recorded, recorded)) self.pack(incremental=True) self.assertEqual(3, self.stats["unchanged"], "mtime+size 相同,默认模式看不出来") self.assertEqual(b"AAAA", self.read("a.txt"), "库里还是旧内容(预期的漏检)") def test_rehash_catches_same_size_same_mtime_edit(self): path = os.path.join(self.src, "a.txt") helpers.write_file(path, b"AAAA") self.pack(incremental=True) vfs = SQLiteVFS(self.db) recorded = vfs.get_file_info("a.txt")["modified_time"].timestamp() vfs.close() helpers.write_file(path, b"BBBB") os.utime(path, (recorded, recorded)) self.pack(incremental=True, rehash=True) self.assertEqual(1, self.stats["updated"], "rehash 必须抓到这次修改") self.assertEqual(b"BBBB", self.read("a.txt")) class IncrementalSnapshotTest(IncrementalTestBase): def test_first_incremental_on_an_empty_database(self): db = self.path("fresh.svfs") vfs = SQLiteVFS(db, compress=True) try: packer = FolderPacker(vfs, fs_name="fresh") with helpers.quiet(): self.assertTrue(packer.pack_folder(self.src, incremental=True)) finally: vfs.close() vfs = SQLiteVFS(db) self.addCleanup(vfs.close) self.assertEqual(3, helpers.count_files(vfs)) self.assertEqual("fresh", vfs.list_snapshots()[0]["name"]) def test_incremental_into_a_new_snapshot_keeps_history(self): """派生新快照后再增量:旧版本原样保留,新版本只有变化的部分被重写。""" vfs = SQLiteVFS(self.db, compress=True) try: parent = vfs._active_snapshot() vfs.create_snapshot("v2", parent_id=parent, copy_entries=True) self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2") os.remove(os.path.join(self.src, "b.txt")) self.assertTrue(self.pack_with(vfs, incremental=True)) finally: vfs.close() self.assertEqual(1, self.stats["updated"]) self.assertEqual(1, self.stats["removed"]) vfs = SQLiteVFS(self.db) self.addCleanup(vfs.close) self.assertEqual(2, len(vfs.list_snapshots())) self.assertEqual(b"aaa-v2", vfs.read_file("a.txt")) self.assertFalse(vfs.exists("b.txt")) vfs.use_snapshot(1) self.assertEqual(b"aaa", vfs.read_file("a.txt"), "旧版本必须原样保留") self.assertEqual(b"bbb", vfs.read_file("b.txt")) def test_incremental_does_not_touch_other_snapshots(self): vfs = SQLiteVFS(self.db, compress=True) try: parent = vfs._active_snapshot() vfs.create_snapshot("v2", parent_id=parent, copy_entries=True) os.remove(os.path.join(self.src, "b.txt")) self.pack_with(vfs, incremental=True) finally: vfs.close() vfs = SQLiteVFS(self.db, snapshot_id=1) self.addCleanup(vfs.close) self.assertTrue(vfs.exists("b.txt"), "v1 不该被 v2 的增量影响") class IncrementalCliTest(helpers.VFSTestCase): def test_cli_incremental(self): src = helpers.make_tree(self.path("src"), {"a.txt": b"aaa", "b.txt": b"bbb"}) db = self.path("cli.svfs") first = helpers.run_cli("sqlite_vfs.cli.packer", src, db, "--compress") self.assertEqual(0, first.returncode, helpers.cli_output(first)) helpers.write_file(os.path.join(src, "new.txt"), b"new") os.remove(os.path.join(src, "b.txt")) second = helpers.run_cli("sqlite_vfs.cli.packer", src, db, "--incremental") self.assertEqual(0, second.returncode, helpers.cli_output(second)) self.assertIn("增量结果", second.stdout) self.assertIn("新增 1", second.stdout) self.assertIn("删除 1", second.stdout) vfs = SQLiteVFS(db) self.addCleanup(vfs.close) self.assertEqual(b"new", vfs.read_file("new.txt")) self.assertFalse(vfs.exists("b.txt")) def test_cli_snapshot_creates_a_new_version(self): src = helpers.make_tree(self.path("src"), {"a.txt": b"aaa"}) db = self.path("cli2.svfs") helpers.run_cli("sqlite_vfs.cli.packer", src, db, "--compress") helpers.write_file(os.path.join(src, "a.txt"), b"aaa-v2") result = helpers.run_cli( "sqlite_vfs.cli.packer", src, db, "--snapshot", "v2", "--incremental" ) self.assertEqual(0, result.returncode, helpers.cli_output(result)) vfs = SQLiteVFS(db) self.addCleanup(vfs.close) self.assertEqual([1, 2], [s["id"] for s in vfs.list_snapshots()]) self.assertEqual(b"aaa-v2", vfs.read_file("a.txt")) vfs.use_snapshot(1) self.assertEqual(b"aaa", vfs.read_file("a.txt")) listing = helpers.run_cli("sqlite_vfs.cli.unpacker", db, self.path("out"), "--list-snapshots") self.assertEqual(0, listing.returncode, helpers.cli_output(listing)) self.assertIn("v2", listing.stdout) if __name__ == "__main__": unittest.main()