338 lines
13 KiB
Python
338 lines
13 KiB
Python
|
"""增量打包。
|
||
|
|
|
||
|
|
约定：
|
||
|
|
* 默认按 ``mtime + size`` 判定变化（不读文件内容，快）；
|
||
|
|
* ``rehash=True`` 时按 sha256 逐文件比对（可靠，但要把文件读一遍）；
|
||
|
|
* 磁盘上消失的路径会被删掉，对应的 blob 也会被回收；
|
||
|
|
* 内容没变的文件不会被重新压缩或重复存储（去重的附带收益）。
|
||
|
|
"""
|
||
|
|
|
||
|
|
import hashlib
|
||
|
|
import os
|
||
|
|
import time
|
||
|
|
import unittest
|
||
|
|
|
||
|
|
import helpers
|
||
|
|
|
||
|
|
from sqlite_vfs.core import SQLiteVFS
|
||
|
|
from sqlite_vfs.folder_packer import FolderPacker
|
||
|
|
|
||
|
|
|
||
|
|
class IncrementalTestBase(helpers.VFSTestCase):
|
||
|
|
def setUp(self):
|
||
|
|
super().setUp()
|
||
|
|
self.src = helpers.make_tree(
|
||
|
|
self.path("src"),
|
||
|
|
{"a.txt": b"aaa", "b.txt": b"bbb", "sub/c.txt": b"ccc"},
|
||
|
|
)
|
||
|
|
self.db = self.path("inc.svfs")
|
||
|
|
self.pack(incremental=False)
|
||
|
|
|
||
|
|
def pack_with(self, vfs, **kwargs):
|
||
|
|
"""用给定的 vfs 打包，并把统计与输出留下来。"""
|
||
|
|
packer = FolderPacker(vfs, fs_name="inc")
|
||
|
|
with helpers.quiet() as output:
|
||
|
|
ok = packer.pack_folder(self.src, **kwargs)
|
||
|
|
self.stats = packer.last_stats
|
||
|
|
self.output = output.getvalue()
|
||
|
|
return ok
|
||
|
|
|
||
|
|
def pack(self, **kwargs):
|
||
|
|
vfs = SQLiteVFS(self.db, compress=True)
|
||
|
|
try:
|
||
|
|
return self.pack_with(vfs, **kwargs)
|
||
|
|
finally:
|
||
|
|
vfs.close()
|
||
|
|
|
||
|
|
def read(self, path):
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
try:
|
||
|
|
return vfs.read_file(path)
|
||
|
|
finally:
|
||
|
|
vfs.close()
|
||
|
|
|
||
|
|
def blob_hashes(self):
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
try:
|
||
|
|
return {row["hash"] for row in vfs.conn.execute("SELECT hash FROM blobs")}
|
||
|
|
finally:
|
||
|
|
vfs.close()
|
||
|
|
|
||
|
|
def hash_for(self, path):
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
try:
|
||
|
|
return vfs.get_blob_info(path)["hash"]
|
||
|
|
finally:
|
||
|
|
vfs.close()
|
||
|
|
|
||
|
|
def touch_later(self, path, data):
|
||
|
|
"""改文件内容并把 mtime 推到 1 秒之后，避免同秒同大小的歧义。"""
|
||
|
|
helpers.write_file(path, data)
|
||
|
|
future = time.time() + 2
|
||
|
|
os.utime(path, (future, future))
|
||
|
|
|
||
|
|
|
||
|
|
class ChangeDetectionTest(IncrementalTestBase):
|
||
|
|
def test_unchanged_tree_writes_nothing(self):
|
||
|
|
self.pack(incremental=True)
|
||
|
|
self.assertEqual(
|
||
|
|
{"added": 0, "updated": 0, "unchanged": 3, "removed": 0},
|
||
|
|
{k: self.stats[k] for k in ("added", "updated", "unchanged", "removed")},
|
||
|
|
)
|
||
|
|
self.assertIn("未变 3", self.output)
|
||
|
|
|
||
|
|
def test_modified_file_is_updated(self):
|
||
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-changed")
|
||
|
|
self.pack(incremental=True)
|
||
|
|
self.assertEqual(1, self.stats["updated"])
|
||
|
|
self.assertEqual(2, self.stats["unchanged"])
|
||
|
|
self.assertEqual(b"aaa-changed", self.read("a.txt"))
|
||
|
|
|
||
|
|
def test_new_file_is_added(self):
|
||
|
|
helpers.write_file(os.path.join(self.src, "new.txt"), b"new")
|
||
|
|
self.pack(incremental=True)
|
||
|
|
self.assertEqual(1, self.stats["added"])
|
||
|
|
self.assertEqual(b"new", self.read("new.txt"))
|
||
|
|
|
||
|
|
def test_deleted_file_is_removed(self):
|
||
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
||
|
|
self.pack(incremental=True)
|
||
|
|
self.assertEqual(1, self.stats["removed"])
|
||
|
|
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertFalse(vfs.exists("b.txt"))
|
||
|
|
self.assertEqual(2, helpers.count_files(vfs))
|
||
|
|
|
||
|
|
def test_new_directory_is_created(self):
|
||
|
|
helpers.make_tree(self.src, {"fresh/deep/x.txt": b"x"})
|
||
|
|
self.pack(incremental=True)
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertTrue(vfs.is_dir("fresh"))
|
||
|
|
self.assertTrue(vfs.is_dir("fresh/deep"))
|
||
|
|
self.assertEqual(b"x", vfs.read_file("fresh/deep/x.txt"))
|
||
|
|
|
||
|
|
def test_deleted_directory_is_removed(self):
|
||
|
|
import shutil
|
||
|
|
|
||
|
|
shutil.rmtree(os.path.join(self.src, "sub"))
|
||
|
|
self.pack(incremental=True)
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertFalse(vfs.exists("sub"))
|
||
|
|
self.assertFalse(vfs.exists("sub/c.txt"))
|
||
|
|
|
||
|
|
def test_file_replaced_by_directory(self):
|
||
|
|
"""同一路径从文件变成目录时必须换掉类型，而不是留着旧条目。"""
|
||
|
|
os.remove(os.path.join(self.src, "a.txt"))
|
||
|
|
helpers.make_tree(self.src, {"a.txt": None, "a.txt/inner.txt": b"inner"})
|
||
|
|
self.pack(incremental=True)
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertTrue(vfs.is_dir("a.txt"))
|
||
|
|
self.assertEqual(b"inner", vfs.read_file("a.txt/inner.txt"))
|
||
|
|
|
||
|
|
def test_mixed_changes_in_one_run(self):
|
||
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2")
|
||
|
|
helpers.write_file(os.path.join(self.src, "new.txt"), b"new")
|
||
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
||
|
|
self.pack(incremental=True)
|
||
|
|
self.assertEqual(1, self.stats["added"])
|
||
|
|
self.assertEqual(1, self.stats["updated"])
|
||
|
|
self.assertEqual(1, self.stats["unchanged"])
|
||
|
|
self.assertEqual(1, self.stats["removed"])
|
||
|
|
|
||
|
|
def test_result_still_verifies(self):
|
||
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2")
|
||
|
|
helpers.write_file(os.path.join(self.src, "new.txt"), b"new")
|
||
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
||
|
|
self.pack(incremental=True)
|
||
|
|
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
report = vfs.verify(check_content=True)
|
||
|
|
self.assertTrue(report["ok"], report["errors"])
|
||
|
|
self.assertEqual(3, report["files_checked"])
|
||
|
|
|
||
|
|
|
||
|
|
class NoRewriteTest(IncrementalTestBase):
|
||
|
|
"""未变的文件不能被重写、重压或重新存储。"""
|
||
|
|
|
||
|
|
def test_unchanged_file_keeps_its_blob_hash(self):
|
||
|
|
before = self.hash_for("sub/c.txt")
|
||
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2")
|
||
|
|
self.pack(incremental=True)
|
||
|
|
self.assertEqual(before, self.hash_for("sub/c.txt"))
|
||
|
|
|
||
|
|
def test_unchanged_files_do_not_add_blobs(self):
|
||
|
|
before = self.blob_hashes()
|
||
|
|
self.pack(incremental=True)
|
||
|
|
self.assertEqual(before, self.blob_hashes())
|
||
|
|
|
||
|
|
def test_dedup_reuses_blob_for_identical_new_file(self):
|
||
|
|
"""新路径但内容与已有文件相同 → 直接复用，不新增 blob。"""
|
||
|
|
before = self.blob_hashes()
|
||
|
|
helpers.write_file(os.path.join(self.src, "copy_of_c.txt"), b"ccc")
|
||
|
|
self.pack(incremental=True)
|
||
|
|
self.assertEqual(1, self.stats["added"])
|
||
|
|
self.assertEqual(before, self.blob_hashes(), "内容相同，不该新增 blob")
|
||
|
|
|
||
|
|
def test_replacing_content_frees_the_old_blob(self):
|
||
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"completely different")
|
||
|
|
self.pack(incremental=True)
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertEqual(3, helpers.blob_count(vfs), "旧的 aaa 已无人引用，应当回收")
|
||
|
|
self.assertEqual(b"completely different", vfs.read_file("a.txt"))
|
||
|
|
|
||
|
|
def test_rehash_skips_files_whose_content_is_identical(self):
|
||
|
|
"""mtime 变了但内容没变：rehash 模式应当判为未变，不做任何写入。"""
|
||
|
|
path = os.path.join(self.src, "a.txt")
|
||
|
|
with open(path, "rb") as f:
|
||
|
|
original = f.read()
|
||
|
|
self.touch_later(path, original) # 只改 mtime
|
||
|
|
|
||
|
|
before = self.hash_for("a.txt")
|
||
|
|
self.pack(incremental=True, rehash=True)
|
||
|
|
self.assertEqual(3, self.stats["unchanged"])
|
||
|
|
self.assertEqual(0, self.stats["updated"])
|
||
|
|
self.assertEqual(before, self.hash_for("a.txt"))
|
||
|
|
|
||
|
|
|
||
|
|
class RehashDetectionTest(IncrementalTestBase):
|
||
|
|
def test_mtime_only_change_is_detected_without_rehash(self):
|
||
|
|
"""同大小 + 同秒修改：默认模式可能漏掉，这正是 rehash 存在的理由。"""
|
||
|
|
path = os.path.join(self.src, "a.txt")
|
||
|
|
helpers.write_file(path, b"AAAA") # 与原始 aaa 大小不同，先制造同大小场景
|
||
|
|
self.pack(incremental=True)
|
||
|
|
|
||
|
|
# 现在 disk 与库里都是 AAAA；把内容换成同样长度的别的字节，
|
||
|
|
# 并把 mtime 强制设回库里记录的那个值
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
recorded = vfs.get_file_info("a.txt")["modified_time"].timestamp()
|
||
|
|
vfs.close()
|
||
|
|
helpers.write_file(path, b"BBBB")
|
||
|
|
os.utime(path, (recorded, recorded))
|
||
|
|
|
||
|
|
self.pack(incremental=True)
|
||
|
|
self.assertEqual(3, self.stats["unchanged"], "mtime+size 相同，默认模式看不出来")
|
||
|
|
self.assertEqual(b"AAAA", self.read("a.txt"), "库里还是旧内容（预期的漏检）")
|
||
|
|
|
||
|
|
def test_rehash_catches_same_size_same_mtime_edit(self):
|
||
|
|
path = os.path.join(self.src, "a.txt")
|
||
|
|
helpers.write_file(path, b"AAAA")
|
||
|
|
self.pack(incremental=True)
|
||
|
|
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
recorded = vfs.get_file_info("a.txt")["modified_time"].timestamp()
|
||
|
|
vfs.close()
|
||
|
|
helpers.write_file(path, b"BBBB")
|
||
|
|
os.utime(path, (recorded, recorded))
|
||
|
|
|
||
|
|
self.pack(incremental=True, rehash=True)
|
||
|
|
self.assertEqual(1, self.stats["updated"], "rehash 必须抓到这次修改")
|
||
|
|
self.assertEqual(b"BBBB", self.read("a.txt"))
|
||
|
|
|
||
|
|
|
||
|
|
class IncrementalSnapshotTest(IncrementalTestBase):
|
||
|
|
def test_first_incremental_on_an_empty_database(self):
|
||
|
|
db = self.path("fresh.svfs")
|
||
|
|
vfs = SQLiteVFS(db, compress=True)
|
||
|
|
try:
|
||
|
|
packer = FolderPacker(vfs, fs_name="fresh")
|
||
|
|
with helpers.quiet():
|
||
|
|
self.assertTrue(packer.pack_folder(self.src, incremental=True))
|
||
|
|
finally:
|
||
|
|
vfs.close()
|
||
|
|
|
||
|
|
vfs = SQLiteVFS(db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertEqual(3, helpers.count_files(vfs))
|
||
|
|
self.assertEqual("fresh", vfs.list_snapshots()[0]["name"])
|
||
|
|
|
||
|
|
def test_incremental_into_a_new_snapshot_keeps_history(self):
|
||
|
|
"""派生新快照后再增量：旧版本原样保留，新版本只有变化的部分被重写。"""
|
||
|
|
vfs = SQLiteVFS(self.db, compress=True)
|
||
|
|
try:
|
||
|
|
parent = vfs._active_snapshot()
|
||
|
|
vfs.create_snapshot("v2", parent_id=parent, copy_entries=True)
|
||
|
|
self.touch_later(os.path.join(self.src, "a.txt"), b"aaa-v2")
|
||
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
||
|
|
self.assertTrue(self.pack_with(vfs, incremental=True))
|
||
|
|
finally:
|
||
|
|
vfs.close()
|
||
|
|
self.assertEqual(1, self.stats["updated"])
|
||
|
|
self.assertEqual(1, self.stats["removed"])
|
||
|
|
|
||
|
|
vfs = SQLiteVFS(self.db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertEqual(2, len(vfs.list_snapshots()))
|
||
|
|
self.assertEqual(b"aaa-v2", vfs.read_file("a.txt"))
|
||
|
|
self.assertFalse(vfs.exists("b.txt"))
|
||
|
|
|
||
|
|
vfs.use_snapshot(1)
|
||
|
|
self.assertEqual(b"aaa", vfs.read_file("a.txt"), "旧版本必须原样保留")
|
||
|
|
self.assertEqual(b"bbb", vfs.read_file("b.txt"))
|
||
|
|
|
||
|
|
def test_incremental_does_not_touch_other_snapshots(self):
|
||
|
|
vfs = SQLiteVFS(self.db, compress=True)
|
||
|
|
try:
|
||
|
|
parent = vfs._active_snapshot()
|
||
|
|
vfs.create_snapshot("v2", parent_id=parent, copy_entries=True)
|
||
|
|
os.remove(os.path.join(self.src, "b.txt"))
|
||
|
|
self.pack_with(vfs, incremental=True)
|
||
|
|
finally:
|
||
|
|
vfs.close()
|
||
|
|
|
||
|
|
vfs = SQLiteVFS(self.db, snapshot_id=1)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertTrue(vfs.exists("b.txt"), "v1 不该被 v2 的增量影响")
|
||
|
|
|
||
|
|
|
||
|
|
class IncrementalCliTest(helpers.VFSTestCase):
|
||
|
|
def test_cli_incremental(self):
|
||
|
|
src = helpers.make_tree(self.path("src"), {"a.txt": b"aaa", "b.txt": b"bbb"})
|
||
|
|
db = self.path("cli.svfs")
|
||
|
|
first = helpers.run_cli("sqlite_vfs.cli.packer", src, db, "--compress")
|
||
|
|
self.assertEqual(0, first.returncode, helpers.cli_output(first))
|
||
|
|
|
||
|
|
helpers.write_file(os.path.join(src, "new.txt"), b"new")
|
||
|
|
os.remove(os.path.join(src, "b.txt"))
|
||
|
|
second = helpers.run_cli("sqlite_vfs.cli.packer", src, db, "--incremental")
|
||
|
|
self.assertEqual(0, second.returncode, helpers.cli_output(second))
|
||
|
|
self.assertIn("增量结果", second.stdout)
|
||
|
|
self.assertIn("新增 1", second.stdout)
|
||
|
|
self.assertIn("删除 1", second.stdout)
|
||
|
|
|
||
|
|
vfs = SQLiteVFS(db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertEqual(b"new", vfs.read_file("new.txt"))
|
||
|
|
self.assertFalse(vfs.exists("b.txt"))
|
||
|
|
|
||
|
|
def test_cli_snapshot_creates_a_new_version(self):
|
||
|
|
src = helpers.make_tree(self.path("src"), {"a.txt": b"aaa"})
|
||
|
|
db = self.path("cli2.svfs")
|
||
|
|
helpers.run_cli("sqlite_vfs.cli.packer", src, db, "--compress")
|
||
|
|
|
||
|
|
helpers.write_file(os.path.join(src, "a.txt"), b"aaa-v2")
|
||
|
|
result = helpers.run_cli(
|
||
|
|
"sqlite_vfs.cli.packer", src, db, "--snapshot", "v2", "--incremental"
|
||
|
|
)
|
||
|
|
self.assertEqual(0, result.returncode, helpers.cli_output(result))
|
||
|
|
|
||
|
|
vfs = SQLiteVFS(db)
|
||
|
|
self.addCleanup(vfs.close)
|
||
|
|
self.assertEqual([1, 2], [s["id"] for s in vfs.list_snapshots()])
|
||
|
|
self.assertEqual(b"aaa-v2", vfs.read_file("a.txt"))
|
||
|
|
vfs.use_snapshot(1)
|
||
|
|
self.assertEqual(b"aaa", vfs.read_file("a.txt"))
|
||
|
|
|
||
|
|
listing = helpers.run_cli("sqlite_vfs.cli.unpacker", db, self.path("out"), "--list-snapshots")
|
||
|
|
self.assertEqual(0, listing.returncode, helpers.cli_output(listing))
|
||
|
|
self.assertIn("v2", listing.stdout)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|