fix+feat: 修复打包/解包主链路，并把存储层重建为内容寻址 + 快照
起始状态：主链路 100% 不可用，而且失败被静默吞掉——打包后库里 0 个文件、 界面仍打印"文件数量: N"；解包导出 0 个文件并在磁盘上生成几百层 sub/sub/sub 垃圾目录。 缺陷修复（4 个 P0 + 12 个 P1，逐条见 BUG_REPORT.md）： - 建表列名 compresse_algo 与 INSERT/read_file 用的 compress_algo 不一致 - list_directory 按 path 而非 parent_path 过滤，导致导出无限自递归 - CLI --ignore-timestamps 把 export_filesystem 赋成引用自身的 lambda - add_file 缺少 commit（直接调用 API 时静默丢数据） - LIKE 前缀误伤兄弟目录（删 doc 连带删 docs）；mv_dir 二次改名 - 失败路径一律返回 exit code 0；--exclude 是子串匹配而非 glob - FolderPacker.add_file 反向调用 export_file_safe，会用库内内容覆盖源文件 存储层（schema v3，格式变更，提供 vfsmigrate 迁移）： - 内容移入 blobs(hash PRIMARY KEY)：相同内容只存一份，跨路径/跨快照共享 - files 的唯一约束改为 UNIQUE(snapshot_id, path)：一个库存多个版本 - 完整性：每文件 sha256 + 快照级摘要 + HMAC-SHA256 签名，只读连接即可验证 - 增量打包：按 mtime+size 或 sha256 只写变化项，未变文件不重写不重压 - get_file_info 不再返回 content（元数据查询不该搬 BLOB） 新增能力： - 可选 FastAPI 静态文件插件（extras，核心仍零第三方依赖） - 内容寻址发布到对象存储 + 逐对象校验的取回客户端 - 压缩默认改回 zlib：实测比自研哈夫曼体积小 1/3~1/170、快数百倍 其它： - P2-4：时间戳显式转字符串，清掉 1240 条 DeprecationWarning（盘上格式不变） - read_file 负数 offset/size 改为报错（原先会走切片语义从末尾取） - requires-python 从 >=3.7 改为按实际 API 推导的 >=3.9 测试：0 → 343 个用例，纯标准库 unittest，有/无 FastAPI 两种环境都跑通。 证据见 docs/evidence-*.txt。
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
"""huffman.py 的往返测试。
|
||||
|
||||
这个模块是独立的、只有标准库依赖的算法实现，理论上应当全绿；
|
||||
它在测试里的作用是：当打包/解包链路出问题时，可以快速排除"压缩算法本身写错了"
|
||||
这一可能，把问题范围锁到 core.py 的集成层。
|
||||
"""
|
||||
|
||||
import random
|
||||
import struct
|
||||
import unittest
|
||||
|
||||
import helpers # noqa: F401 -- 把 src 加入 sys.path
|
||||
|
||||
from sqlite_vfs.huffman import (
|
||||
FREQ_SIZE,
|
||||
MAGIC,
|
||||
compress_bytes,
|
||||
decompress_bytes,
|
||||
)
|
||||
|
||||
|
||||
class RoundTripTest(unittest.TestCase):
|
||||
def assert_roundtrip(self, data, label=""):
|
||||
freq_blob, compressed_blob = compress_bytes(data)
|
||||
|
||||
self.assertEqual(
|
||||
FREQ_SIZE, len(freq_blob), f"{label}: 频率表必须是 256 个 uint32"
|
||||
)
|
||||
self.assertEqual(MAGIC, compressed_blob[:4], f"{label}: 魔数不正确")
|
||||
self.assertEqual(
|
||||
len(data),
|
||||
struct.unpack(">Q", compressed_blob[4:12])[0],
|
||||
f"{label}: 头部记录的原长度不正确",
|
||||
)
|
||||
self.assertEqual(
|
||||
data, decompress_bytes(freq_blob, compressed_blob), f"{label}: 还原结果不一致"
|
||||
)
|
||||
return compressed_blob
|
||||
|
||||
def test_empty(self):
|
||||
freq_blob, compressed_blob = compress_bytes(b"")
|
||||
self.assertEqual(FREQ_SIZE, len(freq_blob))
|
||||
self.assertEqual(b"", decompress_bytes(freq_blob, compressed_blob))
|
||||
|
||||
def test_single_byte(self):
|
||||
self.assert_roundtrip(b"\x00", "单字节")
|
||||
self.assert_roundtrip(b"Z", "单字节")
|
||||
|
||||
def test_single_symbol_repeated(self):
|
||||
self.assert_roundtrip(b"\x7f" * 5000, "单符号重复")
|
||||
|
||||
def test_two_symbols(self):
|
||||
self.assert_roundtrip(b"ab" * 3000, "双符号")
|
||||
|
||||
def test_all_256_byte_values(self):
|
||||
self.assert_roundtrip(bytes(range(256)), "全字节值")
|
||||
self.assert_roundtrip(bytes(range(256)) * 40, "全字节值重复")
|
||||
|
||||
def test_incompressible_random(self):
|
||||
data = random.Random(1).randbytes(20000)
|
||||
compressed_blob = self.assert_roundtrip(data, "随机数据")
|
||||
self.assertGreater(
|
||||
len(compressed_blob),
|
||||
len(data),
|
||||
"随机数据的压缩结果应当大于原始数据（这正是 core 会回退到 none 的原因）",
|
||||
)
|
||||
|
||||
def test_repetitive_text_compresses(self):
|
||||
data = b"the quick brown fox jumps over the lazy dog. " * 2000
|
||||
compressed_blob = self.assert_roundtrip(data, "重复文本")
|
||||
# 该文本有 20 个不同字符，理论码长 ~4.5 bit/字符，压缩后约为原始的 56%
|
||||
self.assertLess(
|
||||
len(compressed_blob),
|
||||
int(len(data) * 0.7),
|
||||
f"高度重复的文本应当被显著压缩，实际 {len(compressed_blob)}/{len(data)}",
|
||||
)
|
||||
|
||||
def test_utf8_chinese(self):
|
||||
self.assert_roundtrip("中文内容，测试哈夫曼编码是否无损。\n".encode("utf-8") * 200, "中文")
|
||||
|
||||
def test_binary_with_nulls_and_high_bytes(self):
|
||||
data = bytes([0, 255, 0, 128, 64, 0, 7]) * 500
|
||||
self.assert_roundtrip(data, "二进制含 0x00/0xFF")
|
||||
|
||||
def test_just_above_core_threshold(self):
|
||||
# core.add_file 的压缩阈值是 1024 字节，边界两侧都要能正确往返
|
||||
self.assert_roundtrip(b"A" * 1024, "1024 字节")
|
||||
self.assert_roundtrip(b"A" * 1025, "1025 字节")
|
||||
|
||||
|
||||
class HeaderValidationTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# 刻意用多符号数据：单符号数据在解压时走快捷路径，不读 payload
|
||||
self.freq_blob, self.compressed_blob = compress_bytes(
|
||||
b"the quick brown fox. " * 500
|
||||
)
|
||||
|
||||
def test_wrong_freq_blob_size_rejected(self):
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
decompress_bytes(self.freq_blob[:-4], self.compressed_blob)
|
||||
self.assertIn("频率表", str(ctx.exception))
|
||||
|
||||
def test_too_short_blob_rejected(self):
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
decompress_bytes(self.freq_blob, b"HUF")
|
||||
self.assertIn("太短", str(ctx.exception))
|
||||
|
||||
def test_bad_magic_rejected(self):
|
||||
broken = b"XXXX" + self.compressed_blob[4:]
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
decompress_bytes(self.freq_blob, broken)
|
||||
self.assertIn("魔数", str(ctx.exception))
|
||||
|
||||
def test_truncated_payload_rejected(self):
|
||||
truncated = self.compressed_blob[:-50]
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
decompress_bytes(self.freq_blob, truncated)
|
||||
self.assertIn("提前结束", str(ctx.exception))
|
||||
|
||||
def test_single_symbol_payload_is_not_validated(self):
|
||||
"""记录既有行为（非缺陷）：单符号数据解压时直接按原长度展开，完全不读 payload，
|
||||
因此 payload 被截断也检测不出来。"""
|
||||
freq_blob, compressed_blob = compress_bytes(b"A" * 10000)
|
||||
self.assertEqual(
|
||||
b"A" * 10000, decompress_bytes(freq_blob, compressed_blob[:20])
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user