123 lines
4.4 KiB
Python
123 lines
4.4 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
"""vfspublish：把 .svfs 发布成"一堆按 hash 命名的对象 + 一份清单"。
|
|||
|
|
|
|||
|
|
产出可以直接 `aws s3 sync` / `rclone copy` 到对象存储或静态托管：
|
|||
|
|
对象名就是内容的 sha256，所以客户端的缓存策略可以是"永久"，
|
|||
|
|
而且每个对象都能端到端校验（CDN 换不掉内容）。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import argparse
|
|||
|
|
import os
|
|||
|
|
import sys
|
|||
|
|
|
|||
|
|
from .. import publish
|
|||
|
|
from ..core import VFSFormatError
|
|||
|
|
|
|||
|
|
|
|||
|
|
def publish_objects():
|
|||
|
|
parser = argparse.ArgumentParser(
|
|||
|
|
description="把虚拟文件系统导出成内容寻址的对象目录 + 清单"
|
|||
|
|
)
|
|||
|
|
parser.add_argument("db_file", help="SQLite数据库文件路径")
|
|||
|
|
parser.add_argument("out_dir", help="输出目录（会创建 blobs/ 与 manifest.json）")
|
|||
|
|
parser.add_argument(
|
|||
|
|
"--snapshot", metavar="ID", help="发布指定快照（默认取最新快照）"
|
|||
|
|
)
|
|||
|
|
parser.add_argument(
|
|||
|
|
"--shard",
|
|||
|
|
type=int,
|
|||
|
|
default=publish.DEFAULT_SHARD,
|
|||
|
|
help=f"blobs 的分片层数，0 表示不分片（默认 {publish.DEFAULT_SHARD}）",
|
|||
|
|
)
|
|||
|
|
parser.add_argument(
|
|||
|
|
"--overwrite",
|
|||
|
|
action="store_true",
|
|||
|
|
help="输出目录非空时也继续（会覆盖同名对象与清单）",
|
|||
|
|
)
|
|||
|
|
parser.add_argument(
|
|||
|
|
"--no-objects",
|
|||
|
|
action="store_true",
|
|||
|
|
help="只生成清单，不导出对象（用于只更新清单的场景）",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
args = parser.parse_args()
|
|||
|
|
|
|||
|
|
if not os.path.exists(args.db_file):
|
|||
|
|
print(f"错误: 数据库文件不存在: {args.db_file}")
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
done = [0]
|
|||
|
|
|
|||
|
|
def progress(current, total):
|
|||
|
|
done[0] = current
|
|||
|
|
if total and (current == total or current % 200 == 0):
|
|||
|
|
print(f" 导出对象: {current}/{total}", flush=True)
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
if args.no_objects:
|
|||
|
|
from ..core import SQLiteVFS
|
|||
|
|
|
|||
|
|
vfs = SQLiteVFS(args.db_file, readonly=True, snapshot_id=args.snapshot)
|
|||
|
|
try:
|
|||
|
|
manifest = publish.build_manifest(
|
|||
|
|
vfs, snapshot_id=args.snapshot, shard=args.shard
|
|||
|
|
)
|
|||
|
|
finally:
|
|||
|
|
vfs.close()
|
|||
|
|
os.makedirs(args.out_dir, exist_ok=True)
|
|||
|
|
import json
|
|||
|
|
|
|||
|
|
path = os.path.join(args.out_dir, publish.MANIFEST_NAME)
|
|||
|
|
payload = json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True)
|
|||
|
|
with open(path, "w", encoding="utf-8") as f:
|
|||
|
|
f.write(payload)
|
|||
|
|
result = {
|
|||
|
|
"out_dir": os.path.abspath(args.out_dir),
|
|||
|
|
"manifest": path,
|
|||
|
|
"manifest_bytes": len(payload.encode("utf-8")),
|
|||
|
|
"objects": 0,
|
|||
|
|
"objects_bytes": 0,
|
|||
|
|
"files": manifest["stats"]["files"],
|
|||
|
|
"directories": manifest["stats"]["directories"],
|
|||
|
|
"dedup_saved": 0,
|
|||
|
|
"snapshot_id": manifest["snapshot"]["id"],
|
|||
|
|
"digest": manifest["snapshot"]["digest"],
|
|||
|
|
}
|
|||
|
|
else:
|
|||
|
|
result = publish.export_objects(
|
|||
|
|
args.db_file,
|
|||
|
|
args.out_dir,
|
|||
|
|
snapshot_id=args.snapshot,
|
|||
|
|
shard=args.shard,
|
|||
|
|
overwrite=args.overwrite,
|
|||
|
|
progress=progress,
|
|||
|
|
)
|
|||
|
|
except VFSFormatError as e:
|
|||
|
|
print(f"错误: {e}")
|
|||
|
|
sys.exit(1)
|
|||
|
|
except FileExistsError as e:
|
|||
|
|
print(f"错误: {e}")
|
|||
|
|
sys.exit(1)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"错误: 发布失败: {e}")
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
print("发布完成!")
|
|||
|
|
print(f" 快照: {result['snapshot_id']} 摘要: {(result['digest'] or '')[:16]}")
|
|||
|
|
print(f" 文件: {result['files']} 目录: {result['directories']}")
|
|||
|
|
print(f" 对象: {result['objects']} 个，共 {result['objects_bytes'] / 1024:.1f} KB（去重后的原始字节）")
|
|||
|
|
if result["dedup_saved"]:
|
|||
|
|
print(f" 去重省下: {result['dedup_saved'] / 1024:.1f} KB（原始总大小 - 去重后）")
|
|||
|
|
if result.get("compression_saved"):
|
|||
|
|
print(
|
|||
|
|
f" 库内实际占用: {result['stored_bytes'] / 1024:.1f} KB"
|
|||
|
|
f"（压缩省下 {result['compression_saved'] / 1024:.1f} KB）"
|
|||
|
|
)
|
|||
|
|
print(f" 清单: {result['manifest']}（{result['manifest_bytes'] / 1024:.1f} KB）")
|
|||
|
|
print(f" 输出目录: {result['out_dir']}")
|
|||
|
|
print(" 下一步: 把整个输出目录同步到对象存储/静态托管即可")
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
publish_objects()
|