add: 补充回归测试(build 开关/资源档位、日志 fork、插件迁移与依赖清单、父进程死亡、worker 疑似失联)

This commit is contained in:
2026-10-13 17:29:38 +08:00
parent fe96fc9b6c
commit 4cdb9c7049
7 changed files with 1181 additions and 0 deletions
+214
View File
@@ -0,0 +1,214 @@
"""
build 命令开关(--encrypt / --license)冒烟测试
两个开关默认都是关的(不加参数 = 明文模块 + 不带授权的内部调试产物),
这里验证"默认关"与"显式打开"在构建产物上都真的生效:
1. 参数解析:默认 encrypt/license 全关,--encrypt / --license 打开对应开关;
--only 里 license_guard 是合法步骤(文档里一直写着,之前被漏在校验表外)
2. 默认(不加密):.mbank 存明文字节码(encrypted=false)、不写 .key(并清掉旧
的)、无 packer_key 记录,且**不需要密钥即可导入**;加 --encrypt 那条路仍然
加密、仍然写 .key,且用 .key 里的密钥能正常导入
3. 默认(不授权):prepare 不要求/不注入授权公钥;加 --license 时缺公钥构建失败
运行: uv run python test/test_build_flags.py
"""
import importlib
import os
import shutil
import sys
import tempfile
from pathlib import Path
SRC = Path(__file__).parent
sys.path.insert(0, str(SRC.parent / "utils" / "cli" / "src"))
from module_bank import PythonToSQLite, layout # noqa: E402
from sqlite_vfs.core import SQLiteVFS # noqa: E402
from chongming_cli import build as b # noqa: E402
failures = []
def check(name, cond, detail=""):
status = "PASS" if cond else "FAIL"
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
if not cond:
failures.append(name)
tmp = Path(tempfile.mkdtemp(prefix="build_flags_test_"))
# ========== 1. 参数解析 ==========
_argv, sys.argv = sys.argv, ["build"]
mode, steps, options = b._parse_args()
check(
"默认参数:pyinstaller + 全流程 + 不加密 + 不授权",
mode == "pyinstaller" and not steps and not options.encrypt and not options.license,
)
sys.argv = ["build", "--encrypt", "--license", "nuitka"]
mode, steps, options = b._parse_args()
check(
"--encrypt/--license 打开对应开关",
mode == "nuitka" and options.encrypt and options.license,
f"mode={mode} encrypt={options.encrypt} license={options.license}",
)
sys.argv = ["build", "--only", "mbank,license_guard"]
mode, steps, options = b._parse_args()
check("--only 接受 license_guard", steps == {"mbank", "license_guard"}, str(steps))
sys.argv = ["build", "--only", "mbank", "--encrypt", "pyinstaller"]
mode, steps, options = b._parse_args()
check(
"位置参数与开关可混用",
mode == "pyinstaller" and steps == {"mbank"} and options.encrypt,
)
sys.argv = ["build", "--unknown-flag"]
try:
b._parse_args()
check("未知参数应退出", False)
except SystemExit:
check("未知参数应退出", True)
sys.argv = _argv
# ========== 2. mbank 加密 / 明文字节码 ==========
def make_source(root: Path) -> Path:
"""造一个最小可导入的包:demo/__init__.py"""
pkg = root / "demo"
pkg.mkdir(parents=True)
(pkg / "__init__.py").write_text("VALUE = 42\n", encoding="utf-8")
return pkg
def bank_meta(bank: Path, module: str) -> dict:
vfs = SQLiteVFS(str(bank), compress=False, readonly=True)
try:
# 包名 → 包内路径由 module_bank 定,别自己拼;明文未混淆的模块不写
# meta.json(默认值即"什么都不用记"),用 read_meta 读缺省
return layout.read_meta(vfs, layout.module_paths(module, True)["meta"])
finally:
vfs.close()
def bank_has_key(bank: Path) -> bool:
vfs = SQLiteVFS(str(bank), compress=False, readonly=True)
try:
return vfs.is_file("packer_key")
finally:
vfs.close()
def import_from_bank(bank: Path, module: str, key):
"""像 utils/launch.py 那样装载:install_importer → import。返回 (模块, 错误)。"""
packer = PythonToSQLite(str(bank))
finder = packer.install_importer(key=key)
try:
mod = importlib.import_module(module)
return mod, None
except Exception as exc: # noqa: BLE001 — 测试就是要区分"能导/不能导"
return None, exc
finally:
if finder in sys.meta_path:
sys.meta_path.remove(finder)
sys.modules.pop(module, None)
plain_dir = make_source(tmp / "plain_src")
plain_bank = tmp / "plain.mbank"
plain_size = b._pack_mbank(
str(plain_bank), str(plain_dir), "demo", obfuscate=False, encrypt=False
)
check("明文打包产出 .mbank", plain_bank.exists(), plain_size)
check("明文打包不写 .key", not plain_bank.with_suffix(".key").exists())
check("明文打包 encrypted=false", bank_meta(plain_bank, "demo")["encrypted"] is False)
check("明文打包无 packer_key 记录", not bank_has_key(plain_bank))
mod, err = import_from_bank(plain_bank, "demo", key=None)
check("明文包无密钥即可导入", mod is not None and getattr(mod, "VALUE", None) == 42, str(err))
enc_dir = make_source(tmp / "enc_src")
enc_bank = tmp / "enc.mbank"
enc_key_path = enc_bank.with_suffix(".key")
enc_key_path.write_text("stale-key-from-previous-build", encoding="utf-8") # 模拟上一次加密构建
enc_size = b._pack_mbank(
str(enc_bank), str(enc_dir), "demo", obfuscate=False, encrypt=False
)
check("默认(不加密)清掉上一次加密构建留下的 .key", not enc_key_path.exists(), enc_size)
encrypted_bank = tmp / "encrypted.mbank"
key = b.Encryption.generate_key()
_orig_gen = b.Encryption.generate_key
b.Encryption.generate_key = staticmethod(lambda: key) # 固定密钥,便于用 .key 验证导入
try:
b._pack_mbank(str(encrypted_bank), str(enc_dir), "demo", obfuscate=False, encrypt=True)
finally:
b.Encryption.generate_key = _orig_gen
check("--encrypt 打包写 .key", encrypted_bank.with_suffix(".key").exists())
check("--encrypt 打包 encrypted=true", bank_meta(encrypted_bank, "demo")["encrypted"] is True)
mod, err = import_from_bank(encrypted_bank, "demo", key=encrypted_bank.with_suffix(".key").read_text())
check("加密包用 .key 里的密钥可导入", mod is not None, str(err))
mod, err = import_from_bank(encrypted_bank, "demo", key=None)
check("加密包缺密钥无法导入(说明 .key 必须随包提供给加载端)", mod is None and isinstance(err, ImportError), str(err))
# ========== 3. 授权公钥注入 / 默认不注入 ==========
def make_project(root: Path) -> Path:
"""造一个最小构建现场:public/{main.py,server.py,config.toml}"""
(root / "public" / "utils").mkdir(parents=True)
(root / "public" / "utils" / "launch.py").write_text("", encoding="utf-8")
(root / "public" / "main.py").write_text("", encoding="utf-8")
(root / "public" / "server.py").write_text("", encoding="utf-8")
(root / "public" / "config.toml").write_text(
'[security]\nsecret_key = "your-secret-key-change-in-production"\n\n'
'[license]\n# 机器码授权公钥(构建时注入)\npublic_key = ""\n',
encoding="utf-8",
)
return root
env_keys = (b.LICENSE_PUBKEY_ENV, b.LICENSE_PUBKEY_FILE_ENV, b.ALLOW_NO_KEY_ENV)
saved_env = {k: os.environ.pop(k, None) for k in env_keys} # 环境变量不该影响测试结论
cwd = Path.cwd()
proj = make_project(tmp / "proj_default")
os.chdir(proj)
try:
msg = b._stage_prepare_build_dir(None, "pyinstaller", b.BuildOptions())
conf = (proj / "build" / "config.toml").read_text(encoding="utf-8")
check("默认(不授权)的 prepare 不报错", msg == "资源准备完成", msg)
check("默认不注入公钥(模板 public_key 保持空)", 'public_key = ""' in conf)
check("JWT 密钥仍会注入", "your-secret-key-change-in-production" not in conf)
try:
b._stage_prepare_build_dir(None, "pyinstaller", b.BuildOptions(license=True))
check("--license 缺授权公钥应失败", False)
except RuntimeError as exc:
check("--license 缺授权公钥应失败", "缺少厂商授权公钥" in str(exc))
os.environ[b.LICENSE_PUBKEY_ENV] = "test-pubkey-b64"
try:
b._stage_prepare_build_dir(None, "pyinstaller", b.BuildOptions(license=True))
conf = (proj / "build" / "config.toml").read_text(encoding="utf-8")
check("--license + 提供公钥后注入 [license] public_key", 'public_key = "test-pubkey-b64"' in conf)
finally:
os.environ.pop(b.LICENSE_PUBKEY_ENV, None)
finally:
os.chdir(cwd)
for k, v in saved_env.items():
if v is not None:
os.environ[k] = v
shutil.rmtree(tmp, ignore_errors=True)
print("\n" + ("=" * 40))
if failures:
print(f"❌ {len(failures)} 项未通过: {failures}")
sys.exit(1)
print("✅ 全部通过")
+100
View File
@@ -0,0 +1,100 @@
"""
fork 安全:文件日志流在子进程重开(reentrant call 修复)测试
背景:gateway 的日志是"队列 handler + 监听线程"结构,文件 handler 的流在父进程
打开。granian/gunicorn 的 worker 由 master **fork** 而来(实测 worker 与 master 的
fd 表完全一致)——子进程会继承父进程的 BufferedWriter,其内部锁/owner 状态在
fork 瞬间可能正被父进程监听线程占用(跨 fork 复制不可靠)。症状:冻结版(Nuitka)
里每个 worker 的**每条文件日志**都报
RuntimeError: reentrant call inside <_io.BufferedWriter ...>
(控制台 handler 正常,文件日志整体失效)。
修复:_after_fork_child 里把文件 handler 的流在子进程重开(新文件描述符)。
本测试验证:
1. fork 后子进程的 handler.stream 不再是继承的那个对象(结构保证)
2. 子进程能真正把记录写进日志文件(行为保证——旧实现下这里写不进去)
3. 控制台等非文件 handler 不被动(无 _open,跳过)
运行: uv run python test_logging_fork.py
"""
import importlib.util
import logging
import os
import sys
import tempfile
from pathlib import Path
tmp = tempfile.TemporaryDirectory()
log_dir = Path(tmp.name)
# 与 test_logging_async.py 相同的加载方式:独立加载 logger 模块并把日志目录重定向
_logger_path = (
Path(__file__).resolve().parent.parent / "gateway" / "src" / "chongming" / "app" / "core" / "logger.py"
)
_spec = importlib.util.spec_from_file_location("chongming.app.core.logger", _logger_path)
logger_mod = importlib.util.module_from_spec(_spec)
sys.modules["chongming.app.core.logger"] = logger_mod
_spec.loader.exec_module(logger_mod) # type: ignore[union-attr]
logger_mod.log_config["file"] = str(log_dir)
failures = []
def check(name, cond, detail=""):
status = "PASS" if cond else "FAIL"
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
if not cond:
failures.append(name)
check("os.register_at_fork 已注册 after_in_child", hasattr(os, "register_at_fork"))
logger = logging.getLogger("forktest")
logger_mod.setup_logging(logger, "forktest")
file_handler = next(
h for h in logger_mod._listener_handlers if isinstance(h, logger_mod.DailyRotatingFileHandler)
)
# 同步直写一条(不经队列),保持监听线程与流都活着,便于接下来 fork
parent_record = logging.LogRecord("forktest", logging.INFO, __file__, 0, "parent-line", (), None)
file_handler.handle(parent_record)
file_handler.flush()
check("父进程日志已落盘", "parent-line" in next(iter(log_dir.glob("forktest-*.log"))).read_text(encoding="utf-8"))
inherited_stream = file_handler.stream
check("父进程流已打开", inherited_stream is not None)
# ---- fork:子进程应换掉继承的流,并能独立写日志 ----
pid = os.fork()
if pid == 0: # 子进程
try:
reopened = file_handler.stream is not inherited_stream
record = logging.LogRecord(
"forktest", logging.INFO, __file__, 0, "child-line", (), None
)
# 直接同步走 handler(不经队列),写入结果立即可断言
file_handler.handle(record)
os._exit(0 if reopened else 3)
except BaseException: # noqa: BLE001 - 子进程异常用退出码传出
os._exit(4)
_, status = os.waitpid(pid, 0)
exit_code = os.waitstatus_to_exitcode(status)
check("fork 后子进程的流已换新(不再是继承对象)", exit_code == 0, f"exit={exit_code}")
check(
"子进程可直接写入日志文件(旧实现下 reentrant 失败)",
"child-line" in next(iter(log_dir.glob("forktest-*.log"))).read_text(encoding="utf-8"),
)
check(
"控制台 handler 不受影响(仍指向 stderr)",
any(isinstance(h, logging.StreamHandler) and not isinstance(h, logger_mod.DailyRotatingFileHandler) for h in logger_mod._listener_handlers),
)
tmp.cleanup()
print("\n" + ("=" * 40))
if failures:
print(f"❌ {len(failures)} 项未通过: {failures}")
sys.exit(1)
print("✅ 全部通过")
+280
View File
@@ -0,0 +1,280 @@
"""
migrate_plugin(内置插件 → 进程插件 一键迁移)测试
覆盖:
1. 前置拦截: 自定义 FastAPI 路由(APIRouter 实例化 / 顶层 router 绑定 /
star-import 再导出)拒绝迁移;--force 可跳过
2. 前置拦截: 网关包导入(import app / from chongming...)拒绝迁移
3. 放行: `from . import router`(子模块,非 APIRouter)不误伤(ip_sound_column 写法)
4. 迁移产物: 项目骨架(pyproject/start.sh/config.toml/README/zenoh 配置)+ 代码/配置
迁移 + [workers] 改写(module=workers.<name>、load_mode=external)+ 保留其余配置
5. 依赖迁移: plugins_deps.toml [<name>] 节 → 新项目 pyproject(按实际 import 过滤、
保留根 pyproject 的 extras 写法、共用节带入);[<name>] 节注释掉、其它节不动
6. --dry-run 不落盘;目标已存在 / 插件不存在时拒绝
运行: uv run python test/test_migrate_plugin.py
"""
import contextlib
import io
import os
import shutil
import sys
import tempfile
import tomllib
from pathlib import Path
ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(ROOT / "utils" / "cli" / "src"))
os.environ.pop("CHONGMING_ROOT", None) # 根目录定位不应受外部环境变量影响
from chongming_cli import migrate_plugin as mp # noqa: E402
failures = []
def check(name, cond, detail=""):
status = "PASS" if cond else "FAIL"
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
if not cond:
failures.append(name)
DEPS_TOML = """\
imports = ["httpx"]
[foo]
imports = [
"volcenginesdkarkruntime",
"chongming_cache",
"orjson",
"totally_missing_pkg",
]
[bar]
imports = ["openai"]
"""
PLUGIN_INIT = '''"""fixture 插件"""
import orjson
import httpx
from chongming_cache import Cache # noqa: F401
from chongming_worker.plugin import Plugin
from volcenginesdkarkruntime import AsyncArk # noqa: F401
from totally_missing_pkg import thing # noqa: F401
from . import router # noqa: F401 触发 @rpc_method 注册
{extra}
class FooPlugin(Plugin):
type = "foo"
db_name = None
async def on_init(self):
pass
plugin = FooPlugin()
create_workers = plugin.create_workers
cleanup = plugin.cleanup
'''
CONFIG_TOML = """\
# foo Worker 插件配置
[workers]
type = "foo"
module = "plugins.foo"
factory = "create_workers"
cleanup = "cleanup"
count = 2
[worker]
name = "foo"
version = "0.1.0"
description = "fixture 插件"
config_version = "1.0"
defaults = { method = "POST", timeout = 30.0 }
[worker.registration]
router_prefix = "/foo"
[workers.config]
data_dir = "./data"
api_key = "PLACEHOLDER"
"""
def make_root(tmp: Path) -> Path:
"""最小仓库现场:根 pyproject(含 extras 依赖)/ config/zenoh_config.json5 副本 / plugins_deps.toml。"""
tmp.mkdir(parents=True, exist_ok=True)
(tmp / "pyproject.toml").write_text(
"[project]\nname = 'fixture'\ndependencies = [\"volcengine-python-sdk[ark]\"]\n",
encoding="utf-8",
)
(tmp / "config").mkdir(parents=True, exist_ok=True)
(tmp / "config" / "zenoh_config.json5").write_text("{}\n", encoding="utf-8")
(tmp / "plugins_deps.toml").write_text(DEPS_TOML, encoding="utf-8")
return tmp
def make_plugin(tmp: Path, name: str = "foo", init_extra: str = "", files: dict | None = None):
"""造内置插件现场:src/plugins/<name>/ + config/workers/<name>/。"""
code = tmp / "src" / "plugins" / name
(code / "router").mkdir(parents=True, exist_ok=True)
(code / "models").mkdir()
(code / "models" / "__init__.py").write_text("", encoding="utf-8")
(code / "__init__.py").write_text(PLUGIN_INIT.format(extra=init_extra), encoding="utf-8")
(code / "router" / "__init__.py").write_text("from . import hello # noqa: F401\n", encoding="utf-8")
(code / "router" / "hello.py").write_text("async def hello():\n return {}\n", encoding="utf-8")
conf = tmp / "config" / "workers" / name
conf.mkdir(parents=True, exist_ok=True)
(conf / "config.toml").write_text(CONFIG_TOML, encoding="utf-8")
(conf / "hello.toml").write_text("[POST]\nsummary = \"hi\"\n", encoding="utf-8")
for rel, text in (files or {}).items():
path = code / rel
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
return tmp
def run(tmp: Path, *args, name: str = "foo") -> tuple[int, str]:
"""chdir 到现场执行迁移,捕获 stdout。"""
buf = io.StringIO()
cwd = Path.cwd()
os.chdir(tmp)
try:
with contextlib.redirect_stdout(buf):
rc = mp.migrate_plugin_package(name, **kwargs_for(args))
finally:
os.chdir(cwd)
return rc, buf.getvalue()
def kwargs_for(args):
kwargs = {}
for a in args:
kwargs[a] = True
return kwargs
ROUTER_FILES = {
"routes.py": (
"from fastapi import APIRouter\n\n"
"router = APIRouter(prefix=\"/foo\")\n"
),
}
tmp_root = Path(tempfile.mkdtemp(prefix="migrate_plugin_test_"))
# ========== 1. 自定义路由拦截(aibox 写法) ==========
case = make_root(tmp_root / "case_router")
make_plugin(case, init_extra="from .routes import router # noqa: F401", files=ROUTER_FILES)
rc, out = run(case)
check("路由挂载(from .routes import router + APIRouter)拒绝迁移", rc == 1, f"rc={rc}")
check("拒绝时给出路由提示", "APIRouter" in out and "拒绝迁移" in out)
check("拒绝时不动任何文件", (case / "src" / "plugins" / "foo").is_dir() and not (case / "workers").exists())
# ========== 2. star-import 再导出的 APIRouter 同样拦截 ==========
case = make_root(tmp_root / "case_star")
make_plugin(
case,
init_extra="from .router import * # noqa: F401,F403",
files={}, # router/ 由 make_plugin 建好,下面覆盖 __init__.py 为 APIRouter 定义
)
(case / "src" / "plugins" / "foo" / "router" / "__init__.py").write_text(
"from fastapi import APIRouter\n\nrouter = APIRouter()\n", encoding="utf-8"
)
rc, out = run(case)
check("star-import 链上的 APIRouter 拒绝迁移", rc == 1 and "APIRouter" in out, f"rc={rc}")
# ========== 3. --force 跳过拦截 ==========
rc, out = run(case, "force")
check("--force 跳过后可迁移", rc == 0 and (case / "workers" / "foo" / "pyproject.toml").is_file(), f"rc={rc}")
check("--force 输出回显跳过", "跳过上述拦截" in out)
check(
"配置缺 module/load_mode 时自动补全",
'module = "workers.foo"' in (case / "workers" / "foo" / "config" / "workers" / "foo" / "config.toml").read_text(),
)
# ========== 4. 网关包导入拦截 ==========
case = make_root(tmp_root / "case_gw")
make_plugin(case, init_extra="from app.core.logger import get_logger # noqa: F401")
rc, out = run(case)
check("网关包导入拒绝迁移", rc == 1 and "网关" in out, f"rc={rc}")
# ========== 5. dry-run 不落盘 ==========
case = make_root(tmp_root / "case_dry")
make_plugin(case)
before = (case / "plugins_deps.toml").read_text(encoding="utf-8")
rc, out = run(case, "dry_run")
check("--dry-run 返回成功且不落盘", rc == 0 and not (case / "workers").exists())
check("--dry-run 不改 plugins_deps.toml", (case / "plugins_deps.toml").read_text(encoding="utf-8") == before)
check("--dry-run 打印演练提示", "未改动任何文件" in out)
# ========== 6. 目标已存在 / 插件不存在 ==========
case = make_root(tmp_root / "case_exists")
make_plugin(case)
(case / "workers" / "foo").mkdir(parents=True)
rc, out = run(case)
check("目标已存在拒绝迁移", rc == 1 and "目标已存在" in out)
case = make_root(tmp_root / "case_missing")
rc, out = run(case)
check("插件不存在拒绝迁移", rc == 1 and "未找到内置插件" in out)
# ========== 7. 完整迁移(happy path) ==========
case = make_root(tmp_root / "case_full")
make_plugin(case)
rc, out = run(case)
project = case / "workers" / "foo"
check("完整迁移成功", rc == 0 and project.is_dir(), f"rc={rc}")
code_dst = project / "workers" / "foo"
conf_dst = project / "config" / "workers" / "foo"
check(
"代码迁移(含 router/ 与 models/)",
(code_dst / "__init__.py").is_file()
and (code_dst / "router" / "hello.py").is_file()
and (code_dst / "models" / "__init__.py").is_file(),
)
check("路由配置迁移", (conf_dst / "hello.toml").is_file())
conf_text = (conf_dst / "config.toml").read_text(encoding="utf-8")
check(
"[workers] 改写: module=workers.foo / load_mode=external / 其余保留",
'module = "workers.foo"' in conf_text
and 'load_mode = "external"' in conf_text
and 'count = 2' in conf_text
and 'api_key = "PLACEHOLDER"' in conf_text,
)
check("项目骨架生成", all((project / f).is_file() for f in ("config.toml", "pyproject.toml", "README.md", "start.sh")))
check("zenoh 配置复制", (project / "config" / "zenoh_config.json5").is_file())
check("start.sh 可执行且入口正确", (project / "start.sh").stat().st_mode & 0o111 and "chongming-worker --type foo" in (project / "start.sh").read_text())
check("README 标注迁移来源", "migrate_plugin foo" in (project / "README.md").read_text())
check("源目录已删除", not (case / "src" / "plugins" / "foo").exists() and not (case / "config" / "workers" / "foo").exists())
pyproject = tomllib.loads((project / "pyproject.toml").read_text(encoding="utf-8"))
deps = pyproject["project"]["dependencies"]
check("新项目依赖: chongming-worker + 迁移带入", "chongming-worker" in deps and "httpx" in deps and "orjson" in deps and "chongming-cache" in deps, str(deps))
check("extras 写法沿用根 pyproject", "volcengine-python-sdk[ark]" in deps, str(deps))
check("未解析/未使用的依赖不进新项目", not any("totally_missing_pkg" in d for d in deps), str(deps))
check("未解析依赖有告警", "totally_missing_pkg" in out)
deps_data = tomllib.loads((case / "plugins_deps.toml").read_text(encoding="utf-8"))
check(
"plugins_deps.toml: [foo] 节注释掉、共用节与其它节不动",
"foo" not in deps_data and deps_data.get("imports") == ["httpx"] and deps_data.get("bar", {}).get("imports") == ["openai"],
str(deps_data),
)
# ========== 8. ip_sound_column 写法(from . import router 子模块)不误伤 ==========
case = make_root(tmp_root / "case_submod")
make_plugin(case, init_extra="from . import router # noqa: F401 触发 @rpc_method 注册")
rc, out = run(case, "dry_run")
check("from . import router(子模块)不误伤", rc == 0, f"rc={rc}")
shutil.rmtree(tmp_root, ignore_errors=True)
print("\n" + ("=" * 40))
if failures:
print(f"❌ {len(failures)} 项未通过: {failures}")
sys.exit(1)
print("✅ 全部通过")
+123
View File
@@ -0,0 +1,123 @@
"""
Nuitka 打包资源档位 / 链接器注入 测试(不触发真实编译)
覆盖 build.py 里 nuitka 打包的三处防坑逻辑:
1. 链接器:装了 ld.lld 时自动经 LDFLAGS 注入 -fuse-ld=lld(默认 GNU ld 链接大工程
峰值数 GB,小内存机器/WSL 会被 OOM 杀掉:"clang: unable to execute command: Killed");
已有 LDFLAGS 时不覆盖(用户在 CHONGMING_NUITKA_PGO 之外自行指定链接器优先)
2. 内存档位:可用内存 ≥ NUITKA_LTO_MIN_GB 完整优化(LTO);低于则去 LTO(LTO 链接
峰值 3 GB+,实测 ld/lld 都被 OOM 杀过);更低(或 CHONGMING_NUITKA_LOW_MEM=1)
进一步 --low-memory + 降并发;CHONGMING_NUITKA_FULL=1 强制完整优化
3. PGO 缺省关闭:--pgo-c 与 standalone/onefile 不兼容且 profiling 运行会等程序退出
(常驻服务卡死),只能由 CHONGMING_NUITKA_PGO=1 显式打开——
这里用源码断言兜住"默认关"这条(防止回潮)
运行: uv run python test/test_nuitka_pack_flags.py
"""
import io
import contextlib
import os
import shutil
import sys
from pathlib import Path
ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(ROOT / "utils" / "cli" / "src"))
from chongming_cli import build as b # noqa: E402
failures = []
def check(name, cond, detail=""):
status = "PASS" if cond else "FAIL"
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
if not cond:
failures.append(name)
def quiet_note():
"""调 _nuitka_env_and_mem_note 时屏蔽它打印的降档提示。"""
with contextlib.redirect_stdout(io.StringIO()):
return b._nuitka_env_and_mem_note()
saved = {k: os.environ.pop(k, None) for k in
(b.NUITKA_LOW_MEM_ENV, b.NUITKA_FULL_ENV, b.NUITKA_PGO_ENV)}
try:
# ========== 1. 链接器注入 ==========
saved_ldflags = os.environ.pop("LDFLAGS", None)
env, note, profile = quiet_note()
if sys.platform.startswith("linux") and shutil.which("ld.lld"):
check("装了 lld 自动注入 -fuse-ld=lld", env.get("LDFLAGS", "").endswith("-fuse-ld=lld"), env.get("LDFLAGS"))
check("成功信息回显链接器", "lld" in note, note)
else:
check("无 lld 时不注入(本机无 lld,跳过)", "-fuse-ld" not in (env.get("LDFLAGS") or ""))
os.environ["LDFLAGS"] = "-Wl,--as-needed"
env, _, _ = quiet_note()
check("已有 LDFLAGS 时追加而不覆盖", env["LDFLAGS"].startswith("-Wl,--as-needed"), env["LDFLAGS"])
if saved_ldflags is None:
os.environ.pop("LDFLAGS", None)
else:
os.environ["LDFLAGS"] = saved_ldflags
# ========== 2. 内存档位 ==========
mem = b._mem_available_gb()
check("_mem_available_gb 可读(Linux)", mem > 0 or not sys.platform.startswith("linux"), f"{mem:.1f} GB")
env, note, profile = quiet_note()
if mem >= 0:
expect = (
"squeeze" if mem < b.NUITKA_SQUEEZE_GB
else "no-lto" if mem < b.NUITKA_LTO_MIN_GB
else "full"
)
check(
"默认档位与内存阈值一致",
profile == expect,
f"{mem:.1f} GB → {profile}(阈值 {b.NUITKA_SQUEEZE_GB}/{b.NUITKA_LTO_MIN_GB} GB)",
)
check(
"非 full 档在提示里说明原因",
(profile == "full" and "LTO" not in note) or (
profile == "no-lto" and "未启用 LTO" in note
) or (profile == "squeeze" and "--low-memory" in note),
note,
)
os.environ[b.NUITKA_LOW_MEM_ENV] = "1"
env, note, profile = quiet_note()
check(f"{b.NUITKA_LOW_MEM_ENV}=1 强制 squeeze 档", profile == "squeeze" and "--low-memory" in note, note)
os.environ.pop(b.NUITKA_LOW_MEM_ENV)
os.environ[b.NUITKA_FULL_ENV] = "1"
env, note, profile = quiet_note()
check(f"{b.NUITKA_FULL_ENV}=1 强制完整优化(覆盖内存自动判断)", profile == "full" and "--low-memory" not in note, note)
os.environ.pop(b.NUITKA_FULL_ENV)
# ========== 3. PGO 缺省关闭(源码兜底,防回潮) ==========
src = (ROOT / "utils" / "cli" / "src" / "chongming_cli" / "build.py").read_text(encoding="utf-8")
check(
"PGO 参数只在 NUITKA_PGO_ENV 开关下出现",
f'os.environ.get(NUITKA_PGO_ENV) == "1"' in src and b.NUITKA_PGO_ENV == "CHONGMING_NUITKA_PGO",
)
check(
"Nuitka 缺 patchelf 时提前失败(链接后才检查,白跑一整轮)",
"patchelf" in src and "缺少 patchelf" in src,
)
check(
"Nuitka 失败会抛错而不是静默返回(构建要非零退出)",
'raise RuntimeError("Nuitka 编译失败' in src,
)
finally:
for k, v in saved.items():
if v is not None:
os.environ[k] = v
print("\n" + ("=" * 40))
if failures:
print(f"❌ {len(failures)} 项未通过: {failures}")
sys.exit(1)
print("✅ 全部通过")
+147
View File
@@ -0,0 +1,147 @@
"""
父进程死亡防护(PR_SET_PDEATHSIG)测试
背景与语义见 gateway/src/chongming/app/core/parent_death.py:进程树 onefile 引导 →
granian master → worker(fork)。上游被 SIGKILL 时 worker 会成孤儿(继续监听
端口、挂在服务发现里,其它网关派发到"半死副本"每次白吃 2 秒超时)。守卫让内核
在父进程死亡时给本进程发 SIGTERM(走既有优雅停机摘除注册)。
验证(三层 fork 复刻真实进程树:本测试(引导) → 中间进程(master) → 孙进程(worker)):
1. 关闭开关 CHONGMING_NO_PARENT_GUARD=1 → 不生效(返回 False)
2. 中间进程 SIGKILL 自己(模拟 master 被 -9,内核不可捕获)→ 已设守卫的孙进程
收到 SIGTERM 并退出
3. 对照:同样结构但不设守卫 → 孙进程收不到信号(证明信号确实来自 PDEATHSIG,
而不是别的东西清理孤儿)
4. 幂等:重复调用不重复 prctl
运行: uv run python test/test_parent_death.py
"""
import importlib.util
import os
import signal
import sys
import tempfile
import time
from pathlib import Path
ROOT = Path(__file__).parent.parent
_PD_PATH = ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "parent_death.py"
# 独立加载(parent_death 无包内依赖,避免连带导入整个 app 包)
_spec = importlib.util.spec_from_file_location("chongming.app.core.parent_death", _PD_PATH)
pd = importlib.util.module_from_spec(_spec)
sys.modules["chongming.app.core.parent_death"] = pd
_spec.loader.exec_module(pd) # type: ignore[union-attr]
failures = []
def check(name, cond, detail=""):
status = "PASS" if cond else "FAIL"
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
if not cond:
failures.append(name)
# ========== 1. 关闭开关 ==========
os.environ[pd.GUARD_OFF_ENV] = "1"
check("关闭开关打开时不生效", pd.guard_parent_death() is False)
os.environ.pop(pd.GUARD_OFF_ENV, None)
# ========== 2/3. 三层 fork:守卫组 / 对照组 ==========
tmp = Path(tempfile.mkdtemp(prefix="parent_death_test_"))
def scenario(name: str, use_guard: bool) -> dict:
"""中间进程 SIGKILL 自己,观察孙进程是否收到 SIGTERM。"""
d = tmp / name
d.mkdir()
ready, dead, pidf = d / "ready", d / "dead", d / "gc.pid"
middle = os.fork()
if middle == 0: # 中间进程(master 角色)
gc = os.fork()
if gc == 0: # 孙进程(worker 角色)
def _on_term(*_):
dead.write_text("term", encoding="utf-8")
os._exit(0)
signal.signal(signal.SIGTERM, _on_term)
if use_guard:
pd.guard_parent_death()
pidf.write_text(str(os.getpid()), encoding="utf-8")
ready.write_text("1", encoding="utf-8")
time.sleep(15) # 等信号;超时说明没收到
os._exit(1)
for _ in range(250): # 等孙进程就绪,再硬杀自己
if ready.exists():
break
time.sleep(0.02)
os.kill(os.getpid(), signal.SIGKILL)
os._exit(1) # 不会走到
os.waitpid(middle, 0) # 中间进程必死于 SIGKILL
gc_pid = int(pidf.read_text(encoding="utf-8"))
for _ in range(100): # 最多等 5s
if dead.exists():
break
time.sleep(0.05)
got = dead.exists()
try: # 清理(对照组孙进程仍活着)
os.kill(gc_pid, signal.SIGKILL)
except ProcessLookupError:
pass
return {"got_term": got, "gc_pid": gc_pid}
guarded = scenario("guarded", use_guard=True)
check(
"master(SIGKILL) 死后,已设守卫的 worker 收到 SIGTERM 退出",
guarded["got_term"],
f"gc={guarded['gc_pid']}",
)
control = scenario("control", use_guard=False)
check(
"对照组(未设守卫)收不到信号 —— 证明退出确实来自 PDEATHSIG",
not control["got_term"],
f"gc={control['gc_pid']}",
)
# ========== 4. 幂等(在子进程里做:避免给测试进程本身设 PDEATHSIG) ==========
pid = os.fork()
if pid == 0:
a = pd.guard_parent_death()
b = pd.guard_parent_death()
os._exit(0 if (a is True and b is True) else 1)
_, status = os.waitpid(pid, 0)
check("重复调用幂等(不重复 prctl)", os.waitstatus_to_exitcode(status) == 0)
# ========== 5. fork 继承回归(master 先设 → worker 仍必须自己设上) ==========
# 背景:若用模块级"已设置"标志位做幂等,fork 的子进程会继承 True 而短路——
# 但内核在 fork 时恰好清零 PDEATHSIG,于是 worker 其实没设上(本轮实测到过)。
# 这里父进程先真实设置,再 fork,断言子进程调用守卫时**内核状态确实被设上**。
libc5 = pd._load_libc()
pd.guard_parent_death() # 父进程(本测试进程)真实设置一次
pid = os.fork()
if pid == 0:
cleared = pd._current_pdeathsig(libc5) == 0 # fork 后内核已清零(不继承)
ok = pd.guard_parent_death() is True # 回归点:不能因父进程设过而短路
set_now = pd._current_pdeathsig(libc5) == int(signal.SIGTERM)
os._exit(0 if (cleared and ok and set_now) else 1)
_, status = os.waitpid(pid, 0)
libc5.prctl(pd._PR_SET_PDEATHSIG, 0) # 清除测试进程自己的设置,避免副作用
check(
"父进程设过后,fork 的子进程仍会真实设置 PDEATHSIG",
os.waitstatus_to_exitcode(status) == 0,
)
import shutil
shutil.rmtree(tmp, ignore_errors=True)
print("\n" + ("=" * 40))
if failures:
print(f"❌ {len(failures)} 项未通过: {failures}")
sys.exit(1)
print("✅ 全部通过")
+226
View File
@@ -0,0 +1,226 @@
"""
插件专属依赖清单(plugins_deps.toml)测试
"插件专属依赖跟插件走、不混进网关构建配置"这条约定的守卫:
1. 仓库清单可解析:doubao 专属(openai / volcengine / bytedance)与共用(httpx)
都能被 build 的读取/合并逻辑读出来
2. 收集清单 = config.toml [pyinstaller_build_import] + plugins_deps.toml,
去重且网关本体依赖在前;文件缺失时不报错、不追加
3. 拆分不回潮:插件专属依赖不再出现在 config.toml 里(防止又手工加回网关清单)
4. 配置写错(imports 非字符串列表 / 顶层多余键 / 插件节缺 imports)直接抛错,
不静默漏收依赖(漏了要到运行期加载插件才 ImportError)
5. venv 里 chongming-cli 版本低于本地源码时告警:打包用的是 venv 里的 CLI,
不先发布升级,plugins_deps.toml 不会生效
运行: uv run python test/test_plugin_deps.py
"""
import importlib.metadata
import os
import shutil
import sys
import tempfile
import tomllib
from pathlib import Path
ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(ROOT / "utils" / "cli" / "src"))
DOUBAO_DEPS = {
"openai",
"volcenginesdkarkruntime",
"bytedance",
"bytedance.volcengine_aicc_sdk",
"bytedance.volcengine_aicc_sdk.secure_http_client",
}
# build 的根目录定位支持 CHONGMING_ROOT 环境变量,测试结论不应受它影响
os.environ.pop("CHONGMING_ROOT", None)
failures = []
def check(name, cond, detail=""):
status = "PASS" if cond else "FAIL"
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
if not cond:
failures.append(name)
try:
from chongming_cli import build as b # noqa: E402
HAVE_CLI = True
except ImportError as exc: # utils/cli 子仓库未就位(干净克隆)时只跑清单一致性检查
HAVE_CLI = False
print(f"[SKIP] chongming_cli 源码不可用({exc}):仅跑清单一致性检查")
cwd = Path.cwd()
def load_repo_deps() -> dict:
"""读仓库的 plugins_deps.toml(不依赖 chongming_cli,一致性检查也可用)。"""
with open(ROOT / "plugins_deps.toml", "rb") as f:
return tomllib.load(f)
def flat_modules(data: dict) -> set[str]:
"""plugins_deps.toml 里的全部模块(共用 imports + 各插件节)。"""
mods = set(data.get("imports", []))
for value in data.values():
if isinstance(value, dict):
mods |= set(value.get("imports", []))
return mods
# ========== 1. 仓库清单可解析(build 的读取逻辑直测) ==========
if HAVE_CLI:
os.chdir(ROOT)
try:
modules, summary = b._load_plugin_imports()
finally:
os.chdir(cwd)
check(
"仓库 plugins_deps.toml 可解析且含 doubao 专属依赖",
DOUBAO_DEPS <= set(modules),
f"缺失: {sorted(DOUBAO_DEPS - set(modules))}",
)
check("共用节(httpx)并入清单", "httpx" in modules)
check("摘要含 doubao 节名", "doubao" in summary, summary)
# ========== 2. 合并:网关本体在前、去重保序 ==========
fake_config = {
"pyinstaller_build_import": {
"base": ["fastapi", "openai"], # openai 故意与插件清单重复,验证去重
"linux": ["uvloop"],
"windows": ["win_only"],
}
}
os.chdir(ROOT)
try:
merged, _ = b._collect_build_imports(fake_config)
finally:
os.chdir(cwd)
check(
"合并后网关本体依赖在前且 openai 只出现一次",
merged[0] == "fastapi" and merged[1] == "openai" and merged.count("openai") == 1,
str(merged[:6]),
)
check(
"插件专属依赖追加在网关清单之后",
"httpx" in merged and merged.index("httpx") > merged.index("openai"),
)
# ========== 3. 拆分不回潮:config.toml 里不应再有插件专属依赖 ==========
repo_modules = flat_modules(load_repo_deps())
with open(ROOT / "config.toml", "rb") as f:
cfg = tomllib.load(f)
bcfg = cfg["pyinstaller_build_import"]
gateway_imports = set(bcfg["base"]) | set(bcfg["linux"]) | set(bcfg["windows"])
check(
"doubao 专属依赖已从 config.toml 移出(只留 plugins_deps.toml)",
not (DOUBAO_DEPS & gateway_imports),
f"回潮: {sorted(DOUBAO_DEPS & gateway_imports)}",
)
check(
"两份清单不重叠(插件专属 vs 网关本体)",
not (repo_modules & gateway_imports),
f"重叠: {sorted(repo_modules & gateway_imports)}",
)
# ========== 4. 文件缺失 / 写错的行为 ==========
tmp = Path(tempfile.mkdtemp(prefix="plugin_deps_test_"))
(tmp / "pyproject.toml").write_text("[project]\nname = 'x'\n", encoding="utf-8")
if HAVE_CLI:
(tmp / "plugins_deps.toml").unlink(missing_ok=True)
cfg_only = {"pyinstaller_build_import": {"base": ["fastapi"], "linux": [], "windows": []}}
os.chdir(tmp)
try:
mods, summary = b._load_plugin_imports()
merged, _ = b._collect_build_imports(cfg_only)
finally:
os.chdir(cwd)
check(
"无 plugins_deps.toml:不报错、不追加",
mods == [] and summary == "" and merged == ["fastapi"],
f"{mods} / {merged}",
)
def expect_error(name: str, text: str) -> None:
(tmp / "plugins_deps.toml").write_text(text, encoding="utf-8")
os.chdir(tmp)
try:
try:
b._load_plugin_imports()
check(name, False, "未抛错")
except ValueError as exc:
check(name, True, str(exc))
finally:
os.chdir(cwd)
expect_error("imports 非列表 → 报错", 'imports = "openai"\n')
expect_error("imports 含非字符串 → 报错", "[doubao]\nimports = [1, 2]\n")
expect_error("插件节缺 imports(如写成 import)→ 报错", "[doubao]\nimport = ['openai']\n")
expect_error("顶层多余标量键 → 报错", "foo = 1\n")
# 合法多节:共用 + 逐插件(去重发生在合并阶段,读取阶段保持原样)
(tmp / "plugins_deps.toml").write_text(
'imports = ["httpx"]\n\n[alpha]\nimports = ["pkg_a"]\n\n'
'[beta]\nimports = ["pkg_b", "pkg_a"]\n',
encoding="utf-8",
)
os.chdir(tmp)
try:
mods, summary = b._load_plugin_imports()
finally:
os.chdir(cwd)
check(
"多节解析:共用 + 逐插件均并入,摘要含各节",
mods == ["httpx", "pkg_a", "pkg_b", "pkg_a"]
and "alpha:1" in summary
and "beta:2" in summary,
f"{mods} / {summary}",
)
shutil.rmtree(tmp, ignore_errors=True)
# ========== 5. venv 里的 CLI 版本(打包装箱用的是它,不是本文件旁边的源码) ==========
local_ver = tomllib.loads((ROOT / "utils" / "cli" / "pyproject.toml").read_text())[
"project"
]["version"]
def as_tuple(v: str):
return tuple(int(x) for x in v.split("."))
try:
installed_ver = importlib.metadata.version("chongming-cli")
except importlib.metadata.PackageNotFoundError:
installed_ver = None
if installed_ver is None:
print("[WARN] venv 未安装 chongming-cli:先 uv sync 或发布后再构建")
else:
try:
outdated = as_tuple(installed_ver) < as_tuple(local_ver)
except ValueError: # 非常规版本号(如 rc/后缀),不猜,直接提示
outdated = True
if outdated:
print(
f"[WARN] venv 里 chongming-cli {installed_ver} < 源码 {local_ver}:"
"plugins_deps.toml 需新版 build 才生效 —— 发布 chongming-cli 后执行 "
"uv lock --upgrade-package chongming-cli && uv sync"
)
else:
check(f"venv chongming-cli {installed_ver} ≥ 源码 {local_ver}", True)
print("\n" + ("=" * 40))
if failures:
print(f"❌ {len(failures)} 项未通过: {failures}")
sys.exit(1)
print("✅ 全部通过")
+91
View File
@@ -0,0 +1,91 @@
"""
worker 超时冷静期(派发负反馈)测试
背景:半死副本(被 SIGKILL 的实例留下的孤儿 worker 等)只要还在服务发现里,
每次被随机选中都会白吃一整轮超时(默认 2s);worker_manager 只做"本次请求换副本
重试",没有跨请求记忆,于是 2 秒惩罚反复发生(实测一轮 31 条超时日志)。
修复:WorkerManager.mark_worker_suspect 给超时/无回复的副本记一段冷静期
(_suspect_until,TTL 30s),三条选副本路径(get_worker / _pick_from_instance /
_pick_other_worker)都优先跳过它,全在冷静期时退回全集(宁慢不缺)。
验证:
1. 标记后的副本跨请求不再被选中
2. 全部候选都处于冷静期时退回全集(不饿死)
3. TTL 到期自动恢复
4. 过期项会被清理(防长期运行膨胀)
5. 换副本路径同样避开冷静期
6. 源码兜底:超时/无回复分支确实调用了标记
运行: uv run python test/test_worker_suspect.py
"""
import asyncio
import sys
import time
from pathlib import Path
ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(ROOT / "gateway" / "src"))
from chongming.app.core.worker_manager import WorkerManager # noqa: E402
failures = []
def check(name, cond, detail=""):
status = "PASS" if cond else "FAIL"
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
if not cond:
failures.append(name)
# 跳过 __init__(依赖 zenoh 会话/缓存),只装配选副本所需的字段
m = WorkerManager.__new__(WorkerManager)
m._suspect_until = {}
m._workers_cache = {"demo": ["w1", "w2", "w3"]}
m._local_workers = {}
def sample(fn, n=40):
return {asyncio.run(fn()) for _ in range(n)}
# ========== 1. 标记后跨请求跳过 ==========
m.mark_worker_suspect("w1")
m.mark_worker_suspect("w2")
picked = sample(lambda: m.get_worker("demo"))
check("冷静期内的副本不再被选中", picked == {"w3"}, str(picked))
# ========== 2. 全在冷静期 → 退回全集 ==========
m.mark_worker_suspect("w3")
picked = sample(lambda: m.get_worker("demo"))
check("全部处于冷静期时退回全集(不饿死)", picked == {"w1", "w2", "w3"}, str(picked))
# ========== 3. TTL 到期恢复 ==========
m._suspect_until = {"w1": time.monotonic() - 0.1}
check("冷静期过期自动恢复", m._filter_suspects(["w1"]) == ["w1"])
# ========== 4. 过期项清理 ==========
m._suspect_until = {}
for i in range(40):
m.mark_worker_suspect(f"x{i}", ttl=-1.0) # 立即过期
check("过期项会被清理(防膨胀)", len(m._suspect_until) <= 32, str(len(m._suspect_until)))
# ========== 5. 换副本路径同样避开 ==========
m._suspect_until = {}
m.mark_worker_suspect("w1")
picked = sample(lambda: m._pick_other_worker("demo", []))
check("换副本路径同样避开冷静期", picked == {"w2", "w3"}, str(picked))
# ========== 6. 源码兜底:失败分支确实标记 ==========
src = (ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py").read_text(
encoding="utf-8"
)
n = src.count("self.mark_worker_suspect(worker_expr)")
check("超时/无回复分支都调用了标记", n >= 2, f"{n} 处")
print("\n" + ("=" * 40))
if failures:
print(f"❌ {len(failures)} 项未通过: {failures}")
sys.exit(1)
print("✅ 全部通过")