modify 0.3.5

This commit is contained in:
2026-10-13 13:27:26 +08:00
parent cd841f4f2d
commit ee8e71a49b
6 changed files with 641 additions and 34 deletions
+26
View File
@@ -10,6 +10,7 @@ chongming 网关的命令脚本集合,独立包维护(从 `src/chongming/scr
| `build_plugin` | 进程插件项目根直接压缩为可部署 zip |
| `gen_models` | 从插件配置 `*.toml` 生成 Pydantic 模型 |
| `gen_plugin` | 生成风格统一的 Worker 插件骨架(`--process` 生成进程插件完整项目) |
| `migrate_plugin` | 内置插件一键迁移为进程插件(`src/plugins/<name>` → `workers/<name>/`) |
另有 `chongming_cli.code_content` 模块:将目录下代码文件汇总为 Markdown(`python -m chongming_cli.code_content <dir>`)。
@@ -27,8 +28,16 @@ uv run gen_plugin myplugin # 生成 inline 插件骨架
uv run gen_plugin myplugin --process # 生成进程插件完整项目(workers/myplugin/)
uv run gen_models config/workers/myplugin src/plugins/myplugin # inline 插件模型
uv run gen_models workers/myplugin/config/workers/myplugin workers/myplugin/workers/myplugin # 进程插件模型
uv run migrate_plugin myplugin --dry-run # 内置插件迁移为进程插件(仅演练)
uv run migrate_plugin myplugin # 实际迁移:建项目 + 迁移代码/配置/依赖 + 删源
```
`migrate_plugin` 前置检查(命中则拒绝迁移,`--force` 跳过):插件挂载了自定义
FastAPI 路由(`APIRouter`,external 插件挂不了,需先改成 TOML `@rpc_method` 端点
或保持 inline);插件导入了网关包(`app` / `chongming`,进程插件与网关零导入关系)。
迁移后依赖从 `plugins_deps.toml` 的 `[<name>]` 节(按代码实际 import 过滤)转入
新项目 `pyproject.toml`,该节自动注释掉 —— 依赖随插件走,不再进网关 exe。
`build --only <step>` 可选步骤:`mbank` / `prepare` / `obfuscate` / `pack` / `clean` / `license_guard` / `vue`,可逗号分隔或重复传多个。
`build` 的两个开关默认**都关**(不加参数即内部调试产物:明文模块、不带授权,横幅与构建报告里会回显状态):
@@ -40,6 +49,23 @@ uv run gen_models workers/myplugin/config/workers/myplugin workers/myplugin/work
内置插件专属依赖(网关本体不 import、只在插件里用)在根目录 `plugins_deps.toml`
(按插件分节,可选文件),`pack` 时自动并入 `--hidden-import` / `--include-module`。
### Nuitka 打包的环境要求(Linux)
- **patchelf 必需**(standalone/onefile 都要):已列进网关 dev 依赖(`uv sync` 自动带),
也可 `sudo apt install patchelf`。Nuitka 直到链接完成后才检查它——缺失会白跑一整轮
编译,build 已改为提前失败提示
- **链接器**:默认 GNU ld 链接大工程峰值数 GB,小内存机器/WSL 上会被 OOM 杀掉
(`clang: unable to execute command: Killed`)。检测到 `ld.lld` 时自动经 `LDFLAGS`
换成 lld(内存与耗时都低得多)
- **内存档位**(按 `MemAvailable` 自动选):≥ 6 GB 完整优化(`--lto=yes`);
< 6 GB 去 LTO(LTO 链接峰值 3 GB+,实测 GNU ld 与 lld 都会被 OOM 杀掉);
< 2.5 GB 再 `--low-memory` 降并发。`CHONGMING_NUITKA_LOW_MEM=1` 强制省内存档、
`CHONGMING_NUITKA_FULL=1` 强制完整优化
- **PGO 默认关闭**:Nuitka 4.3 的 `--pgo-c` 与 standalone/onefile 不兼容(Nuitka 自身
WARNING 后 FATAL),且 profiling 运行会启动程序等它退出——常驻服务会卡死;
能自行退出的程序可设 `CHONGMING_NUITKA_PGO=1` 打开
- 失败自动逐档重试:完整优化 → 去 LTO/UPX → 去 LTO/UPX + `--low-memory`
## 项目根目录定位
脚本需要在项目根目录下读写 `config/`、`src/plugins/`、`workers/` 等路径。
+2 -1
View File
@@ -4,7 +4,7 @@ requires = ["hatchling"]
[project]
name = "chongming-cli"
version = "0.3.4"
version = "0.3.5"
description = "chongming 命令脚本集合:构建打包 / 进程插件打包 / 插件脚手架 / 模型生成 / 机器码授权签发"
readme = "README.md"
requires-python = ">=3.12"
@@ -30,4 +30,5 @@ build = "chongming_cli.build:build"
build_plugin = "chongming_cli.build_plugin:build_plugin"
gen_models = "chongming_cli.gen_models:gen_models"
gen_plugin = "chongming_cli.gen_plugin:gen_plugin"
migrate_plugin = "chongming_cli.migrate_plugin:migrate_plugin"
chongming-license = "chongming_cli.license:license_cmd"
+141 -19
View File
@@ -81,6 +81,16 @@ is_windows = platform.system() == "Windows"
# 插件下线/改 external 进程插件时删掉对应节即可,网关配置不受影响。
PLUGIN_DEPS_FILE = "plugins_deps.toml"
# Nuitka 打包的资源门槛:LTO 链接本仓库规模峰值 3 GB+ —— 实测 GNU ld 与 lld 都会被
# OOM 杀掉("clang: unable to execute command: Killed",机器可用内存不足时)。
# 可用内存 < NUITKA_LTO_MIN_GB 自动去掉 LTO;< NUITKA_SQUEEZE_GB 进一步
# --low-memory + 降并发。环境变量可强制档位(见 _nuitka_env_and_mem_note)。
NUITKA_LTO_MIN_GB = 6.0
NUITKA_SQUEEZE_GB = 2.5
NUITKA_LOW_MEM_ENV = "CHONGMING_NUITKA_LOW_MEM"
NUITKA_FULL_ENV = "CHONGMING_NUITKA_FULL"
NUITKA_PGO_ENV = "CHONGMING_NUITKA_PGO"
# ── 构建结果数据结构 ─────────────────────────────────────────────────────────
@@ -155,8 +165,9 @@ def _run_cmd(
capture: bool = True,
shell: bool | None = None,
echo_on_error: bool = True,
env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess:
"""统一执行外部命令"""
"""统一执行外部命令(env 传给子进程,用于如 LDFLAGS 注入链接器)"""
if shell is None:
shell = is_windows
result = subprocess.run(
@@ -167,6 +178,7 @@ def _run_cmd(
shell=shell,
encoding="utf-8",
errors="replace",
env=env,
)
# 命令失败时自动回显捕获的输出,便于定位失败原因
if echo_on_error and capture and result.returncode != 0:
@@ -652,12 +664,84 @@ def _stage_pyinstaller(config: Config) -> tuple[str, str]:
# ── 阶段 5: Nuitka 打包 ────────────────────────────────────────────────────
def _mem_available_gb() -> float:
"""可用内存(GB,读 /proc/meminfo 的 MemAvailable;非 Linux/读不到返回 -1)。"""
try:
for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines():
if line.startswith("MemAvailable:"):
return int(line.split()[1]) / (1024 * 1024)
except (OSError, ValueError, IndexError):
pass
return -1.0
def _nuitka_env_and_mem_note() -> tuple[dict[str, str], str, str]:
"""Nuitka 打包的运行环境与资源档位。
返回 (子进程 env, 成功信息附加说明, 档位):
- 链接器:默认 GNU ld 链接本仓库规模(1400+ C 文件)峰值数 GB,加 LTO 更高,
WSL/小内存机器上会被 OOM 杀掉(clang: unable to execute command: Killed;
实测 ld 3.2-3.7 GB、lld 3.05 GB 峰值都被杀)。Nuitka 的 Scons 后端读取
LDFLAGS(SconsCompilerSettings)——装了 lld 就换成 lld(--clang 下
-fuse-ld=lld),比 GNU ld 更省内存/更快。
- 档位(按 /proc/meminfo 的 MemAvailable 自动选,env 可强制):
full 可用 ≥ NUITKA_LTO_MIN_GB:--lto=yes --jobs=4
no-lto 可用 < NUITKA_LTO_MIN_GB:去 LTO(链接峰值是 LTO 的大头)
squeeze 可用 < NUITKA_SQUEEZE_GB 或 NUITKA_LOW_MEM_ENV=1:
--low-memory --jobs=2(编译阶段也压低内存)
NUITKA_FULL_ENV=1 可强制 full(已知内存够用/CI 大机器)
"""
env = dict(os.environ)
note = ""
if shutil.which("ld.lld") and "-fuse-ld" not in (env.get("LDFLAGS") or ""):
env["LDFLAGS"] = f"{env.get('LDFLAGS', '')} -fuse-ld=lld".strip()
note += ",链接器 lld"
mem_gb = _mem_available_gb()
profile = "full"
if mem_gb >= 0 and mem_gb < NUITKA_SQUEEZE_GB:
profile = "squeeze"
elif mem_gb >= 0 and mem_gb < NUITKA_LTO_MIN_GB:
profile = "no-lto"
if os.environ.get(NUITKA_LOW_MEM_ENV) == "1":
profile = "squeeze"
if os.environ.get(NUITKA_FULL_ENV) == "1":
profile = "full" # 显式要求完整优化(已知内存够用时)
if profile != "full":
avail = f"{mem_gb:.1f} GB" if mem_gb >= 0 else "未知"
if profile == "no-lto":
print(
f" ⚠️ 可用内存 {avail} < {NUITKA_LTO_MIN_GB:.0f} GB:本次不做 LTO"
"(LTO 链接峰值 3 GB+,实测会被 OOM 杀掉;内存充足时自动恢复完整优化)"
)
note += ",未启用 LTO"
else:
print(
f" ⚠️ 可用内存 {avail} < {NUITKA_SQUEEZE_GB:.0f} GB 或设了 "
f"{NUITKA_LOW_MEM_ENV}=1:本次 --low-memory 降并发打包"
)
note += ",--low-memory"
return env, note, profile
def _stage_nuitka(config: Config) -> tuple[str, str]:
"""执行 Nuitka 打包,返回 (状态消息, 可执行文件路径)"""
"""执行 Nuitka 打包,成功返回 (状态消息, 可执行文件路径),失败抛 RuntimeError。"""
build_imports, plugin_summary = _collect_build_imports(config)
if plugin_summary:
print(f" 📎 并入插件专属依赖 {PLUGIN_DEPS_FILE}({plugin_summary})")
# patchelf 是 Nuitka standalone/onefile 的宿主机依赖,且 Nuitka 直到链接完成后的
# 打包阶段才检查 —— 缺了会白跑一整轮编译(1000+ C 文件),这里提前失败
if not is_windows and shutil.which("patchelf") is None:
raise RuntimeError(
"缺少 patchelf(Nuitka standalone/onefile 在 Linux 必需,且链接完成后才检查):"
"sudo apt install patchelf,或 uv pip install patchelf(注意 uv sync 会清掉后者)"
)
env, note, profile = _nuitka_env_and_mem_note()
cmd = [
sys.executable,
"-m",
@@ -669,16 +753,29 @@ def _stage_nuitka(config: Config) -> tuple[str, str]:
"--static-libpython=yes",
"--remove-output",
"--assume-yes-for-downloads",
"--lto=yes",
"--pgo-c",
"--pgo-python",
]
if profile == "full":
cmd += ["--lto=yes", "--jobs=4"]
# PGO 默认关闭(两个原因,实测确认):
# 1. Nuitka 4.3 对 standalone/onefile + --pgo-c 自身 WARNING
# "Using C level PGO with standalone/onefile mode is not currently working",
# 随后在 C-PGO 收集阶段 FATAL;
# 2. profiling 运行会真正启动程序并等它退出(Popen.communicate 无超时)——
# 网关是常驻服务、永不退出,构建会卡死。
# 能自行退出的程序可设 CHONGMING_NUITKA_PGO=1 打开(--pgo-args 可传参)
if os.environ.get(NUITKA_PGO_ENV) == "1":
cmd += ["--pgo-c", "--pgo-python"]
elif profile == "no-lto":
cmd += ["--jobs=4"]
else: # squeeze:编译阶段也压低内存(low-memory 默认串行,开 2 路兼顾速度)
cmd += ["--low-memory", "--jobs=2"]
cmd += [
"--disable-plugin=tk-inter",
"--output-dir=.",
"--output-filename=chongming",
"main.py",
"--jobs=4",
]
if shutil.which("upx"):
if profile == "full" and shutil.which("upx"):
cmd.append("--enable-plugin=upx")
for mod in build_imports:
@@ -689,17 +786,42 @@ def _stage_nuitka(config: Config) -> tuple[str, str]:
if icon.exists():
cmd.append(f"--windows-icon-from-ico={icon}")
result = _run_cmd(cmd, cwd="build", capture=False)
if result.returncode != 0:
# 降级重试(移除可能不兼容的参数)
print(
f" ⚠️ Nuitka 首次编译失败 (exit={result.returncode}),"
f"尝试移除不兼容参数重试..."
)
fallback = [
arg for arg in cmd if arg not in ("--lto=yes", "--enable-plugin=upx")
# 降级重试链(逐档去掉开销/内存最大的参数):完整优化 → 去 LTO/UPX →
# 去 LTO/PGO/UPX + --low-memory。注意改编译参数会让 Nuitka 的中间目录失效、
# 整轮重编(十几分钟),所以首选档位由内存自动选(见 _nuitka_env_and_mem_note),
# 这条链只兜底。
candidates: list[list[str]] = [
cmd,
[a for a in cmd if a not in ("--lto=yes", "--enable-plugin=upx")],
[
a
for a in cmd
if a not in ("--lto=yes", "--enable-plugin=upx",
"--pgo-c", "--pgo-python", "--jobs=4")
]
_run_cmd(fallback, cwd="build", capture=False)
+ ["--low-memory", "--jobs=2"],
]
attempts: list[list[str]] = []
for candidate in candidates:
if candidate not in attempts:
attempts.append(candidate)
result = None
for i, attempt in enumerate(attempts):
result = _run_cmd(attempt, cwd="build", capture=False, env=env)
if result.returncode == 0:
break
if i + 1 < len(attempts):
print(f" ⚠️ Nuitka 第 {i + 1} 次尝试失败 (exit={result.returncode}),"
"降档重试…")
else:
if result is not None and result.returncode != 0:
print(
" 💡 链接阶段被杀(clang: unable to execute command: Killed)"
"基本都是内存不足:构建前关掉 VS Code / 容器 / dev server 释放内存,"
"或给 WSL 提高上限(在 C:\\Users\\<用户名>\\.wslconfig 加 "
"[wsl2] 段 memory=14GB、swap=8GB 后 `wsl --shutdown` 重进)"
)
exe = Path("build/chongming")
if exe.exists():
@@ -709,9 +831,9 @@ def _stage_nuitka(config: Config) -> tuple[str, str]:
if shutil.which("upx"):
subprocess.run(["upx", "--best", str(exe)], check=False)
size = _format_size(exe)
return f"✅ Nuitka 编译完成 ({size})", str(exe)
return f"✅ Nuitka 编译完成 ({size}{note})", str(exe)
return "❌ Nuitka 编译失败 (未生成可执行文件)", ""
raise RuntimeError("Nuitka 编译失败 (未生成可执行文件,详见上方输出)")
# ── 阶段 6: 清理临时文件 ────────────────────────────────────────────────────
+14 -13
View File
@@ -310,20 +310,21 @@ json5_path = "config/zenoh_config.json5"
"""
def _gen_pyproject_toml(name: str) -> str:
def _gen_pyproject_toml(name: str, extra_deps: list[str] | None = None) -> str:
"""插件 uv 依赖清单(chongming-worker + 扩展依赖,扩展依赖自行追加)。
extra_deps:额外依赖 spec(migrate_plugin 从 plugins_deps.toml 解析而来,
标注来源便于区分脚手架占位与迁移带入)。
注意:不声明 [project.scripts]——插件项目非打包项目(无 build-system),
uv sync 会跳过其入口点;worker 进程入口由 SDK 的 chongming-worker 脚本提供。
"""
deps = [
"chongming-worker",
]
deps_dev = [
"chongming-cli",
]
deps_lines = "\n".join(f' "{d}",' for d in deps)
deps_dev_lines = "\n".join(f' "{d}",' for d in deps_dev)
deps_lines = [' "chongming-worker",']
if extra_deps:
deps_lines.append(" # ↓ 迁移自带(原 plugins_deps.toml 声明,已按代码实际 import 过滤)")
deps_lines.extend(f' "{d}",' for d in extra_deps)
deps = "\n".join(deps_lines)
deps_dev_lines = "\n".join([' "chongming-cli",'])
return f"""# {name} 进程插件依赖清单(扩展依赖直接在此追加,如 "minio")
[project]
name = "{name}-worker"
@@ -331,7 +332,7 @@ version = "0.1.0"
description = "{name} 进程插件(external 模式,独立容器部署)"
requires-python = ">=3.12"
dependencies = [
{deps_lines}
{deps}
]
[[tool.uv.index]]
@@ -348,10 +349,10 @@ dev = [
"""
def _gen_start_sh(name: str) -> str:
def _gen_start_sh(name: str, generated_by: str = "gen_plugin --process") -> str:
"""容器入口脚本:uv 引导依赖 + 启动 worker 进程(SDK 的 chongming-worker 脚本)。"""
return f"""#!/usr/bin/env bash
# {name} 进程插件容器入口(由 gen_plugin --process 生成)
# {name} 进程插件容器入口(由 {generated_by} 生成)
# 要求: 镜像预装 uv(https://docs.astral.sh/uv/)
set -e
cd "$(dirname "$0")"
@@ -961,7 +962,7 @@ logs/
"""
_RUST_BUILD_SH = """#!/usr/bin/env bash
_RUST_BUILD_SH = r"""#!/usr/bin/env bash
# Rust 进程插件打包器(自包含于插件根,与 Python `build_plugin` 对位)
#
# 用法(在插件根目录执行):
+457
View File
@@ -0,0 +1,457 @@
"""
内置插件 → 进程插件 一键迁移(inline → external)
用法:
uv run migrate_plugin <name> [--dry-run] [--force]
把 src/plugins/<name>(inline,网关进程内加载)迁移为 workers/<name>/(自包含的
进程插件项目:开发目录即打包产物,见 build_plugin 与 docs/process-plugin.md)。
在仓库根执行。
迁移动作:
1. 前置检查(不通过则拒绝迁移,给出可操作提示):
- 自定义 FastAPI 路由:网关只对 inline 插件做 getattr(module, "router") 挂载
(且须是 APIRouter 实例),external 插件挂不上 —— 命中则提示先改成 TOML
@rpc_method 端点或保持 inline;确认无影响可加 --force 跳过拦截
- 网关包导入(import app / from chongming...):进程插件与网关零导入关系,
独立进程里这些模块不存在
2. 建项目骨架(复用 gen_plugin 的 --process 模板:config.toml / pyproject.toml /
README.md / start.sh / config/zenoh_config.json5)
3. 迁移代码 src/plugins/<name>/ → workers/<name>/workers/<name>/
4. 迁移配置 config/workers/<name>/ → workers/<name>/config/workers/<name>/
([workers] 节改写:module = "workers.<name>"、load_mode = "external")
5. 迁移依赖:plugins_deps.toml 的 [<name>] 节(按代码实际 import 过滤)解析为
发行包 spec 写入新项目 pyproject.toml,并把该节注释掉 —— 依赖随插件走,
不再进网关 exe
6. 删除源目录(src/plugins/<name>、config/workers/<name>,均可在各自 git 仓库恢复)
7. 打印下一步(uv sync / 本地跑 / build_plugin / 网关侧重打 mbank 等)
注意:迁移后该插件的 HTTP 路由改经 Zenoh 由网关 RouteRegistry 动态挂载,
失去 inline 的进程内直调快路径;[workers.config] 中的相对路径(如 data_dir)
改为相对插件项目根解析,有存量数据需自行搬移。
"""
import argparse
import ast
import re
import shutil
import sys
import tomllib
from pathlib import Path
from . import gen_plugin as gp
from ._root import find_project_root
NAME_RE = re.compile(r"^[a-z][a-z0-9_]*$")
ROUTER_HINT = (
"自定义 FastAPI 路由(APIRouter)只能由网关对 inline 插件挂载,external 插件"
"不参与网关路由发现(迁移后这些端点会 404)。请先把它们改成 TOML @rpc_method "
"形式(网关据此自动注册 HTTP 路由),或保留该插件为 inline;确认无影响可加 "
"--force 跳过拦截。"
)
GATEWAY_IMPORT_HINT = (
"进程插件与网关零导入关系(独立进程/容器里没有网关包,也不该有),"
"请先去掉这些 import,或改用 chongming_* 工具包。"
)
def _rel(root: Path, path: Path) -> str:
"""相对仓库根的路径(不在根下时原样返回,供打印)。"""
try:
return str(path.relative_to(root))
except ValueError:
return str(path)
# ── 源码检查(静态 AST,不 import 插件:导入会触发注册/依赖) ───────────────
def _iter_py(code_dir: Path) -> list[Path]:
return sorted(p for p in code_dir.rglob("*.py") if "__pycache__" not in p.parts)
def _parse(path: Path) -> ast.Module | None:
try:
return ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
except (SyntaxError, UnicodeDecodeError):
return None
def _call_name(node: ast.Call) -> str:
func = node.func
if isinstance(func, ast.Name):
return func.id
if isinstance(func, ast.Attribute):
return func.attr
return ""
def _router_findings(code_dir: Path) -> list[str]:
"""检测自定义 FastAPI 路由的挂载约定(与网关发现逻辑对齐)。
网关侧判据:`getattr(plugins.<name>, "router")` 且 `isinstance(..., APIRouter)`。
静态检测两条:
1. 任意文件实例化 `APIRouter(...)`(本仓库插件协议里 fastapi 路由的唯一用途)
2. 顶层 __init__.py 绑定名 router —— 赋值(排除字面量,如 router = None)或
`from <mod> import router`;`from . import router`(module 为 None)绑定的是
子模块(ip_sound_column 这种触发 @rpc_method 注册的写法),不是 APIRouter,不算
"""
findings: list[str] = []
for path in _iter_py(code_dir):
tree = _parse(path)
if tree is None:
continue
for node in ast.walk(tree):
if isinstance(node, ast.Call) and _call_name(node) == "APIRouter":
findings.append(f"{path.relative_to(code_dir)}:{node.lineno} 实例化 APIRouter")
init = code_dir / "__init__.py"
tree = _parse(init) if init.is_file() else None
if tree is not None:
for node in tree.body:
if isinstance(node, (ast.Assign, ast.AnnAssign)):
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
if not any(isinstance(t, ast.Name) and t.id == "router" for t in targets):
continue
if isinstance(node.value, ast.Constant): # router = None 之类不是 APIRouter
continue
findings.append(f"__init__.py:{node.lineno} 顶层绑定 router 属性")
break
if isinstance(node, ast.ImportFrom) and node.module:
if any((a.asname or a.name) == "router" for a in node.names):
findings.append(f"__init__.py:{node.lineno} from {node.module} import router")
break
return findings
def _gateway_import_findings(code_dir: Path) -> list[str]:
"""检测网关包导入(app / chongming,进程插件侧不存在)。"""
findings: list[str] = []
for path in _iter_py(code_dir):
tree = _parse(path)
if tree is None:
continue
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
if alias.name.split(".")[0] in ("app", "chongming"):
findings.append(f"{path.relative_to(code_dir)}:{node.lineno} import {alias.name}")
elif isinstance(node, ast.ImportFrom) and node.module:
if node.module.split(".")[0] in ("app", "chongming"):
findings.append(
f"{path.relative_to(code_dir)}:{node.lineno} from {node.module} import ..."
)
return findings
def _third_party_modules(code_dir: Path) -> set[str]:
"""插件代码实际 import 的顶层第三方模块名(排除标准库与相对导入)。"""
mods: set[str] = set()
for path in _iter_py(code_dir):
tree = _parse(path)
if tree is None:
continue
for node in ast.walk(tree):
if isinstance(node, ast.Import):
names = [a.name for a in node.names]
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
names = [node.module]
else:
continue
for full in names:
root = full.split(".")[0]
if root and root not in sys.stdlib_module_names:
mods.add(root)
return mods
# ── 依赖解析(plugins_deps.toml 的 import 名 → 新项目 pyproject 的发行包 spec) ──
def _normalize(spec: str) -> str:
"""依赖 spec → 规范化的发行包名(去 extras/版本约束、_→-、小写)。"""
base = re.split(r"[\[<>=!~; ]", spec, maxsplit=1)[0].strip().lower()
return base.replace("_", "-")
def _dist_specs(import_names: list[str], root: Path) -> tuple[list[str], list[str]]:
"""import 名 → 发行包 spec,返回 (已解析, 未解析)。
解析顺序:chongming_<x> → chongming-<x>;其余用 packages_distributions 反查;
命中根 pyproject.toml 里同发行包的写法时优先沿用(保留 extras/版本约束,
如 volcengine-python-sdk[ark])。未解析的(未安装/查不到发行包)交调用方提示。
"""
from importlib.metadata import packages_distributions
root_specs: dict[str, str] = {}
pyproject = root / "pyproject.toml"
if pyproject.is_file():
deps = (
tomllib.loads(pyproject.read_text(encoding="utf-8"))
.get("project", {})
.get("dependencies", [])
)
for spec in deps:
root_specs[_normalize(spec)] = spec
table = packages_distributions()
specs: list[str] = []
unresolved: list[str] = []
for imp in import_names:
root_name = imp.split(".")[0]
if root_name.startswith("chongming_"):
specs.append(root_specs.get(_normalize(root_name), root_name.replace("_", "-")))
continue
dists = table.get(root_name) or []
if not dists:
unresolved.append(imp)
continue
chosen = next((d for d in dists if _normalize(d) in root_specs), dists[0])
specs.append(root_specs.get(_normalize(chosen), chosen))
return list(dict.fromkeys(specs)), unresolved
# ── 配置与依赖清单改写 ─────────────────────────────────────────────────────
def _rewrite_workers_section(text: str, name: str) -> str:
"""改写 [workers] 节:module → workers.<name>、补 load_mode = "external"。
只动该节内的这两行,[worker]/[workers.config] 与注释原样保留。
"""
lines = text.splitlines()
try:
start = next(i for i, line in enumerate(lines) if line.strip() == "[workers]")
except StopIteration:
raise ValueError("config.toml 缺少 [workers] 节") from None
end = next(
(i for i in range(start + 1, len(lines)) if lines[i].lstrip().startswith("[")),
len(lines),
)
module_re = re.compile(r"^\s*module\s*=")
load_re = re.compile(r"^\s*load_mode\s*=")
section: list[str] = []
has_module = has_load = False
for line in lines[start + 1:end]:
if not line.lstrip().startswith("#") and module_re.match(line):
section.append(f'module = "workers.{name}"')
has_module = True
elif not line.lstrip().startswith("#") and load_re.match(line):
section.append('load_mode = "external"')
has_load = True
else:
section.append(line)
if not has_module:
section.insert(0, f'module = "workers.{name}"')
if not has_load:
section.extend([
"# 进程插件:由 Docker/K8s 编排拉起独立进程(容器入口 ./start.sh / "
f"chongming-worker --type {name}),",
"# 经 Zenoh 注册路由,网关动态挂载;与 src/plugins/ 下的 inline 插件可同时存在",
'load_mode = "external"',
])
lines[start + 1:end] = section
return "\n".join(lines) + ("\n" if text.endswith("\n") else "")
def _read_dep_imports(deps_file: Path, name: str) -> tuple[list[str], list[str]]:
"""读 plugins_deps.toml:返回 ([<name>] 节的 imports, 共用 imports)。"""
if not deps_file.is_file():
return [], []
data = tomllib.loads(deps_file.read_text(encoding="utf-8"))
section = data.get(name)
section_imports = list(section.get("imports", [])) if isinstance(section, dict) else []
return section_imports, list(data.get("imports", []) or [])
def _comment_out_section(text: str, name: str) -> tuple[str, bool]:
"""把 [<name>] 节整段注释掉(依赖已随插件迁走),返回 (新文本, 是否命中)。"""
lines = text.splitlines()
header = re.compile(rf"^\s*\[{re.escape(name)}\]\s*(#.*)?$")
start = next((i for i, line in enumerate(lines) if header.match(line)), None)
if start is None:
return text, False
end = next(
(i for i in range(start + 1, len(lines)) if lines[i].lstrip().startswith("[")),
len(lines),
)
for i in range(start, end):
if lines[i].strip():
lines[i] = "# " + lines[i]
return "\n".join(lines) + ("\n" if text.endswith("\n") else ""), True
# ── 主流程 ─────────────────────────────────────────────────────────────────
def migrate_plugin_package(name: str, dry_run: bool = False, force: bool = False) -> int:
"""把内置插件迁移为进程插件项目。返回 0 成功、1 失败/被拦截。"""
root = find_project_root()
if not NAME_RE.match(name):
print(f"❌ 插件名不合法: '{name}'(需小写字母/数字/下划线,且以字母开头)")
return 1
src_code = root / "src" / "plugins" / name
src_conf = root / "config" / "workers" / name
project = root / "workers" / name
deps_file = root / "plugins_deps.toml"
if not src_code.is_dir() or not (src_code / "__init__.py").is_file():
print(f"❌ 未找到内置插件: {_rel(root, src_code)}/(需含 __init__.py)")
return 1
if project.exists():
print(f"❌ 目标已存在: {_rel(root, project)}/(先移除或改名,避免覆盖)")
return 1
if not (src_conf / "config.toml").is_file():
print(f"❌ 未找到插件注册配置: {_rel(root, src_conf)}/config.toml")
return 1
# ── 前置检查:命中则拒绝迁移(--force 跳过) ──
blocked = False
router_hits = _router_findings(src_code)
if router_hits:
print("🚫 检测到自定义 FastAPI 路由(网关只对 inline 插件挂载 APIRouter 实例):")
for hit in router_hits:
print(f" - {hit}")
print(f" {ROUTER_HINT}")
blocked = True
gw_hits = _gateway_import_findings(src_code)
if gw_hits:
print("🚫 检测到网关包导入:")
for hit in gw_hits:
print(f" - {hit}")
print(f" {GATEWAY_IMPORT_HINT}")
blocked = True
if blocked and not force:
print("\n❌ 已拒绝迁移(确认无影响可加 --force 跳过拦截)")
return 1
if blocked:
print("\n⚠️ --force:跳过上述拦截继续迁移")
# 先算好改写后的配置([workers] 节缺失等结构问题在动文件之前暴露)
conf_text = (src_conf / "config.toml").read_text(encoding="utf-8")
try:
conf_new = _rewrite_workers_section(conf_text, name)
except ValueError as e:
print(f"❌ {_rel(root, src_conf)}/config.toml: {e}")
return 1
# 依赖:plugins_deps.toml [<name>] 节 + 共用节,按代码实际 import 过滤后解析
section_imports, shared_imports = _read_dep_imports(deps_file, name)
used = _third_party_modules(src_code)
wanted = [m for m in [*section_imports, *shared_imports] if m.split(".")[0] in used]
unused_declared = [m for m in section_imports if m.split(".")[0] not in used]
specs, unresolved = _dist_specs(wanted, root)
if shared_imports:
used_shared = [m for m in shared_imports if m.split(".")[0] in used]
if used_shared:
print(f"ℹ️ 共用依赖随迁移带入: {', '.join(used_shared)}(共用节保留,供其它插件使用)")
code_dst = project / "workers" / name
conf_dst = project / "config" / "workers" / name
zenoh_src = root / "config" / "zenoh_config.json5"
# 迁移后哪些依赖可能已无使用方(提示清理根 pyproject / plugins_deps.toml)
others = root / "src" / "plugins"
used_by_others: set[str] = set()
if others.is_dir():
for p in sorted(others.iterdir()):
if p.is_dir() and p.name not in ("__pycache__", name):
used_by_others |= _third_party_modules(p)
stale = [m for m in wanted if m.split(".")[0] not in used_by_others]
print(f"\n📦 迁移计划({name}: inline → external):")
print(f" 代码 {_rel(root, src_code)}/ → {_rel(root, code_dst)}/")
print(f" 配置 {_rel(root, src_conf)}/ → {_rel(root, conf_dst)}/([workers]: "
f'module = "workers.{name}", load_mode = "external")')
print(f" 项目 {_rel(root, project)}/:config.toml / pyproject.toml / README.md / "
"start.sh / config/zenoh_config.json5")
if specs:
print(f" 依赖 → 新项目 pyproject.toml: {', '.join(specs)}")
if unused_declared:
print(f" ⚠️ 声明但代码未用到(未带入): {', '.join(unused_declared)}")
if unresolved:
print(f" ⚠️ 无法解析发行包(请手动加进新 pyproject.toml): {', '.join(unresolved)}")
print(f" 清理 源目录删除({_rel(root, src_code)}/、{_rel(root, src_conf)}/,"
"各自 git 仓库可恢复)")
if deps_file.is_file():
print(f" {_rel(root, deps_file)}: 注释掉 [{name}] 节(依赖随插件走,不再进网关 exe)")
if dry_run:
print("\n🔍 --dry-run:仅演练,未改动任何文件")
return 0
# ── 执行 ──
(project / "config").mkdir(parents=True, exist_ok=True)
shutil.copytree(src_code, code_dst, ignore=shutil.ignore_patterns("__pycache__"))
shutil.copytree(src_conf, conf_dst, ignore=shutil.ignore_patterns("__pycache__"))
(conf_dst / "config.toml").write_text(conf_new, encoding="utf-8")
(project / "config.toml").write_text(gp._gen_root_config_toml(), encoding="utf-8")
(project / "pyproject.toml").write_text(
gp._gen_pyproject_toml(name, specs), encoding="utf-8"
)
readme = (
gp._gen_readme(name)
+ f"\n> 由 `uv run migrate_plugin {name}` 自 `src/plugins/{name}` 迁移而来。\n"
)
(project / "README.md").write_text(readme, encoding="utf-8")
start_sh = project / "start.sh"
start_sh.write_text(gp._gen_start_sh(name, generated_by=f"migrate_plugin {name}"), encoding="utf-8")
start_sh.chmod(0o755)
if zenoh_src.is_file():
shutil.copy(zenoh_src, project / "config" / "zenoh_config.json5")
else:
print(
f"⚠️ 未找到 {_rel(root, zenoh_src)},请自行放入 "
f"{_rel(root, project)}/config/zenoh_config.json5"
)
if deps_file.is_file():
new_text, changed = _comment_out_section(deps_file.read_text(encoding="utf-8"), name)
if changed:
deps_file.write_text(new_text, encoding="utf-8")
else:
print(f"ℹ️ {_rel(root, deps_file)} 无 [{name}] 节(依赖可能只在共用 imports 里),未改动")
shutil.rmtree(src_code)
shutil.rmtree(src_conf)
print(f"\n✅ 迁移完成: {_rel(root, src_code)} → {_rel(root, project)}/")
print("下一步:")
print(f" 1. cd {_rel(root, project)} && uv sync && uv run chongming-worker --type {name}"
" # 本地验证(另开终端起网关)")
print(f" 2. cd {_rel(root, project)} && uv run build_plugin {name}"
f" # → dist/{name}.zip(容器部署)")
print(" 3. 网关侧重打:uv run build --only mbank,prepare # src/plugins 已少该插件,"
"mbank/产物需重建")
if stale:
print(f" 4. 以下依赖已无内置使用方,可考虑从根 pyproject.toml 移除: {', '.join(stale)}")
print(" 注意: HTTP 路由改经 Zenoh 动态挂载(失去进程内直调快路径);"
"[workers.config] 中相对路径(如 data_dir)改为相对插件项目根解析")
return 0
def main() -> None:
parser = argparse.ArgumentParser(
description="内置插件(src/plugins/<name>)一键迁移为进程插件(workers/<name>/)",
)
parser.add_argument("name", help="插件名(如 doubao;对应 src/plugins/<name>)")
parser.add_argument("--dry-run", action="store_true", help="仅演练迁移计划,不改动任何文件")
parser.add_argument(
"--force",
action="store_true",
help="跳过自定义路由/网关导入拦截继续迁移(确认无影响时使用)",
)
args = parser.parse_args()
sys.exit(migrate_plugin_package(args.name, dry_run=args.dry_run, force=args.force))
def migrate_plugin():
"""CLI 入口(供 pyproject.toml [project.scripts] 调用)"""
main()
if __name__ == "__main__":
main()
Generated
+1 -1
View File
@@ -238,7 +238,7 @@ wheels = [
[[package]]
name = "chongming-cli"
version = "0.3.4"
version = "0.3.5"
source = { editable = "." }
dependencies = [
{ name = "chongming-config" },