Compare commits

...
2 Commits
Author SHA1 Message Date
chakcy ee8e71a49b modify 0.3.5 2026-10-13 13:27:26 +08:00
chakcy cd841f4f2d modify: 0.3.4 2026-10-13 11:04:18 +08:00
6 changed files with 727 additions and 44 deletions
+30
View File
@@ -10,6 +10,7 @@ chongming 网关的命令脚本集合,独立包维护(从 `src/chongming/scr
| `build_plugin` | 进程插件项目根直接压缩为可部署 zip | | `build_plugin` | 进程插件项目根直接压缩为可部署 zip |
| `gen_models` | 从插件配置 `*.toml` 生成 Pydantic 模型 | | `gen_models` | 从插件配置 `*.toml` 生成 Pydantic 模型 |
| `gen_plugin` | 生成风格统一的 Worker 插件骨架(`--process` 生成进程插件完整项目) | | `gen_plugin` | 生成风格统一的 Worker 插件骨架(`--process` 生成进程插件完整项目) |
| `migrate_plugin` | 内置插件一键迁移为进程插件(`src/plugins/<name>` → `workers/<name>/`) |
另有 `chongming_cli.code_content` 模块:将目录下代码文件汇总为 Markdown(`python -m chongming_cli.code_content <dir>`)。 另有 `chongming_cli.code_content` 模块:将目录下代码文件汇总为 Markdown(`python -m chongming_cli.code_content <dir>`)。
@@ -27,8 +28,16 @@ uv run gen_plugin myplugin # 生成 inline 插件骨架
uv run gen_plugin myplugin --process # 生成进程插件完整项目(workers/myplugin/) uv run gen_plugin myplugin --process # 生成进程插件完整项目(workers/myplugin/)
uv run gen_models config/workers/myplugin src/plugins/myplugin # inline 插件模型 uv run gen_models config/workers/myplugin src/plugins/myplugin # inline 插件模型
uv run gen_models workers/myplugin/config/workers/myplugin workers/myplugin/workers/myplugin # 进程插件模型 uv run gen_models workers/myplugin/config/workers/myplugin workers/myplugin/workers/myplugin # 进程插件模型
uv run migrate_plugin myplugin --dry-run # 内置插件迁移为进程插件(仅演练)
uv run migrate_plugin myplugin # 实际迁移:建项目 + 迁移代码/配置/依赖 + 删源
``` ```
`migrate_plugin` 前置检查(命中则拒绝迁移,`--force` 跳过):插件挂载了自定义
FastAPI 路由(`APIRouter`,external 插件挂不了,需先改成 TOML `@rpc_method` 端点
或保持 inline);插件导入了网关包(`app` / `chongming`,进程插件与网关零导入关系)。
迁移后依赖从 `plugins_deps.toml` 的 `[<name>]` 节(按代码实际 import 过滤)转入
新项目 `pyproject.toml`,该节自动注释掉 —— 依赖随插件走,不再进网关 exe。
`build --only <step>` 可选步骤:`mbank` / `prepare` / `obfuscate` / `pack` / `clean` / `license_guard` / `vue`,可逗号分隔或重复传多个。 `build --only <step>` 可选步骤:`mbank` / `prepare` / `obfuscate` / `pack` / `clean` / `license_guard` / `vue`,可逗号分隔或重复传多个。
`build` 的两个开关默认**都关**(不加参数即内部调试产物:明文模块、不带授权,横幅与构建报告里会回显状态): `build` 的两个开关默认**都关**(不加参数即内部调试产物:明文模块、不带授权,横幅与构建报告里会回显状态):
@@ -36,6 +45,27 @@ uv run gen_models workers/myplugin/config/workers/myplugin workers/myplugin/work
- `--encrypt`:打开加密与混淆 —— `.mbank` 加密存字节码并写 `.key` 密钥文件、做 PyArmor 混淆 - `--encrypt`:打开加密与混淆 —— `.mbank` 加密存字节码并写 `.key` 密钥文件、做 PyArmor 混淆
- `--license`:打开授权 —— 注入厂商授权公钥(缺公钥则构建失败)、跑授权私钥泄漏守卫 - `--license`:打开授权 —— 注入厂商授权公钥(缺公钥则构建失败)、跑授权私钥泄漏守卫
打包收集清单分两处:网关本体依赖在根 `config.toml` `[pyinstaller_build_import]`;
内置插件专属依赖(网关本体不 import、只在插件里用)在根目录 `plugins_deps.toml`
(按插件分节,可选文件),`pack` 时自动并入 `--hidden-import` / `--include-module`。
### Nuitka 打包的环境要求(Linux)
- **patchelf 必需**(standalone/onefile 都要):已列进网关 dev 依赖(`uv sync` 自动带),
也可 `sudo apt install patchelf`。Nuitka 直到链接完成后才检查它——缺失会白跑一整轮
编译,build 已改为提前失败提示
- **链接器**:默认 GNU ld 链接大工程峰值数 GB,小内存机器/WSL 上会被 OOM 杀掉
(`clang: unable to execute command: Killed`)。检测到 `ld.lld` 时自动经 `LDFLAGS`
换成 lld(内存与耗时都低得多)
- **内存档位**(按 `MemAvailable` 自动选):≥ 6 GB 完整优化(`--lto=yes`);
< 6 GB 去 LTO(LTO 链接峰值 3 GB+,实测 GNU ld 与 lld 都会被 OOM 杀掉);
< 2.5 GB 再 `--low-memory` 降并发。`CHONGMING_NUITKA_LOW_MEM=1` 强制省内存档、
`CHONGMING_NUITKA_FULL=1` 强制完整优化
- **PGO 默认关闭**:Nuitka 4.3 的 `--pgo-c` 与 standalone/onefile 不兼容(Nuitka 自身
WARNING 后 FATAL),且 profiling 运行会启动程序等它退出——常驻服务会卡死;
能自行退出的程序可设 `CHONGMING_NUITKA_PGO=1` 打开
- 失败自动逐档重试:完整优化 → 去 LTO/UPX → 去 LTO/UPX + `--low-memory`
## 项目根目录定位 ## 项目根目录定位
脚本需要在项目根目录下读写 `config/`、`src/plugins/`、`workers/` 等路径。 脚本需要在项目根目录下读写 `config/`、`src/plugins/`、`workers/` 等路径。
+2 -1
View File
@@ -4,7 +4,7 @@ requires = ["hatchling"]
[project] [project]
name = "chongming-cli" name = "chongming-cli"
version = "0.3.3" version = "0.3.5"
description = "chongming 命令脚本集合:构建打包 / 进程插件打包 / 插件脚手架 / 模型生成 / 机器码授权签发" description = "chongming 命令脚本集合:构建打包 / 进程插件打包 / 插件脚手架 / 模型生成 / 机器码授权签发"
readme = "README.md" readme = "README.md"
requires-python = ">=3.12" requires-python = ">=3.12"
@@ -30,4 +30,5 @@ build = "chongming_cli.build:build"
build_plugin = "chongming_cli.build_plugin:build_plugin" build_plugin = "chongming_cli.build_plugin:build_plugin"
gen_models = "chongming_cli.gen_models:gen_models" gen_models = "chongming_cli.gen_models:gen_models"
gen_plugin = "chongming_cli.gen_plugin:gen_plugin" gen_plugin = "chongming_cli.gen_plugin:gen_plugin"
migrate_plugin = "chongming_cli.migrate_plugin:migrate_plugin"
chongming-license = "chongming_cli.license:license_cmd" chongming-license = "chongming_cli.license:license_cmd"
+223 -29
View File
@@ -35,6 +35,13 @@
uv run build nuitka --only mbank,prepare uv run build nuitka --only mbank,prepare
uv run build --encrypt --license # 对外分发:加密模块 + 注入授权公钥 uv run build --encrypt --license # 对外分发:加密模块 + 注入授权公钥
uv run build # 内部调试:明文模块、不带授权 uv run build # 内部调试:明文模块、不带授权
依赖收集(--only pack 生效):
打包收集清单 = 根 config.toml [pyinstaller_build_import](网关本体依赖)
+ 根目录 plugins_deps.toml(内置插件专属依赖 —— 网关本体不 import、只在
mbank 内 import 的包,按插件分节声明;文件可选)。
mbank 里的 import 静态分析看不到,两条清单都会被展开为 --hidden-import /
--include-module,缺一个都会在运行期加载插件时才 ImportError。
""" """
import os import os
@@ -45,6 +52,7 @@ import subprocess
import sys import sys
import asyncio import asyncio
import time import time
import tomllib
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
@@ -54,6 +62,8 @@ from sqlite_vfs.core import SQLiteVFS
from sqlite_vfs.folder_packer import FolderPacker from sqlite_vfs.folder_packer import FolderPacker
from chongming_config import get_config, Config from chongming_config import get_config, Config
from ._root import find_project_root
# ── 常量 ──────────────────────────────────────────────────────────────────── # ── 常量 ────────────────────────────────────────────────────────────────────
PYINSTALLER = "pyinstaller" PYINSTALLER = "pyinstaller"
@@ -65,6 +75,22 @@ STEPS = ("mbank", "prepare", "obfuscate", "pack", "clean", "license_guard", "vue
is_windows = platform.system() == "Windows" is_windows = platform.system() == "Windows"
# 插件专属依赖清单(项目根,可选文件):内置插件(src/plugins/*)专属、网关本体
# (src/chongming/)不 import 的第三方依赖,按插件分节声明。放在插件自己的清单里
# 而不是 config.toml [pyinstaller_build_import],避免插件依赖混进网关构建配置:
# 插件下线/改 external 进程插件时删掉对应节即可,网关配置不受影响。
PLUGIN_DEPS_FILE = "plugins_deps.toml"
# Nuitka 打包的资源门槛:LTO 链接本仓库规模峰值 3 GB+ —— 实测 GNU ld 与 lld 都会被
# OOM 杀掉("clang: unable to execute command: Killed",机器可用内存不足时)。
# 可用内存 < NUITKA_LTO_MIN_GB 自动去掉 LTO;< NUITKA_SQUEEZE_GB 进一步
# --low-memory + 降并发。环境变量可强制档位(见 _nuitka_env_and_mem_note)。
NUITKA_LTO_MIN_GB = 6.0
NUITKA_SQUEEZE_GB = 2.5
NUITKA_LOW_MEM_ENV = "CHONGMING_NUITKA_LOW_MEM"
NUITKA_FULL_ENV = "CHONGMING_NUITKA_FULL"
NUITKA_PGO_ENV = "CHONGMING_NUITKA_PGO"
# ── 构建结果数据结构 ───────────────────────────────────────────────────────── # ── 构建结果数据结构 ─────────────────────────────────────────────────────────
@@ -139,8 +165,9 @@ def _run_cmd(
capture: bool = True, capture: bool = True,
shell: bool | None = None, shell: bool | None = None,
echo_on_error: bool = True, echo_on_error: bool = True,
env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess: ) -> subprocess.CompletedProcess:
"""统一执行外部命令""" """统一执行外部命令(env 传给子进程,用于如 LDFLAGS 注入链接器)"""
if shell is None: if shell is None:
shell = is_windows shell = is_windows
result = subprocess.run( result = subprocess.run(
@@ -151,6 +178,7 @@ def _run_cmd(
shell=shell, shell=shell,
encoding="utf-8", encoding="utf-8",
errors="replace", errors="replace",
env=env,
) )
# 命令失败时自动回显捕获的输出,便于定位失败原因 # 命令失败时自动回显捕获的输出,便于定位失败原因
if echo_on_error and capture and result.returncode != 0: if echo_on_error and capture and result.returncode != 0:
@@ -517,16 +545,73 @@ def _stage_pyarmor_obfuscate() -> str:
return "✅ 所有文件混淆成功" return "✅ 所有文件混淆成功"
# ── 打包收集清单(网关本体依赖 + 插件专属依赖) ─────────────────────────────
def _load_plugin_imports() -> tuple[list[str], str]:
"""读取插件专属依赖清单(项目根 plugins_deps.toml,可选),返回 (模块列表, 摘要)。
文件结构(节名 = src/plugins/ 下的插件名):
imports = ["httpx"] # 多个插件共用的插件专属依赖
[doubao]
imports = ["openai", ...] # 单插件专属
文件不存在 = 无插件专属依赖(其他项目复用本 CLI 时不受影响),返回 ([], "")。
配置写错(imports 不是字符串列表、顶层出现非表非 imports 的键)直接抛错:
漏收依赖打包期不报错、运行期加载插件才 ImportError,静默放过的排查成本高得多。
"""
path = find_project_root() / PLUGIN_DEPS_FILE
if not path.exists():
return [], ""
with open(path, "rb") as f:
data = tomllib.load(f)
modules: list[str] = []
notes: list[str] = []
def collect(section: str, entry) -> None:
if not isinstance(entry, list) or not all(isinstance(m, str) for m in entry):
raise ValueError(
f"{PLUGIN_DEPS_FILE} 的 [{section}] imports 必须是字符串列表"
)
modules.extend(entry)
if entry:
notes.append(f"{section}:{len(entry)}")
for name, value in data.items():
if name == "imports":
collect("共用", value)
elif isinstance(value, dict):
collect(name, value.get("imports"))
else:
raise ValueError(
f"{PLUGIN_DEPS_FILE} 顶层 [{name}] 只能是共用 imports 列表或 [插件名] 表"
)
return modules, "、".join(notes)
def _collect_build_imports(config: Config) -> tuple[list[str], str]:
"""打包收集清单 = 网关本体依赖(config.toml)+ 插件专属依赖(plugins_deps.toml)。
去重且保持顺序(网关本体在前);摘要空串 = 未声明插件专属依赖。
"""
base = config.get("pyinstaller_build_import", {})
items = list(base.get("base", []))
items.extend(base.get("windows", []) if is_windows else base.get("linux", []))
plugin_items, summary = _load_plugin_imports()
return list(dict.fromkeys([*items, *plugin_items])), summary
# ── 阶段 4: PyInstaller 打包 ──────────────────────────────────────────────── # ── 阶段 4: PyInstaller 打包 ────────────────────────────────────────────────
def _stage_pyinstaller(config: Config) -> tuple[str, str]: def _stage_pyinstaller(config: Config) -> tuple[str, str]:
"""执行 PyInstaller 打包,返回 (状态消息, 可执行文件路径)""" """执行 PyInstaller 打包,返回 (状态消息, 可执行文件路径)"""
base_imports = config.get("pyinstaller_build_import", {}) build_imports, plugin_summary = _collect_build_imports(config)
build_imports = list(base_imports.get("base", [])) if plugin_summary:
build_imports.extend( print(f" 📎 并入插件专属依赖 {PLUGIN_DEPS_FILE}({plugin_summary})")
base_imports.get("windows", []) if is_windows else base_imports.get("linux", [])
)
icon = Path("../../public/chongming.ico") icon = Path("../../public/chongming.ico")
cmd = [ cmd = [
@@ -579,12 +664,83 @@ def _stage_pyinstaller(config: Config) -> tuple[str, str]:
# ── 阶段 5: Nuitka 打包 ──────────────────────────────────────────────────── # ── 阶段 5: Nuitka 打包 ────────────────────────────────────────────────────
def _stage_nuitka(config: Config) -> tuple[str, str]: def _mem_available_gb() -> float:
"""执行 Nuitka 打包,返回 (状态消息, 可执行文件路径)""" """可用内存(GB,读 /proc/meminfo 的 MemAvailable;非 Linux/读不到返回 -1)。"""
base_imports = config.get("pyinstaller_build_import", {}).get("base", []) try:
extra = config.get("pyinstaller_build_import", {}).get( for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines():
"windows" if is_windows else "linux", [] if line.startswith("MemAvailable:"):
return int(line.split()[1]) / (1024 * 1024)
except (OSError, ValueError, IndexError):
pass
return -1.0
def _nuitka_env_and_mem_note() -> tuple[dict[str, str], str, str]:
"""Nuitka 打包的运行环境与资源档位。
返回 (子进程 env, 成功信息附加说明, 档位):
- 链接器:默认 GNU ld 链接本仓库规模(1400+ C 文件)峰值数 GB,加 LTO 更高,
WSL/小内存机器上会被 OOM 杀掉(clang: unable to execute command: Killed;
实测 ld 3.2-3.7 GB、lld 3.05 GB 峰值都被杀)。Nuitka 的 Scons 后端读取
LDFLAGS(SconsCompilerSettings)——装了 lld 就换成 lld(--clang 下
-fuse-ld=lld),比 GNU ld 更省内存/更快。
- 档位(按 /proc/meminfo 的 MemAvailable 自动选,env 可强制):
full 可用 ≥ NUITKA_LTO_MIN_GB:--lto=yes --jobs=4
no-lto 可用 < NUITKA_LTO_MIN_GB:去 LTO(链接峰值是 LTO 的大头)
squeeze 可用 < NUITKA_SQUEEZE_GB 或 NUITKA_LOW_MEM_ENV=1:
--low-memory --jobs=2(编译阶段也压低内存)
NUITKA_FULL_ENV=1 可强制 full(已知内存够用/CI 大机器)
"""
env = dict(os.environ)
note = ""
if shutil.which("ld.lld") and "-fuse-ld" not in (env.get("LDFLAGS") or ""):
env["LDFLAGS"] = f"{env.get('LDFLAGS', '')} -fuse-ld=lld".strip()
note += ",链接器 lld"
mem_gb = _mem_available_gb()
profile = "full"
if mem_gb >= 0 and mem_gb < NUITKA_SQUEEZE_GB:
profile = "squeeze"
elif mem_gb >= 0 and mem_gb < NUITKA_LTO_MIN_GB:
profile = "no-lto"
if os.environ.get(NUITKA_LOW_MEM_ENV) == "1":
profile = "squeeze"
if os.environ.get(NUITKA_FULL_ENV) == "1":
profile = "full" # 显式要求完整优化(已知内存够用时)
if profile != "full":
avail = f"{mem_gb:.1f} GB" if mem_gb >= 0 else "未知"
if profile == "no-lto":
print(
f" ⚠️ 可用内存 {avail} < {NUITKA_LTO_MIN_GB:.0f} GB:本次不做 LTO"
"(LTO 链接峰值 3 GB+,实测会被 OOM 杀掉;内存充足时自动恢复完整优化)"
) )
note += ",未启用 LTO"
else:
print(
f" ⚠️ 可用内存 {avail} < {NUITKA_SQUEEZE_GB:.0f} GB 或设了 "
f"{NUITKA_LOW_MEM_ENV}=1:本次 --low-memory 降并发打包"
)
note += ",--low-memory"
return env, note, profile
def _stage_nuitka(config: Config) -> tuple[str, str]:
"""执行 Nuitka 打包,成功返回 (状态消息, 可执行文件路径),失败抛 RuntimeError。"""
build_imports, plugin_summary = _collect_build_imports(config)
if plugin_summary:
print(f" 📎 并入插件专属依赖 {PLUGIN_DEPS_FILE}({plugin_summary})")
# patchelf 是 Nuitka standalone/onefile 的宿主机依赖,且 Nuitka 直到链接完成后的
# 打包阶段才检查 —— 缺了会白跑一整轮编译(1000+ C 文件),这里提前失败
if not is_windows and shutil.which("patchelf") is None:
raise RuntimeError(
"缺少 patchelf(Nuitka standalone/onefile 在 Linux 必需,且链接完成后才检查):"
"sudo apt install patchelf,或 uv pip install patchelf(注意 uv sync 会清掉后者)"
)
env, note, profile = _nuitka_env_and_mem_note()
cmd = [ cmd = [
sys.executable, sys.executable,
@@ -597,19 +753,32 @@ def _stage_nuitka(config: Config) -> tuple[str, str]:
"--static-libpython=yes", "--static-libpython=yes",
"--remove-output", "--remove-output",
"--assume-yes-for-downloads", "--assume-yes-for-downloads",
"--lto=yes", ]
"--pgo-c", if profile == "full":
"--pgo-python", cmd += ["--lto=yes", "--jobs=4"]
# PGO 默认关闭(两个原因,实测确认):
# 1. Nuitka 4.3 对 standalone/onefile + --pgo-c 自身 WARNING
# "Using C level PGO with standalone/onefile mode is not currently working",
# 随后在 C-PGO 收集阶段 FATAL;
# 2. profiling 运行会真正启动程序并等它退出(Popen.communicate 无超时)——
# 网关是常驻服务、永不退出,构建会卡死。
# 能自行退出的程序可设 CHONGMING_NUITKA_PGO=1 打开(--pgo-args 可传参)
if os.environ.get(NUITKA_PGO_ENV) == "1":
cmd += ["--pgo-c", "--pgo-python"]
elif profile == "no-lto":
cmd += ["--jobs=4"]
else: # squeeze:编译阶段也压低内存(low-memory 默认串行,开 2 路兼顾速度)
cmd += ["--low-memory", "--jobs=2"]
cmd += [
"--disable-plugin=tk-inter", "--disable-plugin=tk-inter",
"--output-dir=.", "--output-dir=.",
"--output-filename=chongming", "--output-filename=chongming",
"main.py", "main.py",
"--jobs=4",
] ]
if shutil.which("upx"): if profile == "full" and shutil.which("upx"):
cmd.append("--enable-plugin=upx") cmd.append("--enable-plugin=upx")
for mod in list(base_imports) + extra: for mod in build_imports:
cmd.append(f"--include-module={mod}") cmd.append(f"--include-module={mod}")
if is_windows: if is_windows:
@@ -617,17 +786,42 @@ def _stage_nuitka(config: Config) -> tuple[str, str]:
if icon.exists(): if icon.exists():
cmd.append(f"--windows-icon-from-ico={icon}") cmd.append(f"--windows-icon-from-ico={icon}")
result = _run_cmd(cmd, cwd="build", capture=False) # 降级重试链(逐档去掉开销/内存最大的参数):完整优化 → 去 LTO/UPX →
if result.returncode != 0: # 去 LTO/PGO/UPX + --low-memory。注意改编译参数会让 Nuitka 的中间目录失效、
# 降级重试(移除可能不兼容的参数) # 整轮重编(十几分钟),所以首选档位由内存自动选(见 _nuitka_env_and_mem_note),
print( # 这条链只兜底。
f" ⚠️ Nuitka 首次编译失败 (exit={result.returncode})," candidates: list[list[str]] = [
f"尝试移除不兼容参数重试..." cmd,
) [a for a in cmd if a not in ("--lto=yes", "--enable-plugin=upx")],
fallback = [ [
arg for arg in cmd if arg not in ("--lto=yes", "--enable-plugin=upx") a
for a in cmd
if a not in ("--lto=yes", "--enable-plugin=upx",
"--pgo-c", "--pgo-python", "--jobs=4")
] ]
_run_cmd(fallback, cwd="build", capture=False) + ["--low-memory", "--jobs=2"],
]
attempts: list[list[str]] = []
for candidate in candidates:
if candidate not in attempts:
attempts.append(candidate)
result = None
for i, attempt in enumerate(attempts):
result = _run_cmd(attempt, cwd="build", capture=False, env=env)
if result.returncode == 0:
break
if i + 1 < len(attempts):
print(f" ⚠️ Nuitka 第 {i + 1} 次尝试失败 (exit={result.returncode}),"
"降档重试…")
else:
if result is not None and result.returncode != 0:
print(
" 💡 链接阶段被杀(clang: unable to execute command: Killed)"
"基本都是内存不足:构建前关掉 VS Code / 容器 / dev server 释放内存,"
"或给 WSL 提高上限(在 C:\\Users\\<用户名>\\.wslconfig 加 "
"[wsl2] 段 memory=14GB、swap=8GB 后 `wsl --shutdown` 重进)"
)
exe = Path("build/chongming") exe = Path("build/chongming")
if exe.exists(): if exe.exists():
@@ -637,9 +831,9 @@ def _stage_nuitka(config: Config) -> tuple[str, str]:
if shutil.which("upx"): if shutil.which("upx"):
subprocess.run(["upx", "--best", str(exe)], check=False) subprocess.run(["upx", "--best", str(exe)], check=False)
size = _format_size(exe) size = _format_size(exe)
return f"✅ Nuitka 编译完成 ({size})", str(exe) return f"✅ Nuitka 编译完成 ({size}{note})", str(exe)
return "❌ Nuitka 编译失败 (未生成可执行文件)", "" raise RuntimeError("Nuitka 编译失败 (未生成可执行文件,详见上方输出)")
# ── 阶段 6: 清理临时文件 ──────────────────────────────────────────────────── # ── 阶段 6: 清理临时文件 ────────────────────────────────────────────────────
+14 -13
View File
@@ -310,20 +310,21 @@ json5_path = "config/zenoh_config.json5"
""" """
def _gen_pyproject_toml(name: str) -> str: def _gen_pyproject_toml(name: str, extra_deps: list[str] | None = None) -> str:
"""插件 uv 依赖清单(chongming-worker + 扩展依赖,扩展依赖自行追加)。 """插件 uv 依赖清单(chongming-worker + 扩展依赖,扩展依赖自行追加)。
extra_deps:额外依赖 spec(migrate_plugin 从 plugins_deps.toml 解析而来,
标注来源便于区分脚手架占位与迁移带入)。
注意:不声明 [project.scripts]——插件项目非打包项目(无 build-system), 注意:不声明 [project.scripts]——插件项目非打包项目(无 build-system),
uv sync 会跳过其入口点;worker 进程入口由 SDK 的 chongming-worker 脚本提供。 uv sync 会跳过其入口点;worker 进程入口由 SDK 的 chongming-worker 脚本提供。
""" """
deps = [ deps_lines = [' "chongming-worker",']
"chongming-worker", if extra_deps:
] deps_lines.append(" # ↓ 迁移自带(原 plugins_deps.toml 声明,已按代码实际 import 过滤)")
deps_dev = [ deps_lines.extend(f' "{d}",' for d in extra_deps)
"chongming-cli", deps = "\n".join(deps_lines)
] deps_dev_lines = "\n".join([' "chongming-cli",'])
deps_lines = "\n".join(f' "{d}",' for d in deps)
deps_dev_lines = "\n".join(f' "{d}",' for d in deps_dev)
return f"""# {name} 进程插件依赖清单(扩展依赖直接在此追加,如 "minio") return f"""# {name} 进程插件依赖清单(扩展依赖直接在此追加,如 "minio")
[project] [project]
name = "{name}-worker" name = "{name}-worker"
@@ -331,7 +332,7 @@ version = "0.1.0"
description = "{name} 进程插件(external 模式,独立容器部署)" description = "{name} 进程插件(external 模式,独立容器部署)"
requires-python = ">=3.12" requires-python = ">=3.12"
dependencies = [ dependencies = [
{deps_lines} {deps}
] ]
[[tool.uv.index]] [[tool.uv.index]]
@@ -348,10 +349,10 @@ dev = [
""" """
def _gen_start_sh(name: str) -> str: def _gen_start_sh(name: str, generated_by: str = "gen_plugin --process") -> str:
"""容器入口脚本:uv 引导依赖 + 启动 worker 进程(SDK 的 chongming-worker 脚本)。""" """容器入口脚本:uv 引导依赖 + 启动 worker 进程(SDK 的 chongming-worker 脚本)。"""
return f"""#!/usr/bin/env bash return f"""#!/usr/bin/env bash
# {name} 进程插件容器入口(由 gen_plugin --process 生成) # {name} 进程插件容器入口(由 {generated_by} 生成)
# 要求: 镜像预装 uv(https://docs.astral.sh/uv/) # 要求: 镜像预装 uv(https://docs.astral.sh/uv/)
set -e set -e
cd "$(dirname "$0")" cd "$(dirname "$0")"
@@ -961,7 +962,7 @@ logs/
""" """
_RUST_BUILD_SH = """#!/usr/bin/env bash _RUST_BUILD_SH = r"""#!/usr/bin/env bash
# Rust 进程插件打包器(自包含于插件根,与 Python `build_plugin` 对位) # Rust 进程插件打包器(自包含于插件根,与 Python `build_plugin` 对位)
# #
# 用法(在插件根目录执行): # 用法(在插件根目录执行):
+457
View File
@@ -0,0 +1,457 @@
"""
内置插件 → 进程插件 一键迁移(inline → external)
用法:
uv run migrate_plugin <name> [--dry-run] [--force]
把 src/plugins/<name>(inline,网关进程内加载)迁移为 workers/<name>/(自包含的
进程插件项目:开发目录即打包产物,见 build_plugin 与 docs/process-plugin.md)。
在仓库根执行。
迁移动作:
1. 前置检查(不通过则拒绝迁移,给出可操作提示):
- 自定义 FastAPI 路由:网关只对 inline 插件做 getattr(module, "router") 挂载
(且须是 APIRouter 实例),external 插件挂不上 —— 命中则提示先改成 TOML
@rpc_method 端点或保持 inline;确认无影响可加 --force 跳过拦截
- 网关包导入(import app / from chongming...):进程插件与网关零导入关系,
独立进程里这些模块不存在
2. 建项目骨架(复用 gen_plugin 的 --process 模板:config.toml / pyproject.toml /
README.md / start.sh / config/zenoh_config.json5)
3. 迁移代码 src/plugins/<name>/ → workers/<name>/workers/<name>/
4. 迁移配置 config/workers/<name>/ → workers/<name>/config/workers/<name>/
([workers] 节改写:module = "workers.<name>"、load_mode = "external")
5. 迁移依赖:plugins_deps.toml 的 [<name>] 节(按代码实际 import 过滤)解析为
发行包 spec 写入新项目 pyproject.toml,并把该节注释掉 —— 依赖随插件走,
不再进网关 exe
6. 删除源目录(src/plugins/<name>、config/workers/<name>,均可在各自 git 仓库恢复)
7. 打印下一步(uv sync / 本地跑 / build_plugin / 网关侧重打 mbank 等)
注意:迁移后该插件的 HTTP 路由改经 Zenoh 由网关 RouteRegistry 动态挂载,
失去 inline 的进程内直调快路径;[workers.config] 中的相对路径(如 data_dir)
改为相对插件项目根解析,有存量数据需自行搬移。
"""
import argparse
import ast
import re
import shutil
import sys
import tomllib
from pathlib import Path
from . import gen_plugin as gp
from ._root import find_project_root
NAME_RE = re.compile(r"^[a-z][a-z0-9_]*$")
ROUTER_HINT = (
"自定义 FastAPI 路由(APIRouter)只能由网关对 inline 插件挂载,external 插件"
"不参与网关路由发现(迁移后这些端点会 404)。请先把它们改成 TOML @rpc_method "
"形式(网关据此自动注册 HTTP 路由),或保留该插件为 inline;确认无影响可加 "
"--force 跳过拦截。"
)
GATEWAY_IMPORT_HINT = (
"进程插件与网关零导入关系(独立进程/容器里没有网关包,也不该有),"
"请先去掉这些 import,或改用 chongming_* 工具包。"
)
def _rel(root: Path, path: Path) -> str:
"""相对仓库根的路径(不在根下时原样返回,供打印)。"""
try:
return str(path.relative_to(root))
except ValueError:
return str(path)
# ── 源码检查(静态 AST,不 import 插件:导入会触发注册/依赖) ───────────────
def _iter_py(code_dir: Path) -> list[Path]:
return sorted(p for p in code_dir.rglob("*.py") if "__pycache__" not in p.parts)
def _parse(path: Path) -> ast.Module | None:
try:
return ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
except (SyntaxError, UnicodeDecodeError):
return None
def _call_name(node: ast.Call) -> str:
func = node.func
if isinstance(func, ast.Name):
return func.id
if isinstance(func, ast.Attribute):
return func.attr
return ""
def _router_findings(code_dir: Path) -> list[str]:
"""检测自定义 FastAPI 路由的挂载约定(与网关发现逻辑对齐)。
网关侧判据:`getattr(plugins.<name>, "router")` 且 `isinstance(..., APIRouter)`。
静态检测两条:
1. 任意文件实例化 `APIRouter(...)`(本仓库插件协议里 fastapi 路由的唯一用途)
2. 顶层 __init__.py 绑定名 router —— 赋值(排除字面量,如 router = None)或
`from <mod> import router`;`from . import router`(module 为 None)绑定的是
子模块(ip_sound_column 这种触发 @rpc_method 注册的写法),不是 APIRouter,不算
"""
findings: list[str] = []
for path in _iter_py(code_dir):
tree = _parse(path)
if tree is None:
continue
for node in ast.walk(tree):
if isinstance(node, ast.Call) and _call_name(node) == "APIRouter":
findings.append(f"{path.relative_to(code_dir)}:{node.lineno} 实例化 APIRouter")
init = code_dir / "__init__.py"
tree = _parse(init) if init.is_file() else None
if tree is not None:
for node in tree.body:
if isinstance(node, (ast.Assign, ast.AnnAssign)):
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
if not any(isinstance(t, ast.Name) and t.id == "router" for t in targets):
continue
if isinstance(node.value, ast.Constant): # router = None 之类不是 APIRouter
continue
findings.append(f"__init__.py:{node.lineno} 顶层绑定 router 属性")
break
if isinstance(node, ast.ImportFrom) and node.module:
if any((a.asname or a.name) == "router" for a in node.names):
findings.append(f"__init__.py:{node.lineno} from {node.module} import router")
break
return findings
def _gateway_import_findings(code_dir: Path) -> list[str]:
"""检测网关包导入(app / chongming,进程插件侧不存在)。"""
findings: list[str] = []
for path in _iter_py(code_dir):
tree = _parse(path)
if tree is None:
continue
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
if alias.name.split(".")[0] in ("app", "chongming"):
findings.append(f"{path.relative_to(code_dir)}:{node.lineno} import {alias.name}")
elif isinstance(node, ast.ImportFrom) and node.module:
if node.module.split(".")[0] in ("app", "chongming"):
findings.append(
f"{path.relative_to(code_dir)}:{node.lineno} from {node.module} import ..."
)
return findings
def _third_party_modules(code_dir: Path) -> set[str]:
"""插件代码实际 import 的顶层第三方模块名(排除标准库与相对导入)。"""
mods: set[str] = set()
for path in _iter_py(code_dir):
tree = _parse(path)
if tree is None:
continue
for node in ast.walk(tree):
if isinstance(node, ast.Import):
names = [a.name for a in node.names]
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
names = [node.module]
else:
continue
for full in names:
root = full.split(".")[0]
if root and root not in sys.stdlib_module_names:
mods.add(root)
return mods
# ── 依赖解析(plugins_deps.toml 的 import 名 → 新项目 pyproject 的发行包 spec) ──
def _normalize(spec: str) -> str:
"""依赖 spec → 规范化的发行包名(去 extras/版本约束、_→-、小写)。"""
base = re.split(r"[\[<>=!~; ]", spec, maxsplit=1)[0].strip().lower()
return base.replace("_", "-")
def _dist_specs(import_names: list[str], root: Path) -> tuple[list[str], list[str]]:
"""import 名 → 发行包 spec,返回 (已解析, 未解析)。
解析顺序:chongming_<x> → chongming-<x>;其余用 packages_distributions 反查;
命中根 pyproject.toml 里同发行包的写法时优先沿用(保留 extras/版本约束,
如 volcengine-python-sdk[ark])。未解析的(未安装/查不到发行包)交调用方提示。
"""
from importlib.metadata import packages_distributions
root_specs: dict[str, str] = {}
pyproject = root / "pyproject.toml"
if pyproject.is_file():
deps = (
tomllib.loads(pyproject.read_text(encoding="utf-8"))
.get("project", {})
.get("dependencies", [])
)
for spec in deps:
root_specs[_normalize(spec)] = spec
table = packages_distributions()
specs: list[str] = []
unresolved: list[str] = []
for imp in import_names:
root_name = imp.split(".")[0]
if root_name.startswith("chongming_"):
specs.append(root_specs.get(_normalize(root_name), root_name.replace("_", "-")))
continue
dists = table.get(root_name) or []
if not dists:
unresolved.append(imp)
continue
chosen = next((d for d in dists if _normalize(d) in root_specs), dists[0])
specs.append(root_specs.get(_normalize(chosen), chosen))
return list(dict.fromkeys(specs)), unresolved
# ── 配置与依赖清单改写 ─────────────────────────────────────────────────────
def _rewrite_workers_section(text: str, name: str) -> str:
"""改写 [workers] 节:module → workers.<name>、补 load_mode = "external"。
只动该节内的这两行,[worker]/[workers.config] 与注释原样保留。
"""
lines = text.splitlines()
try:
start = next(i for i, line in enumerate(lines) if line.strip() == "[workers]")
except StopIteration:
raise ValueError("config.toml 缺少 [workers] 节") from None
end = next(
(i for i in range(start + 1, len(lines)) if lines[i].lstrip().startswith("[")),
len(lines),
)
module_re = re.compile(r"^\s*module\s*=")
load_re = re.compile(r"^\s*load_mode\s*=")
section: list[str] = []
has_module = has_load = False
for line in lines[start + 1:end]:
if not line.lstrip().startswith("#") and module_re.match(line):
section.append(f'module = "workers.{name}"')
has_module = True
elif not line.lstrip().startswith("#") and load_re.match(line):
section.append('load_mode = "external"')
has_load = True
else:
section.append(line)
if not has_module:
section.insert(0, f'module = "workers.{name}"')
if not has_load:
section.extend([
"# 进程插件:由 Docker/K8s 编排拉起独立进程(容器入口 ./start.sh / "
f"chongming-worker --type {name}),",
"# 经 Zenoh 注册路由,网关动态挂载;与 src/plugins/ 下的 inline 插件可同时存在",
'load_mode = "external"',
])
lines[start + 1:end] = section
return "\n".join(lines) + ("\n" if text.endswith("\n") else "")
def _read_dep_imports(deps_file: Path, name: str) -> tuple[list[str], list[str]]:
"""读 plugins_deps.toml:返回 ([<name>] 节的 imports, 共用 imports)。"""
if not deps_file.is_file():
return [], []
data = tomllib.loads(deps_file.read_text(encoding="utf-8"))
section = data.get(name)
section_imports = list(section.get("imports", [])) if isinstance(section, dict) else []
return section_imports, list(data.get("imports", []) or [])
def _comment_out_section(text: str, name: str) -> tuple[str, bool]:
"""把 [<name>] 节整段注释掉(依赖已随插件迁走),返回 (新文本, 是否命中)。"""
lines = text.splitlines()
header = re.compile(rf"^\s*\[{re.escape(name)}\]\s*(#.*)?$")
start = next((i for i, line in enumerate(lines) if header.match(line)), None)
if start is None:
return text, False
end = next(
(i for i in range(start + 1, len(lines)) if lines[i].lstrip().startswith("[")),
len(lines),
)
for i in range(start, end):
if lines[i].strip():
lines[i] = "# " + lines[i]
return "\n".join(lines) + ("\n" if text.endswith("\n") else ""), True
# ── 主流程 ─────────────────────────────────────────────────────────────────
def migrate_plugin_package(name: str, dry_run: bool = False, force: bool = False) -> int:
"""把内置插件迁移为进程插件项目。返回 0 成功、1 失败/被拦截。"""
root = find_project_root()
if not NAME_RE.match(name):
print(f"❌ 插件名不合法: '{name}'(需小写字母/数字/下划线,且以字母开头)")
return 1
src_code = root / "src" / "plugins" / name
src_conf = root / "config" / "workers" / name
project = root / "workers" / name
deps_file = root / "plugins_deps.toml"
if not src_code.is_dir() or not (src_code / "__init__.py").is_file():
print(f"❌ 未找到内置插件: {_rel(root, src_code)}/(需含 __init__.py)")
return 1
if project.exists():
print(f"❌ 目标已存在: {_rel(root, project)}/(先移除或改名,避免覆盖)")
return 1
if not (src_conf / "config.toml").is_file():
print(f"❌ 未找到插件注册配置: {_rel(root, src_conf)}/config.toml")
return 1
# ── 前置检查:命中则拒绝迁移(--force 跳过) ──
blocked = False
router_hits = _router_findings(src_code)
if router_hits:
print("🚫 检测到自定义 FastAPI 路由(网关只对 inline 插件挂载 APIRouter 实例):")
for hit in router_hits:
print(f" - {hit}")
print(f" {ROUTER_HINT}")
blocked = True
gw_hits = _gateway_import_findings(src_code)
if gw_hits:
print("🚫 检测到网关包导入:")
for hit in gw_hits:
print(f" - {hit}")
print(f" {GATEWAY_IMPORT_HINT}")
blocked = True
if blocked and not force:
print("\n❌ 已拒绝迁移(确认无影响可加 --force 跳过拦截)")
return 1
if blocked:
print("\n⚠️ --force:跳过上述拦截继续迁移")
# 先算好改写后的配置([workers] 节缺失等结构问题在动文件之前暴露)
conf_text = (src_conf / "config.toml").read_text(encoding="utf-8")
try:
conf_new = _rewrite_workers_section(conf_text, name)
except ValueError as e:
print(f"❌ {_rel(root, src_conf)}/config.toml: {e}")
return 1
# 依赖:plugins_deps.toml [<name>] 节 + 共用节,按代码实际 import 过滤后解析
section_imports, shared_imports = _read_dep_imports(deps_file, name)
used = _third_party_modules(src_code)
wanted = [m for m in [*section_imports, *shared_imports] if m.split(".")[0] in used]
unused_declared = [m for m in section_imports if m.split(".")[0] not in used]
specs, unresolved = _dist_specs(wanted, root)
if shared_imports:
used_shared = [m for m in shared_imports if m.split(".")[0] in used]
if used_shared:
print(f"ℹ️ 共用依赖随迁移带入: {', '.join(used_shared)}(共用节保留,供其它插件使用)")
code_dst = project / "workers" / name
conf_dst = project / "config" / "workers" / name
zenoh_src = root / "config" / "zenoh_config.json5"
# 迁移后哪些依赖可能已无使用方(提示清理根 pyproject / plugins_deps.toml)
others = root / "src" / "plugins"
used_by_others: set[str] = set()
if others.is_dir():
for p in sorted(others.iterdir()):
if p.is_dir() and p.name not in ("__pycache__", name):
used_by_others |= _third_party_modules(p)
stale = [m for m in wanted if m.split(".")[0] not in used_by_others]
print(f"\n📦 迁移计划({name}: inline → external):")
print(f" 代码 {_rel(root, src_code)}/ → {_rel(root, code_dst)}/")
print(f" 配置 {_rel(root, src_conf)}/ → {_rel(root, conf_dst)}/([workers]: "
f'module = "workers.{name}", load_mode = "external")')
print(f" 项目 {_rel(root, project)}/:config.toml / pyproject.toml / README.md / "
"start.sh / config/zenoh_config.json5")
if specs:
print(f" 依赖 → 新项目 pyproject.toml: {', '.join(specs)}")
if unused_declared:
print(f" ⚠️ 声明但代码未用到(未带入): {', '.join(unused_declared)}")
if unresolved:
print(f" ⚠️ 无法解析发行包(请手动加进新 pyproject.toml): {', '.join(unresolved)}")
print(f" 清理 源目录删除({_rel(root, src_code)}/、{_rel(root, src_conf)}/,"
"各自 git 仓库可恢复)")
if deps_file.is_file():
print(f" {_rel(root, deps_file)}: 注释掉 [{name}] 节(依赖随插件走,不再进网关 exe)")
if dry_run:
print("\n🔍 --dry-run:仅演练,未改动任何文件")
return 0
# ── 执行 ──
(project / "config").mkdir(parents=True, exist_ok=True)
shutil.copytree(src_code, code_dst, ignore=shutil.ignore_patterns("__pycache__"))
shutil.copytree(src_conf, conf_dst, ignore=shutil.ignore_patterns("__pycache__"))
(conf_dst / "config.toml").write_text(conf_new, encoding="utf-8")
(project / "config.toml").write_text(gp._gen_root_config_toml(), encoding="utf-8")
(project / "pyproject.toml").write_text(
gp._gen_pyproject_toml(name, specs), encoding="utf-8"
)
readme = (
gp._gen_readme(name)
+ f"\n> 由 `uv run migrate_plugin {name}` 自 `src/plugins/{name}` 迁移而来。\n"
)
(project / "README.md").write_text(readme, encoding="utf-8")
start_sh = project / "start.sh"
start_sh.write_text(gp._gen_start_sh(name, generated_by=f"migrate_plugin {name}"), encoding="utf-8")
start_sh.chmod(0o755)
if zenoh_src.is_file():
shutil.copy(zenoh_src, project / "config" / "zenoh_config.json5")
else:
print(
f"⚠️ 未找到 {_rel(root, zenoh_src)},请自行放入 "
f"{_rel(root, project)}/config/zenoh_config.json5"
)
if deps_file.is_file():
new_text, changed = _comment_out_section(deps_file.read_text(encoding="utf-8"), name)
if changed:
deps_file.write_text(new_text, encoding="utf-8")
else:
print(f"ℹ️ {_rel(root, deps_file)} 无 [{name}] 节(依赖可能只在共用 imports 里),未改动")
shutil.rmtree(src_code)
shutil.rmtree(src_conf)
print(f"\n✅ 迁移完成: {_rel(root, src_code)} → {_rel(root, project)}/")
print("下一步:")
print(f" 1. cd {_rel(root, project)} && uv sync && uv run chongming-worker --type {name}"
" # 本地验证(另开终端起网关)")
print(f" 2. cd {_rel(root, project)} && uv run build_plugin {name}"
f" # → dist/{name}.zip(容器部署)")
print(" 3. 网关侧重打:uv run build --only mbank,prepare # src/plugins 已少该插件,"
"mbank/产物需重建")
if stale:
print(f" 4. 以下依赖已无内置使用方,可考虑从根 pyproject.toml 移除: {', '.join(stale)}")
print(" 注意: HTTP 路由改经 Zenoh 动态挂载(失去进程内直调快路径);"
"[workers.config] 中相对路径(如 data_dir)改为相对插件项目根解析")
return 0
def main() -> None:
parser = argparse.ArgumentParser(
description="内置插件(src/plugins/<name>)一键迁移为进程插件(workers/<name>/)",
)
parser.add_argument("name", help="插件名(如 doubao;对应 src/plugins/<name>)")
parser.add_argument("--dry-run", action="store_true", help="仅演练迁移计划,不改动任何文件")
parser.add_argument(
"--force",
action="store_true",
help="跳过自定义路由/网关导入拦截继续迁移(确认无影响时使用)",
)
args = parser.parse_args()
sys.exit(migrate_plugin_package(args.name, dry_run=args.dry_run, force=args.force))
def migrate_plugin():
"""CLI 入口(供 pyproject.toml [project.scripts] 调用)"""
main()
if __name__ == "__main__":
main()
Generated
+1 -1
View File
@@ -238,7 +238,7 @@ wheels = [
[[package]] [[package]]
name = "chongming-cli" name = "chongming-cli"
version = "0.3.2" version = "0.3.5"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "chongming-config" }, { name = "chongming-config" },