Compare commits
11
Commits
bangze
...
4cdb9c7049
| Author | SHA1 | Date | |
|---|---|---|---|
 chakcy
|
4cdb9c7049 | ||
|
chakcy
|
fe96fc9b6c | ||
|
chakcy
|
6d087fa4e3 | ||
|
chakcy
|
ac7ddb7445 | ||
|
chakcy
|
b1fed6f4b5 | ||
|
chakcy
|
c87b7baa64 | ||
|
chakcy
|
c92b34a55f | ||
|
chakcy
|
2df1e7266c | ||
|
chakcy
|
3a359fa485 | ||
|
chakcy
|
a0a65093c4 | ||
|
chakcy
|
b4a5985fe6 |
+4
-1
@@ -41,4 +41,7 @@ config
|
||||
# lib
|
||||
lib
|
||||
auto.crt
|
||||
auto.key
|
||||
auto.key
|
||||
|
||||
*.pem
|
||||
*.key
|
||||
+53
-14
@@ -101,8 +101,17 @@ use_embedded_broker = false
|
||||
config_path = "./config/workers/config.toml"
|
||||
|
||||
[pyinstaller_build_import]
|
||||
# 打包收集清单（PyInstaller --hidden-import / Nuitka --include-module）。
|
||||
# 网关本体依赖随 chongming-gateway 包发布，打包时能沿 `from chongming.app import app`
|
||||
# 静态跟到；本清单保留两类：
|
||||
# - 动态导入、静态分析看不到的（granian 内部子模块、amqtt 插件、
|
||||
# sqlite_vfs.contrib、chongming_worker 子模块等）
|
||||
# - public 骨架自身（server / utils）与服务器可选实现（uvloop 等）
|
||||
# 内置插件专属依赖（网关本体不 import，如 openai / bytedance）在根目录
|
||||
# plugins_deps.toml 按插件分节维护，打包时由 build 自动并入本清单
|
||||
base = [
|
||||
"utils", # public 骨架
|
||||
"httptools",
|
||||
"utils", # public 骨架
|
||||
"utils.launch",
|
||||
"utils.config",
|
||||
"zenoh",
|
||||
@@ -113,22 +122,42 @@ base = [
|
||||
"chongming_request",
|
||||
"chongming_auth",
|
||||
"server",
|
||||
"module_bank", # 模块化打包工具
|
||||
"module_bank", # 模块化打包工具
|
||||
"sqlite_vfs",
|
||||
"sqlite_vfs.core",
|
||||
"sqlite_vfs.contrib",
|
||||
"sqlite_vfs.contrib.fastapi_static",
|
||||
"fastapi", # web 应用依赖
|
||||
"fastapi", # web 应用依赖
|
||||
"fastapi.staticfiles",
|
||||
"fastapi.middleware.cors",
|
||||
"fastapi.middleware.gzip",
|
||||
"uvicorn",
|
||||
# "uvicorn",
|
||||
"granian", # 备选服务器（APP_SERVER=granian，见 public/main.py），含 Rust 内核
|
||||
"granian._compat",
|
||||
"granian._futures",
|
||||
"granian._granian",
|
||||
"granian._imports",
|
||||
"granian._internal",
|
||||
"granian._loops",
|
||||
"granian._signals",
|
||||
"granian._types",
|
||||
"granian.asgi",
|
||||
"granian.constants",
|
||||
"granian.errors",
|
||||
"granian.http",
|
||||
"granian.log",
|
||||
"granian.net",
|
||||
"granian.rsgi",
|
||||
"granian.server",
|
||||
"granian.server.common",
|
||||
"granian.server.mp",
|
||||
"granian.server.mt",
|
||||
"granian.wsgi",
|
||||
"chongming_cache",
|
||||
"chongming_cache.backends",
|
||||
"chongming_cache.backends.sqlite_",
|
||||
"orjson",
|
||||
"python_multipart",
|
||||
"volcenginesdkarkruntime",
|
||||
"chongming_jwt",
|
||||
"chongming_database",
|
||||
"chongming_config",
|
||||
@@ -143,22 +172,32 @@ base = [
|
||||
"aiofiles.os",
|
||||
"sqlmodel",
|
||||
"amqtt",
|
||||
"bytedance",
|
||||
"bytedance.volcengine_aicc_sdk",
|
||||
"bytedance.volcengine_aicc_sdk.secure_http_client",
|
||||
# amqtt 内嵌 Broker 的插件：amqtt 经 entry points 动态加载，静态分析看不到
|
||||
# （Nuitka 尤其如此；缺了会在启动时告警 "Plugin import failed: amqtt.plugins...."）
|
||||
"amqtt.plugins.authentication",
|
||||
"amqtt.plugins.logging_amqtt",
|
||||
"amqtt.plugins.sys.broker",
|
||||
"amqtt.plugins.topic_checking",
|
||||
"aiofiles",
|
||||
"openai",
|
||||
]
|
||||
linux = [
|
||||
"gunicorn.glogging",
|
||||
"gunicorn.app",
|
||||
"gunicorn.app.wsgiapp",
|
||||
# "gunicorn.glogging",
|
||||
# "gunicorn.app",
|
||||
# "gunicorn.app.wsgiapp",
|
||||
"uvloop",
|
||||
]
|
||||
windows = []
|
||||
|
||||
[production.module_system]
|
||||
# 网关本体（chongming-gateway 包）不再打进 .mbank：打包时 PyInstaller/Nuitka 从
|
||||
# site-packages 静态收集（public/server.py 里 `from chongming.app import app`），
|
||||
# 改网关走正常依赖升级（uv lock --upgrade-package chongming-gateway）而不是换 .mbank。
|
||||
# 模块银行只剩内置插件一项：插件仍是加密交付 + 可只换 plugins.mbank 现场更新。
|
||||
path = [
|
||||
["resources/app.mbank", "src/chongming/app", "app"],
|
||||
["resources/plugins.mbank", "src/plugins", "plugins"],
|
||||
[
|
||||
"resources/plugins.mbank",
|
||||
"src/plugins",
|
||||
"plugins",
|
||||
],
|
||||
]
|
||||
type = "module-bank"
|
||||
|
||||
+3
-2
@@ -81,7 +81,7 @@ sudo docker compose down # 停止并移除容器（保留数
|
||||
sudo docker compose down -v # 停止并移除容器 + 数据卷（慎用，会清空数据）
|
||||
```
|
||||
|
||||
### 验证
|
||||
### 健康检查
|
||||
|
||||
```bash
|
||||
sudo docker exec chongming-redis redis-cli ping # 期望 PONG
|
||||
@@ -94,7 +94,8 @@ sudo docker exec chongming-mosquitto mosquitto_sub -p 1883 -t '$SYS/broker/versi
|
||||
|
||||
**Zenoh Router**：项目 [zenoh_config.json5](../config/zenoh_config.json5) 默认以 `peer` 模式连接 `tcp/127.0.0.1:7447`。本 router 容器把 7447 映射到宿主机，同机部署时应用即可直接连到该 router；若跨机部署，需把 `connect.endpoints` 改为 router 机器地址。
|
||||
|
||||
**Mosquitto**：项目 [config.toml](../config.toml) 默认 `use_embedded_broker = true`（内嵌 amqtt）。要切到本外部 broker，需改为：
|
||||
**Mosquitto**：网关默认可内嵌 amqtt Broker（`use_embedded_broker = true`）。要改连本目录的
|
||||
外部 broker，把 `config.toml` 的 `[<env>.mqtt]` 改为：
|
||||
|
||||
```toml
|
||||
[development.mqtt]
|
||||
|
||||
@@ -16,6 +16,10 @@
|
||||
#
|
||||
# 可选：/etc/chongming.env 中写 APP_WORKERS=<N> 覆盖 worker 数（默认取
|
||||
# config.toml [production.server].workers，打包版通常为 4）。
|
||||
# 可选：APP_SERVER=granian|uvicorn|gunicorn 切换 HTTP 服务器（默认 Linux 下
|
||||
# gunicorn，Windows 下 uvicorn）；三者的启动参数同取自 [production.server]。
|
||||
# 可选：CHONGMING_RPC_LOCAL=0 关闭 inline 插件的进程内直调快路径（默认开启；
|
||||
# 仅影响本进程托管的 inline 副本，external 插件始终走 Zenoh）。
|
||||
|
||||
[Unit]
|
||||
Description=Chongming Zenoh Gateway (FastAPI + Zenoh + MQTT)
|
||||
@@ -29,7 +33,8 @@ StartLimitBurst=5
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
# 打包应用入口：Nuitka 编译产物，前台运行 gunicorn master（Linux 分支）
|
||||
# 打包应用入口：Nuitka 编译产物，前台运行所选服务器 master（默认 Linux 分支为
|
||||
# gunicorn，可用 APP_SERVER 切到 granian/uvicorn）
|
||||
ExecStart=/data/chongming/chongming
|
||||
# 所有相对路径（config.toml / config/ / resources/*.mbank / static.svfs /
|
||||
# data/ / logs/ / uploads/）均相对工作目录解析，必须指向安装目录
|
||||
@@ -48,7 +53,7 @@ Environment=PYTHONUNBUFFERED=1
|
||||
# 私有动态库，再补一条 Environment=LD_LIBRARY_PATH=/opt/sophon/sophon-ffmpeg-latest/lib
|
||||
Environment=PATH=/opt/sophon/sophon-ffmpeg-latest/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
# 优雅停机：SIGTERM 由 gunicorn 排空 worker，等在途请求/告警上传完成
|
||||
# 优雅停机：SIGTERM 由服务器 master 排空 worker，等在途请求/告警上传完成
|
||||
KillSignal=SIGTERM
|
||||
KillMode=control-group
|
||||
TimeoutStopSec=30
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
# 多语言插件演进统一规划（Rust 优先）
|
||||
|
||||
> **本文是演进规划记录，不是使用指南**。要开发/使用插件请先看：
|
||||
> [进程插件开发指南（Python）](./process-plugin.md)、
|
||||
> [Rust 插件编写说明](./rust-plugin.md)、
|
||||
> [进程插件线协议规范（语言无关）](./plugin-wire-protocol.md)。
|
||||
|
||||
> 背景：团队技术栈杂糅，Python 在部分计算密集场景性能有限。
|
||||
> 目标：在不破坏现有体系的前提下，开放进程插件多语言实现能力，
|
||||
> **Rust 优先落地**，为后续其他语言（Go / C++ 等）铺路。
|
||||
>
|
||||
> 配套文档：[进程插件线协议规范（语言无关）](./plugin-wire-protocol.md)、
|
||||
> [进程插件开发指南（Python）](./process-plugin.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -60,7 +62,7 @@ chongming-worker-rs/ # 独立仓库（或 utils/worker-rs/，与 Pyth
|
||||
│ # 另有 [workers.config] 运行时读写（保留键校验 + 原子落盘）
|
||||
├── descriptor.rs # 描述符结构体（规范 §2.2 字段表）+ 端点 TOML 解析
|
||||
├── publisher.rs # RoutePublisher 等价：PUT / DELETE / 10s 重发布
|
||||
├── serve.rs # worker_serve 等价：liveliness token + Queryable + 生命周期
|
||||
├── serve.rs # 入口等价：chongming-worker（Python 侧）：liveliness token + Queryable + 生命周期
|
||||
├── runtime.rs # 托管副本模型：ReplicaController（index/epoch/restart/stop/drain）
|
||||
│ # + WorkerPool（按 target 动态扩缩，优雅排空）
|
||||
├── admin.rs # __admin__ Queryable：status/restart/stop/drain/get_config/
|
||||
@@ -159,7 +161,7 @@ Rust 侧用 sqlx 在 `utils/worker-rs` 内提供等价封装（feature `database
|
||||
SQL 占位符遵循各自方言（SQLite `?`、PostgreSQL `$1`），SDK 不做 SQL 重写。
|
||||
- **TLS**：默认不启用 TLS 后端；PostgreSQL 走 SSL 时需追加 `tls-rustls-ring` 等 feature。
|
||||
- **参考实现**：`workers/rsdemo` 的 `db_ping` 端点（写 + 读一表做连通性探测）；`gen_plugin --rust` 模板已内置
|
||||
注释版 `[workers.config.database]` 段与 `database` feature（见 [rsdemo README](../../workers/rsdemo/README.md)）。
|
||||
注释版 `[workers.config.database]` 段与 `database` feature（见 [rsdemo README](../workers/rsdemo/README.md)）。
|
||||
|
||||
---
|
||||
|
||||
@@ -167,7 +169,7 @@ Rust 侧用 sqlx 在 `utils/worker-rs` 内提供等价封装（feature `database
|
||||
|
||||
### 4.1 build_plugin 支持非 Python 插件
|
||||
|
||||
- 现状：`build_plugin` 产物为 pyproject.toml + start.sh（`uv run worker_serve --type <name>`），纯 Python 链路
|
||||
- 现状：`build_plugin` 产物为 pyproject.toml + start.sh（`uv run chongming-worker --type <name>`），纯 Python 链路
|
||||
- 目标：支持声明插件语言/入口，允许打包任意可执行产物 + 自定义 start.sh
|
||||
（Rust 插件 = 编译产物 + `--type` 参数约定），`--type` 语义跨语言保持一致
|
||||
- 约束：打包结构（zip 根 = 容器工作目录）不变
|
||||
|
||||
@@ -237,11 +237,11 @@ worker/<type>/<id>/<subject>
|
||||
| 协议项 | 唯一实现（文件:行号） |
|
||||
| ---- | ---- |
|
||||
| 描述符 keyexpr / route_id 编码 / 发布摘除 | `utils/worker/src/chongming_worker/routes_publisher.py:23-89` |
|
||||
| 描述符字段定义与消费 | `src/chongming/app/core/route_generator.py:82-100`（RouteConfig） |
|
||||
| 描述符字段定义与消费 | `utils/gateway/src/chongming/app/core/route_generator.py:82-100`（RouteConfig） |
|
||||
| 描述符生成约定（TOML → dict） | `utils/worker/src/chongming_worker/route_config.py:25-103` |
|
||||
| 网关订阅 / 防抖 / 幂等 / 冲突 / 筛选 | `src/chongming/app/core/route_registry.py:44-261` |
|
||||
| liveliness 发现 / 负载均衡 / 超时重试 / 缓存 | `src/chongming/app/core/worker_manager.py:73-379` |
|
||||
| 请求信封构造（meta.method / user_info） | `src/chongming/app/core/route_generator.py:241-405` |
|
||||
| 网关订阅 / 防抖 / 幂等 / 冲突 / 筛选 | `utils/gateway/src/chongming/app/core/route_registry.py:44-261` |
|
||||
| liveliness 发现 / 负载均衡 / 超时重试 / 缓存 | `utils/gateway/src/chongming/app/core/worker_manager.py:73-379` |
|
||||
| 请求信封构造（meta.method / user_info） | `utils/gateway/src/chongming/app/core/route_generator.py:241-405` |
|
||||
| 信封解析 / 分派 / 三种响应形态 | `utils/worker/src/chongming_worker/runtime.py:27-220` |
|
||||
| 进程生命周期（10s 重发布 / 优雅退出） | `utils/worker/src/chongming_worker/process.py:35-197` |
|
||||
| 配置读取（json5_path / 插件配置） | `utils/worker/src/chongming_worker/process.py:39-136` |
|
||||
|
||||
+19
-51
@@ -27,9 +27,7 @@
|
||||
|
||||
---
|
||||
|
||||
## 2. 运行原理
|
||||
|
||||
### 2.1 架构
|
||||
## 2. 工作机制（简述）
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
@@ -37,7 +35,7 @@ graph TB
|
||||
REG[网关 RouteRegistry<br/>订阅 chongming/routes/**] -->|动态挂载/摘除| API
|
||||
API --> WM[WorkerManager<br/>负载均衡 + 超时重试]
|
||||
|
||||
subgraph GW[网关容器 chongming]
|
||||
subgraph GW[网关容器]
|
||||
API
|
||||
REG
|
||||
WM
|
||||
@@ -54,36 +52,18 @@ graph TB
|
||||
WS -.->|SIGTERM 时 delete 摘除| REG
|
||||
```
|
||||
|
||||
### 2.2 Worker 进程启动流程（`chongming_worker.process.worker_serve`）
|
||||
`chongming-worker --type demo` 启动后的流程：
|
||||
|
||||
```text
|
||||
chongming-worker --type demo
|
||||
1. 读全局 config.toml（仅 [zenoh] 节）→ 打开 Zenoh 会话
|
||||
2. 自读 config/workers/demo/config.toml 的 [workers] 节（load_plugin_config 展平）
|
||||
3. import 插件模块（plugins.<name> / workers.<name>）→ 触发 @rpc_method 注册
|
||||
4. 进程内契约校验（TOML ↔ 注册表 ↔ 生成模型），硬失败则退出（退出码 1）
|
||||
5. create_workers 创建 Worker 副本（默认 count 个，--count 可覆盖）
|
||||
6. RoutePublisher 发布路由描述符 → 网关动态挂载 HTTP 路由
|
||||
7. 等待退出信号；期间每 10s 周期性重发布（网关后启动 / 样本丢失时自愈）
|
||||
8. SIGTERM/SIGINT → 摘除路由 → cleanup → 取消副本任务 → 关闭会话
|
||||
```
|
||||
1. 读根 `config.toml` 的 `[zenoh]` 节打开会话，自读
|
||||
`config/workers/demo/config.toml` 的 `[workers]` 节（含 `[workers.config]` 运行时配置）
|
||||
2. import 插件模块触发 `@rpc_method` 注册 → **进程内契约校验**（不通过则退出码 1）
|
||||
3. `create_workers` 创建副本（默认 `count` 个，`--count` 可覆盖），注册 liveliness
|
||||
4. 经 Zenoh 把路由描述符 put 到 `chongming/routes/<type>/<route_id>`，网关动态挂载 HTTP 路由；
|
||||
每 10s 重发布（网关后启动或样本丢失时自愈）
|
||||
5. 收到 SIGTERM/SIGINT → delete 描述符（路由摘除）→ cleanup → 关闭会话
|
||||
|
||||
### 2.3 路由注册与摘除协议
|
||||
|
||||
- **发布**：路由描述符 put 到 `chongming/routes/<type>/<route_id>`，其中
|
||||
`route_id = <subject 的 . 替换为 _>__<METHOD>`（如 `demo_hello__GET`），
|
||||
描述符为 TOML 兼容 dict + `schema_version=1` + `type=<插件名>`
|
||||
- **元信息**：插件声明了 `[[workers.mqtt_topics]]` 时，额外发布
|
||||
`chongming/routes/<type>/__meta__`（含 mqtt_topics），网关据此动态订阅外部主题
|
||||
- **自愈**：每 10s 重发布一次全部描述符（`ANNOUNCE_INTERVAL`），网关后启动或样本丢失可恢复
|
||||
- **摘除**：优雅退出时 delete 全部已发布 key，网关 2s 防抖后摘除 HTTP 端点（请求变 404）
|
||||
- **网关侧**：RouteRegistry 常驻订阅 `chongming/routes/**`，与盘扫解耦——
|
||||
**网关容器无需放置 external 插件的配置文件**
|
||||
|
||||
### 2.4 服务发现
|
||||
|
||||
Worker 副本经 liveliness 注册 `worker/<type>/<id>`（与 inline 副本同一机制），
|
||||
WorkerManager 据此做负载均衡，网关侧无需任何改动。
|
||||
要点：网关容器**无需**放置 external 插件的配置文件（RouteRegistry 常驻订阅，与盘扫解耦）；
|
||||
描述符字段与 keyexpr 细节见 [线协议规范](plugin-wire-protocol.md)。
|
||||
|
||||
---
|
||||
|
||||
@@ -114,8 +94,7 @@ workers/<name>/ # 项目根 = zip 根 = 容器工作目录
|
||||
└── start.sh # 容器入口（uv sync + uv run chongming-worker --type <name>）
|
||||
```
|
||||
|
||||
> 参考示例：`workers/demo/`（完整项目，含 5 个端点，配合 `test_worker_isolation.py` /
|
||||
> `test_process_mode_e2e.py` 使用）。
|
||||
> 参考示例：`workers/demo/`（完整项目，含 5 个端点）。
|
||||
|
||||
---
|
||||
|
||||
@@ -291,16 +270,10 @@ uv run --project .. chongming-worker --type demo
|
||||
> 若配置了 `load_mode` 非 `"external"`，会输出告警：网关也会在进程内加载该插件，
|
||||
> 外部进程将与网关内副本**重复服务**。
|
||||
|
||||
### 第 7 步：验证
|
||||
### 第 7 步：联调验证
|
||||
|
||||
```bash
|
||||
# 1. 进程隔离性（阻断 chongming 导入 + AST 检查 import 语句；自动切到 demo 项目根）
|
||||
uv run python test_worker_isolation.py
|
||||
|
||||
# 2. 端到端（需先启动网关 uv run serve）：从 demo 项目根拉起 worker 进程，
|
||||
# 路由动态注册 → 跨进程 RPC → 退出摘除(404) → 重启恢复
|
||||
uv run python test_process_mode_e2e.py
|
||||
```
|
||||
启动网关（`uv run serve`）后调用插件路由即可验证端到端：网关日志可见路由动态挂载，
|
||||
worker 退出后路由自动摘除（请求变 404），重启 worker 后恢复。
|
||||
|
||||
### 第 8 步：打包
|
||||
|
||||
@@ -442,11 +415,8 @@ API 一览（`chongming_worker.storage.StorageClient`）：
|
||||
- `zenoh` / `pydantic` 等第三方库
|
||||
- 插件自身的扩展依赖（项目根 `pyproject.toml` 中声明）
|
||||
|
||||
由 `test_worker_isolation.py` 双重保证：
|
||||
|
||||
1. **运行时阻断**：`sys.modules["chongming"] = None` 后走完整链路
|
||||
（SDK 导入 → 插件配置自读 → 插件 import → 契约校验），任何网关依赖立即 ImportError
|
||||
2. **静态检查**：AST 解析 SDK 各模块源码，确认无 `chongming` / `chongming.*` import 语句
|
||||
隔离性由框架保证：SDK 导入链路中一旦出现 `chongming` / `chongming.*` 的 import，
|
||||
运行时立即 `ImportError`（启动即暴露），无需等到部署阶段。
|
||||
|
||||
---
|
||||
|
||||
@@ -476,7 +446,7 @@ Worker 进程启动时自读 TOML 并**在自己进程内**做三方一致性校
|
||||
| 路由一直 404 | worker 是否已退出（优雅退出会摘除路由）；进程被 SIGKILL 时描述符残留，重启 worker 后重发布自愈覆盖 |
|
||||
| 找不到插件配置 | `chongming-worker` 须在**插件项目根** `workers/<name>/` 下运行（读 `config/workers/<name>/config.toml`，`--config` 可指定全局配置路径） |
|
||||
| storage.* 端点报「storage 未配置」 | 插件 config.toml 未配置 `[workers.config.storage]`（或 `enabled = false` / 连接失败降级）。补齐配置后重启 worker；启动日志可见「对象存储已就绪」或「storage 连接失败」告警 |
|
||||
| 容器内 import `workers.demo` 失败 | 确认在容器工作目录（zip 根）启动，`start.sh` 内含 `cd "$(dirname "$0")"`，worker_serve 会把 CWD 加入 sys.path |
|
||||
| 容器内 import `workers.demo` 失败 | 确认在容器工作目录（zip 根）启动，`start.sh` 内含 `cd "$(dirname "$0")"`，启动入口会把 CWD 加入 sys.path |
|
||||
|
||||
---
|
||||
|
||||
@@ -489,8 +459,6 @@ uv run chongming-worker --type <name> # 本地调试独立 Worker
|
||||
./start.sh # 与容器一致的启动方式（uv sync + 启动）
|
||||
cd .. && uv run build_plugin <name> # 进程插件打包 → dist/<name>.zip
|
||||
uv run gen_models workers/<name>/config/workers/<name> workers/<name>/workers/<name> # 生成模型
|
||||
uv run python test_worker_isolation.py # 进程隔离性测试
|
||||
uv run python test_process_mode_e2e.py # 端到端测试（先启动网关）
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
+9
-9
@@ -281,28 +281,28 @@ unzip myplugin.zip -d /app && cd /app && ./start.sh
|
||||
|
||||
---
|
||||
|
||||
## 9. 本地验证
|
||||
## 9. 本地联调
|
||||
|
||||
```bash
|
||||
# Zenoh Router（替代 zenohd）
|
||||
# 起一个本地 Zenoh Router（替代 zenohd）
|
||||
.venv/bin/python utils/worker-rs/scripts/zenoh_router.py &
|
||||
|
||||
# 协议级 15 项断言（描述符/RPC/流式/错误/摘除/自愈）
|
||||
# 协议级端到端自检（描述符 / RPC / 流式 / 错误 / 摘除 / 自愈）
|
||||
.venv/bin/python utils/worker-rs/scripts/verify_e2e.py \
|
||||
--type myplugin --bin workers/myplugin/target/debug/myplugin
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. 与 Python SDK 的差异（有意为之）
|
||||
## 10. Rust 侧行为要点
|
||||
|
||||
| 差异 | 说明 |
|
||||
| 要点 | 说明 |
|
||||
| ---- | ---- |
|
||||
| 契约校验 | Python 进程内三方校验（硬失败）；Rust 为**发布前自检**：无 handler 端点告警跳过，不发布残缺路由 |
|
||||
| 模型生成 | Rust 用 gen_models 生成强类型结构体 + **启动时指纹校验**（TOML 变更未重生成即告警，比 Python 更强） |
|
||||
| 契约校验 | **发布前自检**：无 handler 的端点告警跳过，不发布残缺路由 |
|
||||
| 模型生成 | `gen_models` 生成强类型结构体 + **启动时指纹校验**（TOML 变更未重生成即告警） |
|
||||
| handler 注入 | worker_id 经 `Request.worker_id`、user_info 经 `Request.user_info` 注入（非函数参数） |
|
||||
| 配置缓存后端 | Rust 支持 sqlite（`local-cache`，本机与 Python 共享同一 .db）+ Redis（`redis-cache`，跨主机）；缺省无缓存回退 |
|
||||
| 数据库 | Rust 走 sqlx（见 §7），配置键与 Python `chongming_database` 对齐 |
|
||||
| 配置缓存后端 | `local-cache`（sqlite，与 Python 共享同一 .db）/ `redis-cache`（跨主机）；缺省无缓存回退 |
|
||||
| 数据库 | 走 sqlx（见 §7），配置键与 `chongming_database` 对齐 |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# Worker 管理与多实例运维
|
||||
|
||||
网关（chongming-gateway）对 Worker 副本的运维能力：管理指令与 REST 端点、扩缩容语义、
|
||||
实例定向调用，以及多网关/多实例部署时的收敛前提。inline 与 external（进程）插件共用同一套
|
||||
管理协议。
|
||||
|
||||
> 使用入口速览见主 [README](../README.md#worker-管理与扩缩容)。
|
||||
|
||||
## 1. 管理协议
|
||||
|
||||
每个用 Worker 框架创建的副本都会注册管理 Queryable：
|
||||
|
||||
```text
|
||||
worker/<type>/<worker_id>/__admin__
|
||||
```
|
||||
|
||||
| 指令 | 说明 |
|
||||
| ---- | ---- |
|
||||
| `status` | 查询副本状态（含 `instance_id`，实例定向据此寻址） |
|
||||
| `restart` | 重启该副本：副本槽位由管理循环驱动，重新生成新 `worker_id` 并更新 liveliness |
|
||||
| `stop` | 停止该副本 |
|
||||
| `drain` | **优雅排空**：摘除 liveliness/业务 Queryable 隔绝新请求，等待在途请求完成后退出，管理循环不重建 |
|
||||
| `rescale` | 调整副本数：不带 `instance_id` 为「全局对称分摊」；带 `instance_id + absolute=true` 且命中本实例时为「逐实例精细扩缩」，target 非本实例则安全忽略 |
|
||||
| `config_set` | 更新 `[workers.config]` 运行时配置（路由配置除外）；成功则写回配置并重启该类型副本，任一失败返回失败明细 |
|
||||
|
||||
> 旧版运行时未声明 `__admin__` 管理能力时，管理指令会超时（HTTP 502，日志有提示）。
|
||||
|
||||
## 2. REST 管理端点（仅超管）
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
| ---- | ---- | ---- |
|
||||
| `GET` | `/api/v1/admin/workers` | 盘点全部副本（inline 本地注册 + external liveliness 发现） |
|
||||
| `POST` | `/api/v1/admin/workers/restart` | 重启单个副本（`worker_type` / `worker_id` 查询参数） |
|
||||
| `POST` | `/api/v1/admin/workers/<type>/stop` | 停止单个副本 / 某插件全部副本 |
|
||||
| `GET` | `/api/v1/admin/workers/<type>/config` | 读取现有 `[workers.config]`（响应含 `source`/`mode` 标注取值来路） |
|
||||
| `POST` | `/api/v1/admin/workers/<type>/config` | 更新配置（body `{updates:{k:v}}`），下发并重启 |
|
||||
| `POST` | `/api/v1/admin/workers/<type>/scale` | 调整副本数（body `{count:<全局总量>}`），全局对称分摊 |
|
||||
| `GET` | `/api/v1/admin/workers/<type>/instances` | 按进程实例（instance_id）分组读取副本 |
|
||||
| `POST` | `/api/v1/admin/workers/<type>/scale_instance` | 逐实例精细扩缩（body `{instance_id, count}`） |
|
||||
|
||||
## 3. 配置热更新
|
||||
|
||||
- 回写插件 `config.toml` 的 `[workers]` 段（保留其他段），拒绝写保留键（`type` / `module` /
|
||||
`count` 等路由与加载控制键）。
|
||||
- 插件按新配置先 `on_cleanup()` 释放旧资源、再 `on_init()` 重建客户端（AI/DB client 等），
|
||||
配置**无需重启网关即生效**。
|
||||
- 仅 index0 副本落盘避免并发写冲突；其余副本同样应用新配置。
|
||||
|
||||
## 4. 扩缩容语义
|
||||
|
||||
- **扩容**：新增副本槽位并注册 liveliness，随即参与负载均衡。
|
||||
- **缩容**：多余副本走 `drain` 优雅排空——先隔绝新请求，等在途请求完成后关闭，避免中断业务。
|
||||
- **持久化**：目标数量写回插件 `config.toml` 的 `[workers].count`，重启后按新数量加载。
|
||||
- **全局对称 vs 逐实例**：`/scale` 调整某类型的**全局总量**（多实例下由各实例均分）；
|
||||
需单独调整**某一个**外部实例时用 `/scale_instance`。
|
||||
|
||||
## 5. 多实例收敛（副本数分摊 / 领导选举 / MQTT Bridge 单例）
|
||||
|
||||
多网关或多进程部署时，以下三件事都以**共享 `[cache]` 为单一事实来源**：
|
||||
|
||||
```toml
|
||||
[cache]
|
||||
backend = "redis" # 多实例必须共享（redis 或共享卷 sqlite）；单机 sqlite 即可
|
||||
launch_worker_count = "toml" # 副本数/运行时配置以 toml（默认）还是 cache 为准
|
||||
```
|
||||
|
||||
**前提**：所有网关与 external 进程指向同一 cache 后端。若各实例各用本机 cache，
|
||||
副本数无法收敛、锁跨不了主机，多实例行为会退化。
|
||||
|
||||
### 5.1 副本数（count）按在线实例分摊
|
||||
|
||||
- `count` 语义为**该类 Worker 的全局目标总量**，各参与者按在线数量自动分摊。
|
||||
- 在线清单与心跳：网关启动登记、每 15s 心跳续约，超 60s 未续约的实例自动清理；关闭时注销。
|
||||
同宿主多网关需显式设置 `CHONGMING_GATEWAY_ID`（缺省回退 hostname）。
|
||||
- external 进程实例同样登记参与分摊，避免多实例把副本数翻倍；实例标识用
|
||||
`--instance` / `CHONGMING_WORKER_INSTANCE`（缺省 `{hostname}:{pid}`，跨重启持续命中请显式指定）。
|
||||
- `/scale` 写全局 count 后向该类型全部存活实例广播 `rescale`，各实例按同一份数据计算 quota，
|
||||
重复下发幂等。
|
||||
|
||||
### 5.2 inline 领导选举（多 worker 兼容）
|
||||
|
||||
- 同一网关的多个服务器 worker（gunicorn `-w` 等）中，同一时刻仅一个进程托管 inline worker
|
||||
（副本数精确等于 `count`，而非 worker 数 × count）；其余进程只做 HTTP 转发。
|
||||
- 持有者崩溃后，其余进程经周期性巡检抢锁接管并拉起 worker（`set_nx` 唯一胜出，无双持有）。
|
||||
- key 按 `gateway_id` 隔离，多网关各自选举、互不抢占。
|
||||
|
||||
### 5.3 MQTT Bridge 单例（多实例免重复响应）
|
||||
|
||||
- Bridge 启动采用共享缓存分布式锁（`set_nx` + 租约续约），全局仅持锁网关消费 MQTT 请求，
|
||||
避免共享外部 Broker 时「一条请求被各消费一次、各回一份响应」。
|
||||
- 持有者失联后租约到期，其余网关经周期巡检（`bridge_supervise_interval`，默认 15s）自动接管重建。
|
||||
- 内嵌 Broker 单例按宿主隔离（`mqtt_broker:<hostname>`）：同宿主多进程仅启动一个 Broker，
|
||||
接管时会清理前任遗留的孤儿 Broker 子进程。
|
||||
|
||||
### 5.4 部署形态速查
|
||||
|
||||
| 部署形态 | MQTT Broker | `[cache]` 后端 |
|
||||
| -------- | ----------- | -------------- |
|
||||
| 单机（含多 worker） | 内嵌 amqtt | 本地 `sqlite` |
|
||||
| 多实例分布式 | **外部 Broker（必须）** | **redis**（或共享卷 `sqlite`） |
|
||||
|
||||
## 6. 逐实例精细扩缩
|
||||
|
||||
同一类型跑多个外部进程（多机/多容器）时，除全局对称扩缩外，可只对**某一个实例**精确扩缩：
|
||||
|
||||
- **语义**：把目标实例的副本数设为绝对值 N，其它实例不动；与全局 `scale` 共存——
|
||||
全局对称是默认/一键复位（`scale` 会清除整张覆盖表回到均分），逐实例是定制覆盖。
|
||||
- **覆盖表**：cache 键 `worker:cm:instcounts:<type>` = `{instance_id: 绝对副本数}`；
|
||||
无记录的实例回退到全局均分。
|
||||
- **命中判断**：`rescale` 命中本实例（`instance_id` 与登记一致）才执行，目标是其它实例则忽略。
|
||||
- **REST / UI**：`POST /admin/workers/<type>/scale_instance` 下发；
|
||||
`GET /admin/workers/<type>/instances` 供前端按实例分组展示。
|
||||
|
||||
## 7. 实例定向调用
|
||||
|
||||
默认逐请求**随机负载均衡**，适合无状态插件。若插件持有只存在于某个进程本地的状态
|
||||
（本地 SQLite/文件、串口、设备连接），可用实例定向把请求固定到该进程：
|
||||
|
||||
- **指定方式**（三入口语义一致）：
|
||||
- HTTP 请求头 `X-Chongming-Instance: <实例id>`
|
||||
- MQTT 信封载荷顶层 `target_instance`（网关读后摘除，业务侧不可见）
|
||||
- 程序内 `worker_manager.call(...)` / `call_raw(...)` / `call_stream(...)` 的 `instance_id=` 参数
|
||||
- **实例 id 取值**：`--instance` → `CHONGMING_WORKER_INSTANCE` → `CHONGMING_GATEWAY_ID` →
|
||||
`{hostname}:{pid}`；容器环境（K8s 注入 `CHONGMING_GATEWAY_ID`）下两侧自动一致，
|
||||
裸机同宿主多网关请显式指定。可读列表见 `GET /api/v1/admin/workers/<type>/instances`。
|
||||
- **粒度**：定向到**进程实例**；实例内有多个副本时，请求仍在这几个副本间随机均衡。
|
||||
- **实例离线时严格报错**（HTTP 503 / MQTT `{"error": ...}`），**不静默回退**到其它实例。
|
||||
- **重试不跨实例**；流式端点不重试，实例离线即失败。
|
||||
- **缓存按实例隔离**：定向且 `cacheable=true` 时缓存键带实例前缀。
|
||||
|
||||
```bash
|
||||
# 把这次请求固定到实例 edge-01 的进程
|
||||
curl -H 'X-Chongming-Instance: edge-01' http://127.0.0.1:8000/api/v1/demo/state
|
||||
|
||||
# MQTT：载荷顶层带 target_instance（网关转发前摘掉）
|
||||
# topic: chongming/demo/state payload: {"meta":{}, "data":{}, "target_instance":"edge-01"}
|
||||
```
|
||||
|
||||
## 8. 前端管理页
|
||||
|
||||
`Worker 管理`（`/workers`）：
|
||||
|
||||
- **登录**：内置 `POST /auth/login` 领取管理 Token（不依赖 auth 插件），仅 `is_superuser` 可操作。
|
||||
- **列表**：副本类型 / 副本 ID / 模式（inline|external）/ 状态，支持关键字、模式、副本数筛选与分页；
|
||||
筛选同时作用于下方逐实例分组表。`worker_type == "auth"` 的配置项不展示、不可改。
|
||||
- **操作**：单副本重启、批量重启、插件级重启、配置编辑（结构化 JSON 校验/格式化）、
|
||||
**调整副本数量**（全局对称）、**逐实例精细扩缩**（按实例分组，仅调该实例）。
|
||||
- **自动刷新**：每 10s 轮询列表与分组表，异步操作结果自动反映。
|
||||
- 调小副本数会触发优雅排空，前端确认提示中会说明。
|
||||
|
||||
## 9. external 插件的可管理性前提
|
||||
|
||||
1. 插件容器运行含管理能力（`__admin__` + `reload` + `rescale`）的 `chongming-worker`；
|
||||
2. `config_set` / `rescale` 由副本在**其进程 cwd** 下写 `config/workers/<type>/config.toml`，
|
||||
容器需挂载与网关同款的配置目录才能写回（含 `[workers].count` 持久化）；
|
||||
3. 其它语言自行实现的进程插件需实现 `__admin__` Queryable + liveliness + 数量管理，
|
||||
否则只能被发现，无法改配置或扩缩容。
|
||||
|
||||
## 10. 多 worker 与 worker 副本的分工
|
||||
|
||||
gunicorn `-w N` 提升的是**网关的 HTTP 接入/转发能力**；插件计算能力由 Worker **副本数**
|
||||
（`count`）或 external 进程数决定。高并发场景按瓶颈扩：转发瓶颈加 `-w`，计算瓶颈加 `count`
|
||||
或改跑 external。
|
||||
@@ -0,0 +1,40 @@
|
||||
# 内置插件专属依赖清单（打包收集用）
|
||||
#
|
||||
# 分工规则：
|
||||
# - 网关本体依赖 → chongming-gateway 包自己的 pyproject.toml（消费者自动获得；
|
||||
# 打包沿 `from chongming.app import app` 静态收集，不必在别处重复声明）
|
||||
# - public 骨架自身 + 动态导入（静态分析看不到，如 granian 子模块）→ 根 config.toml
|
||||
# [pyinstaller_build_import]
|
||||
# - 内置插件（src/plugins/<name>/）专属、网关本体不 import 的依赖 → 本文件，
|
||||
# 按插件分节声明
|
||||
#
|
||||
# `uv run build` 打包时读取本文件，把各节 imports 并入 PyInstaller --hidden-import /
|
||||
# Nuitka --include-module（inline 插件在 mbank 内 import，静态分析看不到，必须显式声明）。
|
||||
# 需 chongming-cli ≥ 0.3.4 的 build 才读取；更早版本会忽略本文件（插件依赖只在
|
||||
# config.toml 里找），升级前不要从那里删依赖。
|
||||
# 插件下线、或改为 external 进程插件（依赖随 workers/<name>/pyproject.toml 自包含）后，
|
||||
# 删掉对应节即可，网关构建配置不受影响。
|
||||
#
|
||||
# 注意：本文件只影响"打进产物 exe 的收集清单"；venv 安装清单仍在根 pyproject.toml
|
||||
# （dev 运行 inline 插件与打包共用同一 venv），两处需同步维护。
|
||||
#
|
||||
# 结构（节名 = src/plugins/ 下的插件名；顶层 imports 为多个插件共用）：
|
||||
# imports = ["xxx"]
|
||||
# [插件名]
|
||||
# imports = ["pkg", "pkg.submodule"]
|
||||
|
||||
# 多个插件共用的插件专属依赖
|
||||
# httpx: aibox_sophon_adapter / auth_oauth / ip_sound_column 的 HTTP 调用
|
||||
imports = [
|
||||
"httpx",
|
||||
]
|
||||
|
||||
[doubao]
|
||||
# 豆包视觉理解：openai SDK + 火山方舟 SDK + AICC 安全通信通道（bytedance 工具包）
|
||||
imports = [
|
||||
"openai",
|
||||
"volcenginesdkarkruntime",
|
||||
"bytedance",
|
||||
"bytedance.volcengine_aicc_sdk",
|
||||
"bytedance.volcengine_aicc_sdk.secure_http_client",
|
||||
]
|
||||
+4
-1
@@ -33,6 +33,8 @@ debug = false
|
||||
env = "production"
|
||||
|
||||
[production.server]
|
||||
# 启动参数对各服务器通用（由 main.py 的转换函数映射，无需按服务器改配置）；
|
||||
# 服务器/副本数可用环境变量覆盖：APP_SERVER=granian|uvicorn|gunicorn、APP_WORKERS=<N>
|
||||
access_log = false
|
||||
host = "0.0.0.0"
|
||||
port = 8000
|
||||
@@ -82,8 +84,9 @@ use_embedded_broker = true
|
||||
# 详细参数参考: https://github.com/Yakifo/amqtt
|
||||
|
||||
[production.module_system]
|
||||
# 运行期模块银行加载清单（frozen 产物用）：网关本体是普通依赖包（chongming-gateway，
|
||||
# 已由 PyInstaller/Nuitka 打进产物），只有内置插件经模块银行加载。
|
||||
path = [
|
||||
"resources/app.mbank",
|
||||
"resources/plugins.mbank",
|
||||
]
|
||||
type = "module-bank"
|
||||
|
||||
+131
-46
@@ -3,23 +3,126 @@
|
||||
"""
|
||||
FastAPI chongming 示例
|
||||
主入口文件，适配 chongming 打包系统
|
||||
|
||||
服务器选择（APP_SERVER 环境变量，默认见 DEFAULT_SERVER）：
|
||||
- gunicorn：现状保留（Linux 默认，uvloop worker + preload 多进程）
|
||||
- uvicorn：Windows 默认（gunicorn 依赖 fork，Windows 不可用）
|
||||
- granian：Rust HTTP 内核，多进程
|
||||
|
||||
启动参数保持 config.toml `[<env>.server]` 现状（host / port / workers /
|
||||
reload / access_log / timeout_keep_alive），由 `_to_uvicorn`、`_to_granian`、
|
||||
`_to_gunicorn_args` 转换为对应服务器的参数。
|
||||
"""
|
||||
import asyncio # noqa: F401
|
||||
import os
|
||||
import sys
|
||||
import platform
|
||||
from utils.launch import launch
|
||||
import sys
|
||||
from multiprocessing import freeze_support
|
||||
|
||||
import server # noqa: F401 # 预加载：master 内完成插件模块银行装载，fork 出的 worker 共享
|
||||
import uvloop # noqa: F401 # 显式导入保证打包时收集 uvloop（uvicorn/granian loop 依赖）
|
||||
from chongming import RELOAD_PATHS # 热重载监听目录（网关包内定义）
|
||||
from chongming.app.core.logger import get_logger
|
||||
from chongming.app.core.parent_death import guard_parent_death
|
||||
from utils.launch import launch
|
||||
from uvicorn.workers import UvicornWorker
|
||||
import server
|
||||
import uvloop
|
||||
import asyncio
|
||||
|
||||
use_shell = platform.system() == "Windows"
|
||||
|
||||
# 未设置 APP_SERVER 时的平台默认（保持现状）
|
||||
# DEFAULT_SERVER = "uvicorn" if use_shell else "gunicorn"
|
||||
DEFAULT_SERVER = "granian"
|
||||
SUPPORTED_SERVERS = ("gunicorn", "uvicorn", "granian")
|
||||
|
||||
|
||||
class UvloopUvicornWorker(UvicornWorker):
|
||||
CONFIG_KWARGS = {"loop": "uvloop"}
|
||||
|
||||
|
||||
def _server_params(app_config: dict) -> dict:
|
||||
"""把 `[<env>.server]` 的现状参数整理为各服务器共用的启动参数。
|
||||
|
||||
只做缺省兜底与运行环境覆盖（打包环境强制禁 reload、APP_WORKERS 覆盖副本数），
|
||||
键名保持 config.toml 现状，改名交给下面的转换函数。
|
||||
"""
|
||||
server_config = app_config.get("server", {})
|
||||
# 打包环境强制禁用重载
|
||||
is_frozen = getattr(sys, "frozen", False)
|
||||
# 支持通过环境变量 APP_WORKERS 覆盖 worker 数量（方便嵌入式设备调优）
|
||||
default_workers = server_config.get("workers", 4)
|
||||
workers = int(os.environ.get("APP_WORKERS", str(default_workers)))
|
||||
return {
|
||||
"host": server_config.get("host", "0.0.0.0"),
|
||||
"port": server_config.get("port", 8000),
|
||||
"workers": workers,
|
||||
"reload": False if is_frozen else server_config.get("reload", False),
|
||||
"access_log": server_config.get("access_log", False),
|
||||
"timeout_keep_alive": server_config.get("timeout_keep_alive", 5),
|
||||
"log_level": "debug" if app_config.get("debug", False) else "info",
|
||||
}
|
||||
|
||||
|
||||
def _to_uvicorn(params: dict) -> dict:
|
||||
"""共用参数 → uvicorn.run 关键字参数（键名一致）。"""
|
||||
return {
|
||||
"host": params["host"],
|
||||
"port": params["port"],
|
||||
"reload": params["reload"],
|
||||
"workers": params["workers"],
|
||||
"log_level": params["log_level"],
|
||||
"access_log": params["access_log"],
|
||||
"timeout_keep_alive": params["timeout_keep_alive"],
|
||||
}
|
||||
|
||||
|
||||
def _to_granian(params: dict) -> dict:
|
||||
"""共用参数 → granian.Granian 关键字参数。
|
||||
|
||||
仅两处改名：host → address、access_log → log_access；
|
||||
timeout_keep_alive 在 granian 无对应项（HTTP/1 keep-alive 只有开关、无空闲
|
||||
超时配置），忽略。
|
||||
"""
|
||||
return {
|
||||
"address": params["host"],
|
||||
"port": params["port"],
|
||||
"interface": "asgi",
|
||||
"workers": params["workers"],
|
||||
"reload": params["reload"],
|
||||
"log_level": params["log_level"],
|
||||
"log_access": params["access_log"],
|
||||
}
|
||||
|
||||
|
||||
def _to_gunicorn_args(params: dict) -> list:
|
||||
"""共用参数 → gunicorn 命令行参数（保持现状：uvloop worker + preload）。"""
|
||||
return [
|
||||
"gunicorn",
|
||||
"server:app",
|
||||
"-k",
|
||||
"__main__.UvloopUvicornWorker",
|
||||
"--preload",
|
||||
"-w",
|
||||
str(params["workers"]),
|
||||
"-b",
|
||||
f"{params['host']}:{params['port']}",
|
||||
]
|
||||
|
||||
|
||||
def _select_server(logger) -> str:
|
||||
"""按 APP_SERVER 环境变量选择服务器，非法值/平台不兼容时回退默认。"""
|
||||
server_kind = (os.environ.get("APP_SERVER") or DEFAULT_SERVER).strip().lower()
|
||||
if server_kind not in SUPPORTED_SERVERS:
|
||||
logger.warning(
|
||||
f"未知 APP_SERVER={server_kind!r}（可选: {', '.join(SUPPORTED_SERVERS)}），"
|
||||
f"回退 {DEFAULT_SERVER}"
|
||||
)
|
||||
server_kind = DEFAULT_SERVER
|
||||
if server_kind == "gunicorn" and use_shell:
|
||||
logger.warning("Windows 不支持 gunicorn（依赖 fork），回退 uvicorn")
|
||||
server_kind = "uvicorn"
|
||||
return server_kind
|
||||
|
||||
|
||||
def app_run(app_config: dict, default_config: dict):
|
||||
"""
|
||||
应用运行函数 - 适配模块银行环境
|
||||
@@ -31,22 +134,10 @@ def app_run(app_config: dict, default_config: dict):
|
||||
Returns:
|
||||
None: 此函数不返回值，直接启动服务器并阻塞运行
|
||||
"""
|
||||
from app.core.logger import get_logger # type: ignore
|
||||
|
||||
logger = get_logger("app")
|
||||
|
||||
# 获取服务器配置
|
||||
server_config = app_config.get("server", {})
|
||||
host = server_config.get("host", "0.0.0.0")
|
||||
port = server_config.get("port", 8000)
|
||||
timeout_keep_alive = server_config.get("timeout_keep_alive", 5)
|
||||
|
||||
# 打包环境强制禁用重载
|
||||
is_frozen = getattr(sys, "frozen", False)
|
||||
reload = False if is_frozen else server_config.get("reload", False)
|
||||
# 支持通过环境变量 APP_WORKERS 覆盖 worker 数量（方便嵌入式设备调优）
|
||||
default_workers = server_config.get("workers", 4)
|
||||
workers = int(os.environ.get("APP_WORKERS", str(default_workers)))
|
||||
params = _server_params(app_config)
|
||||
server_kind = _select_server(logger)
|
||||
|
||||
# 显示启动信息
|
||||
env = app_config.get("env", "development")
|
||||
@@ -54,41 +145,31 @@ def app_run(app_config: dict, default_config: dict):
|
||||
logger.info(f"🚀 启动 {default_config.get('app.name', 'FastAPI 应用')}")
|
||||
logger.info(f"📦 版本: {default_config.get('app.version', '1.0.0')}")
|
||||
logger.info(f"🌍 环境: {env}")
|
||||
logger.info(f"📍 地址: http://{host}:{port}")
|
||||
logger.info(f"📚 文档: http://{host}:{port}/docs")
|
||||
logger.info(f"🖥️ 服务器: {server_kind}")
|
||||
logger.info(f"📍 地址: http://{params['host']}:{params['port']}")
|
||||
logger.info(f"📚 文档: http://{params['host']}:{params['port']}/docs")
|
||||
logger.info(f"🔧 调试: {'启用' if app_config.get('debug', False) else '禁用'}")
|
||||
logger.info(f"🔄 重载: {'启用' if reload else '禁用'}")
|
||||
logger.info(f"👥 Workers: {workers}")
|
||||
logger.info(f"🔄 重载: {'启用' if params['reload'] else '禁用'}")
|
||||
logger.info(f"👥 Workers: {params['workers']}")
|
||||
logger.info("=" * 50)
|
||||
|
||||
# 根据操作系统选择不同的服务器启动方式
|
||||
if use_shell:
|
||||
# 根据所选服务器启动（默认平台分支保持现状）
|
||||
if server_kind == "granian":
|
||||
import granian
|
||||
|
||||
granian.Granian(
|
||||
target="server:app",
|
||||
**_to_granian(params),
|
||||
reload_paths=RELOAD_PATHS,
|
||||
).serve()
|
||||
elif server_kind == "uvicorn":
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(
|
||||
"server:app",
|
||||
host=host,
|
||||
port=port,
|
||||
reload=reload,
|
||||
workers=workers,
|
||||
log_level="debug" if app_config.get("debug", False) else "info",
|
||||
access_log=False,
|
||||
timeout_keep_alive=timeout_keep_alive,
|
||||
)
|
||||
uvicorn.run("server:app", **_to_uvicorn(params))
|
||||
else:
|
||||
from gunicorn.app.wsgiapp import run
|
||||
|
||||
sys.argv = [
|
||||
"gunicorn",
|
||||
"server:app",
|
||||
"-k",
|
||||
"__main__.UvloopUvicornWorker",
|
||||
"--preload",
|
||||
"-w",
|
||||
str(workers),
|
||||
"-b",
|
||||
f"{host}:{port}",
|
||||
]
|
||||
sys.argv = _to_gunicorn_args(params)
|
||||
run()
|
||||
|
||||
|
||||
@@ -99,6 +180,10 @@ def main():
|
||||
Returns:
|
||||
None: 此函数不返回值，调用launch函数启动应用
|
||||
"""
|
||||
# 父进程死亡防护（master 侧）：引导进程（onefile 引导/终端 shell）死亡时
|
||||
# ——含被 SIGKILL——内核给本进程发 SIGTERM，granian 走优雅停机收掉全部 worker；
|
||||
# worker 侧在 app 的 lifespan 里各自设置（PR_SET_PDEATHSIG 不跨 fork 继承）。
|
||||
guard_parent_death(get_logger("app"))
|
||||
freeze_support()
|
||||
launch(app_run, "production")
|
||||
|
||||
|
||||
+6
-7
@@ -3,18 +3,17 @@ import sys
|
||||
from utils.launch import init_module_bank
|
||||
from utils.config import ConfigLoader
|
||||
|
||||
# 加载配置文件并初始化模块银行
|
||||
# 加载配置文件并初始化模块银行（当前只剩内置插件 plugins.mbank）
|
||||
config = ConfigLoader("config.toml")
|
||||
init_module_bank(config)
|
||||
|
||||
# 从数据库动态导入 app 模块
|
||||
# 该模块存储在 SQLite 数据库中，通过自定义导入机制加载
|
||||
# 网关本体是普通依赖包（chongming-gateway），直接导入即可——打包时
|
||||
# PyInstaller/Nuitka 也沿这行静态收集网关及其依赖。
|
||||
# 插件（plugins 包）由上面的模块银行提供，在 lifespan 启动时被网关按配置加载。
|
||||
try:
|
||||
import app # type: ignore
|
||||
|
||||
app_instance = app.app
|
||||
from chongming.app import app as app_instance
|
||||
except ImportError as e:
|
||||
print(f"无法从数据库导入 app 模块: {e}")
|
||||
print(f"无法导入网关（chongming.app）: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# 暴露 ASGI 应用实例，供 uvicorn 服务器使用
|
||||
|
||||
+49
-31
@@ -1,53 +1,71 @@
|
||||
[build-system]
|
||||
build-backend = "hatchling.build"
|
||||
requires = ["hatchling"]
|
||||
|
||||
[project]
|
||||
# 应用/打包项目：本仓负责宿主骨架（public/）、内置插件集成（src/plugins）、构建配置与
|
||||
# 交付产物。网关本体是独立包 chongming-gateway（源码在 utils/gateway，独立仓库；
|
||||
# 发布到私有源后按版本依赖），本文件只声明宿主自己需要的依赖。
|
||||
dependencies = [
|
||||
"chongming-jwt",
|
||||
"chongming-request",
|
||||
"eclipse-zenoh",
|
||||
"fastapi[uvicorn]",
|
||||
"gunicorn",
|
||||
# —— 网关本体（私有源发布版；本地联调见下方 [tool.uv.sources] 注释）——
|
||||
"chongming-gateway",
|
||||
# —— public 骨架（exe 入口）自己的依赖 ——
|
||||
# "granian[reload,uvloop]>=2.8.4",
|
||||
# "gunicorn",
|
||||
# "httptools",
|
||||
"module-bank",
|
||||
# "uvicorn",
|
||||
"uvloop",
|
||||
# —— 共用 venv：内置插件 + 测试所需（dev 运行 inline 插件与打包共用同一 venv）——
|
||||
# 注：注释项由 chongming-gateway 传递引入，保留注释便于查看来源，需要时再显式声明
|
||||
"aiofiles",
|
||||
# "amqtt",
|
||||
"bytedance",
|
||||
# "chongming-auth",
|
||||
# "chongming-cache",
|
||||
# "chongming-jwt",
|
||||
"chongming-mqtt",
|
||||
"chongming-password",
|
||||
"chongming-request",
|
||||
"chongming-stream-manager",
|
||||
# "chongming-worker",
|
||||
# "eclipse-zenoh",
|
||||
# "fastapi",
|
||||
"openai",
|
||||
"python-multipart",
|
||||
"sqlite-vfs",
|
||||
# "sqlite-vfs",
|
||||
"tomli",
|
||||
"typing-extensions",
|
||||
"uvicorn",
|
||||
"volcengine-python-sdk[ark]",
|
||||
"chongming-password",
|
||||
"chongming-worker",
|
||||
"amqtt",
|
||||
"aiofiles",
|
||||
"chongming-stream-manager",
|
||||
"uvloop",
|
||||
"chongming-mqtt",
|
||||
"httptools",
|
||||
"bytedance",
|
||||
"chongming-cache",
|
||||
"chongming-auth",
|
||||
]
|
||||
description = "基于 modules-bank 的 fastapi 打包脚手架"
|
||||
description = "基于 modules-bank 的 fastapi 打包脚手架（宿主项目：骨架 + 插件集成 + 构建）"
|
||||
name = "chongming-zenoh"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
version = "0.1.0"
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["chongming-cli", "nuitka", "pyarmor", "pyinstaller", "ruff"]
|
||||
dev = [
|
||||
"chongming-cli",
|
||||
"greenlet>=3.5.6",
|
||||
# onefile extra：带 zstandard，否则 onefile 产物不做压缩（Nuitka 自身推荐写法）
|
||||
"nuitka[onefile]",
|
||||
# Nuitka standalone/onefile 打包在 Linux 必需（链接完成后才检查，缺了会白跑一整轮编译）
|
||||
"patchelf; sys_platform == 'linux'",
|
||||
"pyarmor",
|
||||
"pyinstaller",
|
||||
"ruff",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/chongming"]
|
||||
[tool.uv]
|
||||
# 宿主项目自身不再出包（网关搬到 chongming-gateway 后本仓无 Python 包），
|
||||
# uv sync 只装依赖与 dev 工具，不再尝试构建/安装本仓 editable。
|
||||
package = false
|
||||
|
||||
# 本地联调网关时取消注释：直接用 utils/gateway 源码（editable），改完无需发版即可被
|
||||
# 本仓（含打包构建）使用；联调结束恢复按发布版依赖，并用 uv lock --upgrade-package
|
||||
# chongming-gateway 刷新版本。
|
||||
# [tool.uv.sources]
|
||||
# chongming-gateway = { path = "utils/gateway", editable = true }
|
||||
|
||||
[[tool.uv.index]]
|
||||
url = "http://124.71.68.6:3000/api/packages/chongming/pypi/simple/"
|
||||
|
||||
[[tool.uv.index]]
|
||||
url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
|
||||
[project.scripts]
|
||||
gunicorn = "chongming:gunicorn_serve"
|
||||
serve = "chongming:serve"
|
||||
worker_serve = "chongming_worker.process:worker_serve"
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
import uvicorn
|
||||
from chongming_config import load_config
|
||||
from .app.core.logger import get_logger
|
||||
|
||||
|
||||
def serve():
|
||||
"""
|
||||
使用 uvicorn 启动服务
|
||||
|
||||
从配置文件中加载服务器配置和日志配置，然后使用 uvicorn 启动应用服务。
|
||||
该函数适用于开发环境或单进程部署场景。
|
||||
|
||||
Returns:
|
||||
None: 该函数不返回值，会阻塞运行直到服务停止
|
||||
"""
|
||||
config = load_config()
|
||||
env = config["default"]["env"]
|
||||
server_config = config[env]["server"]
|
||||
log_config = config[env]["logging"]
|
||||
logger = get_logger("app")
|
||||
logger.info(f"启动服务: {server_config}")
|
||||
logger.info(f"启动日志: {log_config}")
|
||||
uvicorn.run("chongming.app:app", **server_config)
|
||||
|
||||
|
||||
def gunicorn_serve():
|
||||
"""
|
||||
使用 gunicorn 启动服务
|
||||
|
||||
通过 gunicorn 作为进程管理器启动应用，使用 UvicornWorker 作为 worker 类型，
|
||||
支持多进程部署，适用于生产环境。
|
||||
|
||||
从配置文件中读取服务器配置，动态设置 gunicorn 的 worker 数量、绑定地址和端口等参数。
|
||||
|
||||
Returns:
|
||||
None: 该函数不返回值，会阻塞运行直到服务停止
|
||||
"""
|
||||
import sys
|
||||
from gunicorn.app.wsgiapp import run
|
||||
|
||||
config = load_config()
|
||||
env = config["default"]["env"]
|
||||
server_config = config[env]["server"]
|
||||
|
||||
# 构建 gunicorn 命令行参数并启动服务
|
||||
sys.argv = [
|
||||
"gunicorn",
|
||||
"chongming.app:app",
|
||||
"-k",
|
||||
"uvicorn.workers.UvicornWorker",
|
||||
"-w",
|
||||
f"{server_config.get('workers', 4)}",
|
||||
"-b",
|
||||
f"{server_config.get('host', '0.0.0.0')}:{server_config.get('port', 8000)}",
|
||||
]
|
||||
run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
serve()
|
||||
@@ -1,317 +0,0 @@
|
||||
"""
|
||||
chongming FastAPI 应用入口
|
||||
|
||||
重构为配置驱动的微服务网关：
|
||||
- WorkerManager: 统一管理 Zenoh worker 存活检测、负载均衡、超时重试、缓存
|
||||
- RouteGenerator: 根据配置自动生成 FastAPI 端点
|
||||
- 插件工厂函数动态加载
|
||||
"""
|
||||
|
||||
import time
|
||||
import asyncio
|
||||
import itertools
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.middleware.gzip import GZipMiddleware
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from chongming_config import get_config
|
||||
from .core.logger import get_logger
|
||||
# from .core.static_files import SVFSStaticFiles
|
||||
from sqlite_vfs.contrib.fastapi_static import SVFSStaticFiles
|
||||
from .core import lifespan_setup, call_stats
|
||||
from .routes import storage_router, stats_router, worker_admin_router, auth_router
|
||||
|
||||
|
||||
config = get_config()
|
||||
logger = get_logger("app")
|
||||
|
||||
try:
|
||||
env = config["default"]["env"]
|
||||
except KeyError: # pragma: no cover
|
||||
raise ValueError("配置不存在")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# ---- 启动阶段 ----
|
||||
# 1. 本地存储（替代 Nanio S3，省 ~172 MB）
|
||||
lifespan_setup.setup_storage(app)
|
||||
|
||||
# 1.2 内置 auth 库（users 表 + admin 种子）—— auth 域的基础设施部分：
|
||||
# 登录/签发内置在网关（管理平面底线），不依赖 auth 插件是否加载；
|
||||
# 与 auth 插件（用户管理）共用同一张 users 表
|
||||
await lifespan_setup.setup_auth_store(app)
|
||||
|
||||
# 2. Zenoh
|
||||
session = lifespan_setup.setup_zenoh(app)
|
||||
|
||||
# 3. WorkerManager
|
||||
manager = lifespan_setup.setup_worker_manager(app, session)
|
||||
|
||||
# 3.1 内嵌 amqtt Broker（宿主级分布式锁；需在 WorkerManager 就绪后提供共享 cache）
|
||||
await lifespan_setup.setup_mqtt_broker(app, manager)
|
||||
|
||||
# 4.1 全局副本数协调：把当前网关登记进在线列表，供按在线网关分摊 quota
|
||||
manager.register_gateway()
|
||||
|
||||
# 4.2 宿主级 inline worker 领导选举（基于共享 cache 的分布式锁，key 按网关隔离）：
|
||||
# gunicorn 多 worker 下每实例仅一个进程托管 inline worker（避免 N×count 冗余），
|
||||
# 其余进程只做 HTTP 并经 Zenoh liveliness 发现调用；多实例各选各的 leader。
|
||||
is_inline_owner = manager.acquire_inline_owner()
|
||||
app.state.inline_owner_locked = is_inline_owner
|
||||
|
||||
# 5. 插件加载（混合模式：inline 进程内加载；external 由独立进程运行，
|
||||
# 其路由经 Zenoh 动态注册——进程编排交给 Docker/K8s，网关不拉起进程）
|
||||
worker_tasks, custom_routers, inline_types, external_types = (
|
||||
await lifespan_setup.load_workers(
|
||||
app, session, manager, is_inline_owner=is_inline_owner
|
||||
)
|
||||
)
|
||||
|
||||
# 5.1 动态路由注册表常驻启动：订阅 chongming/routes/**，external 插件的
|
||||
# 路由经 Zenoh 注册/摘除，无需网关侧保留插件配置（external_types 仅观测）
|
||||
await lifespan_setup.setup_route_registry(
|
||||
app, session, manager, inline_types=inline_types
|
||||
)
|
||||
|
||||
# 5.5 契约校验（仅 inline 插件；external 插件在自己进程内校验），失败则启动终止
|
||||
lifespan_setup.validate_contracts(app)
|
||||
|
||||
# 6. 注册路由（inline 插件的 TOML 动态路由 + 插件自定义路由）
|
||||
await lifespan_setup.register_routes(
|
||||
app, manager, custom_routers, worker_types=inline_types
|
||||
)
|
||||
|
||||
# 7. 启动 MQTT Bridge（内嵌 Broker + Worker 桥接）
|
||||
await lifespan_setup.setup_mqtt(app, manager)
|
||||
|
||||
# 7.1 启动 MQTT Bridge 持有者自愈巡检（跨所有网关进程，崩溃时自动接管重建）
|
||||
app.state.mqtt_bridge_supervise_task = asyncio.create_task(
|
||||
lifespan_setup.mqtt_bridge_supervisor(app, manager)
|
||||
)
|
||||
|
||||
# 8. 短暂等待，确保 worker 状态同步
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# 打印 worker 就绪状态
|
||||
worker_status = {
|
||||
wtype: len(workers)
|
||||
for wtype, workers in manager._workers_cache.items()
|
||||
}
|
||||
logger.info(f"Worker 就绪状态: {worker_status}")
|
||||
|
||||
# ---- 应用运行 ----
|
||||
yield
|
||||
|
||||
# ---- 关闭阶段 ----
|
||||
logger.info("正在关闭应用...")
|
||||
|
||||
# 1. 停止 MQTT Bridge 持有者自愈巡检，先于 Bridge 关停，避免关停期间再次接管
|
||||
_sup = getattr(app.state, "mqtt_bridge_supervise_task", None)
|
||||
if _sup:
|
||||
_sup.cancel()
|
||||
try:
|
||||
await _sup
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
|
||||
# 1.0 停止 MQTT Bridge
|
||||
await lifespan_setup.shutdown_mqtt(app)
|
||||
|
||||
# 1.5 停止路由注册表订阅（常驻，随应用生命周期启停）
|
||||
registry = getattr(app.state, "route_registry", None)
|
||||
if registry is not None:
|
||||
registry.stop()
|
||||
|
||||
# 2. 清理 inline workers（cleanup_workers 内部按 load_mode 过滤）
|
||||
await lifespan_setup.cleanup_workers(app)
|
||||
|
||||
# 2.2 关闭内置 auth 库
|
||||
auth_db = getattr(app.state, "auth_db", None)
|
||||
if auth_db is not None:
|
||||
await auth_db.dispose()
|
||||
|
||||
# 2.1 释放 inline worker 所有权分布式锁（仅持锁进程，避免误删其他进程获得的新锁）
|
||||
if getattr(app.state, "inline_owner_locked", False):
|
||||
await manager.release_inline_owner()
|
||||
|
||||
# 3. 取消刷新任务
|
||||
refresh_task = getattr(app.state, "refresh_task", None)
|
||||
if refresh_task:
|
||||
refresh_task.cancel()
|
||||
try:
|
||||
await refresh_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# 4. 取消所有 worker 任务（含 self-heal 接管后拉起的任务）
|
||||
all_worker_tasks = list(worker_tasks) + manager.extra_worker_tasks()
|
||||
for task in all_worker_tasks:
|
||||
task.cancel()
|
||||
if all_worker_tasks:
|
||||
await asyncio.gather(*all_worker_tasks, return_exceptions=True)
|
||||
|
||||
# 4.5 取消后台统计/审计任务
|
||||
await call_stats.shutdown()
|
||||
|
||||
# 4.6 全局副本数协调：关闭时把当前网关从在线列表移除（其余网关将重新分摊）
|
||||
manager.unregister_gateway()
|
||||
|
||||
# 5. 关闭 Zenoh 会话
|
||||
session.close()
|
||||
logger.info("Zenoh 会话已关闭")
|
||||
|
||||
logger.info("应用已关闭")
|
||||
|
||||
|
||||
cors = config[env].get("cors", {})
|
||||
app = FastAPI(
|
||||
title=config["default"]["app"]["name"],
|
||||
version=config["default"]["app"]["version"],
|
||||
description=config["default"]["app"]["description"],
|
||||
docs_url="/docs" if config[env]["debug"] else None,
|
||||
redoc_url="/redoc" if config[env]["debug"] else None,
|
||||
openapi_url="/openapi.json" if config[env]["debug"] else None,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(CORSMiddleware, **cors)
|
||||
# GZip 压缩响应（<500B 不压缩；starlette 默认排除 text/event-stream，
|
||||
# 保证 SSE 流式端点即时性；>=128KB 的响应在独立线程压缩，不阻塞事件循环）
|
||||
app.add_middleware(GZipMiddleware)
|
||||
|
||||
|
||||
# access 日志采样率（logging.access_sample_rate，0.1 = 只记录 10% 的成功请求；
|
||||
# 错误 >=400 始终全量记录，保证可追溯；1.0/缺省 = 全量）
|
||||
ACCESS_SAMPLE_RATE = float(config[env].get("logging", {}).get("access_sample_rate", 1.0))
|
||||
# 成功请求采样计数器（每进程独立；错误请求不消耗计数）
|
||||
_access_log_counter = itertools.count()
|
||||
|
||||
|
||||
def _should_log_access(status_code: int, sample_rate: float = ACCESS_SAMPLE_RATE) -> bool:
|
||||
"""错误全量；成功按采样率（采样为确定性计数，无随机数开销）"""
|
||||
if status_code >= 400 or sample_rate >= 1.0:
|
||||
return True
|
||||
denom = max(1, int(round(1.0 / sample_rate)))
|
||||
return next(_access_log_counter) % denom == 0
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def access_log_middleware(request: Request, call_next):
|
||||
"""
|
||||
HTTP访问日志中间件，记录每个请求的详细信息。
|
||||
|
||||
记录的信息包括：
|
||||
- 请求方法和路径
|
||||
- 响应状态码
|
||||
- 请求处理时长（毫秒）
|
||||
- 客户端IP地址
|
||||
|
||||
高吞吐优化：
|
||||
- 采样：成功请求按 access_sample_rate 记录，错误（>=400）全量
|
||||
- 异步：% 参数化消息 + 队列日志（QueueHandler），格式化与文件 I/O
|
||||
在后台监听线程执行，主线程仅入队
|
||||
|
||||
参数:
|
||||
request: FastAPI请求对象
|
||||
call_next: 下一个中间件或路由处理函数
|
||||
|
||||
返回:
|
||||
Response: 处理后的响应对象
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
response = await call_next(request)
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
client = request.client.host if request.client else "-"
|
||||
|
||||
if _should_log_access(response.status_code):
|
||||
logger.info(
|
||||
"%s %s status=%s duration=%.2fms client=%s",
|
||||
request.method, request.url.path, response.status_code, duration, client,
|
||||
extra={"status_code": response.status_code},
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@app.get("/", summary="根路径")
|
||||
async def root():
|
||||
"""
|
||||
根路径端点，返回应用的基本信息。
|
||||
|
||||
返回:
|
||||
dict: 包含应用名称、版本、文档地址和健康检查地址的字典
|
||||
"""
|
||||
return {
|
||||
"message": f"欢迎使用 {config['default']['app']['name']}",
|
||||
"version": config["default"]["app"]["version"],
|
||||
"docs": "/docs" if config[env]["debug"] else None,
|
||||
"health": f"{config['default']['prefix']}/health",
|
||||
}
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_exception_handler(request: Request, exc: RequestValidationError):
|
||||
"""
|
||||
请求验证异常处理器，记录详细的验证错误信息和请求体。
|
||||
帮助快速定位 Pydantic 模型与前端/设备上报数据的不匹配问题。
|
||||
"""
|
||||
body = None
|
||||
try:
|
||||
body = await request.body()
|
||||
body_str = body.decode("utf-8")[:2000] if body else "<empty>"
|
||||
except Exception:
|
||||
body_str = "<unable to read body>"
|
||||
|
||||
errors = exc.errors()
|
||||
error_summary = [
|
||||
f"{' -> '.join(str(loc) for loc in e['loc'])}: {e['msg']}"
|
||||
for e in errors
|
||||
]
|
||||
logger.error(
|
||||
f"请求验证失败 [{request.method} {request.url.path}]:\n"
|
||||
f" 错误: {'; '.join(error_summary)}\n"
|
||||
f" 请求体: {body_str}"
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content={"detail": errors},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def global_exception_handler(_: Request, exc: Exception):
|
||||
"""
|
||||
全局异常处理器，捕获并处理应用中未捕获的异常。
|
||||
|
||||
在调试模式下返回详细的异常信息，在生产模式下返回通用错误提示。
|
||||
|
||||
参数:
|
||||
_: FastAPI请求对象（未使用）
|
||||
exc: 捕获到的异常对象
|
||||
|
||||
返回:
|
||||
JSONResponse: 包含错误信息的JSON响应，状态码为500
|
||||
"""
|
||||
logger.error(f"全局异常: {exc}")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={
|
||||
"error": "内部服务器错误",
|
||||
"detail": str(exc) if config[env]["debug"] else "请查看服务器日志",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# 静态文件服务（可选）
|
||||
if vfs_db_path := config[env].get("file_system", {}).get("path", None):
|
||||
logger.info("静态文件已存在，将使用静态文件服务")
|
||||
static_files_handler = SVFSStaticFiles(vfs_db_path=vfs_db_path)
|
||||
app.mount("/static", static_files_handler, name="static")
|
||||
|
||||
app.include_router(storage_router)
|
||||
app.include_router(stats_router)
|
||||
app.include_router(worker_admin_router)
|
||||
app.include_router(auth_router)
|
||||
@@ -1,85 +0,0 @@
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
from fastapi import Depends, HTTPException, status, Request
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from chongming_jwt import JWT
|
||||
from chongming_cache import Cache
|
||||
from chongming_config import get_config
|
||||
import orjson
|
||||
|
||||
|
||||
security_scheme = HTTPBearer(auto_error=False)
|
||||
|
||||
# JWT 解析共享缓存：与 WorkerManager 共用 [cache] 配置段。
|
||||
# 默认 sqlite（本机）；横向拓展时 config.toml 切 backend = "redis"，
|
||||
# 多网关实例共享同一 Redis 即可全局只解密一次，代码零改动。
|
||||
# 注意：缓存存的是"已验证结果"，信任模型与 worker 回复缓存一致
|
||||
# （缓存基础设施可信；Redis 请配置密码并置于内网）。
|
||||
_jwt_cache = Cache.from_config(get_config()["cache"])
|
||||
# 条目 TTL（秒）：控制存储占用与密钥轮换后的自然清理；
|
||||
# 安全性由命中时的 exp 复核保证，TTL 不承担过期判断
|
||||
_JWT_CACHE_TTL = 60
|
||||
|
||||
|
||||
def _cache_key(token: str, secret: str, algorithm: str) -> str:
|
||||
"""缓存 key：token 内容 + 密钥/算法指纹，不同密钥/算法不串用"""
|
||||
return "jwt:" + hashlib.sha256(f"{secret}:{algorithm}:{token}".encode()).hexdigest()
|
||||
|
||||
|
||||
def _cached_decode(token: str, secret: str, algorithm: str) -> tuple:
|
||||
"""JWT 解密结果缓存（(payload, exp)），共享缓存避免高频请求重复密码学运算。
|
||||
|
||||
- 命中且未过期 -> 直接返回，跳过 RSA/ECDSA 验证与 JSON 解析
|
||||
- 命中但 exp 已过 -> 直接拒绝，不重解密（条目保留至 TTL 到期，
|
||||
过期 token 的重复请求零密码学开销；缓存不延长 token 有效期）
|
||||
- 未命中 -> 完整解码后写入共享缓存（TTL=60s）
|
||||
- decode 失败（无效 token）不缓存，每次都完整验证后拒绝
|
||||
"""
|
||||
key = _cache_key(token, secret, algorithm)
|
||||
entry = _jwt_cache.get(key)
|
||||
if entry is not None:
|
||||
payload = entry["payload"]
|
||||
exp = entry.get("exp", 0)
|
||||
if not exp or exp >= time.time():
|
||||
return payload, exp
|
||||
raise ValueError("Token 已过期")
|
||||
jwt = JWT(secret, algorithm)
|
||||
payload_str = jwt.decode(token)
|
||||
payload = orjson.loads(payload_str.encode("utf-8"))
|
||||
exp = payload.get("exp", 0)
|
||||
_jwt_cache.set(key, {"payload": payload, "exp": exp}, expire=_JWT_CACHE_TTL)
|
||||
return payload, exp
|
||||
|
||||
|
||||
def jwt_auth_dependency(
|
||||
credentials: HTTPAuthorizationCredentials = Depends(security_scheme)
|
||||
) -> dict:
|
||||
"""JWT 校验依赖，返回解码后的 payload，校验失败抛出 401"""
|
||||
config = get_config()
|
||||
secret = config["security"]["secret_key"]
|
||||
algorithm = config["security"]["algorithm"]
|
||||
|
||||
if not credentials:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Missing authorization header",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
token = credentials.credentials
|
||||
try:
|
||||
payload, _ = _cached_decode(token, secret, algorithm)
|
||||
# 缓存后端返回的是独立对象（JSON 反序列化后为新副本），调用方修改不影响缓存
|
||||
return payload
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail=f"Invalid token: {str(e)}",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
async def store_user_to_request(
|
||||
request: Request,
|
||||
user: dict = Depends(jwt_auth_dependency)
|
||||
) -> None:
|
||||
request.state.user = user
|
||||
@@ -1,170 +0,0 @@
|
||||
"""
|
||||
调用统计与审计日志（后台任务执行）
|
||||
|
||||
route_generator 端点在 call_raw 返回后仅做非关键后置操作，全部经
|
||||
spawn() 以 fire-and-forget 后台任务执行，端点立即返回，不增加接口
|
||||
响应延迟、不占用请求协程资源：
|
||||
1. 更新内存调用统计（per worker_type/subject/status 计数 + 延迟聚合 + 最近调用缓冲）
|
||||
2. 写审计日志（logs/audit-YYYY-MM-DD.log，复用 logger 按日切分 + 大小轮转设施）
|
||||
|
||||
record_call 内部全兜底：统计或审计任一步失败仅记 warning，绝不影响响应。
|
||||
统计为进程内视角（gunicorn 多 worker 下各进程独立聚合）。
|
||||
|
||||
后台任务随应用生命周期管理：lifespan 关闭阶段调用 shutdown() 统一取消。
|
||||
"""
|
||||
import asyncio
|
||||
import datetime
|
||||
import orjson
|
||||
import random
|
||||
import time
|
||||
from collections import deque
|
||||
from typing import Any, Coroutine, Dict
|
||||
|
||||
from chongming_config import get_config
|
||||
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger("call_stats")
|
||||
# 审计日志独立文件（audit-YYYY-MM-DD.log），与业务日志分离便于检索
|
||||
audit_logger = get_logger("audit")
|
||||
|
||||
# 最近调用环形缓冲上限
|
||||
MAX_RECENT = 1000
|
||||
|
||||
# 审计日志采样率：与 access 日志同源配置（access_sample_rate，0.1 = 只写 10% 的成功请求；
|
||||
# 错误 status >= 400 始终全量）。压测实测全量审计写盘 8.8MB/4.3 万行，是高吞吐下的隐藏 I/O 大头。
|
||||
_audit_config = get_config()
|
||||
_audit_env = _audit_config["default"]["env"]
|
||||
_AUDIT_SAMPLE_RATE = float(
|
||||
_audit_config[_audit_env].get("logging", {}).get("access_sample_rate", 1.0)
|
||||
)
|
||||
|
||||
|
||||
def _should_record_audit(status: int) -> bool:
|
||||
"""审计日志采样判定（与 access 日志同模式）"""
|
||||
if status >= 400 or _AUDIT_SAMPLE_RATE >= 1.0:
|
||||
return True
|
||||
denom = max(1, int(round(1.0 / _AUDIT_SAMPLE_RATE)))
|
||||
return random.randrange(denom) == 0
|
||||
|
||||
# 后台任务集合（fire-and-forget，shutdown 时统一取消）
|
||||
_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def spawn(coro: Coroutine) -> asyncio.Task:
|
||||
"""以 fire-and-forget 方式创建后台任务并纳入跟踪（shutdown 时取消）"""
|
||||
task = asyncio.create_task(coro)
|
||||
_tasks.add(task)
|
||||
task.add_done_callback(_tasks.discard)
|
||||
return task
|
||||
|
||||
|
||||
async def shutdown() -> None:
|
||||
"""取消所有后台统计/审计任务（应用关闭时调用）"""
|
||||
tasks = list(_tasks)
|
||||
if not tasks:
|
||||
return
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
class CallStats:
|
||||
"""内存调用统计：计数 + 延迟聚合 + 最近调用环形缓冲。
|
||||
|
||||
asyncio 单线程事件循环内访问，无需加锁。
|
||||
"""
|
||||
|
||||
def __init__(self, max_recent: int = MAX_RECENT):
|
||||
self._counts: Dict[tuple, int] = {}
|
||||
self._latency: Dict[str, Dict[str, float]] = {}
|
||||
self._recent: deque = deque(maxlen=max_recent)
|
||||
|
||||
def record(self, entry: Dict[str, Any]) -> None:
|
||||
key = (entry["worker_type"], entry["subject"], entry["status"])
|
||||
self._counts[key] = self._counts.get(key, 0) + 1
|
||||
|
||||
wt = entry["worker_type"]
|
||||
lat = self._latency.setdefault(wt, {"total": 0.0, "count": 0, "max": 0.0})
|
||||
dur = entry.get("duration_ms", 0.0)
|
||||
lat["total"] += dur
|
||||
lat["count"] += 1
|
||||
if dur > lat["max"]:
|
||||
lat["max"] = dur
|
||||
|
||||
self._recent.append(entry)
|
||||
|
||||
def summary(self) -> Dict[str, Any]:
|
||||
"""聚合快照：总量、per worker/subject/status 计数、延迟、最近调用"""
|
||||
calls = [
|
||||
{"worker_type": wt, "subject": subj, "status": status, "count": count}
|
||||
for (wt, subj, status), count in sorted(self._counts.items())
|
||||
]
|
||||
latency = [
|
||||
{
|
||||
"worker_type": wt,
|
||||
"count": lat["count"],
|
||||
"avg_ms": round(lat["total"] / lat["count"], 2) if lat["count"] else 0.0,
|
||||
"max_ms": round(lat["max"], 2),
|
||||
}
|
||||
for wt, lat in sorted(self._latency.items())
|
||||
]
|
||||
return {
|
||||
"total_calls": sum(self._counts.values()),
|
||||
"calls": calls,
|
||||
"latency": latency,
|
||||
"recent": list(self._recent)[-50:],
|
||||
}
|
||||
|
||||
|
||||
_stats = CallStats()
|
||||
|
||||
|
||||
def summary() -> Dict[str, Any]:
|
||||
"""全局统计快照（供 GET /stats 端点使用）"""
|
||||
return _stats.summary()
|
||||
|
||||
|
||||
def _format_audit(entry: Dict[str, Any]) -> str:
|
||||
"""审计日志单行 key=value 格式，便于 grep 检索"""
|
||||
ts = entry.get("ts") or time.time()
|
||||
ts_str = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]
|
||||
user = entry.get("user")
|
||||
if isinstance(user, dict):
|
||||
# JWT payload（auth.py store_user_to_request 注入），orjson 默认紧凑单行输出
|
||||
user_str = orjson.dumps(user).decode()
|
||||
elif isinstance(user, str):
|
||||
user_str = user
|
||||
else:
|
||||
user_str = "-"
|
||||
parts = [
|
||||
"AUDIT",
|
||||
f"ts={ts_str}",
|
||||
f"worker={entry.get('worker_type', '-')}",
|
||||
f"subject={entry.get('subject', '-')}",
|
||||
f"method={entry.get('method', '-')}",
|
||||
f"path={entry.get('path', '-')}",
|
||||
f"user={user_str}",
|
||||
f"client={entry.get('client') or '-'}",
|
||||
f"status={entry.get('status', '-')}",
|
||||
f"duration_ms={entry.get('duration_ms', 0):.3f}",
|
||||
f"req_bytes={entry.get('req_bytes', 0)}",
|
||||
f"resp_bytes={entry.get('resp_bytes', 0)}",
|
||||
]
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
async def record_call(entry: Dict[str, Any]) -> None:
|
||||
"""后台执行体：更新内存统计 + 写审计日志。
|
||||
|
||||
整体兜底：任一步失败仅记 warning，绝不向端点冒泡。
|
||||
"""
|
||||
try:
|
||||
_stats.record(entry)
|
||||
except Exception as e: # pragma: no cover - 防御性兜底
|
||||
logger.warning("调用统计记录失败: %s", e)
|
||||
try:
|
||||
if _should_record_audit(entry.get("status", 200)):
|
||||
audit_logger.info(_format_audit(entry))
|
||||
except Exception as e: # pragma: no cover - 防御性兜底
|
||||
logger.warning("审计日志写入失败: %s", e)
|
||||
@@ -1,64 +0,0 @@
|
||||
"""
|
||||
启动契约校验（网关 inline 模式）
|
||||
|
||||
校验逻辑的唯一实现在 chongming_worker.contracts（网关与 worker 进程共享），
|
||||
本模块只做网关侧的编排：对 inline 插件批量校验、日志汇总、硬失败终止启动。
|
||||
|
||||
external 插件在自己进程内做同样校验（validate_worker_contracts），网关侧跳过。
|
||||
"""
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from chongming_worker import get_all_rpc_methods
|
||||
from chongming_worker.contracts import check_plugin, load_plugin_models
|
||||
from chongming_worker.route_config import load_worker_routes_from_dir
|
||||
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger("contracts")
|
||||
|
||||
|
||||
def validate_contracts(plugins: List[Dict[str, Any]]) -> None:
|
||||
"""
|
||||
对每个发现的 inline 插件执行三方一致性校验。
|
||||
|
||||
挂载于 lifespan 的 load_workers 与 register_routes 之间，此时插件模块已导入
|
||||
（@rpc_method 注册表就绪）且 TOML 路由尚未注册到 FastAPI。
|
||||
|
||||
Args:
|
||||
plugins: _discover_plugins() 返回的 inline 插件配置列表
|
||||
|
||||
Raises:
|
||||
RuntimeError: 存在硬失败项（HTTP 路由缺失 @rpc_method 注册）时
|
||||
"""
|
||||
registry = get_all_rpc_methods()
|
||||
hard_errors: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
for wc in plugins:
|
||||
worker_type = wc.get("type", "")
|
||||
if not worker_type:
|
||||
continue
|
||||
|
||||
toml_routes = load_worker_routes_from_dir(worker_type)
|
||||
registered = registry.get(worker_type, {})
|
||||
model_names = load_plugin_models(worker_type)
|
||||
hard, warns = check_plugin(worker_type, toml_routes, registered, model_names)
|
||||
hard_errors.extend(hard)
|
||||
warnings.extend(warns)
|
||||
|
||||
# 输出结果
|
||||
if warnings:
|
||||
logger.warning("契约校验发现 %d 个警告:", len(warnings))
|
||||
for w in warnings:
|
||||
logger.warning(" ⚠ %s", w)
|
||||
|
||||
if hard_errors:
|
||||
logger.error("契约校验失败，发现 %d 个硬性错误:", len(hard_errors))
|
||||
for e in hard_errors:
|
||||
logger.error(" ✗ %s", e)
|
||||
raise RuntimeError(
|
||||
"启动契约校验失败: "
|
||||
+ "; ".join(hard_errors)
|
||||
)
|
||||
|
||||
logger.info("契约校验通过（TOML 路由 ↔ @rpc_method 注册表一致）")
|
||||
@@ -1,880 +0,0 @@
|
||||
import os
|
||||
import inspect
|
||||
import time
|
||||
import signal
|
||||
import tomllib
|
||||
import glob
|
||||
import multiprocessing
|
||||
import asyncio
|
||||
import socket
|
||||
import sys
|
||||
|
||||
from chongming_cache import Cache, DistributedLock
|
||||
from fastapi import FastAPI
|
||||
import zenoh
|
||||
|
||||
from chongming_config import get_config, get_workers_config
|
||||
from ..core.logger import get_logger
|
||||
from ..core.route_generator import register_dynamic_routes
|
||||
from ..core.worker_manager import WorkerManager
|
||||
from ..core.mqtt_bridge import MQTTBridge
|
||||
|
||||
config = get_config()
|
||||
logger = get_logger("app")
|
||||
|
||||
# 挂载框架/插件父 logger 的 handler：logger 模块只为具名 logger 挂 handler，
|
||||
# 子 logger（chongming_worker.runtime、plugins.doubao 等）经 propagate 到父级才可见
|
||||
get_logger("chongming_worker")
|
||||
get_logger("plugins")
|
||||
|
||||
try:
|
||||
env = config["default"]["env"]
|
||||
except KeyError: # pragma: no cover
|
||||
raise ValueError("配置不存在")
|
||||
|
||||
def is_port_in_use(port: int, host: str = "127.0.0.1") -> bool:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.settimeout(0.5)
|
||||
return s.connect_ex((host, port)) == 0
|
||||
|
||||
|
||||
def _broker_lock_key() -> str:
|
||||
"""内嵌 Broker 排他锁 key。内嵌 Broker 绑定本机端口，故按宿主隔离
|
||||
（多实例共享同一 cache/redis 时，各宿主仍各起各的 Broker）。"""
|
||||
return f"mqtt_broker:{socket.gethostname()}"
|
||||
|
||||
|
||||
def _broker_owner_key() -> str:
|
||||
"""记录内嵌 Broker 子进程实际 PID 的缓存 key（按宿主隔离）。
|
||||
|
||||
持有者在启动 Broker 子进程后登记其 PID；前任持有者崩溃时，接管的进程据此
|
||||
识别并终止孤儿 Broker 子进程，从而真正掌握其生命周期。"""
|
||||
return f"mqtt_broker_owner:{socket.gethostname()}"
|
||||
|
||||
|
||||
def _pid_alive(pid) -> bool:
|
||||
try:
|
||||
os.kill(int(pid), 0)
|
||||
return True
|
||||
except (ProcessLookupError, OSError, ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def _terminate_pid(pid):
|
||||
"""尽力终止进程：先 SIGTERM，宽限后仍存活则 SIGKILL。"""
|
||||
try:
|
||||
os.kill(int(pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, OSError, ValueError, TypeError):
|
||||
return
|
||||
for _ in range(20):
|
||||
if not _pid_alive(pid):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
try:
|
||||
os.kill(int(pid), signal.SIGKILL)
|
||||
except (ProcessLookupError, OSError, ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
def _wait_port_free(port: int, host: str = "127.0.0.1", timeout: float = 3.0):
|
||||
end = time.time() + timeout
|
||||
while time.time() < end and is_port_in_use(port, host):
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def _run_mqtt_broker(host: str, port: int, username: str | None = None,
|
||||
password: str | None = None):
|
||||
"""子进程入口：启动 amqtt Broker（配置了账号密码时启用认证）"""
|
||||
import asyncio as _asyncio
|
||||
from amqtt.broker import Broker
|
||||
|
||||
# amqtt 在 broker 进程内无 handler，WARNING 会经 lastResort 直接刷到 stderr：
|
||||
# - sys_interval=0（禁用 $SYS）被 amqtt 当 falsy 报警
|
||||
# - 客户端连上却不发 CONNECT 时每次刷 2 行 "No data from"/"Failed to initialize..."
|
||||
# 统一压到 ERROR：认证失败本身是 DEBUG 级不受影响，真实故障仍会输出
|
||||
import logging as _logging
|
||||
|
||||
_logging.getLogger("amqtt").setLevel(_logging.ERROR)
|
||||
|
||||
# ---- monkey-patch 1: amqtt StreamWriterAdapter.close() 忽略断开的传输层 ----
|
||||
from amqtt.adapters import StreamWriterAdapter as _SWA
|
||||
|
||||
async def _patched_close(self):
|
||||
if self.is_closed:
|
||||
return
|
||||
self.is_closed = True
|
||||
try:
|
||||
await self._writer.drain()
|
||||
except (OSError, ConnectionError):
|
||||
pass
|
||||
try:
|
||||
if self._writer.can_write_eof():
|
||||
self._writer.write_eof()
|
||||
except OSError:
|
||||
pass # 传输层已断开 (Errno 107)
|
||||
try:
|
||||
self._writer.close()
|
||||
except OSError:
|
||||
pass
|
||||
with __import__("contextlib").suppress(AttributeError):
|
||||
await self._writer.wait_closed()
|
||||
|
||||
_SWA.close = _patched_close
|
||||
|
||||
# ---- monkey-patch 2: 协议版本不匹配时不打 traceback（仅 warning） ----
|
||||
import amqtt.errors as _amqtt_err
|
||||
from amqtt.utils import format_client_message as _fmt_client, gen_client_id as _gen_cid
|
||||
from amqtt.broker import Broker as _Broker
|
||||
from amqtt.mqtt.protocol.broker_handler import BrokerProtocolHandler as _BPH
|
||||
|
||||
async def _patched_init_client(self, reader, writer, remote_address, remote_port):
|
||||
"""与原始方法逻辑一致，但 MQTTError 使用 warning 而非 exception"""
|
||||
try:
|
||||
handler, client_session = await _BPH.init_from_connect(
|
||||
reader, writer, self.plugins_manager
|
||||
)
|
||||
except _amqtt_err.AMQTTError as exc:
|
||||
self.logger.warning(
|
||||
"[MQTT-3.1.0-1] %s: Can't read first packet as CONNECT: %s",
|
||||
_fmt_client(address=remote_address, port=remote_port),
|
||||
exc,
|
||||
)
|
||||
raise
|
||||
except _amqtt_err.MQTTError as exc:
|
||||
self.logger.warning(
|
||||
"%s (client 可能使用 MQTT 5.0，amqtt 仅支持 3.1.1)",
|
||||
str(exc),
|
||||
)
|
||||
try:
|
||||
await writer.close()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
except _amqtt_err.NoDataError as exc:
|
||||
self.logger.warning(
|
||||
"No data from %s: %s",
|
||||
_fmt_client(address=remote_address, port=remote_port),
|
||||
exc,
|
||||
)
|
||||
raise _amqtt_err.AMQTTError(exc) from exc
|
||||
|
||||
if client_session.clean_session:
|
||||
if client_session.client_id is not None and client_session.client_id != "":
|
||||
await self._delete_session(client_session.client_id)
|
||||
else:
|
||||
client_session.client_id = _gen_cid()
|
||||
|
||||
return handler, client_session
|
||||
|
||||
_Broker._initialize_client_session = _patched_init_client
|
||||
|
||||
async def _serve():
|
||||
# 认证插件：配置了账号密码 → FileAuthPlugin（argon2 密码文件，仅此账号可连）；
|
||||
# 未配置 → 匿名（默认行为）
|
||||
auth_plugins = {}
|
||||
passwd_file = None
|
||||
if username and password:
|
||||
import tempfile
|
||||
from pwdlib import PasswordHash
|
||||
|
||||
try:
|
||||
fd, passwd_file = tempfile.mkstemp(prefix="chongming_mqtt_", suffix=".passwd")
|
||||
with os.fdopen(fd, "w") as f:
|
||||
f.write(f"{username}:{PasswordHash.recommended().hash(password)}\n")
|
||||
auth_plugins = {
|
||||
"amqtt.plugins.authentication.FileAuthPlugin": {
|
||||
"password_file": passwd_file,
|
||||
},
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"MQTT 认证初始化失败，回退匿名模式: {e}")
|
||||
if passwd_file:
|
||||
os.unlink(passwd_file)
|
||||
passwd_file = None
|
||||
else:
|
||||
auth_plugins = {
|
||||
"amqtt.plugins.authentication.AnonymousAuthPlugin": {"allow_anonymous": True},
|
||||
}
|
||||
|
||||
cfg = {
|
||||
"listeners": {
|
||||
"default": {
|
||||
"type": "tcp", "bind": f"{host}:{port}", "max_connections": 500,
|
||||
},
|
||||
},
|
||||
"timeout_disconnect_delay": 2,
|
||||
"plugins": {
|
||||
**auth_plugins,
|
||||
"amqtt.plugins.sys.broker.BrokerSysPlugin": {"sys_interval": 0},
|
||||
},
|
||||
}
|
||||
broker = Broker(cfg)
|
||||
try:
|
||||
await broker.start()
|
||||
except Exception:
|
||||
# 端口已被其他 worker 的 Broker 占用，静默退出
|
||||
return
|
||||
finally:
|
||||
# 密码文件仅在启动时读取，启动完成后即可删除
|
||||
if passwd_file:
|
||||
try:
|
||||
os.unlink(passwd_file)
|
||||
except OSError:
|
||||
pass
|
||||
auth_note = f" (认证用户: {username})" if username else " (匿名)"
|
||||
logger.info("MQTT Broker 已启动 @ %s:%s%s", host, port, auth_note)
|
||||
try:
|
||||
while True:
|
||||
await _asyncio.sleep(1)
|
||||
except (KeyboardInterrupt, _asyncio.CancelledError):
|
||||
pass
|
||||
finally:
|
||||
await broker.shutdown()
|
||||
|
||||
try:
|
||||
_asyncio.run(_serve())
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
def setup_storage(app: FastAPI):
|
||||
"""初始化本地文件存储（替代 Nanio S3，省 ~172 MB）"""
|
||||
upload_root = os.path.join(os.getcwd(), "data", "uploads")
|
||||
os.makedirs(upload_root, exist_ok=True)
|
||||
app.state.upload_root = upload_root
|
||||
logger.info(f"本地存储已就绪: {upload_root}")
|
||||
|
||||
|
||||
async def setup_auth_store(app: FastAPI) -> None:
|
||||
"""初始化内置 auth 库（users 表 + admin 种子）—— auth 域的基础设施部分。
|
||||
|
||||
拆分自 plugins.auth：登录/签发内置在网关（管理平面底线），auth 插件
|
||||
缩容到 0 或全部副本故障时登录与 admin API 仍可用。库位置与 auth 插件
|
||||
读同一份配置（config/workers/auth/config.toml 的 [workers.config]：
|
||||
data_dir / database），指向同一张 users 表；插件配置缺失（如彻底移除
|
||||
auth 插件）时回退默认 ./data/auth.db。
|
||||
"""
|
||||
from chongming_auth import open_database, seed_admin
|
||||
|
||||
workers_conf = {}
|
||||
auth_conf_path = "config/workers/auth/config.toml"
|
||||
if os.path.isfile(auth_conf_path):
|
||||
try:
|
||||
with open(auth_conf_path, "rb") as f:
|
||||
workers_conf = tomllib.load(f).get("workers", {}).get("config", {})
|
||||
except Exception as e: # noqa: BLE001 - 配置损坏不阻断启动，走默认路径
|
||||
logger.warning(f"无法读取 auth 插件配置 {auth_conf_path}: {e}（使用默认 data_dir）")
|
||||
|
||||
db = await open_database(workers_conf)
|
||||
created = await seed_admin(db)
|
||||
app.state.auth_db = db
|
||||
logger.info(f"内置 auth 库已就绪 (admin 种子: {'新建' if created else '已存在'})")
|
||||
|
||||
|
||||
def setup_zenoh(app: FastAPI) -> zenoh.Session:
|
||||
"""建立 Zenoh 会话并存入 app.state。"""
|
||||
zenoh_config = config.get("zenoh", {})
|
||||
|
||||
zconfig = zenoh.Config()
|
||||
zconfig.from_file(zenoh_config.get("json5_path", "config/zenoh_config.json5"))
|
||||
|
||||
try:
|
||||
session = zenoh.open(zconfig)
|
||||
logger.info("Zenoh 会话已建立 (TCP 自连接模式)")
|
||||
except Exception as e:
|
||||
logger.error(f"Zenoh 会话建立失败: {e}")
|
||||
raise
|
||||
|
||||
app.state.zenoh_session = session
|
||||
return session
|
||||
|
||||
def setup_worker_manager(app: FastAPI, session: zenoh.Session) -> WorkerManager:
|
||||
"""创建 WorkerManager，启动定期刷新任务，并存入 app.state。"""
|
||||
cache = Cache.from_config(config["cache"])
|
||||
launch_worker_count = str(
|
||||
config["cache"].get("launch_worker_count", "toml")
|
||||
).lower()
|
||||
manager = WorkerManager(session, cache, launch_worker_count=launch_worker_count)
|
||||
app.state.worker_manager = manager
|
||||
|
||||
refresh_task = asyncio.create_task(manager.refresh_workers())
|
||||
app.state.refresh_task = refresh_task
|
||||
return manager
|
||||
|
||||
async def setup_route_registry(
|
||||
app: FastAPI,
|
||||
session: zenoh.Session,
|
||||
manager: WorkerManager,
|
||||
inline_types: list[str] | None = None,
|
||||
):
|
||||
"""
|
||||
启动动态路由注册表（常驻订阅 chongming/routes/**）。
|
||||
|
||||
网关启动即订阅，不依赖网关侧是否登记 external 插件配置：external 插件
|
||||
以独立进程运行（由 Docker/K8s 等编排拉起，网关不负责进程管理），其路由
|
||||
描述符经 Zenoh 动态注册；inline 插件仍在网关进程内加载，两者同时存在
|
||||
（混合模式）。盘扫发现的 external 插件条目仅用于启动日志观测。
|
||||
|
||||
inline_types: 网关进程内加载的插件类型，注册表忽略其描述符以防重复挂载。
|
||||
|
||||
注意：external 插件的自定义 APIRouter（如 multipart 上传端点）无法由
|
||||
网关发现（网关不 import 该插件），此类端点需保持 inline 或等 P2 专门通道。
|
||||
"""
|
||||
from ..core.route_registry import RouteRegistry
|
||||
|
||||
prefix = config["default"]["prefix"]
|
||||
registry = RouteRegistry(app, manager, prefix=prefix)
|
||||
if inline_types:
|
||||
registry.set_inline_types(set(inline_types))
|
||||
await registry.start(session)
|
||||
app.state.route_registry = registry
|
||||
logger.info(
|
||||
f"路由注册表已启动（prefix={prefix}, 外部插件路由动态注册, "
|
||||
f"inline 插件: {sorted(inline_types or []) or '无'}）"
|
||||
)
|
||||
return registry
|
||||
|
||||
|
||||
def _discover_plugins() -> list[dict]:
|
||||
"""
|
||||
发现所有插件配置。
|
||||
|
||||
优先从各插件的独立配置文件 config/workers/<type>/config.toml 的 [workers] 节读取，
|
||||
兼容旧的 config/workers/config.toml 全局配置。
|
||||
|
||||
Returns:
|
||||
[{"type": "doubao", "module": "plugins.doubao", "factory": "create_workers",
|
||||
"cleanup": "cleanup", "count": 3, "config": {...}}, ...]
|
||||
"""
|
||||
plugins = []
|
||||
|
||||
# 1. 自动扫描 config/workers/*/config.toml
|
||||
for path in sorted(glob.glob("config/workers/*/config.toml")):
|
||||
plugin_type = os.path.basename(os.path.dirname(path))
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
plugin_conf = tomllib.load(f)
|
||||
except Exception as e:
|
||||
logger.warning("无法读取插件配置 %s: %s", path, e)
|
||||
continue
|
||||
|
||||
workers_section = plugin_conf.get("workers", {})
|
||||
if not workers_section:
|
||||
continue
|
||||
|
||||
plugin_type = workers_section.get("type", plugin_type)
|
||||
# 插件运行模式（按插件粒度，混合共存）:
|
||||
# "inline"（默认） = 网关进程内加载
|
||||
# "external" = 独立进程运行（Docker/K8s 编排），经 Zenoh 注册路由
|
||||
load_mode = workers_section.get("load_mode", "inline")
|
||||
# 插件声明的 MQTT Bridge 额外订阅主题（[[workers.mqtt_topics]]）
|
||||
# 网关级路由配置，不传给插件运行时；worker 目标即本插件类型。
|
||||
mqtt_topics = []
|
||||
for t in workers_section.get("mqtt_topics", []) or []:
|
||||
if not isinstance(t, dict) or not t.get("topic"):
|
||||
continue
|
||||
t = dict(t)
|
||||
t.setdefault("worker", plugin_type)
|
||||
mqtt_topics.append(t)
|
||||
|
||||
# 合并运行时 config:
|
||||
# 1. plugin_conf 顶层 (worker, worker.registration 等 gen_models 用)
|
||||
# 2. [workers] 节的注册字段展平 (type, module, factory, cleanup, count)
|
||||
# — 其中 type 被 chongming_worker.runtime.create_workers 用于匹配 RPC 方法
|
||||
# 3. [workers.config] 展平到顶层 (api_key, data_dir, heartbeat 等)
|
||||
workers_flat = {k: v for k, v in workers_section.items()
|
||||
if k not in ("config", "mqtt_topics", "load_mode")}
|
||||
runtime_config = {
|
||||
**plugin_conf,
|
||||
**workers_flat,
|
||||
**workers_section.get("config", {}),
|
||||
}
|
||||
|
||||
plugins.append({
|
||||
"type": plugin_type,
|
||||
"module": workers_section.get("module", f"plugins.{plugin_type}"),
|
||||
"factory": workers_section.get("factory", "create_workers"),
|
||||
"cleanup": workers_section.get("cleanup", "cleanup"),
|
||||
"count": workers_section.get("count", 1),
|
||||
"load_mode": load_mode,
|
||||
"mqtt_topics": mqtt_topics,
|
||||
"config": runtime_config,
|
||||
})
|
||||
|
||||
# 2. 兼容旧写法: config/workers/config.toml（仅加载尚未发现的插件）
|
||||
old_config_path = config.get("plugins_worker", {}).get("config_path", "config/workers/config.toml")
|
||||
if os.path.isfile(old_config_path):
|
||||
existing_types = {p["type"] for p in plugins}
|
||||
old_conf = get_workers_config(old_config_path).get("workers", [])
|
||||
for wc in old_conf:
|
||||
wtype = wc.get("type", "")
|
||||
if wtype and wtype not in existing_types:
|
||||
plugins.append({
|
||||
"type": wtype,
|
||||
"module": wc.get("module", ""),
|
||||
"factory": wc.get("factory", ""),
|
||||
"cleanup": wc.get("cleanup", ""),
|
||||
"count": wc.get("count", 1),
|
||||
"config": wc.get("config", {}),
|
||||
})
|
||||
logger.info("从旧配置加载插件: %s (config/workers/config.toml)", wtype)
|
||||
|
||||
return plugins
|
||||
|
||||
|
||||
async def load_workers(app: FastAPI, session: zenoh.Session, manager: WorkerManager,
|
||||
is_inline_owner: bool = True):
|
||||
"""
|
||||
加载 inline 插件，启动对应的 worker 任务（混合模式）。
|
||||
|
||||
自动从 config/workers/<type>/config.toml 发现插件；load_mode="external"
|
||||
的插件不在网关内加载（由独立进程运行，其路由经常驻 RouteRegistry 动态
|
||||
注册，无需网关侧保留插件配置，盘扫发现的 external 条目仅用于观测）。
|
||||
|
||||
同时自动发现插件模块中暴露的 APIRouter（约定：模块顶层名为 `router` 的
|
||||
APIRouter 实例），收集后供 register_custom_routes() 统一注册。
|
||||
|
||||
is_inline_owner: 本进程是否为宿主级 inline worker 持有者。gunicorn 多 worker
|
||||
下仅持有者进程真正拉起 inline worker（避免 N×count 冗余副本）；非持有者进程
|
||||
仍 import 插件模块以便注册自定义路由/契约校验，但不 create_workers，worker 经
|
||||
Zenoh liveliness 被跨进程发现。单进程（uvicorn serve）时为 True，行为不变。
|
||||
|
||||
返回: (worker_tasks, custom_routers, inline_types, external_types)
|
||||
"""
|
||||
workers_conf = _discover_plugins()
|
||||
inline_plugins = [wc for wc in workers_conf if wc.get("load_mode", "inline") != "external"]
|
||||
external_plugins = [wc for wc in workers_conf if wc.get("load_mode", "inline") == "external"]
|
||||
inline_types = sorted({wc["type"] for wc in inline_plugins})
|
||||
external_types = sorted({wc["type"] for wc in external_plugins})
|
||||
logger.info(
|
||||
f"发现 {len(workers_conf)} 个 worker 插件: "
|
||||
f"inline={inline_types or '无'}, external={external_types or '无'} "
|
||||
f"(本进程 inline 持有者: {is_inline_owner})"
|
||||
)
|
||||
|
||||
for wc in external_plugins:
|
||||
logger.info(
|
||||
"插件 %s 为 external 模式，跳过进程内加载（由独立进程经 Zenoh 注册路由）",
|
||||
wc["type"],
|
||||
)
|
||||
|
||||
# 汇总 inline 插件声明的 MQTT Bridge 额外订阅主题，供 setup_mqtt 使用
|
||||
# （external 插件的主题由 RouteRegistry 经 __meta__ 动态推送）
|
||||
app.state.plugin_mqtt_topics = [
|
||||
topic
|
||||
for wc in inline_plugins
|
||||
for topic in (wc.get("mqtt_topics") or [])
|
||||
]
|
||||
|
||||
worker_tasks = []
|
||||
custom_routers = []
|
||||
for wc in inline_plugins:
|
||||
module_name = wc["module"]
|
||||
factory_name = wc["factory"]
|
||||
# count 以 chongming-cache 为单一事实来源：cache 有值时用 cache 的全局总量，
|
||||
# 否则用配置值引导并写回 cache（多网关共享同一 cache 时自动对齐）。
|
||||
count = wc["count"]
|
||||
worker_type = wc["type"]
|
||||
worker_config = wc["config"]
|
||||
|
||||
if not module_name or not factory_name:
|
||||
logger.warning(f"Worker 配置不完整: {wc}")
|
||||
continue
|
||||
|
||||
logger.debug("加载插件 %s: config/workers/%s/config.toml (count=%d)",
|
||||
worker_type, worker_type, count)
|
||||
|
||||
try:
|
||||
module = __import__(module_name, fromlist=[factory_name])
|
||||
|
||||
# 自动发现插件的自定义 FastAPI 路由（约定：模块暴露 `router` 属性）
|
||||
plugin_router = getattr(module, "router", None)
|
||||
if plugin_router is not None:
|
||||
from fastapi import APIRouter
|
||||
if isinstance(plugin_router, APIRouter):
|
||||
custom_routers.append(plugin_router)
|
||||
logger.info(f"发现插件自定义路由: {module_name}.router")
|
||||
|
||||
# 非持有者进程：不立即拉起 inline worker，但仍在下方（build starter 后）
|
||||
# 注册自愈启动器；worker 经 liveliness 被本进程发现调用，@rpc_method 与
|
||||
# 自定义路由/契约校验依赖的模块 import 已在上文完成不依赖拉起。
|
||||
factory = getattr(module, factory_name)
|
||||
|
||||
# 定义"拉起本类型 worker"的协程，供持有者崩溃后的自愈接管复用；
|
||||
# 用默认参数捕获循环变量，避免闭包迟到绑定。
|
||||
def _make_starter(session=session, config=worker_config, cnt=count,
|
||||
mgr=manager, fac=factory):
|
||||
async def starter():
|
||||
if inspect.iscoroutinefunction(fac):
|
||||
return await fac(session, config, cnt, worker_manager=mgr)
|
||||
return fac(session, config, cnt, worker_manager=mgr)
|
||||
return starter
|
||||
|
||||
starter = _make_starter()
|
||||
manager.register_worker_starter(worker_type, starter)
|
||||
|
||||
# 非持有者进程：不拉起 inline worker（启动器已注册，持有者崩溃时由 refresh 自愈接管）
|
||||
if not is_inline_owner:
|
||||
logger.info(
|
||||
"本进程非 %s 的 inline 持有者，跳过创建副本（liveliness 发现即可调用）",
|
||||
worker_type,
|
||||
)
|
||||
continue
|
||||
|
||||
tasks = await starter()
|
||||
worker_tasks.extend(tasks)
|
||||
logger.info(f"已加载 Worker 插件: {module_name}.{factory_name} (副本数: {count})")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
logger.error(
|
||||
f"加载 Worker 插件失败 {module_name}: {e}\n"
|
||||
f" 配置: type={worker_type}, count={count}\n"
|
||||
f" 堆栈:\n{traceback.format_exc()}"
|
||||
)
|
||||
|
||||
app.state.worker_tasks = worker_tasks
|
||||
app.state.inline_types = inline_types
|
||||
app.state.external_types = external_types
|
||||
return worker_tasks, custom_routers, inline_types, external_types
|
||||
|
||||
async def register_routes(
|
||||
app: FastAPI,
|
||||
manager: WorkerManager,
|
||||
custom_routers: list | None = None,
|
||||
worker_types: list[str] | None = None,
|
||||
):
|
||||
"""注册动态路由和插件自定义路由。
|
||||
|
||||
worker_types: 仅注册这些插件类型的 TOML 动态路由（混合模式下只注册
|
||||
inline 插件；external 插件路由由 RouteRegistry 动态挂载）。
|
||||
"""
|
||||
prefix = config["default"]["prefix"]
|
||||
register_dynamic_routes(app, manager, prefix=prefix, worker_types=worker_types)
|
||||
# 注册插件自定义路由（如 FORM-DATA 文件上传等 TOML 无法描述的端点）
|
||||
# 统一加上全局 prefix（如 /api/v1），与 TOML 动态路由保持一致
|
||||
if custom_routers:
|
||||
for r in custom_routers:
|
||||
app.include_router(r, prefix=prefix)
|
||||
logger.info(f"已注册 {len(custom_routers)} 个插件自定义路由（prefix={prefix}）")
|
||||
|
||||
|
||||
def validate_contracts(app: FastAPI | None = None):
|
||||
"""
|
||||
契约校验（TOML 路由 ↔ @rpc_method 注册表 ↔ 生成模型），仅覆盖 inline 插件。
|
||||
|
||||
在 load_workers 之后、register_routes 之前调用：此时插件模块已导入
|
||||
（注册表就绪）。HTTP 路由缺失对应 @rpc_method 时抛 RuntimeError 终止启动，
|
||||
其余不一致以警告日志汇总。
|
||||
|
||||
external 插件在自己进程内做同样校验（validate_worker_contracts），网关侧跳过。
|
||||
"""
|
||||
from ..core.contracts import validate_contracts as _validate
|
||||
|
||||
plugins = _discover_plugins()
|
||||
inline_plugins = [p for p in plugins if p.get("load_mode", "inline") != "external"]
|
||||
_validate(inline_plugins)
|
||||
|
||||
async def setup_mqtt_broker(app: FastAPI, worker_manager: WorkerManager):
|
||||
"""
|
||||
启动内嵌 amqtt Broker 子进程（宿主级分布式锁确保多 worker/多实例仅本宿主启动一个）。
|
||||
|
||||
由配置 [<env>.mqtt].use_embedded_broker 控制：
|
||||
true （默认）→ 网关启动 amqtt Broker 子进程，host/port 为监听地址
|
||||
false → 使用外部 MQTT Broker，网关仅运行 Bridge 客户端，跳过本步
|
||||
配置了 username/password 时 Broker 启用认证，否则保持匿名。
|
||||
|
||||
排他锁基于 chongming-cache 的 DistributedLock，key 按宿主隔离（内嵌 Broker 绑定
|
||||
本机端口）。持有者在缓存登记其 Broker 子进程 PID；前任持有者崩溃/被强杀而留下
|
||||
孤儿 Broker 子进程时，接管的进程借此 PID 识别并终止之，再启动自己托管的子进程，
|
||||
从而真正掌握 Broker 生命周期（可关停、可接管），而非留下不可管控的残留下端点。
|
||||
"""
|
||||
env_config = config.get(env, {})
|
||||
mqtt_config = env_config.get("mqtt", {})
|
||||
if not mqtt_config.get("enabled", True):
|
||||
logger.info("MQTT 已禁用")
|
||||
return None
|
||||
if not mqtt_config.get("use_embedded_broker", True):
|
||||
logger.info(
|
||||
f"use_embedded_broker=false：使用外部 MQTT Broker "
|
||||
f"({mqtt_config.get('host', 'localhost')}:{mqtt_config.get('port', 1883)})，"
|
||||
f"跳过内嵌 amqtt Broker 启动"
|
||||
)
|
||||
return None
|
||||
if worker_manager.cache is None:
|
||||
logger.warning(
|
||||
"MQTT 内嵌 Broker 启用但未配置共享 cache（[cache]），无法获取排他锁；"
|
||||
"请配置 redis 或共享卷 sqlite，或改用外部 Broker"
|
||||
)
|
||||
return None
|
||||
|
||||
host = mqtt_config.get("host", "localhost")
|
||||
port = mqtt_config.get("port", 1883)
|
||||
username = mqtt_config.get("username") or None
|
||||
password = mqtt_config.get("password") or None
|
||||
|
||||
lock = DistributedLock(worker_manager.cache, _broker_lock_key())
|
||||
if not lock.acquire():
|
||||
# 非持有者：等待持有者把 Broker 拉起并监听端口
|
||||
app.state.mqtt_broker_lock_acquired = False
|
||||
app.state.mqtt_broker_lock = None
|
||||
for _ in range(50):
|
||||
if is_port_in_use(port, host):
|
||||
logger.info(f"MQTT Broker 已在 {host}:{port} 运行（由其他进程托管）")
|
||||
return None
|
||||
time.sleep(0.1)
|
||||
logger.warning(f"等待内嵌 MQTT Broker 端口 {port} 就绪超时")
|
||||
return None
|
||||
|
||||
# 本进程赢得宿主锁：持有租约续期，取得 Broker 托管权
|
||||
lock.start()
|
||||
app.state.mqtt_broker_lock = lock
|
||||
app.state.mqtt_broker_lock_acquired = True
|
||||
|
||||
# 端口已被监听：优先识别并终止前任残留的孤儿 Broker 子进程，以真正接管其
|
||||
# 生命周期；否则视为不可托管的既有进程，复用（本进程不持有句柄）。
|
||||
try:
|
||||
prev_pid = int(worker_manager.cache.get(_broker_owner_key()) or 0)
|
||||
except (TypeError, ValueError):
|
||||
prev_pid = None
|
||||
if is_port_in_use(port, host):
|
||||
if prev_pid and _pid_alive(prev_pid) and prev_pid != os.getpid():
|
||||
logger.warning(
|
||||
f"[mqtt-broker] 检出孤儿 Broker 子进程 PID {prev_pid}，终止并接管重启"
|
||||
)
|
||||
_terminate_pid(prev_pid)
|
||||
_wait_port_free(port, host)
|
||||
if is_port_in_use(port, host):
|
||||
logger.info(
|
||||
f"MQTT Broker 端口 {port} 在终止孤儿后仍被占用，属不可托管进程，复用"
|
||||
)
|
||||
return None
|
||||
else:
|
||||
logger.info(f"MQTT Broker 已在 {host}:{port} 运行，复用（不可托管）")
|
||||
return None
|
||||
|
||||
proc = multiprocessing.Process(
|
||||
target=_run_mqtt_broker, args=(host, port, username, password), daemon=False
|
||||
)
|
||||
proc.start()
|
||||
app.state.mqtt_broker_process = proc
|
||||
worker_manager.cache.set(_broker_owner_key(), str(proc.pid))
|
||||
auth_note = f"，认证用户 {username}" if username else "（匿名）"
|
||||
logger.info(f"MQTT Broker 子进程已启动并登记托管 (PID: {proc.pid}){auth_note}")
|
||||
for _ in range(50):
|
||||
if is_port_in_use(port, host):
|
||||
logger.info(f"MQTT Broker 已监听 {port} 端口")
|
||||
return None
|
||||
if not proc.is_alive():
|
||||
raise RuntimeError("MQTT Broker 进程异常退出")
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("MQTT Broker 端口未就绪")
|
||||
|
||||
|
||||
async def shutdown_mqtt_broker(app: FastAPI):
|
||||
"""终止本进程托管的 MQTT Broker 子进程，并释放宿主级分布式排他锁、清理登记。"""
|
||||
proc = getattr(app.state, "mqtt_broker_process", None)
|
||||
if proc and proc.is_alive():
|
||||
proc.terminate()
|
||||
proc.join(timeout=5)
|
||||
if proc.is_alive():
|
||||
proc.kill()
|
||||
logger.info("MQTT Broker 子进程已终止")
|
||||
lock = getattr(app.state, "mqtt_broker_lock", None)
|
||||
if lock is not None:
|
||||
if proc: # 仅真正托管了子进程的进程才清理孤儿登记，避免误删他人登记
|
||||
try:
|
||||
lock._cache.delete(_broker_owner_key())
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug(f"清理 MQTT Broker 登记失败: {e}")
|
||||
await lock.stop()
|
||||
lock.release()
|
||||
app.state.mqtt_broker_lock = None
|
||||
app.state.mqtt_broker_lock_acquired = False
|
||||
|
||||
|
||||
async def _start_mqtt(app: FastAPI, worker_manager: WorkerManager,
|
||||
bridge_lock) -> MQTTBridge | None:
|
||||
"""在已持有分布式锁的前提下构建并启动一个 MQTT Bridge 客户端。
|
||||
|
||||
该逻辑同时被 setup_mqtt（启动时）与 mqtt_bridge_supervisor（持有者自愈接管时）
|
||||
复用。假定调用方已 acquire() 到 "mqtt_bridge" 分布式锁。
|
||||
"""
|
||||
env_config = config.get(env, {})
|
||||
mqtt_config = env_config.get("mqtt", {})
|
||||
host = mqtt_config.get("host", "localhost")
|
||||
port = mqtt_config.get("port", 1883)
|
||||
username = mqtt_config.get("username") or None
|
||||
password = mqtt_config.get("password") or None
|
||||
custom_topics = list(mqtt_config.get("custom_topics", []) or [])
|
||||
custom_topics += list(getattr(app.state, "plugin_mqtt_topics", []) or [])
|
||||
|
||||
bridge = MQTTBridge(
|
||||
worker_manager=worker_manager,
|
||||
host=host,
|
||||
port=port,
|
||||
custom_topics=custom_topics,
|
||||
username=username,
|
||||
password=password,
|
||||
)
|
||||
app.state.mqtt_bridge = bridge
|
||||
await bridge.start()
|
||||
bridge_lock.start() # 后台续约租约，防止持有方宕机/长暂停导致锁被错误抢走
|
||||
app.state.mqtt_bridge_lock = bridge_lock
|
||||
# 把 Bridge 挂接给常驻路由注册表（动态主题 + 补推已到达的元信息）
|
||||
registry = getattr(app.state, "route_registry", None)
|
||||
if registry is not None:
|
||||
registry.attach_bridge(bridge)
|
||||
broker_mode = "内嵌 amqtt" if mqtt_config.get("use_embedded_broker", True) else "外部"
|
||||
auth_note = f"，认证用户 {username}" if username else "（匿名）"
|
||||
logger.info(f"MQTT Bridge 已启动（{broker_mode} Broker {host}:{port}{auth_note}）")
|
||||
return bridge
|
||||
|
||||
|
||||
async def setup_mqtt(app: FastAPI, worker_manager: WorkerManager) -> MQTTBridge | None:
|
||||
"""
|
||||
启动 MQTT Bridge（纯客户端模式）。
|
||||
|
||||
从 config.toml 的环境段读取 host/port/enabled；若 enabled 为 false 则跳过。
|
||||
Broker 来源由 use_embedded_broker 控制：
|
||||
true （默认）= 内嵌 amqtt Broker（lifespan 第 3 步已启动子进程）
|
||||
false = 外部 MQTT Broker（host/port 指向外部地址，网关不启动 Broker）
|
||||
订阅主题 = 全局 [<env>.mqtt].custom_topics（可选兜底）
|
||||
+ 各插件在其 config/workers/<type>/config.toml 中声明的 [[workers.mqtt_topics]]。
|
||||
使用基于共享 cache 的分布式锁确保多进程/多网关中全局只有一个 Bridge 消费。
|
||||
"""
|
||||
env_config = config.get(env, {})
|
||||
mqtt_config = env_config.get("mqtt", {})
|
||||
if not mqtt_config.get("enabled", True):
|
||||
logger.info("MQTT 已禁用")
|
||||
return None
|
||||
|
||||
# 排他锁：基于共享缓存（redis / 共享卷 diskcache），保证全局只有一个 Bridge 消费。
|
||||
# 相比原文件锁，可跨多台主机收敛——多实例共享同一个外部 Broker 时，
|
||||
# 避免一条 MQTT 请求被 N 台网关上各自消费、各回一份响应。
|
||||
bridge_lock = DistributedLock(worker_manager.cache, "mqtt_bridge") # pyright: ignore[reportArgumentType]
|
||||
if not bridge_lock.acquire():
|
||||
logger.info(
|
||||
"MQTT Bridge 已由其他网关实例持有（分布式锁），本网关跳过，"
|
||||
"以规避共享 Broker 下单请求多响应"
|
||||
)
|
||||
return None
|
||||
try:
|
||||
return await _start_mqtt(app, worker_manager, bridge_lock)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"MQTT Bridge 启动失败: {e}")
|
||||
await bridge_lock.stop()
|
||||
bridge_lock.release()
|
||||
app.state.mqtt_bridge = None
|
||||
app.state.mqtt_bridge_lock = None
|
||||
return None
|
||||
|
||||
|
||||
async def mqtt_bridge_supervisor(app: FastAPI, worker_manager: WorkerManager):
|
||||
"""MQTT Bridge 持有者崩溃自愈巡检（跨所有网关进程运行）。
|
||||
|
||||
与 inline 持有者同源的问题：仅靠 gunicorn 复活死掉 slot 无法 failover——新进程
|
||||
可能在旧锁租约未过期时抢锁失败而照样跳过。本巡检在每个网关进程周期性运行：
|
||||
|
||||
- 本进程已持有一存活 Bridge → 正常托管，跳过。
|
||||
- 否则尝试获得 "mqtt_bridge" 分布式锁（set_nx 保证全局唯一胜出，无分裂）；
|
||||
若锁仍被存活持有者把持 → 跳过；若空闲（原持有者租约过期=已死/失联）→ 接管。
|
||||
- 接管时按需拉起内嵌 Broker（setup_mqtt_broker 幂等：端口被占则复用现有进程），
|
||||
再重建 Bridge 客户端。
|
||||
|
||||
运行周期经 [<env>.mqtt].bridge_supervise_interval 配置，默认 15s。
|
||||
"""
|
||||
env_config = config.get(env, {})
|
||||
mqtt_config = env_config.get("mqtt", {})
|
||||
if not mqtt_config.get("enabled", True):
|
||||
return # MQTT 禁用：无需巡检
|
||||
interval = mqtt_config.get("bridge_supervise_interval", 15) or 15
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
bridge = getattr(app.state, "mqtt_bridge", None)
|
||||
# 本进程持有一存活 Bridge：正常托管
|
||||
if bridge is not None and getattr(bridge, "is_running", True):
|
||||
continue
|
||||
|
||||
newly_acquired = False
|
||||
lock = getattr(app.state, "mqtt_bridge_lock", None)
|
||||
if lock is None:
|
||||
# 非持有者：仅当锁空闲（原持有者租约过期）时才能抢得
|
||||
if worker_manager.cache is None:
|
||||
continue
|
||||
candidate = DistributedLock(worker_manager.cache, "mqtt_bridge")
|
||||
if not candidate.acquire():
|
||||
continue # 仍被存活持有者把持
|
||||
candidate.start()
|
||||
app.state.mqtt_bridge_lock = candidate
|
||||
lock = candidate
|
||||
newly_acquired = True
|
||||
|
||||
logger.warning("[mqtt-bridge] 检出持有者失效，接管重建 Broker+Bridge")
|
||||
# 按需拉起内嵌 Broker（幂等；外部 Broker 时 setup_mqtt_broker 内直接返回）
|
||||
try:
|
||||
await setup_mqtt_broker(app, worker_manager)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"[mqtt-bridge] 接管时拉起 Broker 失败: {e}")
|
||||
try:
|
||||
await _start_mqtt(app, worker_manager, lock)
|
||||
logger.info("[mqtt-bridge] 接管完成，Bridge 已重建")
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"[mqtt-bridge] 接管时重建 Bridge 失败: {e}")
|
||||
app.state.mqtt_bridge = None
|
||||
if newly_acquired:
|
||||
await lock.stop()
|
||||
lock.release()
|
||||
app.state.mqtt_bridge_lock = None
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(f"MQTT Bridge 自愈巡检失败: {e}")
|
||||
|
||||
|
||||
async def shutdown_mqtt(app: FastAPI):
|
||||
"""停止 MQTT Bridge 和 Broker 子进程（仅锁持有者释放锁）"""
|
||||
bridge: MQTTBridge | None = getattr(app.state, "mqtt_bridge", None)
|
||||
if bridge and bridge.is_running:
|
||||
await bridge.stop()
|
||||
# 停止续约并受保护释放分布式锁（仅本网关仍持有时删除）
|
||||
bridge_lock = getattr(app.state, "mqtt_bridge_lock", None)
|
||||
if bridge_lock is not None:
|
||||
await bridge_lock.stop()
|
||||
bridge_lock.release()
|
||||
await shutdown_mqtt_broker(app)
|
||||
|
||||
|
||||
async def cleanup_workers(app: FastAPI):
|
||||
"""调用每个 inline worker 配置中定义的 cleanup 函数。
|
||||
|
||||
仅 inline worker 持有者进程执行：非持有者进程未初始化插件（未调用 create_workers，
|
||||
on_init 未执行），cleanup/on_cleanup 无资源可清，且可能引用未创建的对象而误报错。
|
||||
"""
|
||||
# app.state.inline_owner_locked 恒为 bool（lifespan 4.2 设置），缺省按持有处理
|
||||
if not getattr(app.state, "inline_owner_locked", True):
|
||||
logger.info("本进程非 inline worker 持有者，跳过插件 cleanup")
|
||||
return
|
||||
workers_conf = _discover_plugins()
|
||||
|
||||
for wc in workers_conf:
|
||||
if wc.get("load_mode", "inline") == "external":
|
||||
continue
|
||||
module_name = wc.get("module", "")
|
||||
cleanup_name = wc.get("cleanup", "")
|
||||
worker_type = wc.get("type", "")
|
||||
|
||||
if not module_name or not cleanup_name:
|
||||
logger.warning(f"Worker 配置缺少 cleanup: {wc}")
|
||||
continue
|
||||
|
||||
try:
|
||||
module = __import__(module_name, fromlist=[cleanup_name])
|
||||
cleanup = getattr(module, cleanup_name)
|
||||
pid = os.getpid()
|
||||
worker_id = f"worker/{worker_type}/worker-i-{pid}"
|
||||
if inspect.iscoroutinefunction(cleanup):
|
||||
await cleanup(worker_id)
|
||||
else:
|
||||
cleanup(worker_id)
|
||||
except Exception as e:
|
||||
logger.error(f"cleanup 调用失败 {module_name}: {e}")
|
||||
|
||||
@@ -1,396 +0,0 @@
|
||||
import re
|
||||
import os
|
||||
import atexit
|
||||
import queue
|
||||
import logging
|
||||
import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from logging.handlers import RotatingFileHandler, QueueHandler, QueueListener
|
||||
|
||||
from chongming_config import get_config
|
||||
|
||||
|
||||
config = get_config()
|
||||
env = config["default"]["env"]
|
||||
server_config = config[env]["server"]
|
||||
log_config = config[env]["logging"]
|
||||
|
||||
|
||||
class ColoredFormatter(logging.Formatter):
|
||||
"""
|
||||
为 HTTP 状态码添加颜色的日志格式化器
|
||||
|
||||
根据 HTTP 状态码的首位数字为日志消息中的状态码添加不同颜色：
|
||||
- 2xx: 绿色（成功）
|
||||
- 3xx: 青色（重定向）
|
||||
- 4xx: 黄色（客户端错误）
|
||||
- 5xx: 红色（服务器错误）
|
||||
|
||||
Attributes:
|
||||
COLORS: 状态码首位数字到 ANSI 颜色代码的映射字典
|
||||
RESET: ANSI 重置颜色代码
|
||||
"""
|
||||
|
||||
COLORS = {
|
||||
"2": "\033[32m", # 绿色 2xx
|
||||
"3": "\033[36m", # 青色 3xx
|
||||
"4": "\033[33m", # 黄色 4xx
|
||||
"5": "\033[31m", # 红色 5xx
|
||||
}
|
||||
RESET = "\033[0m"
|
||||
|
||||
def format(self, record):
|
||||
"""
|
||||
格式化日志记录，为状态码添加颜色
|
||||
|
||||
如果日志记录包含 status_code 属性，则查找消息中的 "status=数字" 模式
|
||||
并为其添加对应的颜色代码。格式化完成后恢复原始消息以避免影响其他 handler。
|
||||
|
||||
Args:
|
||||
record: 日志记录对象，应包含 msg 属性和可选的 status_code 属性
|
||||
|
||||
Returns:
|
||||
str: 格式化后的日志消息字符串，状态码部分可能包含 ANSI 颜色代码
|
||||
"""
|
||||
# 保存原始消息
|
||||
original_msg = record.msg
|
||||
# 如果日志记录包含 status_code 属性，则为其添加颜色
|
||||
status_code = getattr(record, "status_code", None)
|
||||
if status_code and isinstance(status_code, int):
|
||||
first_digit = str(status_code)[0]
|
||||
color = self.COLORS.get(first_digit, "")
|
||||
colored_status = f"{color}{status_code}{self.RESET}"
|
||||
# 替换消息中的 "status=数字" 为带颜色的版本
|
||||
record.msg = re.sub(
|
||||
rf"status={status_code}\b", f"status={colored_status}", original_msg
|
||||
)
|
||||
result = super().format(record)
|
||||
# 恢复原始消息，避免影响其他 handler
|
||||
record.msg = original_msg
|
||||
return result
|
||||
|
||||
|
||||
class DailyRotatingFileHandler(RotatingFileHandler):
|
||||
"""
|
||||
按自然日切分 + 大小轮转的文件处理器
|
||||
|
||||
日志文件名形如 {log_name}-2026-08-19.log：进程启动时按当天日期命名，
|
||||
跨天后写入时自动切换到新日期文件，避免长时间运行的进程把跨天日志
|
||||
全部堆积到启动当天的文件里。同时保留 RotatingFileHandler 的大小轮转
|
||||
（单文件超限追加 .1/.2 后缀）。
|
||||
|
||||
历史文件（跨天文件与大小备份）按修改时间保留最近 backup_count 个，
|
||||
超出部分在切换日期的瞬间清理。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
log_dir: str,
|
||||
log_name: str,
|
||||
max_bytes: int,
|
||||
backup_count: int,
|
||||
encoding: str = "utf-8",
|
||||
):
|
||||
self.log_dir = Path(log_dir)
|
||||
self.log_name = log_name
|
||||
self._current_date = datetime.date.today()
|
||||
super().__init__(
|
||||
str(self.log_dir / f"{log_name}-{self._current_date}.log"),
|
||||
encoding=encoding,
|
||||
maxBytes=max_bytes,
|
||||
backupCount=backup_count,
|
||||
)
|
||||
|
||||
def _build_filename(self, day: datetime.date) -> str:
|
||||
return str(self.log_dir / f"{self.log_name}-{day}.log")
|
||||
|
||||
def _cleanup_old_files(self) -> None:
|
||||
"""保留最近 backup_count 个日志文件，删除更早的历史文件"""
|
||||
# 按 (日期, 轮转序号) 排序，不依赖 mtime（同日备份可能同时落盘）：
|
||||
# 日期越大越新；同日内主文件(.log)最新，.N 备份按 N 递减更旧
|
||||
pattern = re.compile(
|
||||
rf"^{re.escape(self.log_name)}-(\d{{4}}-\d{{2}}-\d{{2}})\.log(?:\.(\d+))?$"
|
||||
)
|
||||
|
||||
def sort_key(path: Path):
|
||||
match = pattern.match(path.name)
|
||||
if not match:
|
||||
return (datetime.date.min, 0)
|
||||
return (
|
||||
datetime.date.fromisoformat(match.group(1)),
|
||||
-int(match.group(2) or 0),
|
||||
)
|
||||
|
||||
files = sorted(
|
||||
self.log_dir.glob(f"{self.log_name}-*.log*"),
|
||||
key=sort_key,
|
||||
reverse=True,
|
||||
)
|
||||
for old in files[self.backupCount :]:
|
||||
try:
|
||||
old.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def emit(self, record):
|
||||
# 跨天：关闭旧文件流，切换到新日期文件，并清理过期历史文件
|
||||
today = datetime.date.today()
|
||||
if today != self._current_date:
|
||||
self._current_date = today
|
||||
self.close()
|
||||
self.baseFilename = self._build_filename(today)
|
||||
self.stream = self._open()
|
||||
self._cleanup_old_files()
|
||||
super().emit(record)
|
||||
|
||||
|
||||
class LoggerNameFilter(logging.Filter):
|
||||
"""
|
||||
按 log_name 路由日志记录的过滤器
|
||||
|
||||
只放行本 log_name 自身及其子 logger 的记录（如 chongming_worker.runtime
|
||||
归入 chongming_worker），保证共享 QueueListener 下每个 handler 只处理
|
||||
属于自己的记录：控制台每行只打印一次，各日志文件按名隔离。
|
||||
"""
|
||||
|
||||
def __init__(self, log_name: str):
|
||||
super().__init__()
|
||||
self._name = log_name
|
||||
self._prefix = log_name + "."
|
||||
|
||||
def filter(self, record):
|
||||
return record.name == self._name or record.name.startswith(self._prefix)
|
||||
|
||||
|
||||
class ExcludeWatchFilesFilter(logging.Filter):
|
||||
"""
|
||||
日志过滤器，用于排除第三方库的日志输出
|
||||
|
||||
过滤掉来自 watchfiles、aiosqlite、apscheduler、sqlalchemy 等库的日志记录，
|
||||
避免这些库的详细日志污染应用日志输出。
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化过滤器，设置需要过滤的日志名称前缀列表
|
||||
"""
|
||||
super().__init__()
|
||||
self._filter_starts_with_tulpe = (
|
||||
"watchfiles",
|
||||
"aiosqlite",
|
||||
"apscheduler",
|
||||
"sqlalchemy",
|
||||
"sqlitedict",
|
||||
"sqlalchemy.engine.Engine",
|
||||
)
|
||||
|
||||
def filter(self, record):
|
||||
"""
|
||||
判断是否应该保留该日志记录
|
||||
|
||||
Args:
|
||||
record: 日志记录对象，包含 name 属性表示日志来源
|
||||
|
||||
Returns:
|
||||
bool: 如果日志名称不以过滤列表中的任何前缀开头则返回 True（保留），
|
||||
否则返回 False（过滤掉）
|
||||
"""
|
||||
return not record.name.startswith(self._filter_starts_with_tulpe)
|
||||
|
||||
|
||||
# ---- 异步日志队列（QueueHandler + QueueListener）----
|
||||
# 主线程/事件循环只做"入队"（LogRecord 创建 + put，µs 级），文件/控制台 I/O
|
||||
# 由后台监听线程执行，日志记录不再阻塞业务协程。
|
||||
# 队列满时丢弃新日志（日志系统故障不影响业务）；错误日志（>=400）不受采样影响。
|
||||
|
||||
_LOG_QUEUE_SIZE = 10000
|
||||
|
||||
|
||||
class _AsyncQueueHandler(QueueHandler):
|
||||
"""非阻塞入队 + 惰性格式化。
|
||||
|
||||
- 队列满时丢弃（而不是抛 queue.Full 打断业务）
|
||||
- 不预格式化：% 样式消息的格式化推迟到监听线程（formatter 执行），
|
||||
主线程只承担 LogRecord 创建与入队
|
||||
- queue 经模块全局读取：gunicorn preload/fork 后自动指向子进程的新队列
|
||||
"""
|
||||
|
||||
@property
|
||||
def queue(self):
|
||||
return _log_queue
|
||||
|
||||
def __init__(self):
|
||||
# queue 由 property 动态提供（fork 后自动指向新队列）。
|
||||
# 只初始化 Handler 基类（level/filters 等），不走 QueueHandler.__init__
|
||||
# 的 self.queue = queue 赋值（会与 property 冲突）
|
||||
logging.Handler.__init__(self)
|
||||
|
||||
def enqueue(self, record):
|
||||
try:
|
||||
self.queue.put_nowait(record)
|
||||
except queue.Full:
|
||||
pass # 队列满：丢弃该日志
|
||||
|
||||
def prepare(self, record):
|
||||
return record # 格式化留给监听线程的 formatter
|
||||
|
||||
|
||||
class _RegistryListener(QueueListener):
|
||||
"""handler 可动态注册的 QueueListener（每个 log_name 有独立 file handler）"""
|
||||
|
||||
def __init__(self, q: queue.Queue):
|
||||
super().__init__(q)
|
||||
self.handlers: list = [] # 可变列表，运行期可追加
|
||||
|
||||
def register(self, *handlers) -> None:
|
||||
self.handlers.extend(handlers)
|
||||
|
||||
|
||||
_log_queue: queue.Queue = queue.Queue(maxsize=_LOG_QUEUE_SIZE)
|
||||
_listener: Optional[_RegistryListener] = None
|
||||
# 全部真实 handler（文件/控制台），供 fork 后重建监听线程
|
||||
_listener_handlers: List[logging.Handler] = []
|
||||
|
||||
|
||||
def _start_listener() -> None:
|
||||
global _listener
|
||||
_listener = _RegistryListener(_log_queue)
|
||||
_listener.register(*_listener_handlers)
|
||||
_listener.start()
|
||||
|
||||
|
||||
def _after_fork_child() -> None:
|
||||
"""gunicorn preload/fork 场景：线程不跨 fork，子进程重建队列与监听线程。
|
||||
|
||||
队列重建避免 fork 时锁状态不可用导致的死锁；旧队列中的记录在 fork
|
||||
瞬间丢失（启动窗口内可接受）。
|
||||
"""
|
||||
global _log_queue
|
||||
_log_queue = queue.Queue(maxsize=_LOG_QUEUE_SIZE)
|
||||
_start_listener()
|
||||
|
||||
|
||||
if hasattr(os, "register_at_fork"):
|
||||
os.register_at_fork(after_in_child=_after_fork_child)
|
||||
|
||||
|
||||
def shutdown_logging() -> None:
|
||||
"""停止监听线程并冲刷剩余日志（进程退出 / 应用关闭时调用），幂等。
|
||||
|
||||
QueueListener.stop() 停止后会将 _thread 置 None，重复调用需跳过；
|
||||
幂等保证 atexit 与显式调用共存。
|
||||
"""
|
||||
global _listener
|
||||
if _listener is not None:
|
||||
if _listener._thread is not None:
|
||||
_listener.stop() # flush=True：处理完队列中已有记录再停止
|
||||
_listener = None
|
||||
for h in _listener_handlers:
|
||||
try:
|
||||
h.close()
|
||||
except Exception: # pragma: no cover - 关闭失败不影响退出
|
||||
pass
|
||||
|
||||
|
||||
atexit.register(shutdown_logging)
|
||||
|
||||
|
||||
_configured_loggers = set()
|
||||
# 每个进程只打印一次日志配置信息（setup_logging 每个 log_name 都会调用一次）
|
||||
_config_info_printed = False
|
||||
|
||||
|
||||
def setup_logging(logger: logging.Logger, log_name: str = "app"):
|
||||
"""
|
||||
配置日志系统，包括文件和控制台输出
|
||||
|
||||
为指定的 logger 配置以下功能：
|
||||
1. 设置日志级别和格式
|
||||
2. 创建按自然日切分的轮转文件处理器（跨天自动切换文件，支持文件大小限制和备份数量控制）
|
||||
3. 创建带颜色格式化的控制台处理器
|
||||
4. 应用过滤器排除第三方库日志
|
||||
|
||||
该函数具有防重复配置机制，同一个 log_name 只会配置一次。
|
||||
|
||||
Args:
|
||||
logger: 需要配置的 logging.Logger 实例
|
||||
log_name: 日志名称标识，用于区分不同的日志配置和生成日志文件名，默认为 "app"
|
||||
|
||||
Returns:
|
||||
None
|
||||
|
||||
Raises:
|
||||
无显式抛出异常，但可能因文件系统权限问题导致目录创建或文件写入失败
|
||||
"""
|
||||
if log_name in _configured_loggers:
|
||||
return
|
||||
_configured_loggers.add(log_name)
|
||||
log_level = getattr(logging, log_config["level"], logging.INFO)
|
||||
log_format = log_config["format"]
|
||||
# 创建文件夹
|
||||
log_file_path = Path(log_config["file"])
|
||||
if log_file_path != Path("."):
|
||||
log_file_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 配置根日志记录器
|
||||
logger.setLevel(log_level)
|
||||
|
||||
# 创建格式化器
|
||||
plain_formatter = logging.Formatter(log_format)
|
||||
color_formatter = ColoredFormatter(log_format)
|
||||
|
||||
# 创建按自然日切分的轮转文件处理器（跨天自动切文件，单文件超限再按大小轮转）
|
||||
max_bytes = log_config.get("max_size", 10 * 1024 * 1024) # 默认 10MB
|
||||
backup_count = log_config.get("backup_count", 5) # 默认保留 5 个历史文件
|
||||
|
||||
file_handler = DailyRotatingFileHandler(
|
||||
log_config["file"],
|
||||
log_name,
|
||||
max_bytes=max_bytes,
|
||||
backup_count=backup_count,
|
||||
)
|
||||
file_handler.setFormatter(plain_formatter)
|
||||
file_handler.addFilter(ExcludeWatchFilesFilter())
|
||||
file_handler.addFilter(LoggerNameFilter(log_name))
|
||||
|
||||
# 创建控制台处理器并添加过滤器
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setFormatter(color_formatter)
|
||||
console_handler.addFilter(ExcludeWatchFilesFilter())
|
||||
console_handler.addFilter(LoggerNameFilter(log_name))
|
||||
|
||||
# 真实 handler（文件/控制台）挂到后台监听线程；logger 上只挂队列 handler，
|
||||
# 日志调用在主线程仅入队，I/O 由监听线程执行
|
||||
_listener_handlers.extend([file_handler, console_handler])
|
||||
if _listener is None:
|
||||
_start_listener()
|
||||
else:
|
||||
_listener.register(file_handler, console_handler)
|
||||
logger.addHandler(_AsyncQueueHandler())
|
||||
|
||||
global _config_info_printed
|
||||
if not _config_info_printed:
|
||||
_config_info_printed = True
|
||||
print(
|
||||
f"日志系统已配置 - 级别: {logging.getLevelName(log_level)}, 文件: {log_config['file']} "
|
||||
f"(最大：{max_bytes / 1024 / 1024:.1f}MB, 备份：{backup_count})"
|
||||
)
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""
|
||||
获取配置好的日志记录器实例
|
||||
|
||||
根据名称获取或创建 logger 实例，并自动调用 setup_logging 进行配置。
|
||||
如果该名称的 logger 已经配置过，则直接返回已配置的实例。
|
||||
|
||||
Args:
|
||||
name: 日志记录器名称，通常使用模块名如 __name__
|
||||
|
||||
Returns:
|
||||
logging.Logger: 已配置完成的日志记录器实例，可直接用于记录日志
|
||||
"""
|
||||
logger = logging.getLogger(name)
|
||||
setup_logging(logger, name)
|
||||
return logger
|
||||
@@ -1,254 +0,0 @@
|
||||
"""MQTT Bridge — 纯客户端模式，Broker 由子进程管理"""
|
||||
import asyncio
|
||||
import orjson
|
||||
import re
|
||||
from typing import Optional
|
||||
from urllib.parse import quote
|
||||
|
||||
from amqtt.client import MQTTClient
|
||||
from amqtt.mqtt.constants import QOS_1
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger("mqtt")
|
||||
MQTT_TOPIC_PREFIX = "chongming"
|
||||
MQTT_SUBSCRIBE_TOPIC = f"{MQTT_TOPIC_PREFIX}/+/+"
|
||||
|
||||
# 已按"非信封载荷"路由过告警的主题（每主题只告警一次，避免设备 5s 心跳刷屏）
|
||||
_warned_legacy_topics: set = set()
|
||||
|
||||
|
||||
def _compile_topic(topic: str) -> re.Pattern:
|
||||
"""
|
||||
将 MQTT 主题模板（支持 + / # 通配符）编译为正则表达式。
|
||||
|
||||
- ``+`` 匹配单级（编译为捕获组 ``([^/]+)``）
|
||||
- ``#`` 匹配多级（编译为捕获组 ``(.*)``，须位于末尾）
|
||||
捕获组按出现顺序对应，供 {1} {2}... 占位符替换使用。
|
||||
"""
|
||||
parts = topic.split("/")
|
||||
regex = "^"
|
||||
for i, part in enumerate(parts):
|
||||
if i > 0:
|
||||
regex += "/"
|
||||
if part == "#":
|
||||
regex += "(.*)"
|
||||
elif part == "+":
|
||||
regex += "([^/]+)"
|
||||
else:
|
||||
regex += re.escape(part)
|
||||
regex += "$"
|
||||
return re.compile(regex)
|
||||
|
||||
|
||||
class MQTTBridge:
|
||||
def __init__(self, worker_manager, host="localhost", port=1883, custom_topics=None,
|
||||
username=None, password=None):
|
||||
self._wm = worker_manager
|
||||
self._host = host
|
||||
self._port = port
|
||||
self._username = username
|
||||
self._password = password
|
||||
self._client: Optional[MQTTClient] = None
|
||||
self._lt: Optional[asyncio.Task] = None
|
||||
self._running = False
|
||||
# custom_topics: [{"topic","worker","subject"?,"response_topic"?}, ...]
|
||||
# 混合模式双来源：
|
||||
# _static_topics — inline 插件，构造时由网关从配置收集
|
||||
# _dynamic_topics — external 插件，运行时由 RouteRegistry 推送
|
||||
self._static_topics = custom_topics or []
|
||||
self._dynamic_topics: list = []
|
||||
self._compiled: list = []
|
||||
self._rebuild_compiled()
|
||||
self._subscribed_custom: set = set()
|
||||
|
||||
def _rebuild_compiled(self):
|
||||
"""合并静态/动态主题并重编译匹配正则。"""
|
||||
merged = self._static_topics + self._dynamic_topics
|
||||
self._compiled = [
|
||||
(ct, _compile_topic(ct.get("topic", "")))
|
||||
for ct in merged if ct.get("topic")
|
||||
]
|
||||
|
||||
async def start(self):
|
||||
if self._running:
|
||||
return
|
||||
try:
|
||||
await self._sc()
|
||||
except Exception as e:
|
||||
logger.warning(f"MQTT 不可用: {e}")
|
||||
return
|
||||
self._running = True
|
||||
logger.info(f"MQTT Bridge 已连接 {self._host}:{self._port}")
|
||||
|
||||
async def stop(self):
|
||||
if not self._running:
|
||||
return
|
||||
self._running = False
|
||||
await self._sc_stop()
|
||||
|
||||
def _broker_uri(self) -> str:
|
||||
"""构建 Broker URI；凭据经百分号编码（amqtt 连接时自动 unquote）。"""
|
||||
if self._username:
|
||||
user = quote(self._username, safe="")
|
||||
pwd = quote(self._password or "", safe="")
|
||||
return f"mqtt://{user}:{pwd}@{self._host}:{self._port}/"
|
||||
return f"mqtt://{self._host}:{self._port}/"
|
||||
|
||||
async def _sc(self):
|
||||
self._client = MQTTClient(config={"auto_reconnect": False})
|
||||
await self._client.connect(self._broker_uri())
|
||||
topics = [(MQTT_SUBSCRIBE_TOPIC, QOS_1)]
|
||||
topics += [(ct["topic"], QOS_1) for ct, _ in self._compiled]
|
||||
await self._client.subscribe(topics)
|
||||
self._subscribed_custom = {ct["topic"] for ct, _ in self._compiled}
|
||||
logger.info(f"已订阅 {MQTT_SUBSCRIBE_TOPIC}"
|
||||
+ (f" 及 {len(self._compiled)} 个自定义主题" if self._compiled else ""))
|
||||
self._lt = asyncio.create_task(self._loop())
|
||||
|
||||
async def update_dynamic_topics(self, custom_topics: list):
|
||||
"""
|
||||
动态更新 external 插件的自定义订阅主题（RouteRegistry 推送）。
|
||||
|
||||
与构造时的 inline 静态主题合并后增量重订阅（退订已移除主题、订阅新增主题），
|
||||
运行中调用即生效。
|
||||
"""
|
||||
self._dynamic_topics = list(custom_topics or [])
|
||||
self._rebuild_compiled()
|
||||
new_topics = {ct["topic"] for ct, _ in self._compiled}
|
||||
if not (self._running and self._client):
|
||||
self._subscribed_custom = new_topics
|
||||
return
|
||||
removed = self._subscribed_custom - new_topics
|
||||
added = new_topics - self._subscribed_custom
|
||||
if removed:
|
||||
await self._client.unsubscribe(list(removed))
|
||||
if added:
|
||||
await self._client.subscribe([(t, QOS_1) for t in added])
|
||||
self._subscribed_custom = new_topics
|
||||
if added or removed:
|
||||
logger.info(
|
||||
f"自定义订阅主题更新: +{len(added)} -{len(removed)}"
|
||||
f"（当前 {len(new_topics)} 个）"
|
||||
)
|
||||
|
||||
async def _sc_stop(self):
|
||||
# 先断开连接，让 deliver_message() 解锁
|
||||
if self._client:
|
||||
try:
|
||||
await self._client.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
self._client = None
|
||||
# 再取消消息循环
|
||||
if self._lt and not self._lt.done():
|
||||
self._lt.cancel()
|
||||
try:
|
||||
await self._lt
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _loop(self):
|
||||
while self._running:
|
||||
try:
|
||||
m = await self._client.deliver_message() # type: ignore
|
||||
if m is None:
|
||||
break
|
||||
await self._handle(m)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
if self._running:
|
||||
logger.error(f"消息异常: {e}")
|
||||
break
|
||||
|
||||
async def _handle(self, m):
|
||||
t: str = m.topic
|
||||
p = m.data # type: ignore
|
||||
try:
|
||||
pl = orjson.loads(p)
|
||||
except orjson.JSONDecodeError:
|
||||
return
|
||||
|
||||
# 信封协议: {"meta": {...}, "data": {...}}
|
||||
# 兼容非信封载荷（如第三方设备原始报文）: 整包视为业务数据、meta 置空，
|
||||
# 与 worker 运行时 runtime.py 的容错语义一致（method 默认 GET）——
|
||||
# 若在此丢弃，盒子心跳等设备报文将永远到不了 Worker。
|
||||
if not isinstance(pl, dict) or ("meta" not in pl and "data" not in pl):
|
||||
data = pl if isinstance(pl, dict) else {}
|
||||
pl = {"meta": {}, "data": data}
|
||||
if t not in _warned_legacy_topics:
|
||||
_warned_legacy_topics.add(t)
|
||||
logger.warning(
|
||||
"非信封格式载荷（缺少 meta/data）按原始报文路由 [%s]: "
|
||||
"整包视为 data，建议设备侧改用 {'meta': {...}, 'data': {...}} 格式",
|
||||
t,
|
||||
)
|
||||
else:
|
||||
meta = pl.get("meta") or {}
|
||||
data = pl.get("data") or {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
pl = {"meta": meta, "data": data}
|
||||
|
||||
# 1. 优先匹配显式配置的自定义订阅主题（顺序即优先级）
|
||||
for ct, compiled in self._compiled:
|
||||
matched = compiled.match(t)
|
||||
if matched:
|
||||
await self._route_custom(ct, matched, pl, t)
|
||||
return
|
||||
|
||||
# 2. 内置接口 chongming/{worker}/{operation}
|
||||
parts = t.split("/")
|
||||
if len(parts) < 3:
|
||||
return
|
||||
wt, op = parts[1], parts[2]
|
||||
subj = f"{wt}.{op}"
|
||||
rt = f"{MQTT_TOPIC_PREFIX}/{wt}/{op}/response"
|
||||
await self._route(wt, subj, pl, rt)
|
||||
|
||||
def _fill(self, template: str, matched: re.Match) -> str:
|
||||
"""用匹配到的通配符值填充模板中的 {1} {2}... 占位符"""
|
||||
if "{" not in template:
|
||||
return template
|
||||
for i, g in enumerate(matched.groups(), start=1):
|
||||
template = template.replace("{" + str(i) + "}", g)
|
||||
return template
|
||||
|
||||
async def _route_custom(self, ct: dict, matched: re.Match, pl: dict, topic: str):
|
||||
worker = ct.get("worker", "")
|
||||
# subject 默认 "<worker>.<topic 模板最后一段>"，与内置 <插件>.<操作> 约定一致
|
||||
subject_tpl = ct.get("subject")
|
||||
if not subject_tpl:
|
||||
last = ct["topic"].split("/")[-1]
|
||||
subject_tpl = f"{worker}.{last}" if last not in ("+", "#") else worker
|
||||
subject = self._fill(subject_tpl, matched)
|
||||
# response_topic 默认 "<实际匹配主题>/response"
|
||||
rt_tpl = ct.get("response_topic")
|
||||
rt = self._fill(rt_tpl, matched) if rt_tpl else f"{topic}/response"
|
||||
await self._route(worker, subject, pl, rt)
|
||||
|
||||
async def _route(self, wt: str, subj: str, pl: dict, rt: str):
|
||||
logger.info(f"MQTT -> {subj}")
|
||||
meta = pl.get("meta") or {}
|
||||
try:
|
||||
r = await self._wm.call(worker_type=wt, subject=subj, payload=pl,
|
||||
timeout=float(meta.get("timeout", 30)),
|
||||
cacheable=bool(meta.get("cacheable", False)))
|
||||
if r is not None:
|
||||
await self._pub(rt, r)
|
||||
except RuntimeError as e:
|
||||
await self._pub(rt, {"error": str(e)})
|
||||
except Exception as e:
|
||||
await self._pub(rt, {"error": str(e)})
|
||||
|
||||
async def _pub(self, topic, data):
|
||||
try:
|
||||
# data 均为 JSON 可序列化类型（worker 回复解析结果 / {"error": ...}），
|
||||
# orjson.dumps 直接产出 UTF-8 字节，免去 str -> encode 两步
|
||||
await self._client.publish(topic, orjson.dumps(data), QOS_1) # type: ignore
|
||||
except Exception as e:
|
||||
logger.error(f"发布失败 [{topic}]: {e}")
|
||||
|
||||
@property
|
||||
def is_running(self):
|
||||
return self._running
|
||||
@@ -1,826 +0,0 @@
|
||||
"""
|
||||
动态路由生成器
|
||||
|
||||
根据配置文件中的 worker 路由定义，动态生成 FastAPI 端点。
|
||||
为每个端点生成带有正确 Pydantic 类型注解的处理函数，确保 OpenAPI 文档
|
||||
正确反映参数类型、响应结构，并注册到 components/schemas。
|
||||
"""
|
||||
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
|
||||
from inspect import Parameter, Signature
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import orjson
|
||||
from fastapi import FastAPI, HTTPException, APIRouter, Query, Path as FastAPIPath, Request, Depends
|
||||
from fastapi.responses import StreamingResponse, Response
|
||||
from pydantic import create_model, Field, BaseModel
|
||||
|
||||
from chongming_worker.route_config import WORKER_CONFIG_DIR, load_worker_routes_from_dir
|
||||
|
||||
from .worker_manager import WorkerManager, StreamIdleTimeoutError
|
||||
from .logger import get_logger
|
||||
from .auth import store_user_to_request
|
||||
from .call_stats import spawn, record_call
|
||||
|
||||
logger = get_logger("route_generator")
|
||||
|
||||
# 简单类型映射
|
||||
TYPE_MAP: Dict[str, Any] = {
|
||||
"float": float,
|
||||
"int": int,
|
||||
"str": str,
|
||||
"bool": bool,
|
||||
"any": Any,
|
||||
}
|
||||
|
||||
|
||||
def _parse_python_type(type_str: str) -> Any:
|
||||
"""解析配置中的类型字符串为 Python 类型对象"""
|
||||
type_str = type_str.strip()
|
||||
if type_str.startswith("list[") or type_str.startswith("List["):
|
||||
inner = type_str[5:-1].strip()
|
||||
inner_type = _parse_python_type(inner)
|
||||
return List[inner_type] # type: ignore
|
||||
if type_str.startswith("dict[") or type_str.startswith("Dict["):
|
||||
inner = type_str[5:-1].strip()
|
||||
parts = [p.strip() for p in inner.split(",", 1)]
|
||||
if len(parts) == 2:
|
||||
key_type = _parse_python_type(parts[0])
|
||||
val_type = _parse_python_type(parts[1])
|
||||
return Dict[key_type, val_type] # type: ignore
|
||||
return Dict[str, Any] # type: ignore
|
||||
return TYPE_MAP.get(type_str, str)
|
||||
|
||||
def generate_request_model(
|
||||
param_spec: Dict[str, str], model_name: str = "AutoRequest"
|
||||
) -> type[BaseModel]:
|
||||
"""生成 Pydantic 请求体模型"""
|
||||
fields: Dict[str, Any] = {}
|
||||
for param_name, param_type_str in param_spec.items():
|
||||
py_type = _parse_python_type(param_type_str)
|
||||
fields[param_name] = (py_type, Field(...))
|
||||
return create_model(model_name, **fields)
|
||||
|
||||
def generate_response_model(
|
||||
response_spec: Dict[str, Any], model_name: str = "AutoResponse"
|
||||
) -> Optional[type[BaseModel]]:
|
||||
"""生成 Pydantic 响应模型"""
|
||||
if not response_spec:
|
||||
return None
|
||||
fields: Dict[str, Any] = {}
|
||||
for field_name, field_spec in response_spec.items():
|
||||
if isinstance(field_spec, dict):
|
||||
py_type = _parse_python_type(field_spec.get("type", "str"))
|
||||
required = field_spec.get("required", True)
|
||||
default_val = field_spec.get("default", ...)
|
||||
if required:
|
||||
fields[field_name] = (py_type, Field(...))
|
||||
else:
|
||||
fields[field_name] = (py_type, Field(default=default_val if default_val is not ... else None))
|
||||
else:
|
||||
py_type = _parse_python_type(str(field_spec))
|
||||
fields[field_name] = (py_type, Field(...))
|
||||
return create_model(model_name, **fields)
|
||||
|
||||
class RouteConfig:
|
||||
"""单个路由配置模型"""
|
||||
|
||||
def __init__(self, data: Dict[str, Any]):
|
||||
self.subject: str = data.get("subject", "")
|
||||
self.path: str = data.get("path", "")
|
||||
self.method: str = data.get("method", "GET").upper()
|
||||
self.params: List[str] = data.get("params", [])
|
||||
self.response_model: Dict[str, Any] = data.get("response_model", {})
|
||||
self.summary: str = data.get("summary", "")
|
||||
self.docstring: str = data.get("docstring", "")
|
||||
self.timeout: float = data.get("timeout", 3.0)
|
||||
self.cacheable: bool = data.get("cacheable", False)
|
||||
self.auth_required: bool = data.get("auth_required", True)
|
||||
self.internal: bool = data.get("internal", False)
|
||||
# validate=false 时网关不解析/校验请求体，原样透传（校验职责下沉 worker，
|
||||
# 失败经 reply_err 以 503 暴露）；仅对 POST/PUT/PATCH 非流式端点生效
|
||||
self.validate: bool = data.get("validate", True)
|
||||
# 流式端点：以 SSE（text/event-stream）逐块输出
|
||||
self.stream: bool = data.get("stream", False)
|
||||
# 流式空闲超时（秒）：连续未收到数据块的最大间隔（timeout 为流总寿命上限）
|
||||
self.stream_idle_timeout: float = data.get("stream_idle_timeout", 10.0)
|
||||
|
||||
def parse_params(self) -> Dict[str, str]:
|
||||
"""解析 params 列表为 {name: type} 字典"""
|
||||
result: Dict[str, str] = {}
|
||||
for param in self.params:
|
||||
if ":" in param:
|
||||
name, type_str = param.split(":", 1)
|
||||
result[name.strip()] = type_str.strip()
|
||||
else:
|
||||
result[param.strip()] = "str"
|
||||
return result
|
||||
|
||||
def extract_path_params(self) -> List[str]:
|
||||
"""从路径中提取路径参数，如 /add/{a}/{b} -> ['a', 'b']"""
|
||||
return [
|
||||
seg[1:-1] for seg in self.path.split("/")
|
||||
if seg.startswith("{") and seg.endswith("}")
|
||||
]
|
||||
|
||||
|
||||
def register_route_group(
|
||||
app: FastAPI,
|
||||
worker_manager: WorkerManager,
|
||||
worker_type: str,
|
||||
routes: List[Dict[str, Any]],
|
||||
prefix: str = "",
|
||||
extra_schemas: Optional[Dict[str, type[BaseModel]]] = None,
|
||||
raw_body_models: Optional[Dict[Tuple[str, str], str]] = None,
|
||||
response_models: Optional[Dict[Tuple[str, str], str]] = None,
|
||||
) -> List[Any]:
|
||||
"""
|
||||
为一个 worker_type 的一组路由描述符创建并挂载 APIRouter。
|
||||
|
||||
与 register_dynamic_routes（盘扫路径）共用同一注册逻辑，
|
||||
process 模式下 RouteRegistry 也调用本函数做动态挂载。
|
||||
|
||||
Args:
|
||||
app: FastAPI 应用
|
||||
worker_manager: WorkerManager 实例
|
||||
worker_type: worker 类型（OpenAPI 分组标签）
|
||||
routes: 路由描述符列表（TOML 兼容格式，与 load_worker_routes_from_dir 输出一致）
|
||||
prefix: 全局路由前缀（如 /api/v1）
|
||||
extra_schemas: 额外 schema 收集 dict（GET 请求模型注入 OpenAPI 用）
|
||||
raw_body_models: (path, method) -> 请求模型名 的收集 dict
|
||||
（validate=false 端点：请求体文档经 requestBody $ref 注入，见 _inject_schemas_to_openapi）
|
||||
response_models: (path, method) -> 响应模型名 的收集 dict
|
||||
（端点注册不传 response_model 以砍掉注册期模型反射；响应文档经
|
||||
responses[200].content $ref 手动注入，见 _inject_schemas_to_openapi）
|
||||
|
||||
Returns:
|
||||
List[Any]: 本次新增到 app.router.routes 的 APIRoute 对象列表
|
||||
（调用方可记录引用，用于动态移除）
|
||||
"""
|
||||
router = APIRouter(prefix=prefix)
|
||||
for route_conf in routes:
|
||||
_register_single_route(router, worker_manager, worker_type, route_conf, extra_schemas, raw_body_models, response_models)
|
||||
|
||||
# raw_body_models / response_models 记录的 (path, method) 为端点声明路径（不含 prefix），
|
||||
# 而 OpenAPI paths 键为最终挂载路径（prefix + path），这里统一补前缀
|
||||
if prefix:
|
||||
for key in list((raw_body_models or {}).keys()):
|
||||
path, method = key
|
||||
raw_body_models[(prefix + path, method)] = raw_body_models.pop(key)
|
||||
for key in list((response_models or {}).keys()):
|
||||
path, method = key
|
||||
response_models[(prefix + path, method)] = response_models.pop(key)
|
||||
|
||||
before = set(id(r) for r in app.router.routes)
|
||||
app.include_router(router)
|
||||
added = [r for r in app.router.routes if id(r) not in before]
|
||||
logger.info(
|
||||
f"已挂载 worker '{worker_type}' 的路由组: {len(routes)} 个描述符 -> "
|
||||
f"{len(added)} 条 HTTP 路由 (prefix={prefix or '/'})"
|
||||
)
|
||||
return added
|
||||
|
||||
|
||||
def register_dynamic_routes(
|
||||
app: FastAPI,
|
||||
worker_manager: WorkerManager,
|
||||
prefix: str = "",
|
||||
worker_types: Optional[List[str]] = None,
|
||||
):
|
||||
"""
|
||||
根据 config/workers/ 目录结构注册所有动态路由。
|
||||
|
||||
worker_types: 白名单（混合模式下只注册 inline 插件；
|
||||
None = 全部注册，兼容纯 inline 部署）。
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
workers_base = Path(WORKER_CONFIG_DIR)
|
||||
if not workers_base.is_dir():
|
||||
logger.info(f"Worker 配置目录不存在: {WORKER_CONFIG_DIR}")
|
||||
return
|
||||
|
||||
extra_schemas: Dict[str, type[BaseModel]] = {}
|
||||
# validate=false 端点的 (path, method) -> 请求模型名（OpenAPI requestBody 注入用）
|
||||
raw_body_models: Dict[Tuple[str, str], str] = {}
|
||||
# 非流式端点的 (path, method) -> 响应模型名（OpenAPI responses[200] $ref 注入用）
|
||||
response_models: Dict[Tuple[str, str], str] = {}
|
||||
|
||||
# 扫描所有 worker 子目录（每个子目录名 = worker_type）
|
||||
for worker_dir in sorted(workers_base.iterdir()):
|
||||
if not worker_dir.is_dir():
|
||||
continue
|
||||
worker_type = worker_dir.name
|
||||
if worker_types is not None and worker_type not in worker_types:
|
||||
continue
|
||||
routes = load_worker_routes_from_dir(worker_type)
|
||||
if not routes:
|
||||
continue
|
||||
|
||||
register_route_group(app, worker_manager, worker_type, routes, prefix, extra_schemas, raw_body_models, response_models)
|
||||
|
||||
# 将额外 schema 注册到 OpenAPI 的 components/schemas
|
||||
if extra_schemas:
|
||||
_inject_schemas_to_openapi(app, lambda: extra_schemas, lambda: raw_body_models, lambda: response_models)
|
||||
|
||||
logger.info(f"动态路由注册完成，worker 配置目录: {WORKER_CONFIG_DIR}")
|
||||
|
||||
|
||||
def _inject_schemas_to_openapi(app: FastAPI, schemas_provider, raw_body_provider=None, response_provider=None):
|
||||
"""
|
||||
将自定义 Pydantic 模型注册到 FastAPI 的 OpenAPI components/schemas 中。
|
||||
|
||||
通过重写 app.openapi() 方法注入，因为 FastAPI 在生命周期内只生成一次
|
||||
OpenAPI schema，之后会被缓存。
|
||||
|
||||
Args:
|
||||
schemas_provider: 无参 callable，返回 {name: BaseModel} 字典。
|
||||
盘扫路径传静态 dict；process 模式传 RouteRegistry 的动态 dict，
|
||||
使路由表变化后重新生成的 OpenAPI 能反映最新 schema。
|
||||
raw_body_provider: 可选无参 callable，返回 {(path, method): model_name} 字典。
|
||||
validate=false 端点未声明 body 参数（FastAPI 不会自动生成 requestBody），
|
||||
这里按 (path, method) 补注入 requestBody $ref，保持 OpenAPI 请求体文档完整。
|
||||
response_provider: 可选无参 callable，返回 {(path, method): model_name} 字典。
|
||||
端点注册未传 response_model（见 _register_single_route：砍掉注册期模型反射），
|
||||
这里按 (path, method) 补注入 responses[200] 的 $ref，保持响应文档完整。
|
||||
"""
|
||||
original_openapi = app.openapi
|
||||
|
||||
def patched_openapi() -> dict[str, Any]:
|
||||
if app.openapi_schema is not None:
|
||||
return app.openapi_schema
|
||||
schema = original_openapi()
|
||||
if schema is None:
|
||||
return {"openapi": "3.1.0", "info": {"title": "Chongming", "version": "1.0.0"}, "paths": {}}
|
||||
# 注入我们的 schema
|
||||
schemas_dict = schema.setdefault("components", {}).setdefault("schemas", {})
|
||||
for name, model in schemas_provider().items():
|
||||
if name not in schemas_dict:
|
||||
schemas_dict[name] = model.model_json_schema()
|
||||
# validate=false 端点：补注入 requestBody（请求体文档，模型已注册到 components）
|
||||
if raw_body_provider is not None:
|
||||
for (path, method), model_name in raw_body_provider().items():
|
||||
op = schema.get("paths", {}).get(path, {}).get(method.lower())
|
||||
if op is not None:
|
||||
op["requestBody"] = {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {"$ref": f"#/components/schemas/{model_name}"}
|
||||
}
|
||||
},
|
||||
}
|
||||
# 响应模型：补注入 responses[200] 的 $ref（保留 description，仅加 content）
|
||||
if response_provider is not None:
|
||||
for (path, method), model_name in response_provider().items():
|
||||
op = schema.get("paths", {}).get(path, {}).get(method.lower())
|
||||
if op is not None:
|
||||
op.setdefault("responses", {}).setdefault("200", {})
|
||||
op["responses"]["200"].setdefault(
|
||||
"content", {}
|
||||
).setdefault("application/json", {})["schema"] = {
|
||||
"$ref": f"#/components/schemas/{model_name}"
|
||||
}
|
||||
app.openapi_schema = schema
|
||||
return schema
|
||||
|
||||
app.openapi = patched_openapi
|
||||
|
||||
|
||||
def _spawn_call_record(
|
||||
request: Request,
|
||||
worker_type: str,
|
||||
subject: str,
|
||||
rc: RouteConfig,
|
||||
status: int,
|
||||
duration_ms: float,
|
||||
req_bytes: int = 0,
|
||||
resp_bytes: int = 0,
|
||||
) -> None:
|
||||
"""
|
||||
以后台任务记录一次调用（统计 + 审计），非关键后置操作。
|
||||
|
||||
通过 spawn() 以 fire-and-forget 方式执行，端点立即返回，
|
||||
不增加接口响应延迟；后台失败由 record_call 内部兜底。
|
||||
"""
|
||||
entry = {
|
||||
"ts": time.time(),
|
||||
"worker_type": worker_type,
|
||||
"subject": subject,
|
||||
"method": rc.method,
|
||||
"path": request.url.path,
|
||||
"user": getattr(request.state, "user", None),
|
||||
"client": request.client.host if request.client else None,
|
||||
"status": status,
|
||||
"duration_ms": round(duration_ms, 3),
|
||||
"req_bytes": req_bytes,
|
||||
"resp_bytes": resp_bytes,
|
||||
}
|
||||
spawn(record_call(entry))
|
||||
|
||||
|
||||
def _make_body_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, RequestModel: type[BaseModel]):
|
||||
"""创建用于 Body 请求（POST/PUT/PATCH）的端点函数"""
|
||||
path = rc.path
|
||||
subject = rc.subject
|
||||
timeout = rc.timeout
|
||||
http_method = rc.method
|
||||
cacheable = rc.cacheable
|
||||
|
||||
async def endpoint(
|
||||
request: Request,
|
||||
body: RequestModel, # type: ignore
|
||||
) -> Any:
|
||||
start: Optional[float] = None
|
||||
payload_bytes = b""
|
||||
try:
|
||||
# 信封协议: {"meta": {...}, "data": {...}}
|
||||
meta = {"method": http_method}
|
||||
user_info = getattr(request.state, "user", None)
|
||||
if user_info:
|
||||
meta["user_info"] = user_info
|
||||
payload = {"meta": meta, "data": body.model_dump()}
|
||||
payload_bytes = orjson.dumps(payload)
|
||||
start = time.perf_counter()
|
||||
result = await worker_manager.call_raw(
|
||||
worker_type=worker_type,
|
||||
subject=subject,
|
||||
payload=payload_bytes,
|
||||
timeout=timeout,
|
||||
cacheable=cacheable,
|
||||
)
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 200, duration_ms,
|
||||
req_bytes=len(payload_bytes), resp_bytes=len(result),
|
||||
)
|
||||
# 回复为 worker 原始字节，零解析直接转发（response_model 仅用于 OpenAPI 文档）
|
||||
return Response(content=result, media_type="application/json")
|
||||
except RuntimeError as e:
|
||||
duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 503, duration_ms,
|
||||
req_bytes=len(payload_bytes),
|
||||
)
|
||||
logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e)
|
||||
raise HTTPException(status_code=503, detail=str(e))
|
||||
except Exception as e:
|
||||
duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 500, duration_ms,
|
||||
req_bytes=len(payload_bytes),
|
||||
)
|
||||
logger.error(f"路由 {path} 调用异常: {e}")
|
||||
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||
|
||||
return endpoint
|
||||
|
||||
|
||||
def _make_raw_body_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str):
|
||||
"""
|
||||
创建 validate=false 的 body 端点：请求体原样透传（网关零解析/校验）。
|
||||
|
||||
信封结构不变（{"meta": ..., "data": <原始 JSON 值>}），通过字节拼接构造，
|
||||
网关既不解析 body 也不转义；参数校验/类型转换职责下沉到 worker
|
||||
（失败经 reply_err 通道以 503 暴露）。空/空白 body 兜底为 {}。
|
||||
请求模型（params TOML 生成）仅用于 OpenAPI 文档（extra_schemas + requestBody 注入）。
|
||||
"""
|
||||
path = rc.path
|
||||
subject = rc.subject
|
||||
timeout = rc.timeout
|
||||
http_method = rc.method
|
||||
cacheable = rc.cacheable
|
||||
|
||||
async def endpoint(request: Request) -> Any:
|
||||
start: Optional[float] = None
|
||||
payload = b""
|
||||
try:
|
||||
meta = {"method": http_method}
|
||||
user_info = getattr(request.state, "user", None)
|
||||
if user_info:
|
||||
meta["user_info"] = user_info
|
||||
raw = await request.body()
|
||||
data_part = raw.strip() or b"{}"
|
||||
payload = b'{"meta":' + orjson.dumps(meta) + b',"data":' + data_part + b'}'
|
||||
start = time.perf_counter()
|
||||
result = await worker_manager.call_raw(
|
||||
worker_type=worker_type,
|
||||
subject=subject,
|
||||
payload=payload,
|
||||
timeout=timeout,
|
||||
cacheable=cacheable,
|
||||
)
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 200, duration_ms,
|
||||
req_bytes=len(payload), resp_bytes=len(result),
|
||||
)
|
||||
return Response(content=result, media_type="application/json")
|
||||
except RuntimeError as e:
|
||||
duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 503, duration_ms,
|
||||
req_bytes=len(payload),
|
||||
)
|
||||
logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e)
|
||||
raise HTTPException(status_code=503, detail=str(e))
|
||||
except Exception as e:
|
||||
duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 500, duration_ms,
|
||||
req_bytes=len(payload),
|
||||
)
|
||||
logger.error(f"路由 {path} 调用异常: {e}")
|
||||
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||
|
||||
return endpoint
|
||||
|
||||
|
||||
def _make_query_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, path_params: List[str], query_params: Dict[str, str]):
|
||||
"""
|
||||
创建用于 Query 请求（GET/DELETE）的端点函数。
|
||||
|
||||
通过设置函数 __signature__ 和 __annotations__ 为每个参数赋予正确的 Pydantic 类型，
|
||||
使 FastAPI 能在 OpenAPI schema 中生成正确的 type / format 信息。
|
||||
请求模型 schema 通过 extra_schemas 机制单独注入到 components/schemas。
|
||||
"""
|
||||
path = rc.path
|
||||
subject = rc.subject
|
||||
timeout = rc.timeout
|
||||
http_method = rc.method
|
||||
cacheable = rc.cacheable
|
||||
|
||||
# 合并路径参数和查询参数
|
||||
param_spec_combined: Dict[str, str] = {}
|
||||
param_spec_combined.update({k: query_params.get(k, "str") for k in path_params if k in query_params})
|
||||
param_spec_combined.update(query_params)
|
||||
|
||||
# 无参数情况
|
||||
if not param_spec_combined:
|
||||
async def _simple_endpoint(request: Request) -> Any:
|
||||
# 信封协议: {"meta": {...}, "data": {}}
|
||||
meta = {"method": http_method}
|
||||
user_info = getattr(request.state, "user", None)
|
||||
if user_info:
|
||||
meta["user_info"] = user_info
|
||||
payload_bytes = orjson.dumps({"meta": meta, "data": {}})
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
result = await worker_manager.call_raw(
|
||||
worker_type=worker_type,
|
||||
subject=subject,
|
||||
payload=payload_bytes,
|
||||
timeout=timeout,
|
||||
cacheable=cacheable,
|
||||
)
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 200, duration_ms,
|
||||
req_bytes=len(payload_bytes), resp_bytes=len(result),
|
||||
)
|
||||
return Response(content=result, media_type="application/json")
|
||||
except RuntimeError as e:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 503, duration_ms,
|
||||
req_bytes=len(payload_bytes),
|
||||
)
|
||||
logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e)
|
||||
raise HTTPException(status_code=503, detail=str(e))
|
||||
except Exception as e:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 500, duration_ms,
|
||||
req_bytes=len(payload_bytes),
|
||||
)
|
||||
logger.error(f"路由 {path} 调用异常: {e}")
|
||||
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||
return _simple_endpoint
|
||||
|
||||
# 构造参数名列表（用于从 kwargs 中提取）
|
||||
param_names = list(param_spec_combined.keys())
|
||||
|
||||
async def endpoint(request: Request, **kwargs: Any) -> Any:
|
||||
# 信封协议: {"meta": {...}, "data": {...}}
|
||||
meta = {"method": http_method}
|
||||
user_info = getattr(request.state, "user", None)
|
||||
if user_info:
|
||||
meta["user_info"] = user_info
|
||||
payload_bytes = orjson.dumps({"meta": meta, "data": {k: kwargs[k] for k in param_names}})
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
result = await worker_manager.call_raw(
|
||||
worker_type=worker_type,
|
||||
subject=subject,
|
||||
payload=payload_bytes,
|
||||
timeout=timeout,
|
||||
cacheable=cacheable,
|
||||
)
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 200, duration_ms,
|
||||
req_bytes=len(payload_bytes), resp_bytes=len(result),
|
||||
)
|
||||
return Response(content=result, media_type="application/json")
|
||||
except RuntimeError as e:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 503, duration_ms,
|
||||
req_bytes=len(payload_bytes),
|
||||
)
|
||||
logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e)
|
||||
raise HTTPException(status_code=503, detail=str(e))
|
||||
except Exception as e:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 500, duration_ms,
|
||||
req_bytes=len(payload_bytes),
|
||||
)
|
||||
logger.error(f"路由 {path} 调用异常: {e}")
|
||||
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||
|
||||
# 构建带类型注解的 Parameter 列表
|
||||
params: List[Parameter] = [
|
||||
Parameter("request", Parameter.POSITIONAL_OR_KEYWORD, annotation=Request)
|
||||
]
|
||||
annotations: Dict[str, Any] = {}
|
||||
for name, type_str in param_spec_combined.items():
|
||||
py_type = _parse_python_type(type_str)
|
||||
annotations[name] = py_type
|
||||
if name in path_params:
|
||||
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=FastAPIPath(...), annotation=py_type))
|
||||
else:
|
||||
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=Query(...), annotation=py_type))
|
||||
|
||||
# 设置 __annotations__（FastAPI 通过 get_type_hints 读取）
|
||||
endpoint.__annotations__ = annotations
|
||||
# 设置 __signature__（FastAPI 优先于此获取参数列表）
|
||||
endpoint.__signature__ = Signature(params, return_annotation=Dict[str, Any]) # type: ignore
|
||||
|
||||
return endpoint
|
||||
|
||||
|
||||
# 流式端点响应头：禁止缓存；通知反向代理（nginx）关闭缓冲保证 SSE 即时性
|
||||
_STREAM_HEADERS = {
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
}
|
||||
|
||||
|
||||
def _make_stream_endpoint(
|
||||
worker_manager: WorkerManager,
|
||||
rc: RouteConfig,
|
||||
worker_type: str,
|
||||
path_params: List[str] = (),
|
||||
query_params: Optional[Dict[str, str]] = None,
|
||||
RequestModel: Optional[type[BaseModel]] = None,
|
||||
):
|
||||
"""
|
||||
创建流式端点（TOML 中 stream = true）：以 SSE（text/event-stream）逐块输出。
|
||||
|
||||
请求参数形态与普通端点一致（GET 走 query/path 参数，POST/PUT/PATCH 走 body 模型）。
|
||||
错误语义:
|
||||
- 初始错误（无可用 worker 等）-> HTTP 503
|
||||
- 流中错误（Worker 返回 {"error": ...} 收尾块 / 空闲超时）-> SSE error 事件
|
||||
（此时 HTTP 200 已提交，无法改状态码）
|
||||
不校验响应模型：流式输出是逐块 JSON dict。
|
||||
"""
|
||||
path = rc.path
|
||||
subject = rc.subject
|
||||
http_method = rc.method
|
||||
timeout = rc.timeout
|
||||
idle_timeout = rc.stream_idle_timeout
|
||||
param_names = list((query_params or {}).keys())
|
||||
|
||||
def build_payload(request: Request, body=None, kwargs=None) -> Dict[str, Any]:
|
||||
"""构造信封协议 payload（与普通端点一致）"""
|
||||
meta = {"method": http_method}
|
||||
user_info = getattr(request.state, "user", None)
|
||||
if user_info:
|
||||
meta["user_info"] = user_info
|
||||
if body is not None:
|
||||
data = body.model_dump()
|
||||
else:
|
||||
data = {k: kwargs[k] for k in param_names}
|
||||
return {"meta": meta, "data": data}
|
||||
|
||||
def _sse_frame(data: Dict[str, Any]) -> str:
|
||||
return f"data: {orjson.dumps(data).decode('utf-8')}\n\n"
|
||||
|
||||
async def _sse_body(gen, start: float, request: Request, req_bytes: int) -> AsyncGenerator[str, None]:
|
||||
"""SSE 分帧；流中异常转为 SSE error 事件（HTTP 200 已提交）。
|
||||
|
||||
流结束（正常/空闲超时/中断/客户端断开）后在 finally 中后台记录
|
||||
调用统计与审计，时长覆盖整个流。
|
||||
"""
|
||||
status = 200
|
||||
try:
|
||||
async for chunk in gen:
|
||||
yield f"data: {chunk.decode('utf-8')}\n\n"
|
||||
except StreamIdleTimeoutError as e:
|
||||
status = 504
|
||||
logger.warning("路由 %s 流式空闲超时: %s", path, e)
|
||||
yield _sse_frame({"error": str(e)})
|
||||
except asyncio.CancelledError:
|
||||
status = 499 # 客户端断开，流未完成
|
||||
raise
|
||||
except GeneratorExit:
|
||||
status = 499 # 服务端主动关闭流（如连接回收），流未完成
|
||||
raise
|
||||
except Exception as e:
|
||||
status = 500
|
||||
logger.error("路由 %s 流式中断: %s", path, e)
|
||||
yield _sse_frame({"error": "内部服务器错误"})
|
||||
finally:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, status, duration_ms,
|
||||
req_bytes=req_bytes,
|
||||
)
|
||||
|
||||
async def _start_stream(request: Request, payload: Dict[str, Any]) -> StreamingResponse:
|
||||
req_bytes = len(orjson.dumps(payload))
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
gen = await worker_manager.call_stream(
|
||||
worker_type=worker_type,
|
||||
subject=subject,
|
||||
payload=payload,
|
||||
timeout=timeout,
|
||||
idle_timeout=idle_timeout,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
_spawn_call_record(
|
||||
request, worker_type, subject, rc, 503, duration_ms,
|
||||
req_bytes=req_bytes,
|
||||
)
|
||||
logger.warning("路由 %s [%s] 流式调用失败: %s", path, worker_type, e)
|
||||
raise HTTPException(status_code=503, detail=str(e))
|
||||
return StreamingResponse(
|
||||
_sse_body(gen, start, request, req_bytes),
|
||||
media_type="text/event-stream",
|
||||
headers=_STREAM_HEADERS,
|
||||
)
|
||||
|
||||
if RequestModel is not None:
|
||||
async def endpoint(request: Request, body: RequestModel) -> StreamingResponse: # type: ignore
|
||||
return await _start_stream(request, build_payload(request, body=body))
|
||||
return endpoint
|
||||
|
||||
# GET/DELETE 形态
|
||||
if not param_names:
|
||||
async def _simple_endpoint(request: Request) -> StreamingResponse:
|
||||
return await _start_stream(request, build_payload(request, kwargs=None))
|
||||
return _simple_endpoint
|
||||
|
||||
async def endpoint(request: Request, **kwargs: Any) -> StreamingResponse:
|
||||
return await _start_stream(request, build_payload(request, kwargs=kwargs))
|
||||
|
||||
# 附加类型注解与签名（与 _make_query_endpoint 一致），保证 OpenAPI 参数文档正确
|
||||
params: List[Parameter] = [
|
||||
Parameter("request", Parameter.POSITIONAL_OR_KEYWORD, annotation=Request)
|
||||
]
|
||||
annotations: Dict[str, Any] = {}
|
||||
for name, type_str in (query_params or {}).items():
|
||||
py_type = _parse_python_type(type_str)
|
||||
annotations[name] = py_type
|
||||
if name in path_params:
|
||||
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=FastAPIPath(...), annotation=py_type))
|
||||
else:
|
||||
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=Query(...), annotation=py_type))
|
||||
|
||||
endpoint.__annotations__ = annotations
|
||||
endpoint.__signature__ = Signature(params, return_annotation=StreamingResponse) # type: ignore
|
||||
|
||||
return endpoint
|
||||
|
||||
|
||||
def _register_single_route(
|
||||
router: APIRouter,
|
||||
worker_manager: WorkerManager,
|
||||
worker_type: str,
|
||||
route_conf: Dict[str, Any],
|
||||
extra_schemas: Optional[Dict[str, type[BaseModel]]] = None,
|
||||
raw_body_models: Optional[Dict[Tuple[str, str], str]] = None,
|
||||
response_models: Optional[Dict[Tuple[str, str], str]] = None,
|
||||
):
|
||||
"""注册单个路由，生成正确的函数签名以便 OpenAPI 正确生成文档"""
|
||||
rc = RouteConfig(route_conf)
|
||||
|
||||
# MQTT 端点：不注册 HTTP 路由，仅通过 MQTT Bridge 调用
|
||||
if rc.method == "MQTT":
|
||||
logger.info(
|
||||
f"MQTT 端点: subject={rc.subject} (仅 MQTT，不注册 HTTP 路由)"
|
||||
)
|
||||
return
|
||||
|
||||
# internal 端点：不注册 HTTP 路由，仅保留 Zenoh RPC 内部调用能力
|
||||
if rc.internal:
|
||||
logger.info(
|
||||
f"跳过 internal 路由注册: {rc.method} {rc.path} -> worker {worker_type}.{rc.subject}"
|
||||
)
|
||||
return
|
||||
|
||||
param_spec = rc.parse_params()
|
||||
path_params = rc.extract_path_params()
|
||||
query_params = {k: v for k, v in param_spec.items() if k not in path_params}
|
||||
|
||||
# 生成干净的模型名称（替换路径中的 / 和 {} 为 _）
|
||||
# 模型名称包含 HTTP method 以区分同名路径的不同调用类型
|
||||
clean_path = rc.path.replace("/", "_").replace("{", "").replace("}", "").strip("_")
|
||||
if not clean_path:
|
||||
clean_path = "root"
|
||||
method_tag = rc.method.upper()
|
||||
response_model_name = f"{worker_type}_{clean_path}_{method_tag}_Response"
|
||||
request_model_name = f"{worker_type}_{clean_path}_{method_tag}_Request"
|
||||
|
||||
# 生成响应模型
|
||||
if rc.stream:
|
||||
# 流式端点：逐块 SSE 输出，不校验响应模型（OpenAPI 不声明响应 schema）
|
||||
ResponseModel = None
|
||||
if rc.method in ("POST", "PUT", "PATCH"):
|
||||
# body 形态：RequestModel 传给端点做请求体校验
|
||||
RequestModel = generate_request_model(
|
||||
param_spec, request_model_name
|
||||
)
|
||||
endpoint = _make_stream_endpoint(
|
||||
worker_manager, rc, worker_type, path_params, query_params, RequestModel
|
||||
)
|
||||
else:
|
||||
# GET/DELETE 形态：query/path 参数走签名注解；
|
||||
# RequestModel 仅注册到 extra_schemas 供 OpenAPI 展示（与普通 GET 一致）
|
||||
if param_spec:
|
||||
RequestModel = generate_request_model(
|
||||
param_spec, request_model_name
|
||||
)
|
||||
if extra_schemas is not None and request_model_name not in extra_schemas:
|
||||
extra_schemas[request_model_name] = RequestModel
|
||||
endpoint = _make_stream_endpoint(
|
||||
worker_manager, rc, worker_type, path_params, query_params, None
|
||||
)
|
||||
elif rc.method in ("POST", "PUT", "PATCH"):
|
||||
ResponseModel = generate_response_model(
|
||||
rc.response_model, response_model_name
|
||||
)
|
||||
# 响应模型仅注册到 components/schemas + responses[200] $ref 注入，
|
||||
# 不传给 add_api_route：端点返回 Response 对象本就走透传（零运行时校验），
|
||||
# 省掉 FastAPI 注册期对 response_model 的反射解析（external 插件频繁
|
||||
# 挂载/摘除时每次重建都会重复这一开销）
|
||||
if ResponseModel is not None:
|
||||
if extra_schemas is not None and response_model_name not in extra_schemas:
|
||||
extra_schemas[response_model_name] = ResponseModel
|
||||
if response_models is not None:
|
||||
response_models[(rc.path, rc.method)] = response_model_name
|
||||
RequestModel = generate_request_model(
|
||||
param_spec, request_model_name
|
||||
)
|
||||
if rc.validate:
|
||||
endpoint = _make_body_endpoint(worker_manager, rc, worker_type, RequestModel)
|
||||
else:
|
||||
# validate=false：请求体原样透传（网关零解析/校验，校验下沉 worker）。
|
||||
# RequestModel 仅用于 OpenAPI 请求体文档：
|
||||
# 经 extra_schemas 注册 components/schemas + requestBody $ref 注入
|
||||
if extra_schemas is not None and request_model_name not in extra_schemas:
|
||||
extra_schemas[request_model_name] = RequestModel
|
||||
if raw_body_models is not None:
|
||||
raw_body_models[(rc.path, rc.method)] = request_model_name
|
||||
endpoint = _make_raw_body_endpoint(worker_manager, rc, worker_type)
|
||||
else:
|
||||
ResponseModel = generate_response_model(
|
||||
rc.response_model, response_model_name
|
||||
)
|
||||
# 同上：响应模型走 extra_schemas + responses $ref 注入，不传 add_api_route
|
||||
if ResponseModel is not None:
|
||||
if extra_schemas is not None and response_model_name not in extra_schemas:
|
||||
extra_schemas[response_model_name] = ResponseModel
|
||||
if response_models is not None:
|
||||
response_models[(rc.path, rc.method)] = response_model_name
|
||||
# 对 GET 端点也生成 Request 模型并注册到 extra_schemas
|
||||
if param_spec:
|
||||
RequestModel = generate_request_model(
|
||||
param_spec, request_model_name
|
||||
)
|
||||
if extra_schemas is not None and request_model_name not in extra_schemas:
|
||||
extra_schemas[request_model_name] = RequestModel
|
||||
endpoint = _make_query_endpoint(worker_manager, rc, worker_type, path_params, query_params)
|
||||
|
||||
dependencies = []
|
||||
if rc.auth_required:
|
||||
dependencies.append(Depends(store_user_to_request))
|
||||
|
||||
# 注册路由 - 使用 worker_type 作为 OpenAPI 分组标签
|
||||
# 每个 worker 类型自动成为一个独立的分组，显示在 Swagger UI 中
|
||||
# 不传 response_model：端点返回 Response 对象走透传，运行时本就零校验；
|
||||
# 响应文档由 response_models 收集 + _inject_schemas_to_openapi 补 $ref
|
||||
router.add_api_route(
|
||||
path=rc.path,
|
||||
endpoint=endpoint,
|
||||
methods=[rc.method],
|
||||
summary=rc.summary,
|
||||
description=rc.docstring,
|
||||
tags=[worker_type],
|
||||
include_in_schema=True,
|
||||
dependencies=dependencies,
|
||||
)
|
||||
logger.info(f"已注册动态路由: {rc.method} {rc.path} -> worker {worker_type}.{rc.subject} | tags=[{worker_type}]")
|
||||
@@ -1,271 +0,0 @@
|
||||
"""
|
||||
动态路由注册表（process 模式）
|
||||
|
||||
网关订阅 `chongming/routes/**` 命名空间，独立 worker 进程上线时 put 路由描述符、
|
||||
下线时 delete，网关据此动态挂载/摘除 FastAPI 路由并同步 MQTT 自定义订阅主题。
|
||||
|
||||
协议（描述符 schema_version=1）:
|
||||
chongming/routes/<type>/__meta__ 插件级描述: {"schema_version":1,"mqtt_topics":[...]}
|
||||
chongming/routes/<type>/<route_id> 端点描述（TOML 兼容 dict，与 RouteConfig 对位）:
|
||||
{"schema_version":1, "type","subject","method","path","params",
|
||||
"response_model","summary","timeout","cacheable","auth_required","internal"}
|
||||
|
||||
特性:
|
||||
- SampleKind.PUT -> upsert；DELETE -> remove
|
||||
- 同一 route_id 内容相同（hash）视为多副本重复发布，忽略；不同 -> 告警并保留先到者
|
||||
- type 级 2 秒防抖：worker 重启风暴合并为一次路由重建
|
||||
- 迟到一致性：worker 侧周期性重发布描述符（RoutePublisher 由调用方周期调用），
|
||||
网关后启动或样本丢失时，路由在一个发布周期内自动补齐
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import orjson
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import zenoh
|
||||
|
||||
from chongming_worker.routes_publisher import META_KEY, ROUTES_PREFIX, SCHEMA_VERSION
|
||||
|
||||
from .logger import get_logger
|
||||
from .route_generator import register_route_group, _inject_schemas_to_openapi
|
||||
|
||||
logger = get_logger("route_registry")
|
||||
|
||||
# worker 重启风暴防抖窗口（秒）
|
||||
DEBOUNCE_SECONDS = 2.0
|
||||
|
||||
|
||||
def _descriptor_hash(descriptor: Dict[str, Any]) -> str:
|
||||
"""描述符内容 hash（多副本幂等去重用）"""
|
||||
payload = orjson.dumps(descriptor, option=orjson.OPT_SORT_KEYS)
|
||||
return hashlib.sha1(payload).hexdigest()
|
||||
|
||||
|
||||
class RouteRegistry:
|
||||
"""process 模式下的网关侧动态路由注册表。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app,
|
||||
worker_manager,
|
||||
prefix: str = "",
|
||||
):
|
||||
self._app = app
|
||||
self._wm = worker_manager
|
||||
self._prefix = prefix
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
# worker_type -> route_id -> descriptor
|
||||
self._groups: Dict[str, Dict[str, Dict[str, Any]]] = {}
|
||||
# worker_type -> 插件级描述（mqtt_topics 等）
|
||||
self._meta: Dict[str, Dict[str, Any]] = {}
|
||||
# worker_type -> 已挂载的 APIRoute 对象（用于动态摘除）
|
||||
self._route_objs: Dict[str, List[Any]] = {}
|
||||
# OpenAPI 额外 schema（GET 请求模型），随路由表变化
|
||||
self._schemas: Dict[str, Any] = {}
|
||||
# validate=false 端点的 (path, method) -> 请求模型名（OpenAPI requestBody 注入用）
|
||||
self._raw_body_models: Dict[Tuple[str, str], str] = {}
|
||||
# 非流式端点的 (path, method) -> 响应模型名（OpenAPI responses[200] $ref 注入用）
|
||||
self._response_models: Dict[Tuple[str, str], str] = {}
|
||||
# worker_type -> 防抖任务
|
||||
self._debounce: Dict[str, asyncio.Task] = {}
|
||||
|
||||
self._subscriber: Optional[zenoh.Subscriber] = None
|
||||
self._mqtt_bridge = None
|
||||
# 网关进程内加载（inline）的插件类型：忽略其描述符，防止与盘扫注册重复挂载
|
||||
self._inline_types: set = set()
|
||||
|
||||
# 将动态 schema 注入 OpenAPI（provider 实时读取本注册表）
|
||||
_inject_schemas_to_openapi(
|
||||
app,
|
||||
lambda: self._schemas,
|
||||
lambda: self._raw_body_models,
|
||||
lambda: self._response_models,
|
||||
)
|
||||
|
||||
def set_inline_types(self, types: set):
|
||||
"""设置 inline 插件类型集合（混合模式防双注册）。"""
|
||||
self._inline_types = set(types)
|
||||
|
||||
async def start(self, session: zenoh.Session):
|
||||
"""订阅 chongming/routes/**，开始接收路由描述符。"""
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._subscriber = session.declare_subscriber(
|
||||
f"{ROUTES_PREFIX}/**", self._on_sample
|
||||
)
|
||||
logger.info(f"路由注册表已就绪: 订阅 {ROUTES_PREFIX}/**")
|
||||
|
||||
def stop(self):
|
||||
"""停止订阅（shutdown 时调用）。"""
|
||||
if self._subscriber is not None:
|
||||
self._subscriber.undeclare()
|
||||
self._subscriber = None
|
||||
for task in self._debounce.values():
|
||||
task.cancel()
|
||||
|
||||
def attach_bridge(self, mqtt_bridge):
|
||||
"""挂接 MQTT Bridge：后续路由表变化会推送 external 插件的动态订阅主题。"""
|
||||
self._mqtt_bridge = mqtt_bridge
|
||||
# Bridge 晚于路由到达时，立即补推当前已收集的主题
|
||||
topics = self._collect_mqtt_topics()
|
||||
if topics:
|
||||
self._loop.create_task(mqtt_bridge.update_dynamic_topics(topics))
|
||||
|
||||
@property
|
||||
def route_types(self) -> List[str]:
|
||||
"""当前已注册路由的 worker 类型列表（观测用）。"""
|
||||
return sorted(self._groups.keys())
|
||||
|
||||
def ready_summary(self) -> str:
|
||||
"""路由就绪状态摘要（日志/观测用）。"""
|
||||
parts = []
|
||||
for wtype in sorted(self._groups):
|
||||
http = sum(
|
||||
1 for d in self._groups[wtype].values()
|
||||
if d.get("method") not in ("MQTT",) and not d.get("internal")
|
||||
)
|
||||
parts.append(f"{wtype}={len(self._groups[wtype])} 端点/{http} HTTP")
|
||||
return ", ".join(parts) if parts else "(空)"
|
||||
|
||||
# ---- zenoh 回调线程 -> 事件循环 ----
|
||||
|
||||
def _on_sample(self, sample: zenoh.Sample):
|
||||
"""zenoh 后台线程回调：转交事件循环处理。"""
|
||||
if self._loop is None:
|
||||
return
|
||||
key = str(sample.key_expr)
|
||||
kind = sample.kind
|
||||
payload = bytes(sample.payload) if sample.payload else b""
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._handle_sample(key, kind, payload), self._loop
|
||||
)
|
||||
|
||||
async def _handle_sample(self, key: str, kind: zenoh.SampleKind, payload: bytes):
|
||||
"""解析样本并更新内存路由表（在事件循环内执行）。"""
|
||||
parts = key.split("/")
|
||||
if len(parts) < 4:
|
||||
return
|
||||
wtype, route_id = parts[2], parts[3]
|
||||
|
||||
if wtype in self._inline_types:
|
||||
# 该插件由网关进程内加载（盘扫注册路由），忽略外部描述符防双注册
|
||||
return
|
||||
|
||||
if kind == zenoh.SampleKind.DELETE:
|
||||
self._remove(wtype, route_id)
|
||||
return
|
||||
|
||||
try:
|
||||
descriptor = orjson.loads(payload)
|
||||
except orjson.JSONDecodeError:
|
||||
logger.warning("丢弃非法路由描述符 [%s]: JSON 解析失败", key)
|
||||
return
|
||||
if not isinstance(descriptor, dict):
|
||||
logger.warning("丢弃非法路由描述符 [%s]: 非 dict", key)
|
||||
return
|
||||
|
||||
if route_id == META_KEY:
|
||||
self._upsert_meta(wtype, descriptor)
|
||||
else:
|
||||
self._upsert_route(wtype, route_id, descriptor)
|
||||
|
||||
def _upsert_meta(self, wtype: str, descriptor: Dict[str, Any]):
|
||||
old = self._meta.get(wtype)
|
||||
new_hash = _descriptor_hash(descriptor)
|
||||
if old is not None and _descriptor_hash(old) == new_hash:
|
||||
return
|
||||
self._meta[wtype] = descriptor
|
||||
self._schedule_rebuild(wtype)
|
||||
|
||||
def _upsert_route(self, wtype: str, route_id: str, descriptor: Dict[str, Any]):
|
||||
group = self._groups.setdefault(wtype, {})
|
||||
existing = group.get(route_id)
|
||||
if existing is not None:
|
||||
if _descriptor_hash(existing) == _descriptor_hash(descriptor):
|
||||
# 多副本重复发布同一描述符，幂等忽略
|
||||
return
|
||||
logger.warning(
|
||||
"路由描述符冲突 [%s/%s]: 已存在版本保留，忽略新版本",
|
||||
wtype, route_id,
|
||||
)
|
||||
return
|
||||
# 补全描述符字段（发布端缺省时的兜底）
|
||||
descriptor.setdefault("type", wtype)
|
||||
descriptor.setdefault("schema_version", SCHEMA_VERSION)
|
||||
group[route_id] = descriptor
|
||||
logger.info(
|
||||
"收到路由注册: %s %s %s (subject=%s)",
|
||||
descriptor.get("method", "?"),
|
||||
descriptor.get("path", "?"),
|
||||
f"[{wtype}]",
|
||||
descriptor.get("subject", "?"),
|
||||
)
|
||||
self._schedule_rebuild(wtype)
|
||||
|
||||
def _remove(self, wtype: str, route_id: str):
|
||||
if route_id == META_KEY:
|
||||
if self._meta.pop(wtype, None) is not None:
|
||||
self._schedule_rebuild(wtype)
|
||||
return
|
||||
group = self._groups.get(wtype)
|
||||
if group and group.pop(route_id, None) is not None:
|
||||
logger.info("收到路由摘除: [%s] %s", wtype, route_id)
|
||||
self._schedule_rebuild(wtype)
|
||||
|
||||
def _schedule_rebuild(self, wtype: str):
|
||||
"""type 级防抖：合并短时间内的多次变更。"""
|
||||
prev = self._debounce.get(wtype)
|
||||
if prev is not None and not prev.done():
|
||||
prev.cancel()
|
||||
self._debounce[wtype] = asyncio.create_task(self._debounced_rebuild(wtype))
|
||||
|
||||
async def _debounced_rebuild(self, wtype: str):
|
||||
try:
|
||||
await asyncio.sleep(DEBOUNCE_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
self._debounce.pop(wtype, None)
|
||||
await self._rebuild(wtype)
|
||||
|
||||
async def _rebuild(self, wtype: str):
|
||||
"""整组重建该 worker_type 的 HTTP 路由 + 同步 MQTT 自定义主题。"""
|
||||
routes = list(self._groups.get(wtype, {}).values())
|
||||
routes.sort(key=lambda r: (r.get("path", ""), r.get("method", "")))
|
||||
|
||||
# 1. 摘除旧路由（按对象引用从 router.routes 过滤）
|
||||
for r in self._route_objs.pop(wtype, []):
|
||||
if r in self._app.router.routes:
|
||||
self._app.router.routes.remove(r)
|
||||
|
||||
# 2. 挂载新路由（MQTT / internal 端点不注册 HTTP，但保留在路由表中）
|
||||
http_routes = [
|
||||
r for r in routes
|
||||
if r.get("method") not in ("MQTT",) and not r.get("internal")
|
||||
]
|
||||
if http_routes:
|
||||
self._route_objs[wtype] = register_route_group(
|
||||
self._app, self._wm, wtype, http_routes, self._prefix,
|
||||
self._schemas, self._raw_body_models, self._response_models,
|
||||
)
|
||||
else:
|
||||
logger.info("[%s] 无 HTTP 路由可挂载（%d 个描述符，全部为 MQTT/internal）",
|
||||
wtype, len(routes))
|
||||
|
||||
# 路由表变化后 OpenAPI 缓存失效，下一次访问时重建
|
||||
self._app.openapi_schema = None
|
||||
|
||||
# 3. 同步 external 插件的 MQTT 动态订阅主题
|
||||
if self._mqtt_bridge is not None:
|
||||
await self._mqtt_bridge.update_dynamic_topics(self._collect_mqtt_topics())
|
||||
|
||||
def _collect_mqtt_topics(self) -> List[Dict[str, Any]]:
|
||||
"""汇总所有插件 __meta__ 声明的自定义订阅主题。"""
|
||||
topics: List[Dict[str, Any]] = []
|
||||
for wtype in sorted(self._meta):
|
||||
meta = self._meta[wtype]
|
||||
for t in meta.get("mqtt_topics") or []:
|
||||
if isinstance(t, dict) and t.get("topic"):
|
||||
topic = dict(t)
|
||||
topic.setdefault("worker", wtype)
|
||||
topics.append(topic)
|
||||
return topics
|
||||
@@ -1,100 +0,0 @@
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlite_vfs.core import SQLiteVFS
|
||||
|
||||
|
||||
class SVFSStaticFiles(StaticFiles):
|
||||
"""
|
||||
基于 SQLite VFS 的静态文件服务类
|
||||
|
||||
该类继承自 FastAPI 的 StaticFiles，使用 SQLite 虚拟文件系统来存储和提供静态文件。
|
||||
支持单页应用（SPA）的路由回退机制，当请求的文件不存在时，会自动返回 index.html。
|
||||
"""
|
||||
|
||||
def __init__(self, directory, vfs_db_path="static.svfs"):
|
||||
"""
|
||||
初始化 SVFSStaticFiles 实例
|
||||
|
||||
参数:
|
||||
directory (str): 静态文件的根目录路径
|
||||
vfs_db_path (str): SQLite VFS 数据库文件路径，默认为 "static.svfs"
|
||||
|
||||
返回值:
|
||||
无
|
||||
"""
|
||||
vfs_db_path = vfs_db_path
|
||||
self._vfs = SQLiteVFS(vfs_db_path, compress=False)
|
||||
super().__init__(directory=directory)
|
||||
|
||||
async def get_response(self, path: str, scope):
|
||||
"""
|
||||
根据请求路径获取对应的静态文件响应
|
||||
|
||||
该方法处理静态文件的请求逻辑，包括：
|
||||
1. 路径标准化处理
|
||||
2. 文件存在性检查
|
||||
3. SPA 路由回退支持
|
||||
4. 目录索引处理
|
||||
5. MIME 类型自动识别
|
||||
|
||||
参数:
|
||||
path (str): 请求的文件路径
|
||||
scope: ASGI 作用域对象，包含请求上下文信息
|
||||
|
||||
返回值:
|
||||
Response: FastAPI 响应对象，包含文件内容和正确的 Content-Type
|
||||
|
||||
异常:
|
||||
HTTPException: 当文件不存在且无法回退到 index.html 时，抛出 404 错误
|
||||
"""
|
||||
# 统一路径格式：以 / 开头，并将 Windows 反斜杠转为正斜杠
|
||||
normalized_path = "/" + path.replace("\\", "/")
|
||||
|
||||
# 获取文件信息
|
||||
file_info = self._vfs.get_file_info(normalized_path)
|
||||
|
||||
# 处理文件不存在的情况，尝试 SPA 回退机制
|
||||
if file_info is None:
|
||||
# SPA 回退：如果不是静态资源扩展名，尝试返回 index.html
|
||||
if not any(
|
||||
normalized_path.endswith(ext)
|
||||
for ext in [
|
||||
".js",
|
||||
".css",
|
||||
".png",
|
||||
".jpg",
|
||||
".ico",
|
||||
".svg",
|
||||
".webp",
|
||||
".json",
|
||||
]
|
||||
):
|
||||
index_info = self._vfs.get_file_info("/index.html")
|
||||
if index_info:
|
||||
content = self._vfs.read_file("/index.html")
|
||||
return Response(content=content, media_type="text/html")
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
|
||||
# 处理目录请求，返回目录下的 index.html
|
||||
if file_info["is_directory"]:
|
||||
index_info = self._vfs.get_file_info(f"{normalized_path}/index.html")
|
||||
if index_info:
|
||||
content = self._vfs.read_file(f"{normalized_path}/index.html")
|
||||
return Response(content=content, media_type="text/html")
|
||||
raise HTTPException(status_code=404, detail="Directory index not found")
|
||||
|
||||
# 读取文件内容并确定 MIME 类型
|
||||
content = self._vfs.read_file(normalized_path)
|
||||
content_type = file_info.get("content_type") or "application/octet-stream"
|
||||
# 简单根据扩展名补充 MIME 类型（可选）
|
||||
if normalized_path.endswith(".js"):
|
||||
content_type = "application/javascript"
|
||||
elif normalized_path.endswith(".css"):
|
||||
content_type = "text/css"
|
||||
elif normalized_path.endswith(".html"):
|
||||
content_type = "text/html"
|
||||
elif normalized_path.endswith(".ico"):
|
||||
content_type = "image/x-icon"
|
||||
|
||||
return Response(content=content, media_type=content_type)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +0,0 @@
|
||||
from .storage import router as storage_router
|
||||
from .stats import router as stats_router
|
||||
from .worker_admin import router as worker_admin_router
|
||||
from .auth import router as auth_router
|
||||
|
||||
__all__ = ["storage_router", "stats_router", "worker_admin_router", "auth_router"]
|
||||
@@ -1,53 +0,0 @@
|
||||
"""内置登录端点 —— auth 域的基础设施部分（管理平面可用性底线）。
|
||||
|
||||
拆分自 plugins.auth：登录 / token 签发不再依赖 auth 插件是否加载，
|
||||
auth 插件缩容到 0 或全部副本故障时，登录与 admin API 仍可用；
|
||||
用户管理功能（注册 / 资料 / 改密）仍由 auth 插件提供。
|
||||
|
||||
共享契约（Users 模型 / claims / 失败文案）来自 chongming_auth 包，
|
||||
路径与响应 shape 与拆分前完全一致（前端零改动）。
|
||||
"""
|
||||
import time
|
||||
|
||||
from chongming_auth import (
|
||||
DEFAULT_TOKEN_EXPIRES_SECONDS,
|
||||
LoginPostRequest,
|
||||
LoginPostResponse,
|
||||
issue_token,
|
||||
touch_last_login,
|
||||
verify_login,
|
||||
)
|
||||
from chongming_config import get_config
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
config = get_config()
|
||||
|
||||
router = APIRouter(prefix=config["default"]["prefix"], tags=["auth"])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/auth/login",
|
||||
response_model=LoginPostResponse,
|
||||
summary="登录",
|
||||
description="根据用户名和密码进行登录，登录成功返回token，获取请求需带式token进行请求",
|
||||
)
|
||||
async def login(body: LoginPostRequest, request: Request) -> LoginPostResponse:
|
||||
db = request.app.state.auth_db
|
||||
user, message = await verify_login(db, body.username, body.password)
|
||||
if user is None:
|
||||
return LoginPostResponse(
|
||||
result="fail", token="", message=message, timestamp=time.time()
|
||||
)
|
||||
|
||||
await touch_last_login(db, user)
|
||||
|
||||
security = get_config()["security"]
|
||||
token = issue_token(
|
||||
user,
|
||||
security["secret_key"],
|
||||
security["algorithm"],
|
||||
expires_in_seconds=DEFAULT_TOKEN_EXPIRES_SECONDS,
|
||||
)
|
||||
return LoginPostResponse(
|
||||
result="success", token=token, message="登录成功", timestamp=time.time()
|
||||
)
|
||||
@@ -1,23 +0,0 @@
|
||||
# src/chongming/app/routes/stats.py
|
||||
"""
|
||||
网关级调用统计查看端点
|
||||
|
||||
数据来源为 core.call_stats 的内存统计（进程内视角）：
|
||||
动态路由每次调用在后台任务中更新计数/延迟聚合/最近调用缓冲，
|
||||
此处提供只读快照，便于观测接口调用量、错误分布与延迟。
|
||||
"""
|
||||
from fastapi import APIRouter
|
||||
|
||||
from chongming_config import get_config
|
||||
|
||||
from ..core import call_stats
|
||||
|
||||
config = get_config()
|
||||
|
||||
router = APIRouter(prefix=config["default"]["prefix"], tags=["stats"])
|
||||
|
||||
|
||||
@router.get("/stats", summary="调用统计")
|
||||
async def get_call_stats():
|
||||
"""返回内存调用统计快照：总量、per worker/subject/status 计数、延迟聚合、最近调用"""
|
||||
return call_stats.summary()
|
||||
@@ -1,53 +0,0 @@
|
||||
# src/chongming/app/routes/storage.py
|
||||
import os
|
||||
import uuid
|
||||
import urllib.parse
|
||||
import aiofiles
|
||||
|
||||
from fastapi import APIRouter, UploadFile, File, Form, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/v1/storage", tags=["Storage"])
|
||||
|
||||
|
||||
@router.post("/upload", summary="上传文件")
|
||||
async def upload_file(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
worker_type: str = Form(...),
|
||||
):
|
||||
"""上传文件到本地存储"""
|
||||
upload_root = request.app.state.upload_root
|
||||
object_key = f"{worker_type}/{uuid.uuid4()}/{file.filename}"
|
||||
dest = os.path.join(upload_root, object_key)
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
|
||||
async with aiofiles.open(dest, "wb") as afp:
|
||||
while chunk := await file.read(1024 * 1024):
|
||||
await afp.write(chunk)
|
||||
|
||||
file_size = os.path.getsize(dest)
|
||||
return {
|
||||
"file_key": object_key,
|
||||
"bucket": "local",
|
||||
"filename": file.filename,
|
||||
"content_type": file.content_type,
|
||||
"size": file_size,
|
||||
}
|
||||
|
||||
@router.get("/download/{file_key:path}")
|
||||
async def download_file(request: Request, file_key: str):
|
||||
"""从本地存储下载文件"""
|
||||
upload_root = request.app.state.upload_root
|
||||
path = os.path.join(upload_root, file_key)
|
||||
if not os.path.isfile(path):
|
||||
raise HTTPException(404, "文件不存在")
|
||||
|
||||
raw_filename = file_key.split('/')[-1]
|
||||
encoded = urllib.parse.quote(raw_filename, safe='')
|
||||
return FileResponse(
|
||||
path,
|
||||
filename=raw_filename,
|
||||
headers={"Content-Disposition": f"attachment; filename*=UTF-8''{encoded}"},
|
||||
)
|
||||
@@ -1,329 +0,0 @@
|
||||
"""
|
||||
Worker 管理端点（插件 / 单副本重启）
|
||||
|
||||
提供插件实例盘点与"精细化重启单个 worker 副本"的能力：
|
||||
- 经 Zenoh 下发管理指令到目标的 admin Queryable（worker/<type>/<id>/__admin__）
|
||||
- 仅超级管理员（JWT payload.is_superuser）可访问
|
||||
|
||||
协议说明（见 chongming_worker.runtime 单副本管理协议）：
|
||||
- 每个 worker 副本在 worker/<type>/<worker_id>/__admin__ 注册内部 Queryable
|
||||
- {"data": {"cmd": "restart"}} → 副本平滑退出并以新 worker_id 重建
|
||||
- {"data": {"cmd": "stop"}} → 副本平滑退出且不自动重建
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
import pathlib
|
||||
import tomllib
|
||||
|
||||
from chongming_config import get_config
|
||||
|
||||
from ..core.auth import jwt_auth_dependency
|
||||
from ..core.logger import get_logger
|
||||
|
||||
config = get_config()
|
||||
|
||||
logger = get_logger("worker_admin")
|
||||
|
||||
router = APIRouter(prefix=config["default"]["prefix"], tags=["worker_admin"])
|
||||
|
||||
|
||||
def require_admin(
|
||||
payload: dict = Depends(jwt_auth_dependency),
|
||||
) -> dict:
|
||||
"""仅超级管理员可操作（重启属高风险操作，普通登录用户无权限）。"""
|
||||
if not payload.get("is_superuser"):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="需要超级管理员权限 (is_superuser)",
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _get_manager(request: Request):
|
||||
"""从应用状态获取 WorkerManager；未初始化时（lifespan 外）报 503。"""
|
||||
manager = getattr(request.app.state, "worker_manager", None)
|
||||
if manager is None:
|
||||
raise HTTPException(status_code=503, detail="WorkerManager 尚未就绪")
|
||||
return manager
|
||||
|
||||
|
||||
@router.get("/admin/workers", summary="盘点 Worker 实例")
|
||||
async def list_workers(
|
||||
request: Request,
|
||||
_admin: dict = Depends(require_admin),
|
||||
):
|
||||
"""返回全部插件实例（worker_type / worker_id / mode / alive），供前端渲染与选择。"""
|
||||
manager = _get_manager(request)
|
||||
return {"workers": manager.list_workers(), "success": True}
|
||||
|
||||
|
||||
@router.post("/admin/workers/restart", summary="重启单个 Worker 副本")
|
||||
async def restart_worker(
|
||||
request: Request,
|
||||
worker_type: str,
|
||||
worker_id: str,
|
||||
_admin: dict = Depends(require_admin),
|
||||
):
|
||||
"""精细化重启单个 worker 副本：经 Zenoh 下发 restart，副本以新 worker_id 重建。
|
||||
|
||||
- worker_type: 插件类型（如 example）
|
||||
- worker_id: 副本 id（如 worker-0-1-12345），由 GET /admin/workers 提供
|
||||
"""
|
||||
manager = _get_manager(request)
|
||||
try:
|
||||
result = await manager.restart_worker(worker_type, worker_id)
|
||||
except RuntimeError as e:
|
||||
logger.warning(
|
||||
"重启 Worker 副本失败: %s/%s 用户=%s 错误=%s",
|
||||
worker_type, worker_id,
|
||||
_admin.get("username", _admin.get("sub", "-")),
|
||||
e,
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"重启失败: {e}",
|
||||
)
|
||||
logger.info(
|
||||
"重启 Worker 副本: %s/%s 用户=%s 结果=%s",
|
||||
worker_type, worker_id,
|
||||
_admin.get("username", _admin.get("sub", "-")),
|
||||
result,
|
||||
)
|
||||
return {"success": True, "result": result}
|
||||
|
||||
|
||||
@router.post("/admin/workers/{worker_type}/restart", summary="重启整个插件（全部副本）")
|
||||
async def restart_all(
|
||||
request: Request,
|
||||
worker_type: str,
|
||||
_admin: dict = Depends(require_admin),
|
||||
):
|
||||
"""重启某插件的全部副本（逐个下发 restart，各自以新 id 重建）。
|
||||
|
||||
注意：网关进程内加载的 inline 插件副本经本接口重启，仅重建 worker 协程，
|
||||
不会卸载插件；external 插件同理逐个副本重启。
|
||||
"""
|
||||
manager = _get_manager(request)
|
||||
candidates = [
|
||||
w for w in manager.list_workers()
|
||||
if w["worker_type"] == worker_type
|
||||
]
|
||||
if not candidates:
|
||||
raise HTTPException(status_code=404, detail=f"无类型为 {worker_type} 的 worker")
|
||||
results = []
|
||||
failed = []
|
||||
for w in candidates:
|
||||
try:
|
||||
res = await manager.restart_worker(worker_type, w["worker_id"])
|
||||
results.append({"worker_id": w["worker_id"], "ok": True, "result": res})
|
||||
except Exception as e: # noqa: BLE001 - 批量重启不因单个失败中断
|
||||
failed.append({"worker_id": w["worker_id"], "ok": False, "error": str(e)})
|
||||
results.append({"worker_id": w["worker_id"], "ok": False, "error": str(e)})
|
||||
logger.info(
|
||||
"重启插件全部副本: %s (成功 %d/%d) 用户=%s",
|
||||
worker_type, len(results) - len(failed), len(results),
|
||||
_admin.get("username", _admin.get("sub", "-")),
|
||||
)
|
||||
return {"success": not failed, "total": len(results), "failed": len(failed), "results": results}
|
||||
|
||||
|
||||
class ScaleBody(BaseModel):
|
||||
"""副本数量调整请求体：目标副本数。"""
|
||||
count: int = Field(..., ge=0, description="目标副本数（>=0）")
|
||||
|
||||
|
||||
@router.post("/admin/workers/{worker_type}/scale", summary="调整同类 worker 副本数量（扩缩容）")
|
||||
async def scale_workers(
|
||||
request: Request,
|
||||
worker_type: str,
|
||||
body: ScaleBody,
|
||||
_admin: dict = Depends(require_admin),
|
||||
):
|
||||
"""把某类型的副本数调整为 body.count 并持久化到配置。
|
||||
|
||||
- 扩容新增副本；缩容随机关闭副本，并**优雅排空**（摘除 liveliness/业务
|
||||
Queryable 隔绝新请求，等待正在处理的在途请求完成后再关闭）。
|
||||
- inline 直接调整本进程 Worker 池；external 经 Zenoh 下发 rescale 到存活副本，
|
||||
由其进程内池扩缩并写回自身配置。
|
||||
"""
|
||||
manager = _get_manager(request)
|
||||
try:
|
||||
result = await manager.scale_workers(worker_type, body.count)
|
||||
except Exception as e: # noqa: BLE001 - 无副本/失败映射为 4xx
|
||||
logger.warning(
|
||||
"调整副本数量失败: %s -> %d 用户=%s 错误=%s",
|
||||
worker_type, body.count,
|
||||
_admin.get("username", _admin.get("sub", "-")),
|
||||
e,
|
||||
)
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
logger.info(
|
||||
"调整副本数量: %s -> %d 用户=%s 结果=%s",
|
||||
worker_type, body.count,
|
||||
_admin.get("username", _admin.get("sub", "-")),
|
||||
result,
|
||||
)
|
||||
return {"success": True, "result": result}
|
||||
|
||||
|
||||
class ScaleInstanceBody(BaseModel):
|
||||
"""逐实例精细扩缩请求体：目标实例 + 该实例的目标副本数。"""
|
||||
instance_id: str = Field(..., description="外部进程/实例身份（list_instances 返回）")
|
||||
count: int = Field(..., ge=0, description="该实例的目标副本数（绝对数，>=0）")
|
||||
|
||||
|
||||
@router.get(
|
||||
"/admin/workers/{worker_type}/instances",
|
||||
summary="按实例分组列出某类型 worker（精细扩缩寻址）",
|
||||
)
|
||||
async def list_worker_instances(
|
||||
request: Request,
|
||||
worker_type: str,
|
||||
_admin: dict = Depends(require_admin),
|
||||
):
|
||||
"""把某类型的在线副本按「外部进程实例」分组返回。
|
||||
|
||||
每个实例 = 一个可独立承载副本的执行单元（inline=本网关进程/host，
|
||||
external=独立进程/容器），含当前副本数与可选中的 worker_id。
|
||||
"""
|
||||
manager = _get_manager(request)
|
||||
try:
|
||||
instances = await manager.list_instances(worker_type)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("列出实例失败: %s 错误=%s", worker_type, e)
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
return {"success": True, "result": instances}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/admin/workers/{worker_type}/scale_instance",
|
||||
summary="把某实例精确扩缩到指定副本数（精细扩缩容）",
|
||||
)
|
||||
async def scale_worker_instance(
|
||||
request: Request,
|
||||
worker_type: str,
|
||||
body: ScaleInstanceBody,
|
||||
_admin: dict = Depends(require_admin),
|
||||
):
|
||||
"""把指定外部进程/**实例**精确调整到 body.count 个副本（绝对数），不影响其它实例。
|
||||
|
||||
经该实例任一副本下发 rescale{scale, instance_id, absolute:true}，由其进程内
|
||||
WorkerPool 直接扩缩并记入逐实例覆盖表；重启后仍保持该绝对数。
|
||||
"""
|
||||
manager = _get_manager(request)
|
||||
try:
|
||||
result = await manager.scale_instance(
|
||||
worker_type, body.instance_id, body.count
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning(
|
||||
"逐实例扩缩失败: %s instance=%s -> %d 用户=%s 错误=%s",
|
||||
worker_type, body.instance_id, body.count,
|
||||
_admin.get("username", _admin.get("sub", "-")),
|
||||
e,
|
||||
)
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
logger.info(
|
||||
"逐实例扩缩: %s instance=%s -> %d 用户=%s 结果=%s",
|
||||
worker_type, body.instance_id, body.count,
|
||||
_admin.get("username", _admin.get("sub", "-")),
|
||||
result,
|
||||
)
|
||||
return {"success": True, "result": result}
|
||||
|
||||
|
||||
class ConfigUpdateBody(BaseModel):
|
||||
"""配置更新请求体：写目标插件 config.toml 的 [workers.config] 段。"""
|
||||
updates: dict = Field(..., description="待写入 [workers.config] 的键值（路由/加载控制键除外）")
|
||||
|
||||
|
||||
@router.get("/admin/workers/{worker_type}/config", summary="读取插件现有运行时配置")
|
||||
async def read_worker_config(
|
||||
request: Request,
|
||||
worker_type: str,
|
||||
_admin: dict = Depends(require_admin),
|
||||
):
|
||||
"""读取某插件 config.toml 的 [workers.config] 现有值（供前端展示）。
|
||||
|
||||
经 Zenoh 向该类任一存活副本下发 get_config，由其访问自身缓存（或回退 toml 种子）
|
||||
返回 —— 网关不直接接触 external worker 的缓存后端。无存活副本 / 旧版运行时
|
||||
不支持 get_config 时，回退回直读 config/workers/<type>/config.toml 的 [workers].config。
|
||||
路由配置（add.toml 等）不在此范围。
|
||||
|
||||
响应中的 source/mode 供前端如实标注展示值的来路（避免把回退读到的 toml 值当成缓存值）：
|
||||
- source="cache"：值来自运行时缓存；source="toml"：缓存缺失或副本未响应，值取自文件；
|
||||
- mode：该类插件的配置优先来源模式（toml 模式以文件为准、cache 模式以缓存为准）。
|
||||
|
||||
- worker_type: 插件类型（如 example）
|
||||
"""
|
||||
manager = _get_manager(request)
|
||||
try:
|
||||
info = await manager.get_worker_config(worker_type)
|
||||
cfg = info.get("config")
|
||||
if isinstance(cfg, dict) and info.get("status") in ("ok", None):
|
||||
return {
|
||||
"worker_type": worker_type,
|
||||
"worker_id": info.get("worker_id"),
|
||||
"source": info.get("source", "cache"),
|
||||
"mode": info.get("mode") or manager.config_mode,
|
||||
"config": cfg,
|
||||
"success": True,
|
||||
}
|
||||
logger.debug("worker get_config 未返回有效配置（%s），回退读 toml", info)
|
||||
except Exception as e: # noqa: BLE001 - 副本未响应等一律回退读 toml
|
||||
logger.debug("worker get_config 查询失败，回退读 toml: %s", e)
|
||||
|
||||
path = pathlib.Path("config/workers") / worker_type / "config.toml"
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"插件 {worker_type} 的配置文件不存在: {path}")
|
||||
with open(path, "rb") as f:
|
||||
data = tomllib.load(f)
|
||||
workers = data.get("workers") or {}
|
||||
cfg = workers.get("config")
|
||||
return {
|
||||
"worker_type": worker_type,
|
||||
"config": cfg if isinstance(cfg, dict) else {},
|
||||
"source": "toml",
|
||||
"mode": manager.config_mode,
|
||||
"success": True,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/admin/workers/{worker_type}/config", summary="更新插件运行时配置（下发至 worker）")
|
||||
async def update_worker_config(
|
||||
request: Request,
|
||||
worker_type: str,
|
||||
body: ConfigUpdateBody,
|
||||
_admin: dict = Depends(require_admin),
|
||||
):
|
||||
"""更新某插件的运行时配置 `[workers.config]`（路由配置 add.toml 等除外）。
|
||||
|
||||
流程：网关经 Zenoh 下发 config_set 给该类型全部副本 → 由 worker 触发写回各自
|
||||
配置文件（仅 index0 落盘避免同文件并发写）→ 写成功则以新配置重启所有对应副本；
|
||||
任一失败则返回失败明细（error 信息）。
|
||||
|
||||
- worker_type: 插件类型（如 example）
|
||||
- body.updates: 待更新的键值，如 {"data_dir": "./data2"}
|
||||
"""
|
||||
manager = _get_manager(request)
|
||||
if not body.updates:
|
||||
raise HTTPException(status_code=422, detail="updates 不能为空")
|
||||
try:
|
||||
result = await manager.update_worker_config(worker_type, body.updates)
|
||||
except Exception as e: # noqa: BLE001 - 无副本等运行时错误映射为 4xx
|
||||
logger.warning(
|
||||
"更新插件配置失败: %s 用户=%s 错误=%s",
|
||||
worker_type,
|
||||
_admin.get("username", _admin.get("sub", "-")),
|
||||
e,
|
||||
)
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
logger.info(
|
||||
"更新插件配置: %s\n updates=%s\n 结果=成功 %d/%d (失败: %s) 用户=%s",
|
||||
worker_type,
|
||||
body.updates,
|
||||
result["total"] - result["failed"], result["total"],
|
||||
result["errors"],
|
||||
_admin.get("username", _admin.get("sub", "-")),
|
||||
)
|
||||
return result
|
||||
+1
-1
@@ -6,7 +6,7 @@ HTTP 端到端 RPC 压测脚本（chongming 网关全链路）
|
||||
前置启动（均后台运行，需在仓库根目录）:
|
||||
1. Zenoh router: uv run python utils/worker-rs/scripts/zenoh_router.py
|
||||
2. 网关(无热重载): uv run uvicorn chongming.app:app --host 0.0.0.0 --port 8000 --no-access-log
|
||||
3. Worker: uv run worker_serve --type demo --count 2
|
||||
3. Worker: uv run chongming-worker --type demo --count 2
|
||||
|
||||
运行:
|
||||
uv run python test/bench_http.py --concurrency 1,10,50,100,200 --duration 30
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
"""
|
||||
build 命令开关（--encrypt / --license）冒烟测试
|
||||
|
||||
两个开关默认都是关的（不加参数 = 明文模块 + 不带授权的内部调试产物），
|
||||
这里验证"默认关"与"显式打开"在构建产物上都真的生效：
|
||||
1. 参数解析：默认 encrypt/license 全关，--encrypt / --license 打开对应开关；
|
||||
--only 里 license_guard 是合法步骤（文档里一直写着，之前被漏在校验表外）
|
||||
2. 默认（不加密）：.mbank 存明文字节码（encrypted=false）、不写 .key（并清掉旧
|
||||
的）、无 packer_key 记录，且**不需要密钥即可导入**；加 --encrypt 那条路仍然
|
||||
加密、仍然写 .key，且用 .key 里的密钥能正常导入
|
||||
3. 默认（不授权）：prepare 不要求/不注入授权公钥；加 --license 时缺公钥构建失败
|
||||
|
||||
运行: uv run python test/test_build_flags.py
|
||||
"""
|
||||
import importlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
SRC = Path(__file__).parent
|
||||
sys.path.insert(0, str(SRC.parent / "utils" / "cli" / "src"))
|
||||
|
||||
from module_bank import PythonToSQLite, layout # noqa: E402
|
||||
from sqlite_vfs.core import SQLiteVFS # noqa: E402
|
||||
|
||||
from chongming_cli import build as b # noqa: E402
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
tmp = Path(tempfile.mkdtemp(prefix="build_flags_test_"))
|
||||
|
||||
# ========== 1. 参数解析 ==========
|
||||
_argv, sys.argv = sys.argv, ["build"]
|
||||
mode, steps, options = b._parse_args()
|
||||
check(
|
||||
"默认参数：pyinstaller + 全流程 + 不加密 + 不授权",
|
||||
mode == "pyinstaller" and not steps and not options.encrypt and not options.license,
|
||||
)
|
||||
|
||||
sys.argv = ["build", "--encrypt", "--license", "nuitka"]
|
||||
mode, steps, options = b._parse_args()
|
||||
check(
|
||||
"--encrypt/--license 打开对应开关",
|
||||
mode == "nuitka" and options.encrypt and options.license,
|
||||
f"mode={mode} encrypt={options.encrypt} license={options.license}",
|
||||
)
|
||||
|
||||
sys.argv = ["build", "--only", "mbank,license_guard"]
|
||||
mode, steps, options = b._parse_args()
|
||||
check("--only 接受 license_guard", steps == {"mbank", "license_guard"}, str(steps))
|
||||
|
||||
sys.argv = ["build", "--only", "mbank", "--encrypt", "pyinstaller"]
|
||||
mode, steps, options = b._parse_args()
|
||||
check(
|
||||
"位置参数与开关可混用",
|
||||
mode == "pyinstaller" and steps == {"mbank"} and options.encrypt,
|
||||
)
|
||||
|
||||
sys.argv = ["build", "--unknown-flag"]
|
||||
try:
|
||||
b._parse_args()
|
||||
check("未知参数应退出", False)
|
||||
except SystemExit:
|
||||
check("未知参数应退出", True)
|
||||
sys.argv = _argv
|
||||
|
||||
|
||||
# ========== 2. mbank 加密 / 明文字节码 ==========
|
||||
def make_source(root: Path) -> Path:
|
||||
"""造一个最小可导入的包：demo/__init__.py"""
|
||||
pkg = root / "demo"
|
||||
pkg.mkdir(parents=True)
|
||||
(pkg / "__init__.py").write_text("VALUE = 42\n", encoding="utf-8")
|
||||
return pkg
|
||||
|
||||
|
||||
def bank_meta(bank: Path, module: str) -> dict:
|
||||
vfs = SQLiteVFS(str(bank), compress=False, readonly=True)
|
||||
try:
|
||||
# 包名 → 包内路径由 module_bank 定，别自己拼；明文未混淆的模块不写
|
||||
# meta.json（默认值即"什么都不用记"），用 read_meta 读缺省
|
||||
return layout.read_meta(vfs, layout.module_paths(module, True)["meta"])
|
||||
finally:
|
||||
vfs.close()
|
||||
|
||||
|
||||
def bank_has_key(bank: Path) -> bool:
|
||||
vfs = SQLiteVFS(str(bank), compress=False, readonly=True)
|
||||
try:
|
||||
return vfs.is_file("packer_key")
|
||||
finally:
|
||||
vfs.close()
|
||||
|
||||
|
||||
def import_from_bank(bank: Path, module: str, key):
|
||||
"""像 utils/launch.py 那样装载：install_importer → import。返回 (模块, 错误)。"""
|
||||
packer = PythonToSQLite(str(bank))
|
||||
finder = packer.install_importer(key=key)
|
||||
try:
|
||||
mod = importlib.import_module(module)
|
||||
return mod, None
|
||||
except Exception as exc: # noqa: BLE001 — 测试就是要区分"能导/不能导"
|
||||
return None, exc
|
||||
finally:
|
||||
if finder in sys.meta_path:
|
||||
sys.meta_path.remove(finder)
|
||||
sys.modules.pop(module, None)
|
||||
|
||||
|
||||
plain_dir = make_source(tmp / "plain_src")
|
||||
plain_bank = tmp / "plain.mbank"
|
||||
plain_size = b._pack_mbank(
|
||||
str(plain_bank), str(plain_dir), "demo", obfuscate=False, encrypt=False
|
||||
)
|
||||
check("明文打包产出 .mbank", plain_bank.exists(), plain_size)
|
||||
check("明文打包不写 .key", not plain_bank.with_suffix(".key").exists())
|
||||
check("明文打包 encrypted=false", bank_meta(plain_bank, "demo")["encrypted"] is False)
|
||||
check("明文打包无 packer_key 记录", not bank_has_key(plain_bank))
|
||||
|
||||
mod, err = import_from_bank(plain_bank, "demo", key=None)
|
||||
check("明文包无密钥即可导入", mod is not None and getattr(mod, "VALUE", None) == 42, str(err))
|
||||
|
||||
enc_dir = make_source(tmp / "enc_src")
|
||||
enc_bank = tmp / "enc.mbank"
|
||||
enc_key_path = enc_bank.with_suffix(".key")
|
||||
enc_key_path.write_text("stale-key-from-previous-build", encoding="utf-8") # 模拟上一次加密构建
|
||||
enc_size = b._pack_mbank(
|
||||
str(enc_bank), str(enc_dir), "demo", obfuscate=False, encrypt=False
|
||||
)
|
||||
check("默认（不加密）清掉上一次加密构建留下的 .key", not enc_key_path.exists(), enc_size)
|
||||
|
||||
encrypted_bank = tmp / "encrypted.mbank"
|
||||
key = b.Encryption.generate_key()
|
||||
_orig_gen = b.Encryption.generate_key
|
||||
b.Encryption.generate_key = staticmethod(lambda: key) # 固定密钥，便于用 .key 验证导入
|
||||
try:
|
||||
b._pack_mbank(str(encrypted_bank), str(enc_dir), "demo", obfuscate=False, encrypt=True)
|
||||
finally:
|
||||
b.Encryption.generate_key = _orig_gen
|
||||
|
||||
check("--encrypt 打包写 .key", encrypted_bank.with_suffix(".key").exists())
|
||||
check("--encrypt 打包 encrypted=true", bank_meta(encrypted_bank, "demo")["encrypted"] is True)
|
||||
|
||||
mod, err = import_from_bank(encrypted_bank, "demo", key=encrypted_bank.with_suffix(".key").read_text())
|
||||
check("加密包用 .key 里的密钥可导入", mod is not None, str(err))
|
||||
|
||||
mod, err = import_from_bank(encrypted_bank, "demo", key=None)
|
||||
check("加密包缺密钥无法导入（说明 .key 必须随包提供给加载端）", mod is None and isinstance(err, ImportError), str(err))
|
||||
|
||||
|
||||
# ========== 3. 授权公钥注入 / 默认不注入 ==========
|
||||
def make_project(root: Path) -> Path:
|
||||
"""造一个最小构建现场：public/{main.py,server.py,config.toml}"""
|
||||
(root / "public" / "utils").mkdir(parents=True)
|
||||
(root / "public" / "utils" / "launch.py").write_text("", encoding="utf-8")
|
||||
(root / "public" / "main.py").write_text("", encoding="utf-8")
|
||||
(root / "public" / "server.py").write_text("", encoding="utf-8")
|
||||
(root / "public" / "config.toml").write_text(
|
||||
'[security]\nsecret_key = "your-secret-key-change-in-production"\n\n'
|
||||
'[license]\n# 机器码授权公钥（构建时注入）\npublic_key = ""\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
env_keys = (b.LICENSE_PUBKEY_ENV, b.LICENSE_PUBKEY_FILE_ENV, b.ALLOW_NO_KEY_ENV)
|
||||
saved_env = {k: os.environ.pop(k, None) for k in env_keys} # 环境变量不该影响测试结论
|
||||
|
||||
cwd = Path.cwd()
|
||||
proj = make_project(tmp / "proj_default")
|
||||
os.chdir(proj)
|
||||
try:
|
||||
msg = b._stage_prepare_build_dir(None, "pyinstaller", b.BuildOptions())
|
||||
conf = (proj / "build" / "config.toml").read_text(encoding="utf-8")
|
||||
check("默认（不授权）的 prepare 不报错", msg == "资源准备完成", msg)
|
||||
check("默认不注入公钥（模板 public_key 保持空）", 'public_key = ""' in conf)
|
||||
check("JWT 密钥仍会注入", "your-secret-key-change-in-production" not in conf)
|
||||
|
||||
try:
|
||||
b._stage_prepare_build_dir(None, "pyinstaller", b.BuildOptions(license=True))
|
||||
check("--license 缺授权公钥应失败", False)
|
||||
except RuntimeError as exc:
|
||||
check("--license 缺授权公钥应失败", "缺少厂商授权公钥" in str(exc))
|
||||
|
||||
os.environ[b.LICENSE_PUBKEY_ENV] = "test-pubkey-b64"
|
||||
try:
|
||||
b._stage_prepare_build_dir(None, "pyinstaller", b.BuildOptions(license=True))
|
||||
conf = (proj / "build" / "config.toml").read_text(encoding="utf-8")
|
||||
check("--license + 提供公钥后注入 [license] public_key", 'public_key = "test-pubkey-b64"' in conf)
|
||||
finally:
|
||||
os.environ.pop(b.LICENSE_PUBKEY_ENV, None)
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
for k, v in saved_env.items():
|
||||
if v is not None:
|
||||
os.environ[k] = v
|
||||
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
print("\n" + ("=" * 40))
|
||||
if failures:
|
||||
print(f"❌ {len(failures)} 项未通过: {failures}")
|
||||
sys.exit(1)
|
||||
print("✅ 全部通过")
|
||||
+13
-6
@@ -22,8 +22,8 @@ import httpx
|
||||
import orjson
|
||||
from fastapi import FastAPI
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "src"))
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "gateway" / "src"))
|
||||
sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
|
||||
|
||||
# ---- 依赖隔离：桩掉 chongming 包链 / auth / chongming_cache / worker_manager ----
|
||||
@@ -47,14 +47,19 @@ for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub, _wm_stub):
|
||||
sys.modules[mod.__name__] = mod
|
||||
|
||||
# 加载真实 call_stats（route_generator 依赖其 spawn/record_call）
|
||||
_cs_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "call_stats.py"
|
||||
_cs_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "call_stats.py"
|
||||
_cs_spec = importlib.util.spec_from_file_location("chongming.app.core.call_stats", _cs_path)
|
||||
_cs_mod = importlib.util.module_from_spec(_cs_spec)
|
||||
sys.modules["chongming.app.core.call_stats"] = _cs_mod
|
||||
_cs_spec.loader.exec_module(_cs_mod) # type: ignore[union-attr]
|
||||
|
||||
# 审计采样率固定为全量：生产 config 的 access_sample_rate=0.1 下，成功请求的审计行
|
||||
# 本就是按概率写的，断言"几条请求 = 几条审计行"会随机失败。本测试验证的是审计链路
|
||||
# 是否工作，与采样策略无关，故固定为 1.0。
|
||||
_cs_mod._AUDIT_SAMPLE_RATE = 1.0
|
||||
|
||||
# 加载真实 route_generator
|
||||
_rg_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "route_generator.py"
|
||||
_rg_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "route_generator.py"
|
||||
_rg_spec = importlib.util.spec_from_file_location("chongming.app.core.route_generator", _rg_path)
|
||||
_rg_mod = importlib.util.module_from_spec(_rg_spec)
|
||||
sys.modules["chongming.app.core.route_generator"] = _rg_mod
|
||||
@@ -95,12 +100,14 @@ class FakeWorkerManager:
|
||||
self.reply = reply
|
||||
self.error = error
|
||||
|
||||
async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False):
|
||||
async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False,
|
||||
instance_id=None):
|
||||
if self.error == "no_worker":
|
||||
raise RuntimeError(f"类型 '{worker_type}' 没有可用 Worker。")
|
||||
return self.reply
|
||||
|
||||
async def call_stream(self, worker_type, subject, payload, timeout=300.0, idle_timeout=10.0):
|
||||
async def call_stream(self, worker_type, subject, payload, timeout=300.0, idle_timeout=10.0,
|
||||
instance_id=None):
|
||||
if self.error == "no_worker":
|
||||
raise RuntimeError(f"类型 '{worker_type}' 没有可用 Worker。")
|
||||
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
inline 插件 [workers.config] 对齐（网关侧兜底）单测 —— WorkerManager.align_inline_config。
|
||||
|
||||
网关侧在创建 inline 副本之前，按「配置优先来源模式」（根 config.toml 的
|
||||
[cache].launch_worker_count）对齐插件配置：cache 模式取共享缓存里的段，
|
||||
toml 模式以 config/workers/<type>/config.toml 为准并覆盖缓存。目的是让网关与
|
||||
worker 包版本解耦——worker 包仍是旧版时，旧版 on_init 只拿到 toml 展平配置，
|
||||
会出现“前端显示缓存值、实际跑 toml”。
|
||||
|
||||
用 dict 作缓存桩，验证两种模式下插件拿到的配置与缓存/文件两侧的取值一致。
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, os.path.join(REPO, "gateway", "src"))
|
||||
os.chdir(REPO) # 网关模块导入期读根 config.toml / 日志配置
|
||||
|
||||
from chongming.app.core.worker_manager import _CONFIGS_KEY, WorkerManager # noqa: E402
|
||||
|
||||
PASS = 0
|
||||
FAIL = 0
|
||||
WT = "aibox_sophon_adapter"
|
||||
|
||||
# _discover_plugins 展平出来的形态（toml 值）
|
||||
FLAT = {"type": WT, "count": 1, "data_dir": "./data", "mediamtx_addr": "rtsp://toml:8554"}
|
||||
# 该插件 config.toml 的 [workers.config] 原始段
|
||||
SECTION = {"data_dir": "./data", "mediamtx_addr": "rtsp://toml:8554", "heartbeat": {"timeout": 15}}
|
||||
# 前端下发后缓存里的段（改过 mediamtx_addr 与 heartbeat）
|
||||
CACHED = {"data_dir": "./data", "mediamtx_addr": "rtsp://cache:9999", "heartbeat": {"timeout": 30}}
|
||||
|
||||
|
||||
def report(name, cond, detail=""):
|
||||
global PASS, FAIL
|
||||
if cond:
|
||||
PASS += 1
|
||||
print(f"✅ {name}" + (f" — {detail}" if detail else ""))
|
||||
else:
|
||||
FAIL += 1
|
||||
print(f"❌ {name}" + (f" — {detail}" if detail else ""))
|
||||
|
||||
|
||||
def manager(cache, mode):
|
||||
m = WorkerManager.__new__(WorkerManager) # 免会话构造：仅用到 cache + config_mode
|
||||
m.cache = cache
|
||||
m._launch_worker_count = mode
|
||||
return m
|
||||
|
||||
|
||||
def run() -> int:
|
||||
key = _CONFIGS_KEY.format(type=WT)
|
||||
|
||||
# ── 1. cache 模式：缓存有段 → 插件拿到缓存值（本次修复目标）──────────
|
||||
cache = {key: dict(CACHED)}
|
||||
got = manager(cache, "cache").align_inline_config(WT, dict(FLAT), dict(SECTION))
|
||||
report(
|
||||
"cache 模式：插件配置取缓存值（覆盖 toml 同名键）",
|
||||
got.get("mediamtx_addr") == "rtsp://cache:9999"
|
||||
and got.get("heartbeat") == {"timeout": 30},
|
||||
f"mediamtx_addr={got.get('mediamtx_addr')}",
|
||||
)
|
||||
report("cache 模式：不覆盖已存在的缓存段", cache[key] == CACHED, f"cache={cache[key]}")
|
||||
|
||||
# ── 2. cache 模式：缓存缺失 → 用 toml 段引导并写回 ──────────────────
|
||||
cache2 = {}
|
||||
got2 = manager(cache2, "cache").align_inline_config(WT, dict(FLAT), dict(SECTION))
|
||||
report(
|
||||
"cache 模式：缓存缺失用 config.toml 引导并写回",
|
||||
cache2.get(key) == SECTION and got2.get("mediamtx_addr") == "rtsp://toml:8554",
|
||||
f"cache={cache2.get(key)}",
|
||||
)
|
||||
|
||||
# ── 3. toml 模式：以文件为准并覆盖缓存（前端随之显示文件值）─────────
|
||||
cache3 = {key: dict(CACHED)}
|
||||
got3 = manager(cache3, "toml").align_inline_config(WT, dict(FLAT), dict(SECTION))
|
||||
report(
|
||||
"toml 模式：插件配置为 toml 值",
|
||||
got3.get("mediamtx_addr") == "rtsp://toml:8554",
|
||||
f"mediamtx_addr={got3.get('mediamtx_addr')}",
|
||||
)
|
||||
report(
|
||||
"toml 模式：启动时用 toml 覆盖缓存（前端与运行值一致）",
|
||||
cache3.get(key) == SECTION,
|
||||
f"cache={cache3.get(key)}",
|
||||
)
|
||||
|
||||
# ── 4. 边界：无缓存后端 / 无段（旧 config/workers/config.toml 兼容路径）──
|
||||
cache4 = {key: dict(CACHED)}
|
||||
m4 = manager(cache4, "cache")
|
||||
m4.cache = None
|
||||
report(
|
||||
"无缓存后端：原样返回（经典 toml 行为）",
|
||||
m4.align_inline_config(WT, dict(FLAT), dict(SECTION)) == FLAT
|
||||
and cache4[key] == CACHED,
|
||||
)
|
||||
cache5 = {}
|
||||
got5 = manager(cache5, "toml").align_inline_config(WT, dict(FLAT), None)
|
||||
report("无 [workers.config] 段（旧写法）：不写缓存、原样返回", got5 == FLAT and not cache5)
|
||||
|
||||
# ── 5. 与 SDK 侧一致：缓存键必须相同；SDK 为本地版本时才比对解析结果 ──
|
||||
# 注意：venv 里的 chongming-worker 可能是索引里的发布版（uv sync 会把可编辑安装
|
||||
# 换回锁文件版本），新版才有 resolve_startup_config / 正确的 toml 分支语义。
|
||||
sys.path.insert(0, os.path.join(REPO, "utils", "worker", "src"))
|
||||
import chongming_worker.runtime as r # noqa: E402
|
||||
|
||||
report(
|
||||
"缓存键与 SDK 一致（两端必须读写同一键）",
|
||||
r._configs_cache_key(WT) == _CONFIGS_KEY.format(type=WT),
|
||||
f"sdk={r._configs_cache_key(WT)}",
|
||||
)
|
||||
local_src = os.path.join(REPO, "utils", "worker", "src") in os.path.abspath(r.__file__)
|
||||
if not local_src:
|
||||
print(f"⚠️ 跳过解析结果比对：当前 SDK 为 {r.__file__}（非本地源码，缺本次改动）")
|
||||
else:
|
||||
c = {r._configs_cache_key(WT): dict(CACHED)}
|
||||
sdk = r.resolve_runtime_config(WT, dict(FLAT), c, "cache")
|
||||
report(
|
||||
"cache 模式：网关侧兜底与 SDK 解析结果一致",
|
||||
sdk.get("mediamtx_addr") == CACHED["mediamtx_addr"],
|
||||
f"sdk={sdk.get('mediamtx_addr')}",
|
||||
)
|
||||
|
||||
# ── 6. 端到端：网关对齐 + 插件初始化（部署组合：网关新、worker 包为发布版）──
|
||||
# 该断言即线上现象的复现与验证：插件 on_init 拿到的必须是网关对齐后的缓存值，
|
||||
# 与 venv 里 chongming-worker 是否为含本次改动的版本无关。
|
||||
import asyncio
|
||||
|
||||
from chongming_worker.plugin import Plugin # noqa: E402
|
||||
|
||||
seen = {}
|
||||
|
||||
class DemoPlugin(Plugin):
|
||||
type = WT
|
||||
auto_bind = False
|
||||
|
||||
async def _bootstrap(self): # 隔离数据库/对象存储引导，只看配置来源
|
||||
return None
|
||||
|
||||
async def on_init(self):
|
||||
seen["on_init"] = self.config.get("mediamtx_addr")
|
||||
|
||||
def _create_workers_impl(self, session, config, count=1, worker_manager=None):
|
||||
seen["impl"] = (config or {}).get("mediamtx_addr")
|
||||
return []
|
||||
|
||||
aligned = manager({key: dict(CACHED)}, "cache").align_inline_config(
|
||||
WT, dict(FLAT), dict(SECTION)
|
||||
)
|
||||
asyncio.run(DemoPlugin().create_workers(None, aligned, 1))
|
||||
report(
|
||||
"端到端：网关对齐后插件 on_init 读到缓存值（不依赖 worker 包版本）",
|
||||
seen.get("on_init") == CACHED["mediamtx_addr"] == seen.get("impl"),
|
||||
f"on_init={seen.get('on_init')}, impl={seen.get('impl')}, "
|
||||
f"sdk={os.path.abspath(__import__('chongming_worker').__file__)}",
|
||||
)
|
||||
|
||||
print("\n" + "=" * 56)
|
||||
print(f"测试结果: ✅ {PASS} 通过 ❌ {FAIL} 失败")
|
||||
return 0 if FAIL == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
@@ -18,9 +18,9 @@ import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
_HERE = Path(__file__).resolve().parent # chongming-zenoh/
|
||||
_HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/
|
||||
_PROJ = _HERE
|
||||
for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"):
|
||||
for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"):
|
||||
sys.path.insert(0, str(_p))
|
||||
|
||||
from chongming_cache import Cache
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""inline 直调路径 与 zenoh 路径 的一致性测试。
|
||||
|
||||
守护"进程插件 ↔ 内置插件无缝迁移"契约：同一个 worker 副本，分别经
|
||||
A) 进程内直调（chongming_worker.dispatch：网关 inline 快路径用的入口）
|
||||
B) zenoh 查询（session.get → router → queryable 回调 → 原有路径）
|
||||
发起同一批请求，逐字段比对两路结果（仅屏蔽时间戳这类天然不同的字段）。
|
||||
|
||||
覆盖：正常 GET / POST、模型校验失败（reply_err 通道）、未知 subject、
|
||||
非信封旧式载荷、流式多块、流式生成器异常收尾块。
|
||||
|
||||
运行: uv run python test/test_inline_local_dispatch.py
|
||||
（需要 zenoh router 在 config/zenoh_config.json5 指向的端点可达）
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
# 优先本地框架 checkout（与 test_stream_worker.py 同约定）
|
||||
sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
|
||||
sys.path.insert(0, str(REPO_ROOT / "gateway" / "src"))
|
||||
sys.path.insert(0, str(REPO_ROOT / "src")) # plugins 插件源码
|
||||
|
||||
import orjson # noqa: E402
|
||||
import zenoh # noqa: E402
|
||||
|
||||
from chongming_worker.dispatch import get_dispatcher, unregister_dispatcher # noqa: E402
|
||||
from chongming_worker.runtime import create_worker # noqa: E402
|
||||
|
||||
WORKER_TYPE = "example"
|
||||
WORKER_ID = "worker-parity-0"
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
def mask_timestamp(obj):
|
||||
"""屏蔽时间戳类字段（两路调用时刻不同，天然不一致）"""
|
||||
if isinstance(obj, dict):
|
||||
return {k: (0 if k in ("timestamp", "time") else mask_timestamp(v)) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [mask_timestamp(v) for v in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def normalize(payload: bytes):
|
||||
try:
|
||||
return mask_timestamp(orjson.loads(payload))
|
||||
except Exception:
|
||||
return payload
|
||||
|
||||
|
||||
def envelope(method: str, data: dict) -> bytes:
|
||||
return orjson.dumps({"meta": {"method": method, "user_info": None}, "data": data})
|
||||
|
||||
|
||||
async def invoke_zenoh(session, subject: str, payload: bytes, timeout: float = 3.0):
|
||||
"""经 zenoh 查询取首个回复：返回 ("ok", 载荷) / ("err", 载荷) / ("noreply", None)"""
|
||||
key = f"worker/{WORKER_TYPE}/{WORKER_ID}/{subject}"
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _first():
|
||||
for reply in session.get(key, payload=payload,
|
||||
consolidation=zenoh.ConsolidationMode.NONE,
|
||||
timeout=timeout):
|
||||
return reply
|
||||
return None
|
||||
|
||||
reply = await loop.run_in_executor(None, _first)
|
||||
if reply is None:
|
||||
return ("noreply", None)
|
||||
if reply.ok:
|
||||
return ("ok", bytes(reply.ok.payload) if reply.ok.payload else b"")
|
||||
return ("err", bytes(reply.err.payload))
|
||||
|
||||
|
||||
async def collect_zenoh_stream(session, subject: str, payload: bytes, timeout: float = 5.0):
|
||||
key = f"worker/{WORKER_TYPE}/{WORKER_ID}/{subject}"
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _all():
|
||||
out = []
|
||||
for reply in session.get(key, payload=payload,
|
||||
consolidation=zenoh.ConsolidationMode.NONE,
|
||||
timeout=timeout):
|
||||
if reply.ok:
|
||||
out.append(bytes(reply.ok.payload) if reply.ok.payload else b"")
|
||||
else:
|
||||
out.append(("err", bytes(reply.err.payload)))
|
||||
return out
|
||||
|
||||
return await loop.run_in_executor(None, _all)
|
||||
|
||||
|
||||
async def collect_local_stream(dispatcher, subject: str, payload: bytes):
|
||||
out = []
|
||||
async for chunk in dispatcher.invoke_stream(subject, payload):
|
||||
out.append(chunk)
|
||||
return out
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
# ── 起会话 + 一个真实 worker 副本（example 插件，纯计算端点）──
|
||||
import plugins.example.router.add # noqa: F401
|
||||
import plugins.example.router.subtract # noqa: F401
|
||||
import plugins.example.router.stream_chat # noqa: F401
|
||||
|
||||
zconfig = zenoh.Config()
|
||||
zconfig.from_file(str(REPO_ROOT / "config" / "zenoh_config.json5"))
|
||||
session = zenoh.open(zconfig)
|
||||
|
||||
worker_task = asyncio.create_task(
|
||||
create_worker(session, WORKER_TYPE, WORKER_ID, config={"data_dir": "./data"})
|
||||
)
|
||||
await asyncio.sleep(1.5) # 等 liveliness / queryable / 直调注册都就绪
|
||||
|
||||
dispatcher = get_dispatcher(WORKER_TYPE, WORKER_ID)
|
||||
check("直调入口已注册（register_dispatcher）", dispatcher is not None)
|
||||
if dispatcher is None:
|
||||
worker_task.cancel()
|
||||
session.close()
|
||||
return 1
|
||||
|
||||
try:
|
||||
# ── 1) 正常 GET：example.add?a&b ──
|
||||
payload = envelope("GET", {"a": 2, "b": 3})
|
||||
z = await invoke_zenoh(session, "example.add", payload)
|
||||
l = await dispatcher.invoke_unary("example.add", payload)
|
||||
l_norm = ("ok" if l.ok else "err", l.payload)
|
||||
check("GET 两路状态一致", z[0] == l_norm[0], f"zenoh={z[0]} local={l_norm[0]}")
|
||||
check(
|
||||
"GET 两路载荷逐字段一致",
|
||||
normalize(z[1] or b"") == normalize(l_norm[1]),
|
||||
f"zenoh={z[1]!r} local={l_norm[1]!r}",
|
||||
)
|
||||
|
||||
# ── 2) 正常 POST（同 subject 的另一方法）──
|
||||
payload = envelope("POST", {"a": 5, "b": 7})
|
||||
z = await invoke_zenoh(session, "example.add", payload)
|
||||
l = await dispatcher.invoke_unary("example.add", payload)
|
||||
l_norm = ("ok" if l.ok else "err", l.payload)
|
||||
check("POST 两路状态一致", z[0] == l_norm[0], f"zenoh={z[0]} local={l_norm[0]}")
|
||||
check("POST 两路载荷逐字段一致", normalize(z[1] or b"") == normalize(l_norm[1]))
|
||||
|
||||
# ── 3) 模型校验失败 → 框架错误（zenoh reply_err ⇔ local ok=False）──
|
||||
payload = envelope("GET", {"a": "not-a-number"})
|
||||
z = await invoke_zenoh(session, "example.add", payload)
|
||||
l = await dispatcher.invoke_unary("example.add", payload)
|
||||
l_norm = ("ok" if l.ok else "err", l.payload)
|
||||
check("校验失败两路都走错误通道", z[0] == "err" and l_norm[0] == "err",
|
||||
f"zenoh={z[0]} local={l_norm[0]}")
|
||||
z_err, l_err = normalize(z[1] or b""), normalize(l_norm[1])
|
||||
check("错误信封结构一致（operation=error 且含 error 文本）",
|
||||
isinstance(z_err, dict) and isinstance(l_err, dict)
|
||||
and z_err.get("operation") == l_err.get("operation") == "error"
|
||||
and bool(z_err.get("error")) and bool(l_err.get("error")),
|
||||
f"zenoh={z_err} local={l_err}")
|
||||
|
||||
# ── 4) 非信封旧式载荷（method 默认 GET）──
|
||||
payload = orjson.dumps({"a": 1, "b": 1})
|
||||
z = await invoke_zenoh(session, "example.subtract", payload)
|
||||
l = await dispatcher.invoke_unary("example.subtract", payload)
|
||||
l_norm = ("ok" if l.ok else "err", l.payload)
|
||||
check("旧式载荷两路一致", z[0] == l_norm[0] and normalize(z[1] or b"") == normalize(l_norm[1]),
|
||||
f"zenoh={z} local={l_norm}")
|
||||
|
||||
# ── 5) 未知 subject：两路都应失败（路径不同但终态一致：网关侧都会换副本重试后 503）──
|
||||
z = await invoke_zenoh(session, "example.not_exists", envelope("GET", {}), timeout=1.0)
|
||||
l = await dispatcher.invoke_unary("example.not_exists", envelope("GET", {}))
|
||||
check("未知 subject 两路都失败（zenoh 无回复 / 直调返回错误）",
|
||||
z[0] == "noreply" and l.ok is False, f"zenoh={z[0]} local.ok={l.ok}")
|
||||
|
||||
# ── 6) 流式多块 ──
|
||||
payload = envelope("POST", {"prompt": "你好", "tokens": 4})
|
||||
zs = await collect_zenoh_stream(session, "example.stream_chat", payload)
|
||||
ls = await collect_local_stream(dispatcher, "example.stream_chat", payload)
|
||||
check("流式块数一致", len(zs) == len(ls), f"zenoh={len(zs)} local={len(ls)}")
|
||||
check("流式逐块内容一致",
|
||||
[normalize(x) for x in zs] == [normalize(x) for x in ls],
|
||||
f"zenoh={[normalize(x) for x in zs]} local={[normalize(x) for x in ls]}")
|
||||
|
||||
# ── 7) 流式生成器中途异常 → {"error": ...} 收尾块（两路都应以此结束）──
|
||||
# 流终止语义（无 token 的空流）两路一致
|
||||
payload = envelope("POST", {"prompt": "空", "tokens": 0})
|
||||
zs = await collect_zenoh_stream(session, "example.stream_chat", payload)
|
||||
ls = await collect_local_stream(dispatcher, "example.stream_chat", payload)
|
||||
check("空流终止语义一致（不挂起、块数相同）",
|
||||
len(zs) == len(ls), f"zenoh={len(zs)} local={len(ls)}")
|
||||
finally:
|
||||
worker_task.cancel()
|
||||
try:
|
||||
await worker_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
unregister_dispatcher(WORKER_TYPE, WORKER_ID)
|
||||
session.close()
|
||||
|
||||
print(f"\n结果: {len(failures)} 个失败" + (f" -> {failures}" if failures else ""))
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,220 @@
|
||||
"""
|
||||
worker 实例定向（把请求固定到某个进程实例）测试
|
||||
|
||||
验证 WorkerManager 的实例定向语义（HTTP 的 X-Chongming-Instance 头 / MQTT 的
|
||||
target_instance 字段 / 程序调用 instance_id 参数最终都走这里）：
|
||||
1. 不指定实例：与以往一致，在全部候选副本里随机挑
|
||||
2. 指定实例：只在其实例的在线副本里挑，不跨实例
|
||||
3. 身份来源：副本经 admin status 上报 instance_id，网关按需查询并缓存
|
||||
（本地注册的 inline 副本直接用网关自身实例，无需查询）
|
||||
4. 目标实例不在线：报错而不回退到别的实例（定向即严格）
|
||||
5. 调用失败重试：限定在同一实例内换副本，绝不跨实例
|
||||
6. 实例内已无其它副本可换：立即失败，不重复打同一个副本
|
||||
7. 实例维度缓存键：定向调用的缓存与其它实例隔离
|
||||
|
||||
运行: .venv/bin/python test/test_instance_targeting.py
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/
|
||||
_PROJ = _HERE
|
||||
for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"):
|
||||
sys.path.insert(0, str(_p))
|
||||
|
||||
from chongming.app.core.worker_manager import WorkerManager # noqa: E402
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
class _OkPayload:
|
||||
def __init__(self, data: bytes):
|
||||
self.payload = data
|
||||
|
||||
|
||||
class _Ok:
|
||||
def __init__(self, data: bytes):
|
||||
self.ok = _OkPayload(data)
|
||||
|
||||
|
||||
class _Reply:
|
||||
"""一次 get 的回复：ok 与 err 互斥（与 Zenoh reply 的形态一致）。"""
|
||||
|
||||
def __init__(self, data: bytes = b"{}", err: bytes | None = None):
|
||||
if err is None:
|
||||
self.ok = _OkPayload(data)
|
||||
self.err = None
|
||||
else:
|
||||
self.ok = None
|
||||
self.err = _OkPayload(err)
|
||||
|
||||
|
||||
class FakeSession:
|
||||
"""记录每次 get 的 key，并按 key 决定回复。
|
||||
|
||||
admin 查询（worker/<type>/<id>/__admin__）由 admin_instances 决定 instance_id；
|
||||
业务调用（worker/<type>/<id>/<subject>）可按 call_errors 注入错误回复。
|
||||
"""
|
||||
|
||||
def __init__(self, admin_instances: dict | None = None,
|
||||
call_errors: set | None = None):
|
||||
self.admin_instances = admin_instances or {}
|
||||
self.call_errors = set(call_errors or ())
|
||||
self.calls: list = []
|
||||
|
||||
def get(self, key, payload=None, **kwargs):
|
||||
self.calls.append(key)
|
||||
if key.endswith("/__admin__"):
|
||||
wid = key.split("/")[2]
|
||||
iid = self.admin_instances.get(wid)
|
||||
body = (
|
||||
f'{{"worker_id":"{wid}","instance_id":"{iid}","status":"running"}}'.encode()
|
||||
if iid else f'{{"worker_id":"{wid}","status":"running"}}'.encode()
|
||||
)
|
||||
return iter([_Reply(body)])
|
||||
if key in self.call_errors:
|
||||
return iter([_Reply(err=b'{"error":"boom"}')])
|
||||
return iter([_Reply(b'{"ok":true}')])
|
||||
|
||||
def liveliness(self):
|
||||
raise RuntimeError("本测试不触及 liveliness")
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
def new_manager(session, gateway_id="gw-local"):
|
||||
m = WorkerManager(session=session)
|
||||
m._gateway_id = gateway_id
|
||||
return m
|
||||
|
||||
|
||||
async def main():
|
||||
# ── 1. 不定向：与以往一致，全候选集随机 ─────────────────
|
||||
sess = FakeSession(admin_instances={"w-a": "edge-01", "w-b": "edge-02"})
|
||||
m = new_manager(sess)
|
||||
m._workers_cache["demo"] = ["worker/demo/w-a", "worker/demo/w-b"]
|
||||
picked = {await m.get_worker("demo") for _ in range(60)}
|
||||
check("1.1 不定向时可命中全部副本", picked == set(m._workers_cache["demo"]),
|
||||
f"picked={sorted(picked)}")
|
||||
check("1.2 不定向不触发实例身份查询（零额外 RPC）",
|
||||
not [c for c in sess.calls if c.endswith("__admin__")], f"calls={sess.calls}")
|
||||
check("1.3 不定向调用走随机副本",
|
||||
await m.call_raw("demo", "ping", b"{}") == b'{"ok":true}')
|
||||
|
||||
# ── 2. 定向：只在目标实例内挑副本 ────────────────────────
|
||||
sess = FakeSession(admin_instances={"w-a": "edge-01", "w-b": "edge-02",
|
||||
"w-c": "edge-01"})
|
||||
m = new_manager(sess)
|
||||
m._workers_cache["demo"] = [
|
||||
"worker/demo/w-a", "worker/demo/w-b", "worker/demo/w-c",
|
||||
]
|
||||
picked = {await m.get_worker("demo", instance_id="edge-01") for _ in range(60)}
|
||||
check("2.1 定向只命中目标实例的副本",
|
||||
picked == {"worker/demo/w-a", "worker/demo/w-c"}, f"picked={sorted(picked)}")
|
||||
|
||||
# ── 3. 身份查询：按需查询并缓存，实例归属正确 ─────────────
|
||||
sess = FakeSession(admin_instances={"w-a": "edge-01", "w-b": "edge-02",
|
||||
"w-c": "edge-01"})
|
||||
m = new_manager(sess)
|
||||
m._workers_cache["demo"] = [
|
||||
"worker/demo/w-a", "worker/demo/w-b", "worker/demo/w-c",
|
||||
]
|
||||
index = await m._instance_workers("demo", "edge-01")
|
||||
check("3.1 实例→副本反向索引正确",
|
||||
sorted(index) == ["worker/demo/w-a", "worker/demo/w-c"], f"index={sorted(index)}")
|
||||
admin_calls = [c for c in sess.calls if c.endswith("__admin__")]
|
||||
check("3.2 每个副本只查询一次身份", len(admin_calls) == 3, f"n={len(admin_calls)}")
|
||||
await m._refresh_instance_map("demo")
|
||||
check("3.3 重复刷新复用缓存不重复查询",
|
||||
len([c for c in sess.calls if c.endswith("__admin__")]) == len(admin_calls))
|
||||
|
||||
# ── 3b. inline 本地注册副本：直接用网关实例，零查询 ──────
|
||||
sess = FakeSession()
|
||||
m = new_manager(sess, gateway_id="gw-1")
|
||||
m.register_worker("demo", "worker-0-1-999")
|
||||
index = await m._instance_workers("demo", "gw-1")
|
||||
check("3.4 inline 副本归入网关实例", index == ["worker/demo/worker-0-1-999"],
|
||||
f"index={index}")
|
||||
check("3.5 inline 副本身份查询零 RPC",
|
||||
not [c for c in sess.calls if c.endswith("__admin__")], f"calls={sess.calls}")
|
||||
|
||||
# ── 4. 定向实例不在线：严格报错，不回退 ──────────────────
|
||||
sess = FakeSession(admin_instances={"w-a": "edge-01"})
|
||||
m = new_manager(sess)
|
||||
m._workers_cache["demo"] = ["worker/demo/w-a"]
|
||||
check("4.1 目标实例无副本时 get_worker 返回 None",
|
||||
await m.get_worker("demo", instance_id="edge-99") is None)
|
||||
try:
|
||||
await m.call_raw("demo", "ping", b"{}", instance_id="edge-99")
|
||||
check("4.2 定向到离线实例应报错", False, "未抛错")
|
||||
except RuntimeError as e:
|
||||
msg = str(e)
|
||||
check("4.2 定向到离线实例报错且说明不跨实例回退",
|
||||
"edge-99" in msg and "不" in msg, f"msg={msg[:90]}")
|
||||
|
||||
# ── 5. 重试锚定同实例：绝不跨实例 ────────────────────────
|
||||
sess = FakeSession(
|
||||
admin_instances={"w-a": "edge-01", "w-b": "edge-02", "w-c": "edge-01"},
|
||||
call_errors={"worker/demo/w-b/ping"}, # 只让别实例的副本出错，验证不会被选中
|
||||
)
|
||||
m = new_manager(sess)
|
||||
m._workers_cache["demo"] = [
|
||||
"worker/demo/w-a", "worker/demo/w-b", "worker/demo/w-c",
|
||||
]
|
||||
# 目标实例 edge-01 内的 w-a 失败 -> 应换到同实例的 w-c，不碰 edge-02 的 w-b
|
||||
sess.call_errors = {"worker/demo/w-a/ping"}
|
||||
out = await m.call_raw("demo", "ping", b"{}", instance_id="edge-01", retries=2)
|
||||
business = [c for c in sess.calls if not c.endswith("__admin__")]
|
||||
check("5.1 重试不跨实例（未打其它实例的副本）",
|
||||
all("/w-b/" not in c for c in business), f"calls={business}")
|
||||
check("5.2 同实例内换副本后成功", out == b'{"ok":true}', f"out={out!r}")
|
||||
|
||||
# ── 6. 实例内无其它副本可换：立即失败 ────────────────────
|
||||
sess = FakeSession(admin_instances={"w-a": "edge-01"},
|
||||
call_errors={"worker/demo/w-a/ping"})
|
||||
m = new_manager(sess)
|
||||
m._workers_cache["demo"] = ["worker/demo/w-a", "worker/demo/w-b"]
|
||||
sess.admin_instances["w-b"] = "edge-02"
|
||||
raised = None
|
||||
try:
|
||||
await m.call_raw("demo", "ping", b"{}", instance_id="edge-01", retries=2)
|
||||
except RuntimeError as e:
|
||||
raised = e
|
||||
check("6.1 实例内无副本可换应报错", raised is not None, f"raised={raised!r}")
|
||||
n = len([c for c in sess.calls if not c.endswith("__admin__")])
|
||||
check("6.2 不重复打同一个副本（只调用一次）", n == 1, f"业务调用次数={n}")
|
||||
check("6.3 错误信息点名实例与原因",
|
||||
raised is not None and "edge-01" in str(raised), f"msg={str(raised)[:90]}")
|
||||
|
||||
# ── 7. 定向调用的缓存按实例隔离 ──────────────────────────
|
||||
sess = FakeSession(admin_instances={"w-a": "edge-01", "w-b": "edge-02"})
|
||||
m = new_manager(sess)
|
||||
m._workers_cache["demo"] = ["worker/demo/w-a", "worker/demo/w-b"]
|
||||
r1 = await m.call_raw("demo", "state", b"{}", cacheable=True, instance_id="edge-01")
|
||||
r2 = await m.call_raw("demo", "state", b"{}", cacheable=True, instance_id="edge-02")
|
||||
keys = [f"{iid}@demo:state:" for iid in ("edge-01", "edge-02")]
|
||||
check("7.1 不同实例的定向缓存键互不影响",
|
||||
all(any(k.startswith(p) for k in m._rpc_mem_cache._d) for p in keys),
|
||||
f"cache_keys={list(m._rpc_mem_cache._d)}")
|
||||
check("7.2 定向调用结果正常返回", r1 == b'{"ok":true}' and r2 == b'{"ok":true}',
|
||||
f"r1={r1!r}, r2={r2!r}")
|
||||
|
||||
print()
|
||||
if failures:
|
||||
print(f"FAILED: {len(failures)} 项 -> {failures}")
|
||||
return 1
|
||||
print("全部通过")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,207 @@
|
||||
"""
|
||||
HTTP 实例定向头（X-Chongming-Instance）透传测试
|
||||
|
||||
依赖隔离：桩掉 chongming 包链、auth 与 chongming_cache，直接加载真实
|
||||
route_generator.py 与 worker_manager.py，用 FastAPI TestClient 验证:
|
||||
1. 携带 X-Chongming-Instance 的请求，该实例 id 透传到 call_raw / call_stream
|
||||
2. 未携带该头时 instance_id 为 None（行为与改动前完全一致，兼容旧调用方）
|
||||
3. 覆盖全部端点形态：GET query / POST body / validate=false 原样透传 / SSE 流式
|
||||
4. 头值为空白时视为未指定
|
||||
5. 定向到离线实例（RuntimeError）-> HTTP 503，错误信息透出
|
||||
|
||||
运行: .venv/bin/python test/test_instance_targeting_http.py
|
||||
"""
|
||||
import importlib.util
|
||||
import logging
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import orjson
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
TEST_DIR = Path(__file__).resolve().parent
|
||||
REPO_ROOT = TEST_DIR.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "gateway" / "src"))
|
||||
sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
|
||||
|
||||
# ---- 依赖隔离：桩掉 chongming 包链 / auth / chongming_cache ----
|
||||
_pkg = types.ModuleType("chongming")
|
||||
_pkg.__path__ = []
|
||||
_app = types.ModuleType("chongming.app")
|
||||
_app.__path__ = []
|
||||
_core = types.ModuleType("chongming.app.core")
|
||||
_core.__path__ = []
|
||||
_logger_stub = types.ModuleType("chongming.app.core.logger")
|
||||
_logger_stub.get_logger = lambda name: logging.getLogger(name)
|
||||
_auth_stub = types.ModuleType("chongming.app.core.auth")
|
||||
_auth_stub.store_user_to_request = lambda x: x
|
||||
_cache_stub = types.ModuleType("chongming_cache")
|
||||
_cache_stub.Cache = object
|
||||
_cache_stub.DistributedLock = object # worker_manager 顶层导入（本测试不触达）
|
||||
for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub):
|
||||
sys.modules[mod.__name__] = mod
|
||||
|
||||
_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py"
|
||||
_wm_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.worker_manager", _wm_path
|
||||
)
|
||||
_wm_mod = importlib.util.module_from_spec(_wm_spec)
|
||||
sys.modules["chongming.app.core.worker_manager"] = _wm_mod
|
||||
_wm_spec.loader.exec_module(_wm_mod) # type: ignore[union-attr]
|
||||
|
||||
_cs_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "call_stats.py"
|
||||
_cs_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.call_stats", _cs_path
|
||||
)
|
||||
_cs_mod = importlib.util.module_from_spec(_cs_spec)
|
||||
sys.modules["chongming.app.core.call_stats"] = _cs_mod
|
||||
_cs_spec.loader.exec_module(_cs_mod) # type: ignore[union-attr]
|
||||
|
||||
_rg_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "route_generator.py"
|
||||
_rg_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.route_generator", _rg_path
|
||||
)
|
||||
_rg_mod = importlib.util.module_from_spec(_rg_spec)
|
||||
sys.modules["chongming.app.core.route_generator"] = _rg_mod
|
||||
_rg_spec.loader.exec_module(_rg_mod) # type: ignore[union-attr]
|
||||
|
||||
register_route_group = _rg_mod.register_route_group
|
||||
INSTANCE_HEADER = _rg_mod.INSTANCE_HEADER
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
class FakeWorkerManager:
|
||||
"""记录每次调用收到的 instance_id；可指定定点失败以验证 503 映射。"""
|
||||
|
||||
def __init__(self, fail_instance=None):
|
||||
self.fail_instance = fail_instance # 指定实例报错（模拟实例离线）
|
||||
self.raw_instances = [] # call_raw 收到的 instance_id
|
||||
self.stream_instances = [] # call_stream 收到的 instance_id
|
||||
|
||||
async def call_raw(self, worker_type, subject, payload, timeout=3.0,
|
||||
retries=1, cacheable=False, instance_id=None):
|
||||
self.raw_instances.append(instance_id)
|
||||
if instance_id and instance_id == self.fail_instance:
|
||||
raise RuntimeError(
|
||||
f"类型 '{worker_type}' 的实例 '{instance_id}' 没有在线副本"
|
||||
"（定向调用不跨实例回退）。"
|
||||
)
|
||||
return orjson.dumps({"op": subject, "instance": instance_id})
|
||||
|
||||
async def call_stream(self, worker_type, subject, payload, timeout=300.0,
|
||||
idle_timeout=10.0, instance_id=None):
|
||||
self.stream_instances.append(instance_id)
|
||||
if instance_id and instance_id == self.fail_instance:
|
||||
raise RuntimeError(f"实例 '{instance_id}' 没有在线副本")
|
||||
|
||||
async def _gen():
|
||||
yield orjson.dumps({"instance": instance_id})
|
||||
|
||||
return _gen()
|
||||
|
||||
|
||||
ROUTES = [
|
||||
{"subject": "t.get", "path": "/calc/add", "method": "GET",
|
||||
"params": ["a: float", "b: float"], "summary": "add",
|
||||
"timeout": 3.0, "auth_required": False},
|
||||
{"subject": "t.post", "path": "/calc/echo", "method": "POST",
|
||||
"params": ["msg: str"], "summary": "echo",
|
||||
"timeout": 3.0, "auth_required": False},
|
||||
{"subject": "t.raw", "path": "/calc/raw", "method": "POST",
|
||||
"params": ["payload: any"], "summary": "原样透传",
|
||||
"validate": False, "timeout": 3.0, "auth_required": False},
|
||||
{"subject": "t.stream", "path": "/stream/chat", "method": "POST",
|
||||
"params": ["prompt: str"], "summary": "sse",
|
||||
"stream": True, "timeout": 60.0, "stream_idle_timeout": 1.0,
|
||||
"auth_required": False},
|
||||
]
|
||||
|
||||
|
||||
def build_client(fwm):
|
||||
app = FastAPI()
|
||||
register_route_group(app, fwm, "demotype", ROUTES, prefix="/api/v1")
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def main():
|
||||
# ── 1. GET query 端点 ────────────────────────────────────
|
||||
fwm = FakeWorkerManager()
|
||||
c = build_client(fwm)
|
||||
r = c.get("/api/v1/calc/add?a=1&b=2", headers={INSTANCE_HEADER: "edge-01"})
|
||||
check("1.1 GET 携带头 -> 200", r.status_code == 200, f"status={r.status_code}")
|
||||
check("1.2 GET 头透传到 call_raw", fwm.raw_instances == ["edge-01"],
|
||||
f"instances={fwm.raw_instances}")
|
||||
check("1.3 响应体回带实例（证明打到目标实例）",
|
||||
r.json().get("instance") == "edge-01", f"body={r.json()}")
|
||||
|
||||
# ── 2. 未携带头：兼容旧调用方 ────────────────────────────
|
||||
fwm = FakeWorkerManager()
|
||||
c = build_client(fwm)
|
||||
r = c.get("/api/v1/calc/add?a=1&b=2")
|
||||
check("2.1 未携带头 -> 200（默认随机均衡路径不变）", r.status_code == 200)
|
||||
check("2.2 未携带头时 instance_id=None", fwm.raw_instances == [None],
|
||||
f"instances={fwm.raw_instances}")
|
||||
check("2.3 响应体不含实例", r.json().get("instance") is None, f"body={r.json()}")
|
||||
|
||||
# ── 3. POST body 端点 ────────────────────────────────────
|
||||
fwm = FakeWorkerManager()
|
||||
c = build_client(fwm)
|
||||
r = c.post("/api/v1/calc/echo", json={"msg": "hi"},
|
||||
headers={INSTANCE_HEADER: "edge-02"})
|
||||
check("3.1 POST body 头透传", fwm.raw_instances == ["edge-02"],
|
||||
f"instances={fwm.raw_instances}")
|
||||
|
||||
# ── 4. validate=false 原样透传端点 ───────────────────────
|
||||
fwm = FakeWorkerManager()
|
||||
c = build_client(fwm)
|
||||
r = c.post("/api/v1/calc/raw", content=b'{"anything": [1, 2]}',
|
||||
headers={INSTANCE_HEADER: "edge-03",
|
||||
"content-type": "application/json"})
|
||||
check("4.1 原样透传端点头透传", fwm.raw_instances == ["edge-03"],
|
||||
f"instances={fwm.raw_instances}")
|
||||
|
||||
# ── 5. SSE 流式端点 ──────────────────────────────────────
|
||||
fwm = FakeWorkerManager()
|
||||
c = build_client(fwm)
|
||||
with c.stream("POST", "/api/v1/stream/chat", json={"prompt": "x"},
|
||||
headers={INSTANCE_HEADER: "edge-04"}) as r:
|
||||
body = b"".join(r.iter_bytes())
|
||||
check("5.1 流式端点头透传", fwm.stream_instances == ["edge-04"],
|
||||
f"instances={fwm.stream_instances}")
|
||||
check("5.2 流式响应含目标实例", b"edge-04" in body, f"body={body[:80]!r}")
|
||||
|
||||
# ── 6. 空白头值视为未指定 ────────────────────────────────
|
||||
fwm = FakeWorkerManager()
|
||||
c = build_client(fwm)
|
||||
r = c.get("/api/v1/calc/add?a=1&b=2", headers={INSTANCE_HEADER: " "})
|
||||
check("6.1 空白头值 -> instance_id=None", fwm.raw_instances == [None],
|
||||
f"instances={fwm.raw_instances}")
|
||||
|
||||
# ── 7. 定向实例离线 -> 503 ───────────────────────────────
|
||||
fwm = FakeWorkerManager(fail_instance="edge-gone")
|
||||
c = build_client(fwm)
|
||||
r = c.get("/api/v1/calc/add?a=1&b=2", headers={INSTANCE_HEADER: "edge-gone"})
|
||||
check("7.1 定向实例离线 -> 503", r.status_code == 503, f"status={r.status_code}")
|
||||
check("7.2 503 详情点名实例", "edge-gone" in str(r.json().get("detail", "")),
|
||||
f"detail={r.json().get('detail')}")
|
||||
|
||||
print()
|
||||
if failures:
|
||||
print(f"FAILED: {len(failures)} 项 -> {failures}")
|
||||
return 1
|
||||
print("全部通过")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -17,7 +17,7 @@ import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "gateway" / "src"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "utils" / "worker" / "src"))
|
||||
|
||||
import orjson
|
||||
|
||||
@@ -17,7 +17,7 @@ import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "gateway" / "src"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "utils" / "worker" / "src"))
|
||||
|
||||
_tmp = tempfile.TemporaryDirectory()
|
||||
@@ -25,7 +25,7 @@ _tmp = tempfile.TemporaryDirectory()
|
||||
# 预加载真实 logger 模块（不触发 chongming 包链导入），把日志目录重定向到
|
||||
# 临时目录，之后 import chongming.app 时 get_logger 即用新路径
|
||||
_logger_path = (
|
||||
Path(__file__).resolve().parent.parent / "src" / "chongming" / "app" / "core" / "logger.py"
|
||||
Path(__file__).resolve().parent.parent / "gateway" / "src" / "chongming" / "app" / "core" / "logger.py"
|
||||
)
|
||||
_logger_spec = importlib.util.spec_from_file_location("chongming.app.core.logger", _logger_path)
|
||||
logger_mod = importlib.util.module_from_spec(_logger_spec)
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
fork 安全：文件日志流在子进程重开（reentrant call 修复）测试
|
||||
|
||||
背景：gateway 的日志是"队列 handler + 监听线程"结构，文件 handler 的流在父进程
|
||||
打开。granian/gunicorn 的 worker 由 master **fork** 而来（实测 worker 与 master 的
|
||||
fd 表完全一致）——子进程会继承父进程的 BufferedWriter，其内部锁/owner 状态在
|
||||
fork 瞬间可能正被父进程监听线程占用（跨 fork 复制不可靠）。症状：冻结版（Nuitka）
|
||||
里每个 worker 的**每条文件日志**都报
|
||||
RuntimeError: reentrant call inside <_io.BufferedWriter ...>
|
||||
（控制台 handler 正常，文件日志整体失效）。
|
||||
|
||||
修复：_after_fork_child 里把文件 handler 的流在子进程重开（新文件描述符）。
|
||||
本测试验证：
|
||||
1. fork 后子进程的 handler.stream 不再是继承的那个对象（结构保证）
|
||||
2. 子进程能真正把记录写进日志文件（行为保证——旧实现下这里写不进去）
|
||||
3. 控制台等非文件 handler 不被动（无 _open，跳过）
|
||||
|
||||
运行: uv run python test_logging_fork.py
|
||||
"""
|
||||
import importlib.util
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
log_dir = Path(tmp.name)
|
||||
|
||||
# 与 test_logging_async.py 相同的加载方式：独立加载 logger 模块并把日志目录重定向
|
||||
_logger_path = (
|
||||
Path(__file__).resolve().parent.parent / "gateway" / "src" / "chongming" / "app" / "core" / "logger.py"
|
||||
)
|
||||
_spec = importlib.util.spec_from_file_location("chongming.app.core.logger", _logger_path)
|
||||
logger_mod = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["chongming.app.core.logger"] = logger_mod
|
||||
_spec.loader.exec_module(logger_mod) # type: ignore[union-attr]
|
||||
logger_mod.log_config["file"] = str(log_dir)
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
check("os.register_at_fork 已注册 after_in_child", hasattr(os, "register_at_fork"))
|
||||
|
||||
logger = logging.getLogger("forktest")
|
||||
logger_mod.setup_logging(logger, "forktest")
|
||||
|
||||
file_handler = next(
|
||||
h for h in logger_mod._listener_handlers if isinstance(h, logger_mod.DailyRotatingFileHandler)
|
||||
)
|
||||
# 同步直写一条（不经队列），保持监听线程与流都活着，便于接下来 fork
|
||||
parent_record = logging.LogRecord("forktest", logging.INFO, __file__, 0, "parent-line", (), None)
|
||||
file_handler.handle(parent_record)
|
||||
file_handler.flush()
|
||||
|
||||
check("父进程日志已落盘", "parent-line" in next(iter(log_dir.glob("forktest-*.log"))).read_text(encoding="utf-8"))
|
||||
|
||||
inherited_stream = file_handler.stream
|
||||
check("父进程流已打开", inherited_stream is not None)
|
||||
|
||||
# ---- fork：子进程应换掉继承的流，并能独立写日志 ----
|
||||
pid = os.fork()
|
||||
if pid == 0: # 子进程
|
||||
try:
|
||||
reopened = file_handler.stream is not inherited_stream
|
||||
record = logging.LogRecord(
|
||||
"forktest", logging.INFO, __file__, 0, "child-line", (), None
|
||||
)
|
||||
# 直接同步走 handler（不经队列），写入结果立即可断言
|
||||
file_handler.handle(record)
|
||||
os._exit(0 if reopened else 3)
|
||||
except BaseException: # noqa: BLE001 - 子进程异常用退出码传出
|
||||
os._exit(4)
|
||||
|
||||
_, status = os.waitpid(pid, 0)
|
||||
exit_code = os.waitstatus_to_exitcode(status)
|
||||
check("fork 后子进程的流已换新（不再是继承对象）", exit_code == 0, f"exit={exit_code}")
|
||||
check(
|
||||
"子进程可直接写入日志文件（旧实现下 reentrant 失败）",
|
||||
"child-line" in next(iter(log_dir.glob("forktest-*.log"))).read_text(encoding="utf-8"),
|
||||
)
|
||||
check(
|
||||
"控制台 handler 不受影响（仍指向 stderr）",
|
||||
any(isinstance(h, logging.StreamHandler) and not isinstance(h, logger_mod.DailyRotatingFileHandler) for h in logger_mod._listener_handlers),
|
||||
)
|
||||
|
||||
tmp.cleanup()
|
||||
|
||||
print("\n" + ("=" * 40))
|
||||
if failures:
|
||||
print(f"❌ {len(failures)} 项未通过: {failures}")
|
||||
sys.exit(1)
|
||||
print("✅ 全部通过")
|
||||
@@ -0,0 +1,280 @@
|
||||
"""
|
||||
migrate_plugin（内置插件 → 进程插件 一键迁移）测试
|
||||
|
||||
覆盖:
|
||||
1. 前置拦截: 自定义 FastAPI 路由（APIRouter 实例化 / 顶层 router 绑定 /
|
||||
star-import 再导出）拒绝迁移；--force 可跳过
|
||||
2. 前置拦截: 网关包导入（import app / from chongming...）拒绝迁移
|
||||
3. 放行: `from . import router`（子模块，非 APIRouter）不误伤（ip_sound_column 写法）
|
||||
4. 迁移产物: 项目骨架（pyproject/start.sh/config.toml/README/zenoh 配置）+ 代码/配置
|
||||
迁移 + [workers] 改写（module=workers.<name>、load_mode=external）+ 保留其余配置
|
||||
5. 依赖迁移: plugins_deps.toml [<name>] 节 → 新项目 pyproject（按实际 import 过滤、
|
||||
保留根 pyproject 的 extras 写法、共用节带入）；[<name>] 节注释掉、其它节不动
|
||||
6. --dry-run 不落盘；目标已存在 / 插件不存在时拒绝
|
||||
|
||||
运行: uv run python test/test_migrate_plugin.py
|
||||
"""
|
||||
import contextlib
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(ROOT / "utils" / "cli" / "src"))
|
||||
|
||||
os.environ.pop("CHONGMING_ROOT", None) # 根目录定位不应受外部环境变量影响
|
||||
|
||||
from chongming_cli import migrate_plugin as mp # noqa: E402
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
DEPS_TOML = """\
|
||||
imports = ["httpx"]
|
||||
|
||||
[foo]
|
||||
imports = [
|
||||
"volcenginesdkarkruntime",
|
||||
"chongming_cache",
|
||||
"orjson",
|
||||
"totally_missing_pkg",
|
||||
]
|
||||
|
||||
[bar]
|
||||
imports = ["openai"]
|
||||
"""
|
||||
|
||||
PLUGIN_INIT = '''"""fixture 插件"""
|
||||
import orjson
|
||||
import httpx
|
||||
from chongming_cache import Cache # noqa: F401
|
||||
from chongming_worker.plugin import Plugin
|
||||
from volcenginesdkarkruntime import AsyncArk # noqa: F401
|
||||
from totally_missing_pkg import thing # noqa: F401
|
||||
|
||||
from . import router # noqa: F401 触发 @rpc_method 注册
|
||||
{extra}
|
||||
|
||||
class FooPlugin(Plugin):
|
||||
type = "foo"
|
||||
db_name = None
|
||||
|
||||
async def on_init(self):
|
||||
pass
|
||||
|
||||
|
||||
plugin = FooPlugin()
|
||||
create_workers = plugin.create_workers
|
||||
cleanup = plugin.cleanup
|
||||
'''
|
||||
|
||||
CONFIG_TOML = """\
|
||||
# foo Worker 插件配置
|
||||
[workers]
|
||||
type = "foo"
|
||||
module = "plugins.foo"
|
||||
factory = "create_workers"
|
||||
cleanup = "cleanup"
|
||||
count = 2
|
||||
|
||||
[worker]
|
||||
name = "foo"
|
||||
version = "0.1.0"
|
||||
description = "fixture 插件"
|
||||
config_version = "1.0"
|
||||
defaults = { method = "POST", timeout = 30.0 }
|
||||
|
||||
[worker.registration]
|
||||
router_prefix = "/foo"
|
||||
|
||||
[workers.config]
|
||||
data_dir = "./data"
|
||||
api_key = "PLACEHOLDER"
|
||||
"""
|
||||
|
||||
|
||||
def make_root(tmp: Path) -> Path:
|
||||
"""最小仓库现场：根 pyproject（含 extras 依赖）/ config/zenoh_config.json5 副本 / plugins_deps.toml。"""
|
||||
tmp.mkdir(parents=True, exist_ok=True)
|
||||
(tmp / "pyproject.toml").write_text(
|
||||
"[project]\nname = 'fixture'\ndependencies = [\"volcengine-python-sdk[ark]\"]\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp / "config").mkdir(parents=True, exist_ok=True)
|
||||
(tmp / "config" / "zenoh_config.json5").write_text("{}\n", encoding="utf-8")
|
||||
(tmp / "plugins_deps.toml").write_text(DEPS_TOML, encoding="utf-8")
|
||||
return tmp
|
||||
|
||||
|
||||
def make_plugin(tmp: Path, name: str = "foo", init_extra: str = "", files: dict | None = None):
|
||||
"""造内置插件现场：src/plugins/<name>/ + config/workers/<name>/。"""
|
||||
code = tmp / "src" / "plugins" / name
|
||||
(code / "router").mkdir(parents=True, exist_ok=True)
|
||||
(code / "models").mkdir()
|
||||
(code / "models" / "__init__.py").write_text("", encoding="utf-8")
|
||||
(code / "__init__.py").write_text(PLUGIN_INIT.format(extra=init_extra), encoding="utf-8")
|
||||
(code / "router" / "__init__.py").write_text("from . import hello # noqa: F401\n", encoding="utf-8")
|
||||
(code / "router" / "hello.py").write_text("async def hello():\n return {}\n", encoding="utf-8")
|
||||
conf = tmp / "config" / "workers" / name
|
||||
conf.mkdir(parents=True, exist_ok=True)
|
||||
(conf / "config.toml").write_text(CONFIG_TOML, encoding="utf-8")
|
||||
(conf / "hello.toml").write_text("[POST]\nsummary = \"hi\"\n", encoding="utf-8")
|
||||
for rel, text in (files or {}).items():
|
||||
path = code / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return tmp
|
||||
|
||||
|
||||
def run(tmp: Path, *args, name: str = "foo") -> tuple[int, str]:
|
||||
"""chdir 到现场执行迁移，捕获 stdout。"""
|
||||
buf = io.StringIO()
|
||||
cwd = Path.cwd()
|
||||
os.chdir(tmp)
|
||||
try:
|
||||
with contextlib.redirect_stdout(buf):
|
||||
rc = mp.migrate_plugin_package(name, **kwargs_for(args))
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
return rc, buf.getvalue()
|
||||
|
||||
|
||||
def kwargs_for(args):
|
||||
kwargs = {}
|
||||
for a in args:
|
||||
kwargs[a] = True
|
||||
return kwargs
|
||||
|
||||
|
||||
ROUTER_FILES = {
|
||||
"routes.py": (
|
||||
"from fastapi import APIRouter\n\n"
|
||||
"router = APIRouter(prefix=\"/foo\")\n"
|
||||
),
|
||||
}
|
||||
|
||||
tmp_root = Path(tempfile.mkdtemp(prefix="migrate_plugin_test_"))
|
||||
|
||||
# ========== 1. 自定义路由拦截（aibox 写法） ==========
|
||||
case = make_root(tmp_root / "case_router")
|
||||
make_plugin(case, init_extra="from .routes import router # noqa: F401", files=ROUTER_FILES)
|
||||
rc, out = run(case)
|
||||
check("路由挂载（from .routes import router + APIRouter）拒绝迁移", rc == 1, f"rc={rc}")
|
||||
check("拒绝时给出路由提示", "APIRouter" in out and "拒绝迁移" in out)
|
||||
check("拒绝时不动任何文件", (case / "src" / "plugins" / "foo").is_dir() and not (case / "workers").exists())
|
||||
|
||||
# ========== 2. star-import 再导出的 APIRouter 同样拦截 ==========
|
||||
case = make_root(tmp_root / "case_star")
|
||||
make_plugin(
|
||||
case,
|
||||
init_extra="from .router import * # noqa: F401,F403",
|
||||
files={}, # router/ 由 make_plugin 建好，下面覆盖 __init__.py 为 APIRouter 定义
|
||||
)
|
||||
(case / "src" / "plugins" / "foo" / "router" / "__init__.py").write_text(
|
||||
"from fastapi import APIRouter\n\nrouter = APIRouter()\n", encoding="utf-8"
|
||||
)
|
||||
rc, out = run(case)
|
||||
check("star-import 链上的 APIRouter 拒绝迁移", rc == 1 and "APIRouter" in out, f"rc={rc}")
|
||||
|
||||
# ========== 3. --force 跳过拦截 ==========
|
||||
rc, out = run(case, "force")
|
||||
check("--force 跳过后可迁移", rc == 0 and (case / "workers" / "foo" / "pyproject.toml").is_file(), f"rc={rc}")
|
||||
check("--force 输出回显跳过", "跳过上述拦截" in out)
|
||||
check(
|
||||
"配置缺 module/load_mode 时自动补全",
|
||||
'module = "workers.foo"' in (case / "workers" / "foo" / "config" / "workers" / "foo" / "config.toml").read_text(),
|
||||
)
|
||||
|
||||
# ========== 4. 网关包导入拦截 ==========
|
||||
case = make_root(tmp_root / "case_gw")
|
||||
make_plugin(case, init_extra="from app.core.logger import get_logger # noqa: F401")
|
||||
rc, out = run(case)
|
||||
check("网关包导入拒绝迁移", rc == 1 and "网关" in out, f"rc={rc}")
|
||||
|
||||
# ========== 5. dry-run 不落盘 ==========
|
||||
case = make_root(tmp_root / "case_dry")
|
||||
make_plugin(case)
|
||||
before = (case / "plugins_deps.toml").read_text(encoding="utf-8")
|
||||
rc, out = run(case, "dry_run")
|
||||
check("--dry-run 返回成功且不落盘", rc == 0 and not (case / "workers").exists())
|
||||
check("--dry-run 不改 plugins_deps.toml", (case / "plugins_deps.toml").read_text(encoding="utf-8") == before)
|
||||
check("--dry-run 打印演练提示", "未改动任何文件" in out)
|
||||
|
||||
# ========== 6. 目标已存在 / 插件不存在 ==========
|
||||
case = make_root(tmp_root / "case_exists")
|
||||
make_plugin(case)
|
||||
(case / "workers" / "foo").mkdir(parents=True)
|
||||
rc, out = run(case)
|
||||
check("目标已存在拒绝迁移", rc == 1 and "目标已存在" in out)
|
||||
case = make_root(tmp_root / "case_missing")
|
||||
rc, out = run(case)
|
||||
check("插件不存在拒绝迁移", rc == 1 and "未找到内置插件" in out)
|
||||
|
||||
# ========== 7. 完整迁移（happy path） ==========
|
||||
case = make_root(tmp_root / "case_full")
|
||||
make_plugin(case)
|
||||
rc, out = run(case)
|
||||
project = case / "workers" / "foo"
|
||||
check("完整迁移成功", rc == 0 and project.is_dir(), f"rc={rc}")
|
||||
|
||||
code_dst = project / "workers" / "foo"
|
||||
conf_dst = project / "config" / "workers" / "foo"
|
||||
check(
|
||||
"代码迁移（含 router/ 与 models/）",
|
||||
(code_dst / "__init__.py").is_file()
|
||||
and (code_dst / "router" / "hello.py").is_file()
|
||||
and (code_dst / "models" / "__init__.py").is_file(),
|
||||
)
|
||||
check("路由配置迁移", (conf_dst / "hello.toml").is_file())
|
||||
conf_text = (conf_dst / "config.toml").read_text(encoding="utf-8")
|
||||
check(
|
||||
"[workers] 改写: module=workers.foo / load_mode=external / 其余保留",
|
||||
'module = "workers.foo"' in conf_text
|
||||
and 'load_mode = "external"' in conf_text
|
||||
and 'count = 2' in conf_text
|
||||
and 'api_key = "PLACEHOLDER"' in conf_text,
|
||||
)
|
||||
check("项目骨架生成", all((project / f).is_file() for f in ("config.toml", "pyproject.toml", "README.md", "start.sh")))
|
||||
check("zenoh 配置复制", (project / "config" / "zenoh_config.json5").is_file())
|
||||
check("start.sh 可执行且入口正确", (project / "start.sh").stat().st_mode & 0o111 and "chongming-worker --type foo" in (project / "start.sh").read_text())
|
||||
check("README 标注迁移来源", "migrate_plugin foo" in (project / "README.md").read_text())
|
||||
check("源目录已删除", not (case / "src" / "plugins" / "foo").exists() and not (case / "config" / "workers" / "foo").exists())
|
||||
|
||||
pyproject = tomllib.loads((project / "pyproject.toml").read_text(encoding="utf-8"))
|
||||
deps = pyproject["project"]["dependencies"]
|
||||
check("新项目依赖: chongming-worker + 迁移带入", "chongming-worker" in deps and "httpx" in deps and "orjson" in deps and "chongming-cache" in deps, str(deps))
|
||||
check("extras 写法沿用根 pyproject", "volcengine-python-sdk[ark]" in deps, str(deps))
|
||||
check("未解析/未使用的依赖不进新项目", not any("totally_missing_pkg" in d for d in deps), str(deps))
|
||||
check("未解析依赖有告警", "totally_missing_pkg" in out)
|
||||
|
||||
deps_data = tomllib.loads((case / "plugins_deps.toml").read_text(encoding="utf-8"))
|
||||
check(
|
||||
"plugins_deps.toml: [foo] 节注释掉、共用节与其它节不动",
|
||||
"foo" not in deps_data and deps_data.get("imports") == ["httpx"] and deps_data.get("bar", {}).get("imports") == ["openai"],
|
||||
str(deps_data),
|
||||
)
|
||||
|
||||
# ========== 8. ip_sound_column 写法（from . import router 子模块）不误伤 ==========
|
||||
case = make_root(tmp_root / "case_submod")
|
||||
make_plugin(case, init_extra="from . import router # noqa: F401 触发 @rpc_method 注册")
|
||||
rc, out = run(case, "dry_run")
|
||||
check("from . import router（子模块）不误伤", rc == 0, f"rc={rc}")
|
||||
|
||||
shutil.rmtree(tmp_root, ignore_errors=True)
|
||||
|
||||
print("\n" + ("=" * 40))
|
||||
if failures:
|
||||
print(f"❌ {len(failures)} 项未通过: {failures}")
|
||||
sys.exit(1)
|
||||
print("✅ 全部通过")
|
||||
@@ -23,8 +23,8 @@ sys.path.insert(0, str(Path(__file__).parent))
|
||||
from amqtt.client import MQTTClient
|
||||
from amqtt.errors import ConnectError
|
||||
|
||||
from src.chongming.app.core.lifespan_setup import _run_mqtt_broker
|
||||
from src.chongming.app.core.mqtt_bridge import MQTTBridge
|
||||
from chongming.app.core.lifespan_setup import _run_mqtt_broker
|
||||
from chongming.app.core.mqtt_bridge import MQTTBridge
|
||||
|
||||
HOST = "127.0.0.1"
|
||||
USERNAME = "cm_user"
|
||||
|
||||
@@ -19,9 +19,9 @@ import tempfile
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
_HERE = Path(__file__).resolve().parent # chongming-zenoh/
|
||||
_HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/
|
||||
_PROJ = _HERE
|
||||
for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"):
|
||||
for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"):
|
||||
sys.path.insert(0, str(_p))
|
||||
|
||||
import chongming.app.core.lifespan_setup as ls
|
||||
|
||||
@@ -22,9 +22,9 @@ import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
_HERE = Path(__file__).resolve().parent # chongming-zenoh/
|
||||
_HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/
|
||||
_PROJ = _HERE
|
||||
for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"):
|
||||
for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"):
|
||||
sys.path.insert(0, str(_p))
|
||||
|
||||
import chongming.app.core.lifespan_setup as ls
|
||||
|
||||
@@ -12,7 +12,7 @@ import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "gateway" / "src"))
|
||||
|
||||
from chongming.app.core.mqtt_bridge import (
|
||||
MQTTBridge,
|
||||
@@ -33,7 +33,8 @@ class FakeWM:
|
||||
self.calls = []
|
||||
self.result = {"result": "ok"}
|
||||
|
||||
async def call(self, worker_type, subject, payload, timeout=30, cacheable=False):
|
||||
async def call(self, worker_type, subject, payload, timeout=30, cacheable=False,
|
||||
instance_id=None):
|
||||
self.calls.append(
|
||||
{
|
||||
"worker_type": worker_type,
|
||||
@@ -41,6 +42,7 @@ class FakeWM:
|
||||
"payload": payload,
|
||||
"timeout": timeout,
|
||||
"cacheable": cacheable,
|
||||
"instance_id": instance_id,
|
||||
}
|
||||
)
|
||||
return self.result
|
||||
@@ -153,6 +155,41 @@ async def test_flat_payload_routed():
|
||||
print("✅ 非信封（扁平）载荷按原始报文路由通过")
|
||||
|
||||
|
||||
async def test_target_instance():
|
||||
"""实例定向：载荷顶层 target_instance 作为控制字段摘除并透传给 worker_manager.call"""
|
||||
wm = FakeWM()
|
||||
bridge, published = make_bridge(
|
||||
wm, [{"topic": "devices/+/telemetry", "worker": "device"}]
|
||||
)
|
||||
# 1. 带 target_instance：透传实例名，且控制字段不进业务 payload
|
||||
await bridge._handle(make_msg(
|
||||
"devices/d1/telemetry",
|
||||
{"meta": {}, "data": {"v": 1}, "target_instance": "edge-01"},
|
||||
))
|
||||
assert wm.calls[-1]["instance_id"] == "edge-01", wm.calls[-1]
|
||||
assert "target_instance" not in wm.calls[-1]["payload"], wm.calls[-1]["payload"]
|
||||
assert wm.calls[-1]["payload"]["data"] == {"v": 1}
|
||||
|
||||
# 2. 不带该字段：instance_id 为 None（跨实例随机均衡，与改动前一致）
|
||||
await bridge._handle(make_msg("devices/d1/telemetry", {"meta": {}, "data": {"v": 1}}))
|
||||
assert wm.calls[-1]["instance_id"] is None, wm.calls[-1]
|
||||
|
||||
# 3. 空白值视为未指定
|
||||
await bridge._handle(make_msg(
|
||||
"devices/d1/telemetry",
|
||||
{"meta": {}, "data": {"v": 1}, "target_instance": " "},
|
||||
))
|
||||
assert wm.calls[-1]["instance_id"] is None, wm.calls[-1]
|
||||
|
||||
# 4. 内置 chongming/... 主题同样支持
|
||||
await bridge._handle(make_msg(
|
||||
f"{MQTT_TOPIC_PREFIX}/example/add",
|
||||
{"meta": {}, "data": {"a": 1}, "target_instance": "edge-02"},
|
||||
))
|
||||
assert wm.calls[-1]["instance_id"] == "edge-02", wm.calls[-1]
|
||||
print("✅ 实例定向（target_instance 控制字段）透传通过")
|
||||
|
||||
|
||||
async def test_non_json_dropped():
|
||||
wm = FakeWM()
|
||||
bridge, published = make_bridge(
|
||||
@@ -219,6 +256,7 @@ async def main():
|
||||
await test_custom_routing()
|
||||
await test_envelope_meta_fields()
|
||||
await test_flat_payload_routed()
|
||||
await test_target_instance()
|
||||
await test_non_json_dropped()
|
||||
await test_board_ping_topic()
|
||||
test_plugin_config_driven_collection()
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
Nuitka 打包资源档位 / 链接器注入 测试（不触发真实编译）
|
||||
|
||||
覆盖 build.py 里 nuitka 打包的三处防坑逻辑：
|
||||
1. 链接器：装了 ld.lld 时自动经 LDFLAGS 注入 -fuse-ld=lld（默认 GNU ld 链接大工程
|
||||
峰值数 GB，小内存机器/WSL 会被 OOM 杀掉："clang: unable to execute command: Killed"）；
|
||||
已有 LDFLAGS 时不覆盖（用户在 CHONGMING_NUITKA_PGO 之外自行指定链接器优先）
|
||||
2. 内存档位：可用内存 ≥ NUITKA_LTO_MIN_GB 完整优化（LTO）；低于则去 LTO（LTO 链接
|
||||
峰值 3 GB+，实测 ld/lld 都被 OOM 杀过）；更低（或 CHONGMING_NUITKA_LOW_MEM=1）
|
||||
进一步 --low-memory + 降并发；CHONGMING_NUITKA_FULL=1 强制完整优化
|
||||
3. PGO 缺省关闭：--pgo-c 与 standalone/onefile 不兼容且 profiling 运行会等程序退出
|
||||
（常驻服务卡死），只能由 CHONGMING_NUITKA_PGO=1 显式打开——
|
||||
这里用源码断言兜住"默认关"这条（防止回潮）
|
||||
|
||||
运行: uv run python test/test_nuitka_pack_flags.py
|
||||
"""
|
||||
import io
|
||||
import contextlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(ROOT / "utils" / "cli" / "src"))
|
||||
|
||||
from chongming_cli import build as b # noqa: E402
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
def quiet_note():
|
||||
"""调 _nuitka_env_and_mem_note 时屏蔽它打印的降档提示。"""
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
return b._nuitka_env_and_mem_note()
|
||||
|
||||
|
||||
saved = {k: os.environ.pop(k, None) for k in
|
||||
(b.NUITKA_LOW_MEM_ENV, b.NUITKA_FULL_ENV, b.NUITKA_PGO_ENV)}
|
||||
|
||||
try:
|
||||
# ========== 1. 链接器注入 ==========
|
||||
saved_ldflags = os.environ.pop("LDFLAGS", None)
|
||||
env, note, profile = quiet_note()
|
||||
if sys.platform.startswith("linux") and shutil.which("ld.lld"):
|
||||
check("装了 lld 自动注入 -fuse-ld=lld", env.get("LDFLAGS", "").endswith("-fuse-ld=lld"), env.get("LDFLAGS"))
|
||||
check("成功信息回显链接器", "lld" in note, note)
|
||||
else:
|
||||
check("无 lld 时不注入（本机无 lld，跳过）", "-fuse-ld" not in (env.get("LDFLAGS") or ""))
|
||||
|
||||
os.environ["LDFLAGS"] = "-Wl,--as-needed"
|
||||
env, _, _ = quiet_note()
|
||||
check("已有 LDFLAGS 时追加而不覆盖", env["LDFLAGS"].startswith("-Wl,--as-needed"), env["LDFLAGS"])
|
||||
if saved_ldflags is None:
|
||||
os.environ.pop("LDFLAGS", None)
|
||||
else:
|
||||
os.environ["LDFLAGS"] = saved_ldflags
|
||||
|
||||
# ========== 2. 内存档位 ==========
|
||||
mem = b._mem_available_gb()
|
||||
check("_mem_available_gb 可读（Linux）", mem > 0 or not sys.platform.startswith("linux"), f"{mem:.1f} GB")
|
||||
|
||||
env, note, profile = quiet_note()
|
||||
if mem >= 0:
|
||||
expect = (
|
||||
"squeeze" if mem < b.NUITKA_SQUEEZE_GB
|
||||
else "no-lto" if mem < b.NUITKA_LTO_MIN_GB
|
||||
else "full"
|
||||
)
|
||||
check(
|
||||
"默认档位与内存阈值一致",
|
||||
profile == expect,
|
||||
f"{mem:.1f} GB → {profile}（阈值 {b.NUITKA_SQUEEZE_GB}/{b.NUITKA_LTO_MIN_GB} GB）",
|
||||
)
|
||||
check(
|
||||
"非 full 档在提示里说明原因",
|
||||
(profile == "full" and "LTO" not in note) or (
|
||||
profile == "no-lto" and "未启用 LTO" in note
|
||||
) or (profile == "squeeze" and "--low-memory" in note),
|
||||
note,
|
||||
)
|
||||
|
||||
os.environ[b.NUITKA_LOW_MEM_ENV] = "1"
|
||||
env, note, profile = quiet_note()
|
||||
check(f"{b.NUITKA_LOW_MEM_ENV}=1 强制 squeeze 档", profile == "squeeze" and "--low-memory" in note, note)
|
||||
|
||||
os.environ.pop(b.NUITKA_LOW_MEM_ENV)
|
||||
os.environ[b.NUITKA_FULL_ENV] = "1"
|
||||
env, note, profile = quiet_note()
|
||||
check(f"{b.NUITKA_FULL_ENV}=1 强制完整优化（覆盖内存自动判断）", profile == "full" and "--low-memory" not in note, note)
|
||||
os.environ.pop(b.NUITKA_FULL_ENV)
|
||||
|
||||
# ========== 3. PGO 缺省关闭（源码兜底，防回潮） ==========
|
||||
src = (ROOT / "utils" / "cli" / "src" / "chongming_cli" / "build.py").read_text(encoding="utf-8")
|
||||
check(
|
||||
"PGO 参数只在 NUITKA_PGO_ENV 开关下出现",
|
||||
f'os.environ.get(NUITKA_PGO_ENV) == "1"' in src and b.NUITKA_PGO_ENV == "CHONGMING_NUITKA_PGO",
|
||||
)
|
||||
check(
|
||||
"Nuitka 缺 patchelf 时提前失败（链接后才检查，白跑一整轮）",
|
||||
"patchelf" in src and "缺少 patchelf" in src,
|
||||
)
|
||||
check(
|
||||
"Nuitka 失败会抛错而不是静默返回（构建要非零退出）",
|
||||
'raise RuntimeError("Nuitka 编译失败' in src,
|
||||
)
|
||||
finally:
|
||||
for k, v in saved.items():
|
||||
if v is not None:
|
||||
os.environ[k] = v
|
||||
|
||||
print("\n" + ("=" * 40))
|
||||
if failures:
|
||||
print(f"❌ {len(failures)} 项未通过: {failures}")
|
||||
sys.exit(1)
|
||||
print("✅ 全部通过")
|
||||
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
父进程死亡防护（PR_SET_PDEATHSIG）测试
|
||||
|
||||
背景与语义见 gateway/src/chongming/app/core/parent_death.py：进程树 onefile 引导 →
|
||||
granian master → worker（fork）。上游被 SIGKILL 时 worker 会成孤儿（继续监听
|
||||
端口、挂在服务发现里，其它网关派发到"半死副本"每次白吃 2 秒超时）。守卫让内核
|
||||
在父进程死亡时给本进程发 SIGTERM（走既有优雅停机摘除注册）。
|
||||
|
||||
验证（三层 fork 复刻真实进程树：本测试(引导) → 中间进程(master) → 孙进程(worker)）：
|
||||
1. 关闭开关 CHONGMING_NO_PARENT_GUARD=1 → 不生效（返回 False）
|
||||
2. 中间进程 SIGKILL 自己（模拟 master 被 -9，内核不可捕获）→ 已设守卫的孙进程
|
||||
收到 SIGTERM 并退出
|
||||
3. 对照：同样结构但不设守卫 → 孙进程收不到信号（证明信号确实来自 PDEATHSIG，
|
||||
而不是别的东西清理孤儿）
|
||||
4. 幂等：重复调用不重复 prctl
|
||||
|
||||
运行: uv run python test/test_parent_death.py
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent.parent
|
||||
_PD_PATH = ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "parent_death.py"
|
||||
|
||||
# 独立加载（parent_death 无包内依赖，避免连带导入整个 app 包）
|
||||
_spec = importlib.util.spec_from_file_location("chongming.app.core.parent_death", _PD_PATH)
|
||||
pd = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["chongming.app.core.parent_death"] = pd
|
||||
_spec.loader.exec_module(pd) # type: ignore[union-attr]
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
# ========== 1. 关闭开关 ==========
|
||||
os.environ[pd.GUARD_OFF_ENV] = "1"
|
||||
check("关闭开关打开时不生效", pd.guard_parent_death() is False)
|
||||
os.environ.pop(pd.GUARD_OFF_ENV, None)
|
||||
|
||||
# ========== 2/3. 三层 fork：守卫组 / 对照组 ==========
|
||||
tmp = Path(tempfile.mkdtemp(prefix="parent_death_test_"))
|
||||
|
||||
|
||||
def scenario(name: str, use_guard: bool) -> dict:
|
||||
"""中间进程 SIGKILL 自己，观察孙进程是否收到 SIGTERM。"""
|
||||
d = tmp / name
|
||||
d.mkdir()
|
||||
ready, dead, pidf = d / "ready", d / "dead", d / "gc.pid"
|
||||
|
||||
middle = os.fork()
|
||||
if middle == 0: # 中间进程（master 角色）
|
||||
gc = os.fork()
|
||||
if gc == 0: # 孙进程（worker 角色）
|
||||
def _on_term(*_):
|
||||
dead.write_text("term", encoding="utf-8")
|
||||
os._exit(0)
|
||||
|
||||
signal.signal(signal.SIGTERM, _on_term)
|
||||
if use_guard:
|
||||
pd.guard_parent_death()
|
||||
pidf.write_text(str(os.getpid()), encoding="utf-8")
|
||||
ready.write_text("1", encoding="utf-8")
|
||||
time.sleep(15) # 等信号；超时说明没收到
|
||||
os._exit(1)
|
||||
for _ in range(250): # 等孙进程就绪，再硬杀自己
|
||||
if ready.exists():
|
||||
break
|
||||
time.sleep(0.02)
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
os._exit(1) # 不会走到
|
||||
|
||||
os.waitpid(middle, 0) # 中间进程必死于 SIGKILL
|
||||
gc_pid = int(pidf.read_text(encoding="utf-8"))
|
||||
for _ in range(100): # 最多等 5s
|
||||
if dead.exists():
|
||||
break
|
||||
time.sleep(0.05)
|
||||
got = dead.exists()
|
||||
try: # 清理（对照组孙进程仍活着）
|
||||
os.kill(gc_pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
return {"got_term": got, "gc_pid": gc_pid}
|
||||
|
||||
|
||||
guarded = scenario("guarded", use_guard=True)
|
||||
check(
|
||||
"master(SIGKILL) 死后，已设守卫的 worker 收到 SIGTERM 退出",
|
||||
guarded["got_term"],
|
||||
f"gc={guarded['gc_pid']}",
|
||||
)
|
||||
|
||||
control = scenario("control", use_guard=False)
|
||||
check(
|
||||
"对照组（未设守卫）收不到信号 —— 证明退出确实来自 PDEATHSIG",
|
||||
not control["got_term"],
|
||||
f"gc={control['gc_pid']}",
|
||||
)
|
||||
|
||||
# ========== 4. 幂等（在子进程里做：避免给测试进程本身设 PDEATHSIG） ==========
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
a = pd.guard_parent_death()
|
||||
b = pd.guard_parent_death()
|
||||
os._exit(0 if (a is True and b is True) else 1)
|
||||
_, status = os.waitpid(pid, 0)
|
||||
check("重复调用幂等（不重复 prctl）", os.waitstatus_to_exitcode(status) == 0)
|
||||
|
||||
# ========== 5. fork 继承回归（master 先设 → worker 仍必须自己设上） ==========
|
||||
# 背景：若用模块级"已设置"标志位做幂等，fork 的子进程会继承 True 而短路——
|
||||
# 但内核在 fork 时恰好清零 PDEATHSIG，于是 worker 其实没设上（本轮实测到过）。
|
||||
# 这里父进程先真实设置，再 fork，断言子进程调用守卫时**内核状态确实被设上**。
|
||||
libc5 = pd._load_libc()
|
||||
pd.guard_parent_death() # 父进程（本测试进程）真实设置一次
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
cleared = pd._current_pdeathsig(libc5) == 0 # fork 后内核已清零（不继承）
|
||||
ok = pd.guard_parent_death() is True # 回归点：不能因父进程设过而短路
|
||||
set_now = pd._current_pdeathsig(libc5) == int(signal.SIGTERM)
|
||||
os._exit(0 if (cleared and ok and set_now) else 1)
|
||||
_, status = os.waitpid(pid, 0)
|
||||
libc5.prctl(pd._PR_SET_PDEATHSIG, 0) # 清除测试进程自己的设置，避免副作用
|
||||
check(
|
||||
"父进程设过后，fork 的子进程仍会真实设置 PDEATHSIG",
|
||||
os.waitstatus_to_exitcode(status) == 0,
|
||||
)
|
||||
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
print("\n" + ("=" * 40))
|
||||
if failures:
|
||||
print(f"❌ {len(failures)} 项未通过: {failures}")
|
||||
sys.exit(1)
|
||||
print("✅ 全部通过")
|
||||
@@ -16,12 +16,13 @@
|
||||
D. WorkerManager 单测（FakeSession）:
|
||||
9. call_raw 零解析返回原始字节；call() 仍解析为 dict（MQTT 语义保留）
|
||||
10. cacheable: 缓存原始 bytes、二次命中不再发查询、业务错误不缓存
|
||||
11. reply.err -> 重试 -> RuntimeError；空回复兜底 b"{}"
|
||||
11. reply.err -> 换副本重试/无副本可换时立即 RuntimeError；空回复兜底 b"{}"
|
||||
|
||||
运行: uv run python test_passthrough_optimization.py
|
||||
"""
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import random
|
||||
import logging
|
||||
import sys
|
||||
import types
|
||||
@@ -32,8 +33,8 @@ import orjson
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "src"))
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "gateway" / "src"))
|
||||
sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
|
||||
|
||||
# ---- 依赖隔离：桩掉 chongming 包链 / auth / chongming_cache ----
|
||||
@@ -49,11 +50,12 @@ _auth_stub = types.ModuleType("chongming.app.core.auth")
|
||||
_auth_stub.store_user_to_request = lambda x: x # 仅用于 Depends，桩掉鉴权
|
||||
_cache_stub = types.ModuleType("chongming_cache")
|
||||
_cache_stub.Cache = object
|
||||
_cache_stub.DistributedLock = object # worker_manager 顶层导入（本测试不触达）
|
||||
for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub):
|
||||
sys.modules[mod.__name__] = mod
|
||||
|
||||
# 加载真实 worker_manager（route_generator 依赖其 StreamIdleTimeoutError）
|
||||
_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py"
|
||||
_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py"
|
||||
_wm_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.worker_manager", _wm_path
|
||||
)
|
||||
@@ -62,7 +64,7 @@ sys.modules["chongming.app.core.worker_manager"] = _wm_mod
|
||||
_wm_spec.loader.exec_module(_wm_mod) # type: ignore[union-attr]
|
||||
|
||||
# 加载真实 call_stats（route_generator 依赖其 spawn/record_call）
|
||||
_cs_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "call_stats.py"
|
||||
_cs_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "call_stats.py"
|
||||
_cs_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.call_stats", _cs_path
|
||||
)
|
||||
@@ -71,7 +73,7 @@ sys.modules["chongming.app.core.call_stats"] = _cs_mod
|
||||
_cs_spec.loader.exec_module(_cs_mod) # type: ignore[union-attr]
|
||||
|
||||
# 加载真实 route_generator
|
||||
_rg_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "route_generator.py"
|
||||
_rg_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "route_generator.py"
|
||||
_rg_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.route_generator", _rg_path
|
||||
)
|
||||
@@ -101,7 +103,8 @@ class FakeWorkerManager:
|
||||
self.error = error
|
||||
self.call_raw_payloads = [] # 捕获的预序列化请求字节
|
||||
|
||||
async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False):
|
||||
async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False,
|
||||
instance_id=None):
|
||||
self.call_raw_payloads.append(payload)
|
||||
if self.error == "no_worker":
|
||||
raise RuntimeError(f"类型 '{worker_type}' 没有可用 Worker。")
|
||||
@@ -282,16 +285,27 @@ def test_openapi_request_body_injected():
|
||||
# ---- D. WorkerManager 单测（FakeSession）----
|
||||
|
||||
class FakeSession:
|
||||
"""模拟 zenoh Session：get() 返回单回复迭代器"""
|
||||
"""模拟 zenoh Session：get() 返回单回复迭代器
|
||||
|
||||
def __init__(self, payload=b'{"result": 5}', err_payload=None):
|
||||
err_payload 非空时整体报错；err_target 给定时只对该 key 前缀报错（其余正常回复），
|
||||
用于验证「失败换副本重试」。
|
||||
"""
|
||||
|
||||
def __init__(self, payload=b'{"result": 5}', err_payload=None, err_target=None):
|
||||
self.payload = payload
|
||||
self.err_payload = err_payload
|
||||
self.err_target = err_target
|
||||
self.get_calls = 0
|
||||
self.call_keys = []
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
self.get_calls += 1
|
||||
if self.err_payload is not None:
|
||||
key = args[0] if args else kwargs.get("key_expr", "")
|
||||
self.call_keys.append(str(key))
|
||||
failing = self.err_payload is not None and (
|
||||
self.err_target is None or str(key).startswith(self.err_target)
|
||||
)
|
||||
if failing:
|
||||
reply = SimpleNamespace(ok=None, err=SimpleNamespace(payload=self.err_payload))
|
||||
else:
|
||||
reply = SimpleNamespace(ok=SimpleNamespace(payload=self.payload), err=None)
|
||||
@@ -341,14 +355,49 @@ async def test_wm_cache_bytes():
|
||||
|
||||
|
||||
async def test_wm_reply_err_retry():
|
||||
# 单副本：无其它副本可换 —— 立即失败（不空转重试同一个坏副本），保留具体错误
|
||||
session = FakeSession(err_payload=b'{"error": "boom"}')
|
||||
wm = make_wm(session)
|
||||
try:
|
||||
await wm.call_raw("demo", "calc.add", b'{}')
|
||||
check("reply.err 应抛 RuntimeError", False)
|
||||
except RuntimeError as e:
|
||||
check("reply.err 重试后抛 RuntimeError", "boom" in str(e), str(e))
|
||||
check("reply.err 触发重试（2 次查询）", session.get_calls == 2, str(session.get_calls))
|
||||
check("reply.err 抛 RuntimeError 且保留具体错误", "boom" in str(e), str(e))
|
||||
check("单副本无其它可换时不空转重试（1 次查询）",
|
||||
session.get_calls == 1, str(session.get_calls))
|
||||
|
||||
|
||||
async def test_wm_reply_err_switches_worker():
|
||||
"""多副本：先打的副本报错则换另一个副本重试，且不重复打已失败的副本。
|
||||
|
||||
首个副本由 random.choice 决定，故对多个 seed 试跑：必有一个覆盖「先失败后成功」
|
||||
路径（另一个覆盖「首个即成功」），两条路径都不允许重复打同一个失败的副本。
|
||||
"""
|
||||
ok, detail = False, "无 seed 覆盖到失败换副本路径"
|
||||
for seed in (0, 1):
|
||||
random.seed(seed)
|
||||
session = FakeSession(
|
||||
payload=b'{"result": 5}', err_payload=b'{"error": "boom"}',
|
||||
err_target="worker/demo/w-dead",
|
||||
)
|
||||
wm = make_wm(session)
|
||||
wm._workers_cache = {"demo": ["worker/demo/w-dead", "worker/demo/w-alive"]}
|
||||
try:
|
||||
result = await wm.call_raw("demo", "calc.add", b'{}', retries=2)
|
||||
except RuntimeError as e: # 未换到正常副本（不该发生）
|
||||
detail = f"seed={seed} 意外失败: {e}"
|
||||
continue
|
||||
assert result == b'{"result": 5}', result
|
||||
dead_calls = session.call_keys.count("worker/demo/w-dead/calc.add")
|
||||
if dead_calls > 1:
|
||||
detail = f"seed={seed} 重复打失败副本: {session.call_keys}"
|
||||
continue
|
||||
if session.call_keys == [
|
||||
"worker/demo/w-dead/calc.add",
|
||||
"worker/demo/w-alive/calc.add",
|
||||
]:
|
||||
ok, detail = True, f"seed={seed} keys={session.call_keys}"
|
||||
check("失败副本 -> 换副本重试成功（且不重复打失败副本）", ok, detail)
|
||||
|
||||
|
||||
async def test_wm_empty_reply_fallback():
|
||||
@@ -375,6 +424,7 @@ def run():
|
||||
asyncio.run(test_wm_call_dict_semantics())
|
||||
asyncio.run(test_wm_cache_bytes())
|
||||
asyncio.run(test_wm_reply_err_retry())
|
||||
asyncio.run(test_wm_reply_err_switches_worker())
|
||||
asyncio.run(test_wm_empty_reply_fallback())
|
||||
|
||||
print(f"\n结果: {len(failures)} 个失败")
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
"""
|
||||
插件专属依赖清单（plugins_deps.toml）测试
|
||||
|
||||
"插件专属依赖跟插件走、不混进网关构建配置"这条约定的守卫：
|
||||
1. 仓库清单可解析：doubao 专属（openai / volcengine / bytedance）与共用（httpx）
|
||||
都能被 build 的读取/合并逻辑读出来
|
||||
2. 收集清单 = config.toml [pyinstaller_build_import] + plugins_deps.toml，
|
||||
去重且网关本体依赖在前；文件缺失时不报错、不追加
|
||||
3. 拆分不回潮：插件专属依赖不再出现在 config.toml 里（防止又手工加回网关清单）
|
||||
4. 配置写错（imports 非字符串列表 / 顶层多余键 / 插件节缺 imports）直接抛错，
|
||||
不静默漏收依赖（漏了要到运行期加载插件才 ImportError）
|
||||
5. venv 里 chongming-cli 版本低于本地源码时告警：打包用的是 venv 里的 CLI，
|
||||
不先发布升级，plugins_deps.toml 不会生效
|
||||
|
||||
运行: uv run python test/test_plugin_deps.py
|
||||
"""
|
||||
import importlib.metadata
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(ROOT / "utils" / "cli" / "src"))
|
||||
|
||||
DOUBAO_DEPS = {
|
||||
"openai",
|
||||
"volcenginesdkarkruntime",
|
||||
"bytedance",
|
||||
"bytedance.volcengine_aicc_sdk",
|
||||
"bytedance.volcengine_aicc_sdk.secure_http_client",
|
||||
}
|
||||
|
||||
# build 的根目录定位支持 CHONGMING_ROOT 环境变量，测试结论不应受它影响
|
||||
os.environ.pop("CHONGMING_ROOT", None)
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
try:
|
||||
from chongming_cli import build as b # noqa: E402
|
||||
|
||||
HAVE_CLI = True
|
||||
except ImportError as exc: # utils/cli 子仓库未就位（干净克隆）时只跑清单一致性检查
|
||||
HAVE_CLI = False
|
||||
print(f"[SKIP] chongming_cli 源码不可用（{exc}）：仅跑清单一致性检查")
|
||||
|
||||
cwd = Path.cwd()
|
||||
|
||||
|
||||
def load_repo_deps() -> dict:
|
||||
"""读仓库的 plugins_deps.toml（不依赖 chongming_cli，一致性检查也可用）。"""
|
||||
with open(ROOT / "plugins_deps.toml", "rb") as f:
|
||||
return tomllib.load(f)
|
||||
|
||||
|
||||
def flat_modules(data: dict) -> set[str]:
|
||||
"""plugins_deps.toml 里的全部模块（共用 imports + 各插件节）。"""
|
||||
mods = set(data.get("imports", []))
|
||||
for value in data.values():
|
||||
if isinstance(value, dict):
|
||||
mods |= set(value.get("imports", []))
|
||||
return mods
|
||||
|
||||
|
||||
# ========== 1. 仓库清单可解析（build 的读取逻辑直测） ==========
|
||||
if HAVE_CLI:
|
||||
os.chdir(ROOT)
|
||||
try:
|
||||
modules, summary = b._load_plugin_imports()
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
|
||||
check(
|
||||
"仓库 plugins_deps.toml 可解析且含 doubao 专属依赖",
|
||||
DOUBAO_DEPS <= set(modules),
|
||||
f"缺失: {sorted(DOUBAO_DEPS - set(modules))}",
|
||||
)
|
||||
check("共用节（httpx）并入清单", "httpx" in modules)
|
||||
check("摘要含 doubao 节名", "doubao" in summary, summary)
|
||||
|
||||
# ========== 2. 合并：网关本体在前、去重保序 ==========
|
||||
fake_config = {
|
||||
"pyinstaller_build_import": {
|
||||
"base": ["fastapi", "openai"], # openai 故意与插件清单重复，验证去重
|
||||
"linux": ["uvloop"],
|
||||
"windows": ["win_only"],
|
||||
}
|
||||
}
|
||||
os.chdir(ROOT)
|
||||
try:
|
||||
merged, _ = b._collect_build_imports(fake_config)
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
|
||||
check(
|
||||
"合并后网关本体依赖在前且 openai 只出现一次",
|
||||
merged[0] == "fastapi" and merged[1] == "openai" and merged.count("openai") == 1,
|
||||
str(merged[:6]),
|
||||
)
|
||||
check(
|
||||
"插件专属依赖追加在网关清单之后",
|
||||
"httpx" in merged and merged.index("httpx") > merged.index("openai"),
|
||||
)
|
||||
|
||||
# ========== 3. 拆分不回潮：config.toml 里不应再有插件专属依赖 ==========
|
||||
repo_modules = flat_modules(load_repo_deps())
|
||||
with open(ROOT / "config.toml", "rb") as f:
|
||||
cfg = tomllib.load(f)
|
||||
bcfg = cfg["pyinstaller_build_import"]
|
||||
gateway_imports = set(bcfg["base"]) | set(bcfg["linux"]) | set(bcfg["windows"])
|
||||
|
||||
check(
|
||||
"doubao 专属依赖已从 config.toml 移出（只留 plugins_deps.toml）",
|
||||
not (DOUBAO_DEPS & gateway_imports),
|
||||
f"回潮: {sorted(DOUBAO_DEPS & gateway_imports)}",
|
||||
)
|
||||
check(
|
||||
"两份清单不重叠（插件专属 vs 网关本体）",
|
||||
not (repo_modules & gateway_imports),
|
||||
f"重叠: {sorted(repo_modules & gateway_imports)}",
|
||||
)
|
||||
|
||||
# ========== 4. 文件缺失 / 写错的行为 ==========
|
||||
tmp = Path(tempfile.mkdtemp(prefix="plugin_deps_test_"))
|
||||
(tmp / "pyproject.toml").write_text("[project]\nname = 'x'\n", encoding="utf-8")
|
||||
|
||||
|
||||
if HAVE_CLI:
|
||||
(tmp / "plugins_deps.toml").unlink(missing_ok=True)
|
||||
cfg_only = {"pyinstaller_build_import": {"base": ["fastapi"], "linux": [], "windows": []}}
|
||||
os.chdir(tmp)
|
||||
try:
|
||||
mods, summary = b._load_plugin_imports()
|
||||
merged, _ = b._collect_build_imports(cfg_only)
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
check(
|
||||
"无 plugins_deps.toml：不报错、不追加",
|
||||
mods == [] and summary == "" and merged == ["fastapi"],
|
||||
f"{mods} / {merged}",
|
||||
)
|
||||
|
||||
def expect_error(name: str, text: str) -> None:
|
||||
(tmp / "plugins_deps.toml").write_text(text, encoding="utf-8")
|
||||
os.chdir(tmp)
|
||||
try:
|
||||
try:
|
||||
b._load_plugin_imports()
|
||||
check(name, False, "未抛错")
|
||||
except ValueError as exc:
|
||||
check(name, True, str(exc))
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
|
||||
expect_error("imports 非列表 → 报错", 'imports = "openai"\n')
|
||||
expect_error("imports 含非字符串 → 报错", "[doubao]\nimports = [1, 2]\n")
|
||||
expect_error("插件节缺 imports（如写成 import）→ 报错", "[doubao]\nimport = ['openai']\n")
|
||||
expect_error("顶层多余标量键 → 报错", "foo = 1\n")
|
||||
|
||||
# 合法多节：共用 + 逐插件（去重发生在合并阶段，读取阶段保持原样）
|
||||
(tmp / "plugins_deps.toml").write_text(
|
||||
'imports = ["httpx"]\n\n[alpha]\nimports = ["pkg_a"]\n\n'
|
||||
'[beta]\nimports = ["pkg_b", "pkg_a"]\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chdir(tmp)
|
||||
try:
|
||||
mods, summary = b._load_plugin_imports()
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
check(
|
||||
"多节解析：共用 + 逐插件均并入，摘要含各节",
|
||||
mods == ["httpx", "pkg_a", "pkg_b", "pkg_a"]
|
||||
and "alpha:1" in summary
|
||||
and "beta:2" in summary,
|
||||
f"{mods} / {summary}",
|
||||
)
|
||||
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
# ========== 5. venv 里的 CLI 版本（打包装箱用的是它，不是本文件旁边的源码） ==========
|
||||
local_ver = tomllib.loads((ROOT / "utils" / "cli" / "pyproject.toml").read_text())[
|
||||
"project"
|
||||
]["version"]
|
||||
|
||||
|
||||
def as_tuple(v: str):
|
||||
return tuple(int(x) for x in v.split("."))
|
||||
|
||||
|
||||
try:
|
||||
installed_ver = importlib.metadata.version("chongming-cli")
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
installed_ver = None
|
||||
|
||||
if installed_ver is None:
|
||||
print("[WARN] venv 未安装 chongming-cli：先 uv sync 或发布后再构建")
|
||||
else:
|
||||
try:
|
||||
outdated = as_tuple(installed_ver) < as_tuple(local_ver)
|
||||
except ValueError: # 非常规版本号（如 rc/后缀），不猜，直接提示
|
||||
outdated = True
|
||||
if outdated:
|
||||
print(
|
||||
f"[WARN] venv 里 chongming-cli {installed_ver} < 源码 {local_ver}："
|
||||
"plugins_deps.toml 需新版 build 才生效 —— 发布 chongming-cli 后执行 "
|
||||
"uv lock --upgrade-package chongming-cli && uv sync"
|
||||
)
|
||||
else:
|
||||
check(f"venv chongming-cli {installed_ver} ≥ 源码 {local_ver}", True)
|
||||
|
||||
print("\n" + ("=" * 40))
|
||||
if failures:
|
||||
print(f"❌ {len(failures)} 项未通过: {failures}")
|
||||
sys.exit(1)
|
||||
print("✅ 全部通过")
|
||||
@@ -6,7 +6,7 @@
|
||||
2. demo 进程插件（workers/demo/，load_mode = "external"）
|
||||
|
||||
本脚本从 demo 插件项目根（= 打包 zip 根）拉起/停止独立 worker 进程
|
||||
（worker_serve --type demo），验证:
|
||||
（chongming-worker --type demo），验证:
|
||||
1. worker 启动后 → HTTP 路由动态出现且跨进程 RPC 可调
|
||||
2. worker 优雅退出 → 路由动态摘除（404）
|
||||
3. worker 重启 → 路由恢复
|
||||
|
||||
@@ -18,7 +18,7 @@ import zenoh
|
||||
from fastapi import FastAPI
|
||||
|
||||
from chongming_worker.routes_publisher import RoutePublisher
|
||||
from src.chongming.app.core.route_registry import RouteRegistry
|
||||
from chongming.app.core.route_registry import RouteRegistry
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
@@ -24,9 +24,9 @@ from pathlib import Path
|
||||
import orjson
|
||||
import zenoh
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
|
||||
sys.path.insert(0, str(REPO_ROOT / "src"))
|
||||
sys.path.insert(0, str(REPO_ROOT / "gateway" / "src"))
|
||||
|
||||
# ---- 网关依赖隔离（同 test_stream_gateway）：桩掉 chongming 包链与 chongming_cache ----
|
||||
_pkg = types.ModuleType("chongming")
|
||||
@@ -39,10 +39,11 @@ _logger_stub = types.ModuleType("chongming.app.core.logger")
|
||||
_logger_stub.get_logger = lambda name: logging.getLogger(name)
|
||||
_cache_stub = types.ModuleType("chongming_cache")
|
||||
_cache_stub.Cache = object
|
||||
_cache_stub.DistributedLock = object # worker_manager 顶层导入（本测试不触达）
|
||||
for mod in (_pkg, _app, _core, _logger_stub, _cache_stub):
|
||||
sys.modules[mod.__name__] = mod
|
||||
|
||||
_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py"
|
||||
_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py"
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.worker_manager", _wm_path
|
||||
)
|
||||
@@ -67,7 +68,8 @@ def check(name, cond, detail=""):
|
||||
failures.append(name)
|
||||
|
||||
|
||||
LISTEN_PORT = 7447
|
||||
import os
|
||||
LISTEN_PORT = int(os.environ.get("CM_STREAM_TEST_PORT", "7447"))
|
||||
LISTEN = f"tcp/127.0.0.1:{LISTEN_PORT}"
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
网关 call_stream 单测（依赖隔离：桩掉 chongming 包链与 chongming_cache）
|
||||
|
||||
直接加载 src/chongming/app/core/worker_manager.py 模块（避开 chongming.app
|
||||
直接加载 gateway/src/chongming/app/core/worker_manager.py 模块（避开 chongming.app
|
||||
整链导入与 config.toml 依赖），验证:
|
||||
1. 多块顺序产出（ConsolidationMode.NONE 逐块消费）
|
||||
2. Worker 错误收尾块 {"error": ...}：转发后终止流
|
||||
@@ -24,7 +24,7 @@ from types import SimpleNamespace
|
||||
import orjson
|
||||
import zenoh
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
# ---- 依赖隔离：桩掉网关包链（chongming.app 顶层导入 fastapi/uvicorn/config）----
|
||||
_pkg = types.ModuleType("chongming")
|
||||
@@ -37,10 +37,11 @@ _logger_stub = types.ModuleType("chongming.app.core.logger")
|
||||
_logger_stub.get_logger = lambda name: logging.getLogger(name)
|
||||
_cache_stub = types.ModuleType("chongming_cache")
|
||||
_cache_stub.Cache = object # WorkerManager 仅作类型标注使用
|
||||
_cache_stub.DistributedLock = object # 同上（set_if_not_exists 用的分布式锁）
|
||||
for mod in (_pkg, _app, _core, _logger_stub, _cache_stub):
|
||||
sys.modules[mod.__name__] = mod
|
||||
|
||||
_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py"
|
||||
_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py"
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.worker_manager", _wm_path
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
网关路由层流式端点单测（依赖隔离：桩掉 chongming 包链、auth 与 chongming_cache）
|
||||
|
||||
直接加载 src/chongming/app/core/route_generator.py，用 FastAPI TestClient 验证:
|
||||
直接加载 gateway/src/chongming/app/core/route_generator.py，用 FastAPI TestClient 验证:
|
||||
1. 流式 POST 端点：SSE 输出（media_type / 逐块 data: 帧 / 响应头）
|
||||
2. 流式 GET 端点：query 参数正确透传到 payload
|
||||
3. 初始错误（无可用 worker）-> HTTP 503
|
||||
@@ -21,8 +21,8 @@ import orjson
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "src"))
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "gateway" / "src"))
|
||||
sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
|
||||
|
||||
# ---- 依赖隔离：桩掉 chongming 包链 / auth / chongming_cache ----
|
||||
@@ -38,11 +38,12 @@ _auth_stub = types.ModuleType("chongming.app.core.auth")
|
||||
_auth_stub.store_user_to_request = lambda x: x # 仅用于 Depends，桩掉鉴权
|
||||
_cache_stub = types.ModuleType("chongming_cache")
|
||||
_cache_stub.Cache = object
|
||||
_cache_stub.DistributedLock = object # worker_manager 顶层导入（本测试不触达）
|
||||
for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub):
|
||||
sys.modules[mod.__name__] = mod
|
||||
|
||||
# 加载真实 worker_manager（route_generator 依赖其 StreamIdleTimeoutError）
|
||||
_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py"
|
||||
_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py"
|
||||
_wm_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.worker_manager", _wm_path
|
||||
)
|
||||
@@ -51,7 +52,7 @@ sys.modules["chongming.app.core.worker_manager"] = _wm_mod
|
||||
_wm_spec.loader.exec_module(_wm_mod) # type: ignore[union-attr]
|
||||
|
||||
# 加载真实 call_stats（route_generator 依赖其 spawn/record_call）
|
||||
_cs_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "call_stats.py"
|
||||
_cs_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "call_stats.py"
|
||||
_cs_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.call_stats", _cs_path
|
||||
)
|
||||
@@ -60,7 +61,7 @@ sys.modules["chongming.app.core.call_stats"] = _cs_mod
|
||||
_cs_spec.loader.exec_module(_cs_mod) # type: ignore[union-attr]
|
||||
|
||||
# 加载真实 route_generator
|
||||
_rg_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "route_generator.py"
|
||||
_rg_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "route_generator.py"
|
||||
_rg_spec = importlib.util.spec_from_file_location(
|
||||
"chongming.app.core.route_generator", _rg_path
|
||||
)
|
||||
@@ -90,18 +91,21 @@ class FakeWorkerManager:
|
||||
self.call_payloads = []
|
||||
self.stream_payloads = []
|
||||
|
||||
async def call(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False):
|
||||
async def call(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False,
|
||||
instance_id=None):
|
||||
self.call_payloads.append((worker_type, subject, payload))
|
||||
data = payload["data"]
|
||||
return {"result": data.get("a", 0) + data.get("b", 0)}
|
||||
|
||||
async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False):
|
||||
async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False,
|
||||
instance_id=None):
|
||||
# 网关透传路径：payload 为预序列化字节，回复原始字节
|
||||
self.call_payloads.append((worker_type, subject, orjson.loads(payload)))
|
||||
data = orjson.loads(payload)["data"]
|
||||
return orjson.dumps({"result": data.get("a", 0) + data.get("b", 0)})
|
||||
|
||||
async def call_stream(self, worker_type, subject, payload, timeout=300.0, idle_timeout=10.0):
|
||||
async def call_stream(self, worker_type, subject, payload, timeout=300.0, idle_timeout=10.0,
|
||||
instance_id=None):
|
||||
if self.stream_error == "no_worker":
|
||||
raise RuntimeError(f"类型 '{worker_type}' 没有可用 Worker。")
|
||||
self.stream_payloads.append((worker_type, subject, payload))
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""
|
||||
worker 超时冷静期（派发负反馈）测试
|
||||
|
||||
背景：半死副本（被 SIGKILL 的实例留下的孤儿 worker 等）只要还在服务发现里，
|
||||
每次被随机选中都会白吃一整轮超时（默认 2s）；worker_manager 只做"本次请求换副本
|
||||
重试"，没有跨请求记忆，于是 2 秒惩罚反复发生（实测一轮 31 条超时日志）。
|
||||
|
||||
修复：WorkerManager.mark_worker_suspect 给超时/无回复的副本记一段冷静期
|
||||
（_suspect_until，TTL 30s），三条选副本路径（get_worker / _pick_from_instance /
|
||||
_pick_other_worker）都优先跳过它，全在冷静期时退回全集（宁慢不缺）。
|
||||
|
||||
验证：
|
||||
1. 标记后的副本跨请求不再被选中
|
||||
2. 全部候选都处于冷静期时退回全集（不饿死）
|
||||
3. TTL 到期自动恢复
|
||||
4. 过期项会被清理（防长期运行膨胀）
|
||||
5. 换副本路径同样避开冷静期
|
||||
6. 源码兜底：超时/无回复分支确实调用了标记
|
||||
|
||||
运行: uv run python test/test_worker_suspect.py
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(ROOT / "gateway" / "src"))
|
||||
|
||||
from chongming.app.core.worker_manager import WorkerManager # noqa: E402
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
# 跳过 __init__（依赖 zenoh 会话/缓存），只装配选副本所需的字段
|
||||
m = WorkerManager.__new__(WorkerManager)
|
||||
m._suspect_until = {}
|
||||
m._workers_cache = {"demo": ["w1", "w2", "w3"]}
|
||||
m._local_workers = {}
|
||||
|
||||
|
||||
def sample(fn, n=40):
|
||||
return {asyncio.run(fn()) for _ in range(n)}
|
||||
|
||||
|
||||
# ========== 1. 标记后跨请求跳过 ==========
|
||||
m.mark_worker_suspect("w1")
|
||||
m.mark_worker_suspect("w2")
|
||||
picked = sample(lambda: m.get_worker("demo"))
|
||||
check("冷静期内的副本不再被选中", picked == {"w3"}, str(picked))
|
||||
|
||||
# ========== 2. 全在冷静期 → 退回全集 ==========
|
||||
m.mark_worker_suspect("w3")
|
||||
picked = sample(lambda: m.get_worker("demo"))
|
||||
check("全部处于冷静期时退回全集（不饿死）", picked == {"w1", "w2", "w3"}, str(picked))
|
||||
|
||||
# ========== 3. TTL 到期恢复 ==========
|
||||
m._suspect_until = {"w1": time.monotonic() - 0.1}
|
||||
check("冷静期过期自动恢复", m._filter_suspects(["w1"]) == ["w1"])
|
||||
|
||||
# ========== 4. 过期项清理 ==========
|
||||
m._suspect_until = {}
|
||||
for i in range(40):
|
||||
m.mark_worker_suspect(f"x{i}", ttl=-1.0) # 立即过期
|
||||
check("过期项会被清理（防膨胀）", len(m._suspect_until) <= 32, str(len(m._suspect_until)))
|
||||
|
||||
# ========== 5. 换副本路径同样避开 ==========
|
||||
m._suspect_until = {}
|
||||
m.mark_worker_suspect("w1")
|
||||
picked = sample(lambda: m._pick_other_worker("demo", []))
|
||||
check("换副本路径同样避开冷静期", picked == {"w2", "w3"}, str(picked))
|
||||
|
||||
# ========== 6. 源码兜底：失败分支确实标记 ==========
|
||||
src = (ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
n = src.count("self.mark_worker_suspect(worker_expr)")
|
||||
check("超时/无回复分支都调用了标记", n >= 2, f"{n} 处")
|
||||
|
||||
print("\n" + ("=" * 40))
|
||||
if failures:
|
||||
print(f"❌ {len(failures)} 项未通过: {failures}")
|
||||
sys.exit(1)
|
||||
print("✅ 全部通过")
|
||||
@@ -2,7 +2,7 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SRC = Path(__file__).parent / "src"
|
||||
SRC = Path(__file__).resolve().parent.parent / "src" # 仓库 src/（plugins 包源码所在）
|
||||
sys.path.insert(0, str(SRC))
|
||||
|
||||
failures = []
|
||||
|
||||
Reference in New Issue
Block a user