init: chongming-gateway 0.1.0(网关本体独立包:FastAPI + Zenoh 配置驱动微服务网关)
This commit is contained in:
+72
@@ -0,0 +1,72 @@
|
|||||||
|
/target
|
||||||
|
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
.pytest_cache/
|
||||||
|
*.py[cod]
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
.venv/
|
||||||
|
env/
|
||||||
|
bin/
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
include/
|
||||||
|
man/
|
||||||
|
venv/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
pip-selfcheck.json
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.coverage
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
|
||||||
|
# Mr Developer
|
||||||
|
.mr.developer.cfg
|
||||||
|
.project
|
||||||
|
.pydevproject
|
||||||
|
|
||||||
|
# Rope
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyCharm
|
||||||
|
.idea/
|
||||||
|
|
||||||
|
# VSCode
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
# Pyenv
|
||||||
|
.python-version
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
# chongming-gateway
|
||||||
|
|
||||||
|
chongming 网关本体:基于 **FastAPI + Zenoh** 的配置驱动微服务网关。
|
||||||
|
|
||||||
|
把宿主项目里的插件(inline 进程内 / external 独立进程)发布的路由统一挂载为 HTTP 端点,
|
||||||
|
并提供 Worker 服务发现、负载均衡、超时重试、实例定向、MQTT Bridge 与内置登录鉴权。
|
||||||
|
|
||||||
|
网关只提供**代码**:配置、插件与打包骨架由宿主项目提供(见「运行时约定」)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 安装
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 作为依赖(宿主项目,从私有源安装)
|
||||||
|
uv add chongming-gateway
|
||||||
|
|
||||||
|
# 开发期可编辑安装(在 chongming-core 仓库内)
|
||||||
|
uv pip install -e utils/gateway
|
||||||
|
```
|
||||||
|
|
||||||
|
网关自带完整依赖清单(`fastapi` / `granian` / `zenoh` / `chongming-worker` 等),
|
||||||
|
宿主项目无需重复声明;需要哪些第三方包,见本包 `pyproject.toml`。
|
||||||
|
|
||||||
|
## 快速开始
|
||||||
|
|
||||||
|
宿主项目(cwd)下准备 `config.toml` 与 `config/workers/<type>/config.toml`,
|
||||||
|
并保证插件包(`plugins.<type>`)可导入。之后任选一种启动方式:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 按 config.toml [<env>.server] 启动(三个入口等价,仅默认服务器不同)
|
||||||
|
uv run serve # uvicorn
|
||||||
|
uv run granian # granian
|
||||||
|
uv run gunicorn # gunicorn + uvicorn worker
|
||||||
|
|
||||||
|
# 或把 ASGI 应用交给任意服务器
|
||||||
|
uvicorn chongming.app:app --host 0.0.0.0 --port 8000
|
||||||
|
```
|
||||||
|
|
||||||
|
网关按 `config/workers/<name>/config.toml` 的 `[workers]` 节逐插件读取加载方式
|
||||||
|
(`load_mode = "inline" | "external"`)与副本数,启动时自动发现并挂载路由。
|
||||||
|
|
||||||
|
## 运行时约定
|
||||||
|
|
||||||
|
- **配置**:cwd 下 `config.toml`(环境、服务器、安全、MQTT、缓存等)与
|
||||||
|
`config/workers/**`(插件注册与路由定义),运行期读取,随宿主项目部署。
|
||||||
|
- **插件**:网关按 `plugins.<type>` 约定导入插件包,来源不限——`.mbank` 模块银行、
|
||||||
|
源码目录(开发态 `src/plugins`)或已安装包均可。
|
||||||
|
- **工作目录**:日志、上传、数据等相对路径均以 cwd 为基准。
|
||||||
|
|
||||||
|
## 目录结构
|
||||||
|
|
||||||
|
```text
|
||||||
|
src/chongming/
|
||||||
|
├── __init__.py # 启动入口(serve / granian / gunicorn)与服务器参数转换
|
||||||
|
└── app/
|
||||||
|
├── __init__.py # FastAPI 应用装配(lifespan、中间件、异常处理)
|
||||||
|
├── core/ # WorkerManager / RouteGenerator / MQTT Bridge / 日志 / 鉴权
|
||||||
|
└── routes/ # 内置管理端点(stats / worker_admin / auth)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 开发与发布
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 在 chongming-core 仓库内改代码后,让 venv 用本地源码(联调用)
|
||||||
|
uv pip install -e utils/gateway
|
||||||
|
|
||||||
|
# 发版:构建 wheel 并推私有源
|
||||||
|
uv build # dist/chongming_gateway-<version>-py3-none-any.whl
|
||||||
|
uv run twine upload --repository <私有源> dist/*
|
||||||
|
```
|
||||||
|
|
||||||
|
发版后依赖方执行 `uv lock --upgrade-package chongming-gateway` 刷新版本 pin。
|
||||||
|
注意 `uv sync` 会用锁文件版本替换可编辑安装——打包/发版前先确认 venv 指向目标版本。
|
||||||
|
|
||||||
|
## 相关文档
|
||||||
|
|
||||||
|
- [项目主 README](../../README.md):整体架构、使用与构建部署
|
||||||
|
- [Worker 管理与多实例运维](../../docs/worker-management.md):管理端点、扩缩容、实例定向
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
[build-system]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
requires = ["hatchling"]
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "chongming-gateway"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "chongming 网关:FastAPI + Zenoh 的配置驱动微服务网关(插件路由动态挂载、Worker 服务发现与负载均衡)"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.12"
|
||||||
|
dependencies = [
|
||||||
|
# chongming 家族工具包
|
||||||
|
"chongming-auth",
|
||||||
|
"chongming-cache",
|
||||||
|
"chongming-config",
|
||||||
|
"chongming-jwt",
|
||||||
|
"chongming-worker",
|
||||||
|
# Web / 服务器
|
||||||
|
"fastapi",
|
||||||
|
"granian[reload,uvloop]>=2.8.4",
|
||||||
|
"gunicorn",
|
||||||
|
"httptools",
|
||||||
|
"starlette",
|
||||||
|
"uvicorn",
|
||||||
|
"uvloop",
|
||||||
|
# 运行时
|
||||||
|
"amqtt",
|
||||||
|
"eclipse-zenoh",
|
||||||
|
"orjson",
|
||||||
|
"pwdlib",
|
||||||
|
"pydantic",
|
||||||
|
"sqlite-vfs",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
|
gunicorn = "chongming:gunicorn_serve"
|
||||||
|
granian = "chongming:granian_serve"
|
||||||
|
serve = "chongming:serve"
|
||||||
|
|
||||||
|
[tool.hatch.build.targets.wheel]
|
||||||
|
packages = ["src/chongming"]
|
||||||
|
|
||||||
|
[[tool.uv.index]]
|
||||||
|
url = "http://124.71.68.6:3000/api/packages/chongming/pypi/simple/"
|
||||||
|
|
||||||
|
[[tool.uv.index]]
|
||||||
|
url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||||
|
|
||||||
|
[dependency-groups]
|
||||||
|
dev = [
|
||||||
|
"twine>=7.0.0",
|
||||||
|
]
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
"""chongming 网关启动入口(uvicorn / granian / gunicorn)。
|
||||||
|
|
||||||
|
config.toml 中 `[<env>.server]` 的启动参数保持现状(host / port / workers /
|
||||||
|
reload / access_log / timeout_keep_alive),由 `_to_uvicorn`、`_to_granian` 在
|
||||||
|
启动时转换为对应服务器的关键字参数,切换服务器不必改配置格式。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import uvicorn
|
||||||
|
from chongming_config import Server, load_config
|
||||||
|
|
||||||
|
from .app.core.logger import get_logger
|
||||||
|
|
||||||
|
# 开发期热重载(reload=true)监听的源码目录(granian 用;uvicorn 走其默认扫描)。
|
||||||
|
# 网关自身源码目录用 __file__ 推导:editable 安装时指向仓库内的源码,wheel 安装时
|
||||||
|
# 指向 site-packages(打包产物强制禁 reload,该目录只在开发态有意义);
|
||||||
|
# 插件目录固定为宿主项目的 src/plugins。
|
||||||
|
RELOAD_PATHS = [Path(__file__).resolve().parent, Path("src/plugins")]
|
||||||
|
|
||||||
|
|
||||||
|
def _server_config_view(server_config: Server) -> Mapping[str, Any]:
|
||||||
|
"""[<env>.server] 的宽松视图:实际 TOML 可能带 Server 之外的键
|
||||||
|
(如 timeout_keep_alive,chongming_config.Server 尚未声明)。"""
|
||||||
|
return server_config
|
||||||
|
|
||||||
|
|
||||||
|
def _to_uvicorn(server_config: Server) -> dict:
|
||||||
|
"""把 [<env>.server] 的现状参数转为 uvicorn.run 的关键字参数。
|
||||||
|
|
||||||
|
uvicorn 的参数名与配置项一致,这里只做缺省兜底。
|
||||||
|
"""
|
||||||
|
cfg = _server_config_view(server_config)
|
||||||
|
converted: dict = {
|
||||||
|
"host": cfg.get("host", "0.0.0.0"),
|
||||||
|
"port": cfg.get("port", 8000),
|
||||||
|
"reload": cfg.get("reload", False),
|
||||||
|
}
|
||||||
|
for key in ("workers", "access_log", "timeout_keep_alive"):
|
||||||
|
if key in cfg:
|
||||||
|
converted[key] = cfg[key]
|
||||||
|
return converted
|
||||||
|
|
||||||
|
|
||||||
|
def _to_granian(server_config: Server) -> dict:
|
||||||
|
"""把 [<env>.server] 的现状参数转为 granian.Granian 的关键字参数。
|
||||||
|
|
||||||
|
仅两处改名:host → address、access_log → log_access;
|
||||||
|
timeout_keep_alive 在 granian 无对应项(HTTP/1 keep-alive 只有开关、无空闲
|
||||||
|
超时配置),忽略。
|
||||||
|
"""
|
||||||
|
cfg = _server_config_view(server_config)
|
||||||
|
converted: dict = {
|
||||||
|
"address": cfg.get("host", "0.0.0.0"),
|
||||||
|
"port": cfg.get("port", 8000),
|
||||||
|
"interface": "asgi",
|
||||||
|
}
|
||||||
|
for key in ("reload", "workers"):
|
||||||
|
if key in cfg:
|
||||||
|
converted[key] = cfg[key]
|
||||||
|
if "access_log" in cfg:
|
||||||
|
converted["log_access"] = cfg["access_log"]
|
||||||
|
return converted
|
||||||
|
|
||||||
|
|
||||||
|
def serve():
|
||||||
|
"""
|
||||||
|
使用 uvicorn 启动服务
|
||||||
|
|
||||||
|
从配置文件中加载服务器配置和日志配置,然后使用 uvicorn 启动应用服务。
|
||||||
|
该函数适用于开发环境或单进程部署场景。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None: 该函数不返回值,会阻塞运行直到服务停止
|
||||||
|
"""
|
||||||
|
config = load_config()
|
||||||
|
env = config["default"]["env"]
|
||||||
|
server_config = config[env]["server"]
|
||||||
|
log_config = config[env]["logging"]
|
||||||
|
logger = get_logger("app")
|
||||||
|
logger.info(f"启动服务: {server_config}")
|
||||||
|
logger.info(f"启动日志: {log_config}")
|
||||||
|
uvicorn.run("chongming.app:app", **_to_uvicorn(server_config))
|
||||||
|
|
||||||
|
|
||||||
|
def granian_serve():
|
||||||
|
"""
|
||||||
|
使用 granian 启动服务
|
||||||
|
|
||||||
|
从配置文件中加载服务器配置和日志配置,然后使用 granian 启动应用服务。
|
||||||
|
该函数适用于开发环境或单进程部署场景。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None: 该函数不返回值,会阻塞运行直到服务停止
|
||||||
|
"""
|
||||||
|
# 延迟导入:未选用 granian 的部署环境不必安装它
|
||||||
|
import granian
|
||||||
|
|
||||||
|
config = load_config()
|
||||||
|
env = config["default"]["env"]
|
||||||
|
server_config = config[env]["server"]
|
||||||
|
log_config = config[env]["logging"]
|
||||||
|
logger = get_logger("app")
|
||||||
|
logger.info(f"启动服务: {server_config}")
|
||||||
|
logger.info(f"启动日志: {log_config}")
|
||||||
|
granian.Granian(
|
||||||
|
target="chongming.app:app",
|
||||||
|
**_to_granian(server_config),
|
||||||
|
reload_paths=RELOAD_PATHS,
|
||||||
|
).serve()
|
||||||
|
|
||||||
|
|
||||||
|
def gunicorn_serve():
|
||||||
|
"""
|
||||||
|
使用 gunicorn 启动服务
|
||||||
|
|
||||||
|
通过 gunicorn 作为进程管理器启动应用,使用 UvicornWorker 作为 worker 类型,
|
||||||
|
支持多进程部署,适用于生产环境。
|
||||||
|
|
||||||
|
从配置文件中读取服务器配置,动态设置 gunicorn 的 worker 数量、绑定地址和端口等参数。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None: 该函数不返回值,会阻塞运行直到服务停止
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from gunicorn.app.wsgiapp import run
|
||||||
|
|
||||||
|
config = load_config()
|
||||||
|
env = config["default"]["env"]
|
||||||
|
server_config = config[env]["server"]
|
||||||
|
|
||||||
|
# 构建 gunicorn 命令行参数并启动服务
|
||||||
|
sys.argv = [
|
||||||
|
"gunicorn",
|
||||||
|
"chongming.app:app",
|
||||||
|
"-k",
|
||||||
|
"uvicorn.workers.UvicornWorker",
|
||||||
|
"-w",
|
||||||
|
f"{server_config.get('workers', 4)}",
|
||||||
|
"-b",
|
||||||
|
f"{server_config.get('host', '0.0.0.0')}:{server_config.get('port', 8000)}",
|
||||||
|
]
|
||||||
|
run()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
@@ -0,0 +1,354 @@
|
|||||||
|
"""
|
||||||
|
chongming FastAPI 应用入口
|
||||||
|
|
||||||
|
重构为配置驱动的微服务网关:
|
||||||
|
- WorkerManager: 统一管理 Zenoh worker 存活检测、负载均衡、超时重试、缓存
|
||||||
|
- RouteGenerator: 根据配置自动生成 FastAPI 端点
|
||||||
|
- 插件工厂函数动态加载
|
||||||
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
import asyncio
|
||||||
|
import itertools
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
|
from fastapi import FastAPI, Request
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
from fastapi.middleware.gzip import GZipMiddleware
|
||||||
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
|
from chongming_config import get_config
|
||||||
|
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||||
|
from .core.logger import get_logger
|
||||||
|
from sqlite_vfs.contrib.fastapi_static import SVFSStaticFiles
|
||||||
|
from .core import lifespan_setup, call_stats
|
||||||
|
from .core.parent_death import guard_parent_death
|
||||||
|
from .routes import stats_router, worker_admin_router, auth_router
|
||||||
|
|
||||||
|
|
||||||
|
config = get_config()
|
||||||
|
logger = get_logger("app")
|
||||||
|
|
||||||
|
try:
|
||||||
|
env = config["default"]["env"]
|
||||||
|
except KeyError: # pragma: no cover
|
||||||
|
raise ValueError("配置不存在")
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(app: FastAPI):
|
||||||
|
# ---- 启动阶段 ----
|
||||||
|
# 0. 父进程死亡防护:每个 granian worker 各设一次(PR_SET_PDEATHSIG 不跨 fork
|
||||||
|
# 继承)——master 被 SIGKILL 时内核直接给 worker 发 SIGTERM,走下面的优雅
|
||||||
|
# 停机摘除路由与副本注册,避免留下继续监听端口/挂在服务发现里的孤儿
|
||||||
|
guard_parent_death(logger)
|
||||||
|
|
||||||
|
# 1. 本地存储(替代 Nanio S3,省 ~172 MB)
|
||||||
|
lifespan_setup.setup_storage(app)
|
||||||
|
|
||||||
|
# 1.2 内置 auth 库(users 表 + admin 种子)—— auth 域的基础设施部分:
|
||||||
|
# 登录/签发内置在网关(管理平面底线),不依赖 auth 插件是否加载;
|
||||||
|
# 与 auth 插件(用户管理)共用同一张 users 表
|
||||||
|
await lifespan_setup.setup_auth_store(app)
|
||||||
|
|
||||||
|
# 2. Zenoh
|
||||||
|
session = lifespan_setup.setup_zenoh(app)
|
||||||
|
|
||||||
|
# 3. WorkerManager
|
||||||
|
manager = lifespan_setup.setup_worker_manager(app, session)
|
||||||
|
|
||||||
|
# 3.1 内嵌 amqtt Broker(宿主级分布式锁;需在 WorkerManager 就绪后提供共享 cache)
|
||||||
|
await lifespan_setup.setup_mqtt_broker(app, manager)
|
||||||
|
|
||||||
|
# 4.1 全局副本数协调:把当前网关登记进在线列表,供按在线网关分摊 quota
|
||||||
|
manager.register_gateway()
|
||||||
|
|
||||||
|
# 4.2 宿主级 inline worker 领导选举(基于共享 cache 的分布式锁,key 按网关隔离):
|
||||||
|
# gunicorn 多 worker 下每实例仅一个进程托管 inline worker(避免 N×count 冗余),
|
||||||
|
# 其余进程只做 HTTP 并经 Zenoh liveliness 发现调用;多实例各选各的 leader。
|
||||||
|
is_inline_owner = manager.acquire_inline_owner()
|
||||||
|
app.state.inline_owner_locked = is_inline_owner
|
||||||
|
|
||||||
|
# 5. 插件加载(混合模式:inline 进程内加载;external 由独立进程运行,
|
||||||
|
# 其路由经 Zenoh 动态注册——进程编排交给 Docker/K8s,网关不拉起进程)
|
||||||
|
worker_tasks, custom_routers, inline_types, external_types = (
|
||||||
|
await lifespan_setup.load_workers(
|
||||||
|
app, session, manager, is_inline_owner=is_inline_owner
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# 5.1 动态路由注册表常驻启动:订阅 chongming/routes/**,external 插件的
|
||||||
|
# 路由经 Zenoh 注册/摘除,无需网关侧保留插件配置(external_types 仅观测)
|
||||||
|
await lifespan_setup.setup_route_registry(
|
||||||
|
app, session, manager, inline_types=inline_types
|
||||||
|
)
|
||||||
|
|
||||||
|
# 5.5 契约校验(仅 inline 插件;external 插件在自己进程内校验),失败则启动终止
|
||||||
|
lifespan_setup.validate_contracts(app)
|
||||||
|
|
||||||
|
# 6. 注册路由(inline 插件的 TOML 动态路由 + 插件自定义路由)
|
||||||
|
await lifespan_setup.register_routes(
|
||||||
|
app, manager, custom_routers, worker_types=inline_types
|
||||||
|
)
|
||||||
|
|
||||||
|
# 7. 启动 MQTT Bridge(内嵌 Broker + Worker 桥接)
|
||||||
|
await lifespan_setup.setup_mqtt(app, manager)
|
||||||
|
|
||||||
|
# 7.1 启动 MQTT Bridge 持有者自愈巡检(跨所有网关进程,崩溃时自动接管重建)
|
||||||
|
app.state.mqtt_bridge_supervise_task = asyncio.create_task(
|
||||||
|
lifespan_setup.mqtt_bridge_supervisor(app, manager)
|
||||||
|
)
|
||||||
|
|
||||||
|
# 8. 短暂等待,确保 worker 状态同步
|
||||||
|
await asyncio.sleep(0.5)
|
||||||
|
|
||||||
|
# 打印 worker 就绪状态
|
||||||
|
worker_status = {
|
||||||
|
wtype: len(workers)
|
||||||
|
for wtype, workers in manager._workers_cache.items()
|
||||||
|
}
|
||||||
|
logger.info(f"Worker 就绪状态: {worker_status}")
|
||||||
|
|
||||||
|
# ---- 应用运行 ----
|
||||||
|
yield
|
||||||
|
|
||||||
|
# ---- 关闭阶段 ----
|
||||||
|
logger.info("正在关闭应用...")
|
||||||
|
|
||||||
|
# 1. 停止 MQTT Bridge 持有者自愈巡检,先于 Bridge 关停,避免关停期间再次接管
|
||||||
|
_sup = getattr(app.state, "mqtt_bridge_supervise_task", None)
|
||||||
|
if _sup:
|
||||||
|
_sup.cancel()
|
||||||
|
try:
|
||||||
|
await _sup
|
||||||
|
except (asyncio.CancelledError, Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 1.0 停止 MQTT Bridge
|
||||||
|
await lifespan_setup.shutdown_mqtt(app)
|
||||||
|
|
||||||
|
# 1.5 停止路由注册表订阅(常驻,随应用生命周期启停)
|
||||||
|
registry = getattr(app.state, "route_registry", None)
|
||||||
|
if registry is not None:
|
||||||
|
registry.stop()
|
||||||
|
|
||||||
|
# 2. 清理 inline workers(cleanup_workers 内部按 load_mode 过滤)
|
||||||
|
await lifespan_setup.cleanup_workers(app)
|
||||||
|
|
||||||
|
# 2.2 关闭内置 auth 库
|
||||||
|
auth_db = getattr(app.state, "auth_db", None)
|
||||||
|
if auth_db is not None:
|
||||||
|
await auth_db.dispose()
|
||||||
|
|
||||||
|
# 2.1 释放 inline worker 所有权分布式锁(仅持锁进程,避免误删其他进程获得的新锁)
|
||||||
|
if getattr(app.state, "inline_owner_locked", False):
|
||||||
|
await manager.release_inline_owner()
|
||||||
|
|
||||||
|
# 3. 取消刷新任务
|
||||||
|
refresh_task = getattr(app.state, "refresh_task", None)
|
||||||
|
if refresh_task:
|
||||||
|
refresh_task.cancel()
|
||||||
|
try:
|
||||||
|
await refresh_task
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 4. 取消所有 worker 任务(含 self-heal 接管后拉起的任务)
|
||||||
|
all_worker_tasks = list(worker_tasks) + manager.extra_worker_tasks()
|
||||||
|
for task in all_worker_tasks:
|
||||||
|
task.cancel()
|
||||||
|
if all_worker_tasks:
|
||||||
|
await asyncio.gather(*all_worker_tasks, return_exceptions=True)
|
||||||
|
|
||||||
|
# 4.5 取消后台统计/审计任务
|
||||||
|
await call_stats.shutdown()
|
||||||
|
|
||||||
|
# 4.6 全局副本数协调:关闭时把当前网关从在线列表移除(其余网关将重新分摊)
|
||||||
|
manager.unregister_gateway()
|
||||||
|
|
||||||
|
# 5. 关闭 Zenoh 会话
|
||||||
|
# 退出阶段等对端应答可能超时(zenoh.ZError: close operation timed out),此时
|
||||||
|
# 进程马上就要退出,记录告警即可——不让它把 lifespan 关闭流程带崩、打出一串
|
||||||
|
# [ERROR] traceback(granian 停机日志里很吓人,但无实际影响)
|
||||||
|
try:
|
||||||
|
session.close()
|
||||||
|
logger.info("Zenoh 会话已关闭")
|
||||||
|
except Exception as e: # noqa: BLE001 - zenoh close 超时属退出阶段的正常竞争
|
||||||
|
logger.warning("Zenoh 会话关闭超时/异常(退出阶段忽略): %s", e)
|
||||||
|
|
||||||
|
logger.info("应用已关闭")
|
||||||
|
|
||||||
|
|
||||||
|
cors = config[env].get("cors", {})
|
||||||
|
app = FastAPI(
|
||||||
|
title=config["default"]["app"]["name"],
|
||||||
|
version=config["default"]["app"]["version"],
|
||||||
|
description=config["default"]["app"]["description"],
|
||||||
|
docs_url="/docs" if config[env]["debug"] else None,
|
||||||
|
redoc_url="/redoc" if config[env]["debug"] else None,
|
||||||
|
openapi_url="/openapi.json" if config[env]["debug"] else None,
|
||||||
|
lifespan=lifespan,
|
||||||
|
)
|
||||||
|
|
||||||
|
app.add_middleware(CORSMiddleware, **cors)
|
||||||
|
# GZip 压缩响应(<500B 不压缩;starlette 默认排除 text/event-stream,
|
||||||
|
# 保证 SSE 流式端点即时性;>=128KB 的响应在独立线程压缩,不阻塞事件循环)
|
||||||
|
app.add_middleware(GZipMiddleware)
|
||||||
|
|
||||||
|
|
||||||
|
# access 日志采样率(logging.access_sample_rate,0.1 = 只记录 10% 的成功请求;
|
||||||
|
# 错误 >=400 始终全量记录,保证可追溯;1.0/缺省 = 全量)
|
||||||
|
ACCESS_SAMPLE_RATE = float(config[env].get("logging", {}).get("access_sample_rate", 1.0))
|
||||||
|
# 成功请求采样计数器(每进程独立;错误请求不消耗计数)
|
||||||
|
_access_log_counter = itertools.count()
|
||||||
|
|
||||||
|
|
||||||
|
def _should_log_access(status_code: int, sample_rate: float = ACCESS_SAMPLE_RATE) -> bool:
|
||||||
|
"""错误全量;成功按采样率(采样为确定性计数,无随机数开销)"""
|
||||||
|
if status_code >= 400 or sample_rate >= 1.0:
|
||||||
|
return True
|
||||||
|
denom = max(1, int(round(1.0 / sample_rate)))
|
||||||
|
return next(_access_log_counter) % denom == 0
|
||||||
|
|
||||||
|
|
||||||
|
def _log_access(scope: Scope, status_code: int, start: float) -> None:
|
||||||
|
"""按采样规则写一条访问日志(成功按 access_sample_rate,错误 >=400 全量)"""
|
||||||
|
if not _should_log_access(status_code):
|
||||||
|
return
|
||||||
|
client = scope.get("client")
|
||||||
|
logger.info(
|
||||||
|
"%s %s status=%s duration=%.2fms client=%s",
|
||||||
|
scope.get("method", "-"), scope.get("path", "-"), status_code,
|
||||||
|
(time.perf_counter() - start) * 1000, client[0] if client else "-",
|
||||||
|
extra={"status_code": status_code},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class AccessLogMiddleware:
|
||||||
|
"""
|
||||||
|
HTTP访问日志中间件(纯 ASGI 实现),记录每个请求的详细信息。
|
||||||
|
|
||||||
|
记录的信息包括:
|
||||||
|
- 请求方法和路径
|
||||||
|
- 响应状态码
|
||||||
|
- 请求处理时长(毫秒,按慢路径:从进入中间件到响应头产出)
|
||||||
|
- 客户端IP地址
|
||||||
|
|
||||||
|
高吞吐优化:
|
||||||
|
- 采样:成功请求按 access_sample_rate 记录,错误(>=400)全量
|
||||||
|
- 异步:% 参数化消息 + 队列日志(QueueHandler),格式化与文件 I/O
|
||||||
|
在后台监听线程执行,主线程仅入队
|
||||||
|
- 不用 BaseHTTPMiddleware(@app.middleware("http")):它会为每请求建 anyio
|
||||||
|
task group 并把响应体经内存流过一手,压测实测高并发下吃掉 30~50% 吞吐;
|
||||||
|
这里只包一层 send 截获 response.start,无缓冲、无额外任务
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, app: ASGIApp) -> None:
|
||||||
|
self.app = app
|
||||||
|
|
||||||
|
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||||
|
if scope["type"] != "http":
|
||||||
|
await self.app(scope, receive, send)
|
||||||
|
return
|
||||||
|
|
||||||
|
start = time.perf_counter()
|
||||||
|
logged = False
|
||||||
|
|
||||||
|
async def send_wrapper(message: Message) -> None:
|
||||||
|
nonlocal logged
|
||||||
|
# 响应头产出即记录(流式响应因此只计到首字节,不计整段流)
|
||||||
|
if message["type"] == "http.response.start" and not logged:
|
||||||
|
logged = True
|
||||||
|
_log_access(scope, message["status"], start)
|
||||||
|
await send(message)
|
||||||
|
|
||||||
|
try:
|
||||||
|
await self.app(scope, receive, send_wrapper)
|
||||||
|
except Exception:
|
||||||
|
# 未产出响应就异常(外层 500):补一条全量错误日志后原样抛出
|
||||||
|
if not logged:
|
||||||
|
_log_access(scope, 500, start)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
app.add_middleware(AccessLogMiddleware)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/", summary="根路径")
|
||||||
|
async def root():
|
||||||
|
"""
|
||||||
|
根路径端点,返回应用的基本信息。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
dict: 包含应用名称、版本、文档地址和健康检查地址的字典
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"message": f"欢迎使用 {config['default']['app']['name']}",
|
||||||
|
"version": config["default"]["app"]["version"],
|
||||||
|
"docs": "/docs" if config[env]["debug"] else None,
|
||||||
|
"health": f"{config['default']['prefix']}/health",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(RequestValidationError)
|
||||||
|
async def validation_exception_handler(request: Request, exc: RequestValidationError):
|
||||||
|
"""
|
||||||
|
请求验证异常处理器,记录详细的验证错误信息和请求体。
|
||||||
|
帮助快速定位 Pydantic 模型与前端/设备上报数据的不匹配问题。
|
||||||
|
"""
|
||||||
|
body = None
|
||||||
|
try:
|
||||||
|
body = await request.body()
|
||||||
|
body_str = body.decode("utf-8")[:2000] if body else "<empty>"
|
||||||
|
except Exception:
|
||||||
|
body_str = "<unable to read body>"
|
||||||
|
|
||||||
|
errors = exc.errors()
|
||||||
|
error_summary = [
|
||||||
|
f"{' -> '.join(str(loc) for loc in e['loc'])}: {e['msg']}"
|
||||||
|
for e in errors
|
||||||
|
]
|
||||||
|
logger.error(
|
||||||
|
f"请求验证失败 [{request.method} {request.url.path}]:\n"
|
||||||
|
f" 错误: {'; '.join(error_summary)}\n"
|
||||||
|
f" 请求体: {body_str}"
|
||||||
|
)
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=422,
|
||||||
|
content={"detail": errors},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(Exception)
|
||||||
|
async def global_exception_handler(_: Request, exc: Exception):
|
||||||
|
"""
|
||||||
|
全局异常处理器,捕获并处理应用中未捕获的异常。
|
||||||
|
|
||||||
|
在调试模式下返回详细的异常信息,在生产模式下返回通用错误提示。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
_: FastAPI请求对象(未使用)
|
||||||
|
exc: 捕获到的异常对象
|
||||||
|
|
||||||
|
返回:
|
||||||
|
JSONResponse: 包含错误信息的JSON响应,状态码为500
|
||||||
|
"""
|
||||||
|
logger.error(f"全局异常: {exc}")
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=500,
|
||||||
|
content={
|
||||||
|
"error": "内部服务器错误",
|
||||||
|
"detail": str(exc) if config[env]["debug"] else "请查看服务器日志",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# 静态文件服务(可选)
|
||||||
|
if vfs_db_path := config[env].get("file_system", {}).get("path", None):
|
||||||
|
logger.info("静态文件已存在,将使用静态文件服务")
|
||||||
|
static_files_handler = SVFSStaticFiles(vfs_db_path=vfs_db_path)
|
||||||
|
app.mount("/static", static_files_handler, name="static")
|
||||||
|
|
||||||
|
app.include_router(stats_router)
|
||||||
|
app.include_router(worker_admin_router)
|
||||||
|
app.include_router(auth_router)
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from collections import OrderedDict
|
||||||
|
|
||||||
|
from fastapi import Depends, HTTPException, status, Request
|
||||||
|
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||||
|
from chongming_jwt import JWT
|
||||||
|
from chongming_cache import Cache
|
||||||
|
from chongming_config import get_config
|
||||||
|
import orjson
|
||||||
|
|
||||||
|
|
||||||
|
security_scheme = HTTPBearer(auto_error=False)
|
||||||
|
|
||||||
|
# JWT 解析共享缓存:与 WorkerManager 共用 [cache] 配置段。
|
||||||
|
# 默认 sqlite(本机);横向拓展时 config.toml 切 backend = "redis",
|
||||||
|
# 多网关实例共享同一 Redis 即可全局只解密一次,代码零改动。
|
||||||
|
# 注意:缓存存的是"已验证结果",信任模型与 worker 回复缓存一致
|
||||||
|
# (缓存基础设施可信;Redis 请配置密码并置于内网)。
|
||||||
|
_jwt_cache = Cache.from_config(get_config()["cache"])
|
||||||
|
# 条目 TTL(秒):控制存储占用与密钥轮换后的自然清理;
|
||||||
|
# 安全性由命中时的 exp 复核保证,TTL 不承担过期判断
|
||||||
|
_JWT_CACHE_TTL = 60
|
||||||
|
|
||||||
|
# ── 一级:进程内 JWT 结果缓存(LRU + TTL)────────────────────────────
|
||||||
|
# 鉴权依赖是同步依赖(def),每次请求都要走一遍共享缓存——backend=redis 时
|
||||||
|
# 即"每请求一次同步网络往返 + FastAPI 线程池 hop"。同一 token 在 TTL 内被
|
||||||
|
# 反复使用是常态,故在进程内加一层 L1:命中直接返回,不碰网络。
|
||||||
|
# 语义与 L2 完全一致:key 含密钥/算法指纹;命中仍复核 exp(过期=拒绝而非放行);
|
||||||
|
# 条目按 TTL 惰性清理并限长。CHONGMING_JWT_L1=0 可关闭(便于 A/B 与排障)。
|
||||||
|
_JWT_L1_ENABLED = os.environ.get("CHONGMING_JWT_L1", "1") != "0"
|
||||||
|
_JWT_L1_MAX = 1024
|
||||||
|
# 存 orjson 字节而非 dict:命中时反序列化出新副本,保证"调用方修改返回值
|
||||||
|
# 不污染缓存"这一 L2 既有契约(L2 同样是 JSON 往返后返回新对象)
|
||||||
|
_jwt_l1: "OrderedDict[str, tuple[bytes, float, float]]" = OrderedDict() # key → (payload_json, exp, 写入时刻)
|
||||||
|
|
||||||
|
|
||||||
|
def _l1_get(key: str):
|
||||||
|
"""命中返回 (payload_json: bytes, exp);条目超过共享缓存 TTL(运行时可调,便于测试/运维)视为过期"""
|
||||||
|
entry = _jwt_l1.get(key)
|
||||||
|
if entry is None:
|
||||||
|
return None
|
||||||
|
_payload_json, _exp, ts = entry
|
||||||
|
if time.time() - ts >= _JWT_CACHE_TTL:
|
||||||
|
_jwt_l1.pop(key, None)
|
||||||
|
return None
|
||||||
|
_jwt_l1.move_to_end(key)
|
||||||
|
return entry
|
||||||
|
|
||||||
|
|
||||||
|
def _l1_set(key: str, payload_json: bytes, exp: float) -> None:
|
||||||
|
_jwt_l1[key] = (payload_json, exp, time.time())
|
||||||
|
_jwt_l1.move_to_end(key)
|
||||||
|
while len(_jwt_l1) > _JWT_L1_MAX:
|
||||||
|
_jwt_l1.popitem(last=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _l1_sweep(now: float) -> None:
|
||||||
|
"""惰性清理:超 TTL 的整条丢弃(token 永不重复时防堆积),只在表较大时调用"""
|
||||||
|
stale = [k for k, v in _jwt_l1.items() if now - v[2] >= _JWT_CACHE_TTL]
|
||||||
|
for k in stale:
|
||||||
|
_jwt_l1.pop(k, None)
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_key(token: str, secret: str, algorithm: str) -> str:
|
||||||
|
"""缓存 key:token 内容 + 密钥/算法指纹,不同密钥/算法不串用"""
|
||||||
|
return "jwt:" + hashlib.sha256(f"{secret}:{algorithm}:{token}".encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _cached_decode(token: str, secret: str, algorithm: str) -> tuple:
|
||||||
|
"""JWT 解密结果缓存((payload, exp)):一级进程内 LRU,二级共享缓存。
|
||||||
|
|
||||||
|
- 一级命中且未过期 -> 直接返回(零网络、零密码学);过期 -> 直接拒绝
|
||||||
|
- 一级未命中 -> 查二级共享缓存;命中且未过期 -> 回填一级后返回
|
||||||
|
- 二级也未命中 -> 完整解码(RSA/ECDSA 验证)后写二级 + 一级(TTL=60s/30s)
|
||||||
|
- decode 失败(无效 token)不缓存,每次都完整验证后拒绝
|
||||||
|
"""
|
||||||
|
key = _cache_key(token, secret, algorithm)
|
||||||
|
now = time.time()
|
||||||
|
if _JWT_L1_ENABLED:
|
||||||
|
hit = _l1_get(key)
|
||||||
|
if hit is not None:
|
||||||
|
payload_json, exp, _ts = hit
|
||||||
|
if not exp or exp >= now:
|
||||||
|
# 反序列化出新副本:调用方修改不影响缓存条目(与 L2 契约一致)
|
||||||
|
return orjson.loads(payload_json), exp
|
||||||
|
raise ValueError("Token 已过期")
|
||||||
|
if len(_jwt_l1) > _JWT_L1_MAX // 2: # 仅大表时惰性清理,避免常态 O(n)
|
||||||
|
_l1_sweep(now)
|
||||||
|
|
||||||
|
entry = _jwt_cache.get(key)
|
||||||
|
if entry is not None:
|
||||||
|
payload = entry["payload"]
|
||||||
|
exp = entry.get("exp", 0)
|
||||||
|
if not exp or exp >= time.time():
|
||||||
|
if _JWT_L1_ENABLED:
|
||||||
|
_l1_set(key, orjson.dumps(payload), exp)
|
||||||
|
return payload, exp
|
||||||
|
raise ValueError("Token 已过期")
|
||||||
|
jwt = JWT(secret, algorithm)
|
||||||
|
payload_str = jwt.decode(token)
|
||||||
|
payload = orjson.loads(payload_str.encode("utf-8"))
|
||||||
|
exp = payload.get("exp", 0)
|
||||||
|
_jwt_cache.set(key, {"payload": payload, "exp": exp}, expire=_JWT_CACHE_TTL)
|
||||||
|
if _JWT_L1_ENABLED:
|
||||||
|
_l1_set(key, orjson.dumps(payload), exp)
|
||||||
|
return payload, exp
|
||||||
|
|
||||||
|
|
||||||
|
def jwt_auth_dependency(
|
||||||
|
credentials: HTTPAuthorizationCredentials = Depends(security_scheme)
|
||||||
|
) -> dict:
|
||||||
|
"""JWT 校验依赖,返回解码后的 payload,校验失败抛出 401"""
|
||||||
|
config = get_config()
|
||||||
|
secret = config["security"]["secret_key"]
|
||||||
|
algorithm = config["security"]["algorithm"]
|
||||||
|
|
||||||
|
if not credentials:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||||
|
detail="Missing authorization header",
|
||||||
|
headers={"WWW-Authenticate": "Bearer"},
|
||||||
|
)
|
||||||
|
token = credentials.credentials
|
||||||
|
try:
|
||||||
|
payload, _ = _cached_decode(token, secret, algorithm)
|
||||||
|
# 缓存后端返回的是独立对象(JSON 反序列化后为新副本),调用方修改不影响缓存
|
||||||
|
return payload
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||||
|
detail=f"Invalid token: {str(e)}",
|
||||||
|
headers={"WWW-Authenticate": "Bearer"},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def store_user_to_request(
|
||||||
|
request: Request,
|
||||||
|
user: dict = Depends(jwt_auth_dependency)
|
||||||
|
) -> None:
|
||||||
|
request.state.user = user
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
"""
|
||||||
|
调用统计与审计日志(后台任务执行)
|
||||||
|
|
||||||
|
route_generator 端点在 call_raw 返回后仅做非关键后置操作,全部经
|
||||||
|
spawn() 以 fire-and-forget 后台任务执行,端点立即返回,不增加接口
|
||||||
|
响应延迟、不占用请求协程资源:
|
||||||
|
1. 更新内存调用统计(per worker_type/subject/status 计数 + 延迟聚合 + 最近调用缓冲)
|
||||||
|
2. 写审计日志(logs/audit-YYYY-MM-DD.log,复用 logger 按日切分 + 大小轮转设施)
|
||||||
|
|
||||||
|
record_call 内部全兜底:统计或审计任一步失败仅记 warning,绝不影响响应。
|
||||||
|
统计为进程内视角(gunicorn 多 worker 下各进程独立聚合)。
|
||||||
|
|
||||||
|
后台任务随应用生命周期管理:lifespan 关闭阶段调用 shutdown() 统一取消。
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import datetime
|
||||||
|
import orjson
|
||||||
|
import random
|
||||||
|
import time
|
||||||
|
from collections import deque
|
||||||
|
from typing import Any, Coroutine, Dict
|
||||||
|
|
||||||
|
from chongming_config import get_config
|
||||||
|
|
||||||
|
from .logger import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("call_stats")
|
||||||
|
# 审计日志独立文件(audit-YYYY-MM-DD.log),与业务日志分离便于检索
|
||||||
|
audit_logger = get_logger("audit")
|
||||||
|
|
||||||
|
# 最近调用环形缓冲上限
|
||||||
|
MAX_RECENT = 1000
|
||||||
|
|
||||||
|
# 审计日志采样率:与 access 日志同源配置(access_sample_rate,0.1 = 只写 10% 的成功请求;
|
||||||
|
# 错误 status >= 400 始终全量)。压测实测全量审计写盘 8.8MB/4.3 万行,是高吞吐下的隐藏 I/O 大头。
|
||||||
|
_audit_config = get_config()
|
||||||
|
_audit_env = _audit_config["default"]["env"]
|
||||||
|
_AUDIT_SAMPLE_RATE = float(
|
||||||
|
_audit_config[_audit_env].get("logging", {}).get("access_sample_rate", 1.0)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _should_record_audit(status: int) -> bool:
|
||||||
|
"""审计日志采样判定(与 access 日志同模式)"""
|
||||||
|
if status >= 400 or _AUDIT_SAMPLE_RATE >= 1.0:
|
||||||
|
return True
|
||||||
|
denom = max(1, int(round(1.0 / _AUDIT_SAMPLE_RATE)))
|
||||||
|
return random.randrange(denom) == 0
|
||||||
|
|
||||||
|
# 后台任务集合(fire-and-forget,shutdown 时统一取消)
|
||||||
|
_tasks: set[asyncio.Task] = set()
|
||||||
|
|
||||||
|
|
||||||
|
def spawn(coro: Coroutine) -> asyncio.Task:
|
||||||
|
"""以 fire-and-forget 方式创建后台任务并纳入跟踪(shutdown 时取消)"""
|
||||||
|
task = asyncio.create_task(coro)
|
||||||
|
_tasks.add(task)
|
||||||
|
task.add_done_callback(_tasks.discard)
|
||||||
|
return task
|
||||||
|
|
||||||
|
|
||||||
|
async def shutdown() -> None:
|
||||||
|
"""取消所有后台统计/审计任务(应用关闭时调用)"""
|
||||||
|
tasks = list(_tasks)
|
||||||
|
if not tasks:
|
||||||
|
return
|
||||||
|
for task in tasks:
|
||||||
|
task.cancel()
|
||||||
|
await asyncio.gather(*tasks, return_exceptions=True)
|
||||||
|
|
||||||
|
|
||||||
|
class CallStats:
|
||||||
|
"""内存调用统计:计数 + 延迟聚合 + 最近调用环形缓冲。
|
||||||
|
|
||||||
|
asyncio 单线程事件循环内访问,无需加锁。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, max_recent: int = MAX_RECENT):
|
||||||
|
self._counts: Dict[tuple, int] = {}
|
||||||
|
self._latency: Dict[str, Dict[str, float]] = {}
|
||||||
|
self._recent: deque = deque(maxlen=max_recent)
|
||||||
|
|
||||||
|
def record(self, entry: Dict[str, Any]) -> None:
|
||||||
|
key = (entry["worker_type"], entry["subject"], entry["status"])
|
||||||
|
self._counts[key] = self._counts.get(key, 0) + 1
|
||||||
|
|
||||||
|
wt = entry["worker_type"]
|
||||||
|
lat = self._latency.setdefault(wt, {"total": 0.0, "count": 0, "max": 0.0})
|
||||||
|
dur = entry.get("duration_ms", 0.0)
|
||||||
|
lat["total"] += dur
|
||||||
|
lat["count"] += 1
|
||||||
|
if dur > lat["max"]:
|
||||||
|
lat["max"] = dur
|
||||||
|
|
||||||
|
self._recent.append(entry)
|
||||||
|
|
||||||
|
def summary(self) -> Dict[str, Any]:
|
||||||
|
"""聚合快照:总量、per worker/subject/status 计数、延迟、最近调用"""
|
||||||
|
calls = [
|
||||||
|
{"worker_type": wt, "subject": subj, "status": status, "count": count}
|
||||||
|
for (wt, subj, status), count in sorted(self._counts.items())
|
||||||
|
]
|
||||||
|
latency = [
|
||||||
|
{
|
||||||
|
"worker_type": wt,
|
||||||
|
"count": lat["count"],
|
||||||
|
"avg_ms": round(lat["total"] / lat["count"], 2) if lat["count"] else 0.0,
|
||||||
|
"max_ms": round(lat["max"], 2),
|
||||||
|
}
|
||||||
|
for wt, lat in sorted(self._latency.items())
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"total_calls": sum(self._counts.values()),
|
||||||
|
"calls": calls,
|
||||||
|
"latency": latency,
|
||||||
|
"recent": list(self._recent)[-50:],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_stats = CallStats()
|
||||||
|
|
||||||
|
|
||||||
|
def summary() -> Dict[str, Any]:
|
||||||
|
"""全局统计快照(供 GET /stats 端点使用)"""
|
||||||
|
return _stats.summary()
|
||||||
|
|
||||||
|
|
||||||
|
def _format_audit(entry: Dict[str, Any]) -> str:
|
||||||
|
"""审计日志单行 key=value 格式,便于 grep 检索"""
|
||||||
|
ts = entry.get("ts") or time.time()
|
||||||
|
ts_str = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]
|
||||||
|
user = entry.get("user")
|
||||||
|
if isinstance(user, dict):
|
||||||
|
# JWT payload(auth.py store_user_to_request 注入),orjson 默认紧凑单行输出
|
||||||
|
user_str = orjson.dumps(user).decode()
|
||||||
|
elif isinstance(user, str):
|
||||||
|
user_str = user
|
||||||
|
else:
|
||||||
|
user_str = "-"
|
||||||
|
parts = [
|
||||||
|
"AUDIT",
|
||||||
|
f"ts={ts_str}",
|
||||||
|
f"worker={entry.get('worker_type', '-')}",
|
||||||
|
f"subject={entry.get('subject', '-')}",
|
||||||
|
f"method={entry.get('method', '-')}",
|
||||||
|
f"path={entry.get('path', '-')}",
|
||||||
|
f"user={user_str}",
|
||||||
|
f"client={entry.get('client') or '-'}",
|
||||||
|
f"status={entry.get('status', '-')}",
|
||||||
|
f"duration_ms={entry.get('duration_ms', 0):.3f}",
|
||||||
|
f"req_bytes={entry.get('req_bytes', 0)}",
|
||||||
|
f"resp_bytes={entry.get('resp_bytes', 0)}",
|
||||||
|
]
|
||||||
|
return " ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
async def record_call(entry: Dict[str, Any]) -> None:
|
||||||
|
"""后台执行体:更新内存统计 + 写审计日志。
|
||||||
|
|
||||||
|
整体兜底:任一步失败仅记 warning,绝不向端点冒泡。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
_stats.record(entry)
|
||||||
|
except Exception as e: # pragma: no cover - 防御性兜底
|
||||||
|
logger.warning("调用统计记录失败: %s", e)
|
||||||
|
try:
|
||||||
|
if _should_record_audit(entry.get("status", 200)):
|
||||||
|
audit_logger.info(_format_audit(entry))
|
||||||
|
except Exception as e: # pragma: no cover - 防御性兜底
|
||||||
|
logger.warning("审计日志写入失败: %s", e)
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""
|
||||||
|
启动契约校验(网关 inline 模式)
|
||||||
|
|
||||||
|
校验逻辑的唯一实现在 chongming_worker.contracts(网关与 worker 进程共享),
|
||||||
|
本模块只做网关侧的编排:对 inline 插件批量校验、日志汇总、硬失败终止启动。
|
||||||
|
|
||||||
|
external 插件在自己进程内做同样校验(validate_worker_contracts),网关侧跳过。
|
||||||
|
"""
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from chongming_worker import get_all_rpc_methods
|
||||||
|
from chongming_worker.contracts import check_plugin, load_plugin_models
|
||||||
|
from chongming_worker.route_config import load_worker_routes_from_dir
|
||||||
|
|
||||||
|
from .logger import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("contracts")
|
||||||
|
|
||||||
|
|
||||||
|
def validate_contracts(plugins: List[Dict[str, Any]]) -> None:
|
||||||
|
"""
|
||||||
|
对每个发现的 inline 插件执行三方一致性校验。
|
||||||
|
|
||||||
|
挂载于 lifespan 的 load_workers 与 register_routes 之间,此时插件模块已导入
|
||||||
|
(@rpc_method 注册表就绪)且 TOML 路由尚未注册到 FastAPI。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
plugins: _discover_plugins() 返回的 inline 插件配置列表
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
RuntimeError: 存在硬失败项(HTTP 路由缺失 @rpc_method 注册)时
|
||||||
|
"""
|
||||||
|
registry = get_all_rpc_methods()
|
||||||
|
hard_errors: List[str] = []
|
||||||
|
warnings: List[str] = []
|
||||||
|
|
||||||
|
for wc in plugins:
|
||||||
|
worker_type = wc.get("type", "")
|
||||||
|
if not worker_type:
|
||||||
|
continue
|
||||||
|
|
||||||
|
toml_routes = load_worker_routes_from_dir(worker_type)
|
||||||
|
registered = registry.get(worker_type, {})
|
||||||
|
model_names = load_plugin_models(worker_type)
|
||||||
|
hard, warns = check_plugin(worker_type, toml_routes, registered, model_names)
|
||||||
|
hard_errors.extend(hard)
|
||||||
|
warnings.extend(warns)
|
||||||
|
|
||||||
|
# 输出结果
|
||||||
|
if warnings:
|
||||||
|
logger.warning("契约校验发现 %d 个警告:", len(warnings))
|
||||||
|
for w in warnings:
|
||||||
|
logger.warning(" ⚠ %s", w)
|
||||||
|
|
||||||
|
if hard_errors:
|
||||||
|
logger.error("契约校验失败,发现 %d 个硬性错误:", len(hard_errors))
|
||||||
|
for e in hard_errors:
|
||||||
|
logger.error(" ✗ %s", e)
|
||||||
|
raise RuntimeError(
|
||||||
|
"启动契约校验失败: "
|
||||||
|
+ "; ".join(hard_errors)
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("契约校验通过(TOML 路由 ↔ @rpc_method 注册表一致)")
|
||||||
@@ -0,0 +1,915 @@
|
|||||||
|
import os
|
||||||
|
import inspect
|
||||||
|
import time
|
||||||
|
import signal
|
||||||
|
import tomllib
|
||||||
|
import glob
|
||||||
|
import importlib
|
||||||
|
import multiprocessing
|
||||||
|
import asyncio
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from chongming_cache import Cache, DistributedLock
|
||||||
|
from fastapi import FastAPI
|
||||||
|
import zenoh
|
||||||
|
|
||||||
|
from chongming_config import get_config, get_workers_config
|
||||||
|
from ..core.logger import get_logger
|
||||||
|
from ..core.parent_death import guard_parent_death
|
||||||
|
from ..core.route_generator import register_dynamic_routes
|
||||||
|
from ..core.worker_manager import WorkerManager
|
||||||
|
from ..core.mqtt_bridge import MQTTBridge
|
||||||
|
|
||||||
|
config = get_config()
|
||||||
|
logger = get_logger("app")
|
||||||
|
|
||||||
|
# 挂载框架/插件父 logger 的 handler:logger 模块只为具名 logger 挂 handler,
|
||||||
|
# 子 logger(chongming_worker.runtime、plugins.doubao 等)经 propagate 到父级才可见
|
||||||
|
get_logger("chongming_worker")
|
||||||
|
get_logger("plugins")
|
||||||
|
|
||||||
|
try:
|
||||||
|
env = config["default"]["env"]
|
||||||
|
except KeyError: # pragma: no cover
|
||||||
|
raise ValueError("配置不存在")
|
||||||
|
|
||||||
|
def is_port_in_use(port: int, host: str = "127.0.0.1") -> bool:
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||||
|
s.settimeout(0.5)
|
||||||
|
return s.connect_ex((host, port)) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def _broker_lock_key() -> str:
|
||||||
|
"""内嵌 Broker 排他锁 key。内嵌 Broker 绑定本机端口,故按宿主隔离
|
||||||
|
(多实例共享同一 cache/redis 时,各宿主仍各起各的 Broker)。"""
|
||||||
|
return f"mqtt_broker:{socket.gethostname()}"
|
||||||
|
|
||||||
|
|
||||||
|
def _broker_owner_key() -> str:
|
||||||
|
"""记录内嵌 Broker 子进程实际 PID 的缓存 key(按宿主隔离)。
|
||||||
|
|
||||||
|
持有者在启动 Broker 子进程后登记其 PID;前任持有者崩溃时,接管的进程据此
|
||||||
|
识别并终止孤儿 Broker 子进程,从而真正掌握其生命周期。"""
|
||||||
|
return f"mqtt_broker_owner:{socket.gethostname()}"
|
||||||
|
|
||||||
|
|
||||||
|
def _pid_alive(pid) -> bool:
|
||||||
|
try:
|
||||||
|
os.kill(int(pid), 0)
|
||||||
|
return True
|
||||||
|
except (ProcessLookupError, OSError, ValueError, TypeError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _terminate_pid(pid):
|
||||||
|
"""尽力终止进程:先 SIGTERM,宽限后仍存活则 SIGKILL。"""
|
||||||
|
try:
|
||||||
|
os.kill(int(pid), signal.SIGTERM)
|
||||||
|
except (ProcessLookupError, OSError, ValueError, TypeError):
|
||||||
|
return
|
||||||
|
for _ in range(20):
|
||||||
|
if not _pid_alive(pid):
|
||||||
|
return
|
||||||
|
time.sleep(0.1)
|
||||||
|
try:
|
||||||
|
os.kill(int(pid), signal.SIGKILL)
|
||||||
|
except (ProcessLookupError, OSError, ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_port_free(port: int, host: str = "127.0.0.1", timeout: float = 3.0):
|
||||||
|
end = time.time() + timeout
|
||||||
|
while time.time() < end and is_port_in_use(port, host):
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_mqtt_broker(host: str, port: int, username: str | None = None,
|
||||||
|
password: str | None = None):
|
||||||
|
"""子进程入口:启动 amqtt Broker(配置了账号密码时启用认证)"""
|
||||||
|
# 父进程死亡防护:宿主 worker 死亡(含被 SIGKILL)时本子进程收 SIGTERM 退出,
|
||||||
|
# 不留孤儿——即便漏网,下次启动的孤儿接管逻辑也会兜底(见 setup_mqtt_broker)
|
||||||
|
guard_parent_death()
|
||||||
|
import asyncio as _asyncio
|
||||||
|
from amqtt.broker import Broker
|
||||||
|
|
||||||
|
# amqtt 在 broker 进程内无 handler,WARNING 会经 lastResort 直接刷到 stderr:
|
||||||
|
# - sys_interval=0(禁用 $SYS)被 amqtt 当 falsy 报警
|
||||||
|
# - 客户端连上却不发 CONNECT 时每次刷 2 行 "No data from"/"Failed to initialize..."
|
||||||
|
# 统一压到 ERROR:认证失败本身是 DEBUG 级不受影响,真实故障仍会输出
|
||||||
|
import logging as _logging
|
||||||
|
|
||||||
|
_logging.getLogger("amqtt").setLevel(_logging.ERROR)
|
||||||
|
|
||||||
|
# ---- monkey-patch 1: amqtt StreamWriterAdapter.close() 忽略断开的传输层 ----
|
||||||
|
from amqtt.adapters import StreamWriterAdapter as _SWA
|
||||||
|
|
||||||
|
async def _patched_close(self):
|
||||||
|
if self.is_closed:
|
||||||
|
return
|
||||||
|
self.is_closed = True
|
||||||
|
try:
|
||||||
|
await self._writer.drain()
|
||||||
|
except (OSError, ConnectionError):
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
if self._writer.can_write_eof():
|
||||||
|
self._writer.write_eof()
|
||||||
|
except OSError:
|
||||||
|
pass # 传输层已断开 (Errno 107)
|
||||||
|
try:
|
||||||
|
self._writer.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
with __import__("contextlib").suppress(AttributeError):
|
||||||
|
await self._writer.wait_closed()
|
||||||
|
|
||||||
|
_SWA.close = _patched_close
|
||||||
|
|
||||||
|
# ---- monkey-patch 2: 协议版本不匹配时不打 traceback(仅 warning) ----
|
||||||
|
import amqtt.errors as _amqtt_err
|
||||||
|
from amqtt.utils import format_client_message as _fmt_client, gen_client_id as _gen_cid
|
||||||
|
from amqtt.broker import Broker as _Broker
|
||||||
|
from amqtt.mqtt.protocol.broker_handler import BrokerProtocolHandler as _BPH
|
||||||
|
|
||||||
|
async def _patched_init_client(self, reader, writer, remote_address, remote_port):
|
||||||
|
"""与原始方法逻辑一致,但 MQTTError 使用 warning 而非 exception"""
|
||||||
|
try:
|
||||||
|
handler, client_session = await _BPH.init_from_connect(
|
||||||
|
reader, writer, self.plugins_manager
|
||||||
|
)
|
||||||
|
except _amqtt_err.AMQTTError as exc:
|
||||||
|
self.logger.warning(
|
||||||
|
"[MQTT-3.1.0-1] %s: Can't read first packet as CONNECT: %s",
|
||||||
|
_fmt_client(address=remote_address, port=remote_port),
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
except _amqtt_err.MQTTError as exc:
|
||||||
|
self.logger.warning(
|
||||||
|
"%s (client 可能使用 MQTT 5.0,amqtt 仅支持 3.1.1)",
|
||||||
|
str(exc),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
await writer.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
except _amqtt_err.NoDataError as exc:
|
||||||
|
self.logger.warning(
|
||||||
|
"No data from %s: %s",
|
||||||
|
_fmt_client(address=remote_address, port=remote_port),
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
raise _amqtt_err.AMQTTError(exc) from exc
|
||||||
|
|
||||||
|
if client_session.clean_session:
|
||||||
|
if client_session.client_id is not None and client_session.client_id != "":
|
||||||
|
await self._delete_session(client_session.client_id)
|
||||||
|
else:
|
||||||
|
client_session.client_id = _gen_cid()
|
||||||
|
|
||||||
|
return handler, client_session
|
||||||
|
|
||||||
|
_Broker._initialize_client_session = _patched_init_client
|
||||||
|
|
||||||
|
async def _serve():
|
||||||
|
# 认证插件:配置了账号密码 → FileAuthPlugin(argon2 密码文件,仅此账号可连);
|
||||||
|
# 未配置 → 匿名(默认行为)
|
||||||
|
auth_plugins = {}
|
||||||
|
passwd_file = None
|
||||||
|
if username and password:
|
||||||
|
import tempfile
|
||||||
|
from pwdlib import PasswordHash
|
||||||
|
|
||||||
|
try:
|
||||||
|
fd, passwd_file = tempfile.mkstemp(prefix="chongming_mqtt_", suffix=".passwd")
|
||||||
|
with os.fdopen(fd, "w") as f:
|
||||||
|
f.write(f"{username}:{PasswordHash.recommended().hash(password)}\n")
|
||||||
|
auth_plugins = {
|
||||||
|
"amqtt.plugins.authentication.FileAuthPlugin": {
|
||||||
|
"password_file": passwd_file,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"MQTT 认证初始化失败,回退匿名模式: {e}")
|
||||||
|
if passwd_file:
|
||||||
|
os.unlink(passwd_file)
|
||||||
|
passwd_file = None
|
||||||
|
else:
|
||||||
|
auth_plugins = {
|
||||||
|
"amqtt.plugins.authentication.AnonymousAuthPlugin": {"allow_anonymous": True},
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg = {
|
||||||
|
"listeners": {
|
||||||
|
"default": {
|
||||||
|
"type": "tcp", "bind": f"{host}:{port}", "max_connections": 500,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"timeout_disconnect_delay": 2,
|
||||||
|
"plugins": {
|
||||||
|
**auth_plugins,
|
||||||
|
"amqtt.plugins.sys.broker.BrokerSysPlugin": {"sys_interval": 0},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
broker = Broker(cfg)
|
||||||
|
try:
|
||||||
|
await broker.start()
|
||||||
|
except Exception:
|
||||||
|
# 端口已被其他 worker 的 Broker 占用,静默退出
|
||||||
|
return
|
||||||
|
finally:
|
||||||
|
# 密码文件仅在启动时读取,启动完成后即可删除
|
||||||
|
if passwd_file:
|
||||||
|
try:
|
||||||
|
os.unlink(passwd_file)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
auth_note = f" (认证用户: {username})" if username else " (匿名)"
|
||||||
|
logger.info("MQTT Broker 已启动 @ %s:%s%s", host, port, auth_note)
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
await _asyncio.sleep(1)
|
||||||
|
except (KeyboardInterrupt, _asyncio.CancelledError):
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
await broker.shutdown()
|
||||||
|
|
||||||
|
try:
|
||||||
|
_asyncio.run(_serve())
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def setup_storage(app: FastAPI):
|
||||||
|
"""初始化本地文件存储(替代 Nanio S3,省 ~172 MB)"""
|
||||||
|
upload_root = os.path.join(os.getcwd(), "data", "uploads")
|
||||||
|
os.makedirs(upload_root, exist_ok=True)
|
||||||
|
app.state.upload_root = upload_root
|
||||||
|
logger.info(f"本地存储已就绪: {upload_root}")
|
||||||
|
|
||||||
|
|
||||||
|
async def setup_auth_store(app: FastAPI) -> None:
|
||||||
|
"""初始化内置 auth 库(users 表 + admin 种子)—— auth 域的基础设施部分。
|
||||||
|
|
||||||
|
拆分自 plugins.auth:登录/签发内置在网关(管理平面底线),auth 插件
|
||||||
|
缩容到 0 或全部副本故障时登录与 admin API 仍可用。库位置与 auth 插件
|
||||||
|
读同一份配置(config/workers/auth/config.toml 的 [workers.config]:
|
||||||
|
data_dir / database),指向同一张 users 表;插件配置缺失(如彻底移除
|
||||||
|
auth 插件)时回退默认 ./data/auth.db。
|
||||||
|
"""
|
||||||
|
from chongming_auth import open_database, seed_admin
|
||||||
|
|
||||||
|
workers_conf = {}
|
||||||
|
auth_conf_path = "config/workers/auth/config.toml"
|
||||||
|
if os.path.isfile(auth_conf_path):
|
||||||
|
try:
|
||||||
|
with open(auth_conf_path, "rb") as f:
|
||||||
|
workers_conf = tomllib.load(f).get("workers", {}).get("config", {})
|
||||||
|
except Exception as e: # noqa: BLE001 - 配置损坏不阻断启动,走默认路径
|
||||||
|
logger.warning(f"无法读取 auth 插件配置 {auth_conf_path}: {e}(使用默认 data_dir)")
|
||||||
|
|
||||||
|
db = await open_database(workers_conf)
|
||||||
|
created = await seed_admin(db)
|
||||||
|
app.state.auth_db = db
|
||||||
|
logger.info(f"内置 auth 库已就绪 (admin 种子: {'新建' if created else '已存在'})")
|
||||||
|
|
||||||
|
|
||||||
|
def setup_zenoh(app: FastAPI) -> zenoh.Session:
|
||||||
|
"""建立 Zenoh 会话并存入 app.state。"""
|
||||||
|
zenoh_config = config.get("zenoh", {})
|
||||||
|
|
||||||
|
zconfig = zenoh.Config()
|
||||||
|
zconfig.from_file(zenoh_config.get("json5_path", "config/zenoh_config.json5"))
|
||||||
|
|
||||||
|
try:
|
||||||
|
session = zenoh.open(zconfig)
|
||||||
|
logger.info("Zenoh 会话已建立 (TCP 自连接模式)")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Zenoh 会话建立失败: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
app.state.zenoh_session = session
|
||||||
|
return session
|
||||||
|
|
||||||
|
def setup_worker_manager(app: FastAPI, session: zenoh.Session) -> WorkerManager:
|
||||||
|
"""创建 WorkerManager,启动定期刷新任务,并存入 app.state。"""
|
||||||
|
cache = Cache.from_config(config["cache"])
|
||||||
|
launch_worker_count = str(
|
||||||
|
config["cache"].get("launch_worker_count", "toml")
|
||||||
|
).lower()
|
||||||
|
manager = WorkerManager(session, cache, launch_worker_count=launch_worker_count)
|
||||||
|
app.state.worker_manager = manager
|
||||||
|
|
||||||
|
refresh_task = asyncio.create_task(manager.refresh_workers())
|
||||||
|
app.state.refresh_task = refresh_task
|
||||||
|
return manager
|
||||||
|
|
||||||
|
async def setup_route_registry(
|
||||||
|
app: FastAPI,
|
||||||
|
session: zenoh.Session,
|
||||||
|
manager: WorkerManager,
|
||||||
|
inline_types: list[str] | None = None,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
启动动态路由注册表(常驻订阅 chongming/routes/**)。
|
||||||
|
|
||||||
|
网关启动即订阅,不依赖网关侧是否登记 external 插件配置:external 插件
|
||||||
|
以独立进程运行(由 Docker/K8s 等编排拉起,网关不负责进程管理),其路由
|
||||||
|
描述符经 Zenoh 动态注册;inline 插件仍在网关进程内加载,两者同时存在
|
||||||
|
(混合模式)。盘扫发现的 external 插件条目仅用于启动日志观测。
|
||||||
|
|
||||||
|
inline_types: 网关进程内加载的插件类型,注册表忽略其描述符以防重复挂载。
|
||||||
|
|
||||||
|
注意:external 插件的自定义 APIRouter(如 multipart 上传端点)无法由
|
||||||
|
网关发现(网关不 import 该插件),此类端点需保持 inline 或等 P2 专门通道。
|
||||||
|
"""
|
||||||
|
from ..core.route_registry import RouteRegistry
|
||||||
|
|
||||||
|
prefix = config["default"]["prefix"]
|
||||||
|
registry = RouteRegistry(app, manager, prefix=prefix)
|
||||||
|
if inline_types:
|
||||||
|
registry.set_inline_types(set(inline_types))
|
||||||
|
await registry.start(session)
|
||||||
|
app.state.route_registry = registry
|
||||||
|
logger.info(
|
||||||
|
f"路由注册表已启动(prefix={prefix}, 外部插件路由动态注册, "
|
||||||
|
f"inline 插件: {sorted(inline_types or []) or '无'})"
|
||||||
|
)
|
||||||
|
return registry
|
||||||
|
|
||||||
|
|
||||||
|
def _discover_plugins() -> list[dict]:
|
||||||
|
"""
|
||||||
|
发现所有插件配置。
|
||||||
|
|
||||||
|
优先从各插件的独立配置文件 config/workers/<type>/config.toml 的 [workers] 节读取,
|
||||||
|
兼容旧的 config/workers/config.toml 全局配置。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
[{"type": "doubao", "module": "plugins.doubao", "factory": "create_workers",
|
||||||
|
"cleanup": "cleanup", "count": 3, "config": {...}}, ...]
|
||||||
|
"""
|
||||||
|
plugins = []
|
||||||
|
|
||||||
|
# 1. 自动扫描 config/workers/*/config.toml
|
||||||
|
for path in sorted(glob.glob("config/workers/*/config.toml")):
|
||||||
|
plugin_type = os.path.basename(os.path.dirname(path))
|
||||||
|
try:
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
plugin_conf = tomllib.load(f)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("无法读取插件配置 %s: %s", path, e)
|
||||||
|
continue
|
||||||
|
|
||||||
|
workers_section = plugin_conf.get("workers", {})
|
||||||
|
if not workers_section:
|
||||||
|
continue
|
||||||
|
|
||||||
|
plugin_type = workers_section.get("type", plugin_type)
|
||||||
|
# 插件运行模式(按插件粒度,混合共存):
|
||||||
|
# "inline"(默认) = 网关进程内加载
|
||||||
|
# "external" = 独立进程运行(Docker/K8s 编排),经 Zenoh 注册路由
|
||||||
|
load_mode = workers_section.get("load_mode", "inline")
|
||||||
|
# 插件声明的 MQTT Bridge 额外订阅主题([[workers.mqtt_topics]])
|
||||||
|
# 网关级路由配置,不传给插件运行时;worker 目标即本插件类型。
|
||||||
|
mqtt_topics = []
|
||||||
|
for t in workers_section.get("mqtt_topics", []) or []:
|
||||||
|
if not isinstance(t, dict) or not t.get("topic"):
|
||||||
|
continue
|
||||||
|
t = dict(t)
|
||||||
|
t.setdefault("worker", plugin_type)
|
||||||
|
mqtt_topics.append(t)
|
||||||
|
|
||||||
|
# 合并运行时 config:
|
||||||
|
# 1. plugin_conf 顶层 (worker, worker.registration 等 gen_models 用)
|
||||||
|
# 2. [workers] 节的注册字段展平 (type, module, factory, cleanup, count)
|
||||||
|
# — 其中 type 被 chongming_worker.runtime.create_workers 用于匹配 RPC 方法
|
||||||
|
# 3. [workers.config] 展平到顶层 (api_key, data_dir, heartbeat 等)
|
||||||
|
workers_flat = {k: v for k, v in workers_section.items()
|
||||||
|
if k not in ("config", "mqtt_topics", "load_mode")}
|
||||||
|
runtime_config = {
|
||||||
|
**plugin_conf,
|
||||||
|
**workers_flat,
|
||||||
|
**workers_section.get("config", {}),
|
||||||
|
}
|
||||||
|
|
||||||
|
plugins.append({
|
||||||
|
"type": plugin_type,
|
||||||
|
"module": workers_section.get("module", f"plugins.{plugin_type}"),
|
||||||
|
"factory": workers_section.get("factory", "create_workers"),
|
||||||
|
"cleanup": workers_section.get("cleanup", "cleanup"),
|
||||||
|
"count": workers_section.get("count", 1),
|
||||||
|
"load_mode": load_mode,
|
||||||
|
"mqtt_topics": mqtt_topics,
|
||||||
|
"config": runtime_config,
|
||||||
|
# [workers.config] 展平前的原始段:供 load_workers 按「配置优先来源模式」
|
||||||
|
# 引导/覆盖共享缓存,使 inline 插件初始化即拿到缓存值(cache 模式)
|
||||||
|
"config_section": dict(workers_section.get("config") or {}),
|
||||||
|
})
|
||||||
|
|
||||||
|
# 2. 兼容旧写法: config/workers/config.toml(仅加载尚未发现的插件)
|
||||||
|
old_config_path = config.get("plugins_worker", {}).get("config_path", "config/workers/config.toml")
|
||||||
|
if os.path.isfile(old_config_path):
|
||||||
|
existing_types = {p["type"] for p in plugins}
|
||||||
|
old_conf = get_workers_config(old_config_path).get("workers", [])
|
||||||
|
for wc in old_conf:
|
||||||
|
wtype = wc.get("type", "")
|
||||||
|
if wtype and wtype not in existing_types:
|
||||||
|
plugins.append({
|
||||||
|
"type": wtype,
|
||||||
|
"module": wc.get("module", ""),
|
||||||
|
"factory": wc.get("factory", ""),
|
||||||
|
"cleanup": wc.get("cleanup", ""),
|
||||||
|
"count": wc.get("count", 1),
|
||||||
|
"config": wc.get("config", {}),
|
||||||
|
})
|
||||||
|
logger.info("从旧配置加载插件: %s (config/workers/config.toml)", wtype)
|
||||||
|
|
||||||
|
return plugins
|
||||||
|
|
||||||
|
|
||||||
|
def _import_plugin_module(module_name: str):
|
||||||
|
"""导入插件模块(`plugins.<type>` 约定)。
|
||||||
|
|
||||||
|
生产/打包环境里 plugins 包由模块银行(plugins.mbank)或宿主项目安装提供;
|
||||||
|
开发态宿主通常是 src 布局(cwd/src/plugins 就在仓库里),此时首次导入可能
|
||||||
|
还不认识 plugins——这里兜底把 cwd/src 加入 sys.path 重试一次。兜底在本进程
|
||||||
|
的 import 处执行(非启动器),多 worker 各自生效,不依赖 fork/spawn 继承。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return importlib.import_module(module_name)
|
||||||
|
except ImportError:
|
||||||
|
src = Path("src")
|
||||||
|
if module_name.split(".")[0] != "plugins" or not (src / "plugins").is_dir():
|
||||||
|
raise
|
||||||
|
src_resolved = str(src.resolve())
|
||||||
|
if src_resolved not in sys.path:
|
||||||
|
sys.path.insert(0, src_resolved)
|
||||||
|
logger.debug("开发态兜底:%s 加入 sys.path(宿主 src 布局)", src_resolved)
|
||||||
|
return importlib.import_module(module_name)
|
||||||
|
|
||||||
|
|
||||||
|
async def load_workers(app: FastAPI, session: zenoh.Session, manager: WorkerManager,
|
||||||
|
is_inline_owner: bool = True):
|
||||||
|
"""
|
||||||
|
加载 inline 插件,启动对应的 worker 任务(混合模式)。
|
||||||
|
|
||||||
|
自动从 config/workers/<type>/config.toml 发现插件;load_mode="external"
|
||||||
|
的插件不在网关内加载(由独立进程运行,其路由经常驻 RouteRegistry 动态
|
||||||
|
注册,无需网关侧保留插件配置,盘扫发现的 external 条目仅用于观测)。
|
||||||
|
|
||||||
|
同时自动发现插件模块中暴露的 APIRouter(约定:模块顶层名为 `router` 的
|
||||||
|
APIRouter 实例),收集后供 register_custom_routes() 统一注册。
|
||||||
|
|
||||||
|
is_inline_owner: 本进程是否为宿主级 inline worker 持有者。gunicorn 多 worker
|
||||||
|
下仅持有者进程真正拉起 inline worker(避免 N×count 冗余副本);非持有者进程
|
||||||
|
仍 import 插件模块以便注册自定义路由/契约校验,但不 create_workers,worker 经
|
||||||
|
Zenoh liveliness 被跨进程发现。单进程(uvicorn serve)时为 True,行为不变。
|
||||||
|
|
||||||
|
返回: (worker_tasks, custom_routers, inline_types, external_types)
|
||||||
|
"""
|
||||||
|
workers_conf = _discover_plugins()
|
||||||
|
inline_plugins = [wc for wc in workers_conf if wc.get("load_mode", "inline") != "external"]
|
||||||
|
external_plugins = [wc for wc in workers_conf if wc.get("load_mode", "inline") == "external"]
|
||||||
|
inline_types = sorted({wc["type"] for wc in inline_plugins})
|
||||||
|
external_types = sorted({wc["type"] for wc in external_plugins})
|
||||||
|
logger.info(
|
||||||
|
f"发现 {len(workers_conf)} 个 worker 插件: "
|
||||||
|
f"inline={inline_types or '无'}, external={external_types or '无'} "
|
||||||
|
f"(本进程 inline 持有者: {is_inline_owner})"
|
||||||
|
)
|
||||||
|
|
||||||
|
for wc in external_plugins:
|
||||||
|
logger.info(
|
||||||
|
"插件 %s 为 external 模式,跳过进程内加载(由独立进程经 Zenoh 注册路由)",
|
||||||
|
wc["type"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# 汇总 inline 插件声明的 MQTT Bridge 额外订阅主题,供 setup_mqtt 使用
|
||||||
|
# (external 插件的主题由 RouteRegistry 经 __meta__ 动态推送)
|
||||||
|
app.state.plugin_mqtt_topics = [
|
||||||
|
topic
|
||||||
|
for wc in inline_plugins
|
||||||
|
for topic in (wc.get("mqtt_topics") or [])
|
||||||
|
]
|
||||||
|
|
||||||
|
worker_tasks = []
|
||||||
|
custom_routers = []
|
||||||
|
for wc in inline_plugins:
|
||||||
|
module_name = wc["module"]
|
||||||
|
factory_name = wc["factory"]
|
||||||
|
# count 以 chongming-cache 为单一事实来源:cache 有值时用 cache 的全局总量,
|
||||||
|
# 否则用配置值引导并写回 cache(多网关共享同一 cache 时自动对齐)。
|
||||||
|
count = wc["count"]
|
||||||
|
worker_type = wc["type"]
|
||||||
|
# [workers.config] 单一事实来源:按配置优先来源模式对齐(cache 模式取缓存值、
|
||||||
|
# toml 模式以文件为准并覆盖缓存),必须在注册 starter/创建副本之前完成——
|
||||||
|
# 插件实例(on_init 重建客户端等)用的就是这份配置。
|
||||||
|
worker_config = manager.align_inline_config(
|
||||||
|
worker_type, wc["config"], wc.get("config_section")
|
||||||
|
)
|
||||||
|
|
||||||
|
if not module_name or not factory_name:
|
||||||
|
logger.warning(f"Worker 配置不完整: {wc}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.debug("加载插件 %s: config/workers/%s/config.toml (count=%d)",
|
||||||
|
worker_type, worker_type, count)
|
||||||
|
|
||||||
|
try:
|
||||||
|
module = _import_plugin_module(module_name)
|
||||||
|
|
||||||
|
# 自动发现插件的自定义 FastAPI 路由(约定:模块暴露 `router` 属性)
|
||||||
|
plugin_router = getattr(module, "router", None)
|
||||||
|
if plugin_router is not None:
|
||||||
|
from fastapi import APIRouter
|
||||||
|
if isinstance(plugin_router, APIRouter):
|
||||||
|
custom_routers.append(plugin_router)
|
||||||
|
logger.info(f"发现插件自定义路由: {module_name}.router")
|
||||||
|
|
||||||
|
# 非持有者进程:不立即拉起 inline worker,但仍在下方(build starter 后)
|
||||||
|
# 注册自愈启动器;worker 经 liveliness 被本进程发现调用,@rpc_method 与
|
||||||
|
# 自定义路由/契约校验依赖的模块 import 已在上文完成不依赖拉起。
|
||||||
|
factory = getattr(module, factory_name)
|
||||||
|
|
||||||
|
# 定义"拉起本类型 worker"的协程,供持有者崩溃后的自愈接管复用;
|
||||||
|
# 用默认参数捕获循环变量,避免闭包迟到绑定。
|
||||||
|
def _make_starter(session=session, config=worker_config, cnt=count,
|
||||||
|
mgr=manager, fac=factory):
|
||||||
|
async def starter():
|
||||||
|
if inspect.iscoroutinefunction(fac):
|
||||||
|
return await fac(session, config, cnt, worker_manager=mgr)
|
||||||
|
return fac(session, config, cnt, worker_manager=mgr)
|
||||||
|
return starter
|
||||||
|
|
||||||
|
starter = _make_starter()
|
||||||
|
manager.register_worker_starter(worker_type, starter)
|
||||||
|
|
||||||
|
# 非持有者进程:不拉起 inline worker(启动器已注册,持有者崩溃时由 refresh 自愈接管)
|
||||||
|
if not is_inline_owner:
|
||||||
|
logger.info(
|
||||||
|
"本进程非 %s 的 inline 持有者,跳过创建副本(liveliness 发现即可调用)",
|
||||||
|
worker_type,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
tasks = await starter()
|
||||||
|
worker_tasks.extend(tasks)
|
||||||
|
logger.info(f"已加载 Worker 插件: {module_name}.{factory_name} (副本数: {count})")
|
||||||
|
except Exception as e:
|
||||||
|
import traceback
|
||||||
|
logger.error(
|
||||||
|
f"加载 Worker 插件失败 {module_name}: {e}\n"
|
||||||
|
f" 配置: type={worker_type}, count={count}\n"
|
||||||
|
f" 堆栈:\n{traceback.format_exc()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
app.state.worker_tasks = worker_tasks
|
||||||
|
app.state.inline_types = inline_types
|
||||||
|
app.state.external_types = external_types
|
||||||
|
return worker_tasks, custom_routers, inline_types, external_types
|
||||||
|
|
||||||
|
async def register_routes(
|
||||||
|
app: FastAPI,
|
||||||
|
manager: WorkerManager,
|
||||||
|
custom_routers: list | None = None,
|
||||||
|
worker_types: list[str] | None = None,
|
||||||
|
):
|
||||||
|
"""注册动态路由和插件自定义路由。
|
||||||
|
|
||||||
|
worker_types: 仅注册这些插件类型的 TOML 动态路由(混合模式下只注册
|
||||||
|
inline 插件;external 插件路由由 RouteRegistry 动态挂载)。
|
||||||
|
"""
|
||||||
|
prefix = config["default"]["prefix"]
|
||||||
|
register_dynamic_routes(app, manager, prefix=prefix, worker_types=worker_types)
|
||||||
|
# 注册插件自定义路由(如 FORM-DATA 文件上传等 TOML 无法描述的端点)
|
||||||
|
# 统一加上全局 prefix(如 /api/v1),与 TOML 动态路由保持一致
|
||||||
|
if custom_routers:
|
||||||
|
for r in custom_routers:
|
||||||
|
app.include_router(r, prefix=prefix)
|
||||||
|
logger.info(f"已注册 {len(custom_routers)} 个插件自定义路由(prefix={prefix})")
|
||||||
|
|
||||||
|
|
||||||
|
def validate_contracts(app: FastAPI | None = None):
|
||||||
|
"""
|
||||||
|
契约校验(TOML 路由 ↔ @rpc_method 注册表 ↔ 生成模型),仅覆盖 inline 插件。
|
||||||
|
|
||||||
|
在 load_workers 之后、register_routes 之前调用:此时插件模块已导入
|
||||||
|
(注册表就绪)。HTTP 路由缺失对应 @rpc_method 时抛 RuntimeError 终止启动,
|
||||||
|
其余不一致以警告日志汇总。
|
||||||
|
|
||||||
|
external 插件在自己进程内做同样校验(validate_worker_contracts),网关侧跳过。
|
||||||
|
"""
|
||||||
|
from ..core.contracts import validate_contracts as _validate
|
||||||
|
|
||||||
|
plugins = _discover_plugins()
|
||||||
|
inline_plugins = [p for p in plugins if p.get("load_mode", "inline") != "external"]
|
||||||
|
_validate(inline_plugins)
|
||||||
|
|
||||||
|
async def setup_mqtt_broker(app: FastAPI, worker_manager: WorkerManager):
|
||||||
|
"""
|
||||||
|
启动内嵌 amqtt Broker 子进程(宿主级分布式锁确保多 worker/多实例仅本宿主启动一个)。
|
||||||
|
|
||||||
|
由配置 [<env>.mqtt].use_embedded_broker 控制:
|
||||||
|
true (默认)→ 网关启动 amqtt Broker 子进程,host/port 为监听地址
|
||||||
|
false → 使用外部 MQTT Broker,网关仅运行 Bridge 客户端,跳过本步
|
||||||
|
配置了 username/password 时 Broker 启用认证,否则保持匿名。
|
||||||
|
|
||||||
|
排他锁基于 chongming-cache 的 DistributedLock,key 按宿主隔离(内嵌 Broker 绑定
|
||||||
|
本机端口)。持有者在缓存登记其 Broker 子进程 PID;前任持有者崩溃/被强杀而留下
|
||||||
|
孤儿 Broker 子进程时,接管的进程借此 PID 识别并终止之,再启动自己托管的子进程,
|
||||||
|
从而真正掌握 Broker 生命周期(可关停、可接管),而非留下不可管控的残留下端点。
|
||||||
|
"""
|
||||||
|
env_config = config.get(env, {})
|
||||||
|
mqtt_config = env_config.get("mqtt", {})
|
||||||
|
if not mqtt_config.get("enabled", True):
|
||||||
|
logger.info("MQTT 已禁用")
|
||||||
|
return None
|
||||||
|
if not mqtt_config.get("use_embedded_broker", True):
|
||||||
|
logger.info(
|
||||||
|
f"use_embedded_broker=false:使用外部 MQTT Broker "
|
||||||
|
f"({mqtt_config.get('host', 'localhost')}:{mqtt_config.get('port', 1883)}),"
|
||||||
|
f"跳过内嵌 amqtt Broker 启动"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
if worker_manager.cache is None:
|
||||||
|
logger.warning(
|
||||||
|
"MQTT 内嵌 Broker 启用但未配置共享 cache([cache]),无法获取排他锁;"
|
||||||
|
"请配置 redis 或共享卷 sqlite,或改用外部 Broker"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
host = mqtt_config.get("host", "localhost")
|
||||||
|
port = mqtt_config.get("port", 1883)
|
||||||
|
username = mqtt_config.get("username") or None
|
||||||
|
password = mqtt_config.get("password") or None
|
||||||
|
|
||||||
|
lock = DistributedLock(worker_manager.cache, _broker_lock_key())
|
||||||
|
if not lock.acquire():
|
||||||
|
# 非持有者:等待持有者把 Broker 拉起并监听端口
|
||||||
|
app.state.mqtt_broker_lock_acquired = False
|
||||||
|
app.state.mqtt_broker_lock = None
|
||||||
|
for _ in range(50):
|
||||||
|
if is_port_in_use(port, host):
|
||||||
|
logger.info(f"MQTT Broker 已在 {host}:{port} 运行(由其他进程托管)")
|
||||||
|
return None
|
||||||
|
time.sleep(0.1)
|
||||||
|
logger.warning(f"等待内嵌 MQTT Broker 端口 {port} 就绪超时")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# 本进程赢得宿主锁:持有租约续期,取得 Broker 托管权
|
||||||
|
lock.start()
|
||||||
|
app.state.mqtt_broker_lock = lock
|
||||||
|
app.state.mqtt_broker_lock_acquired = True
|
||||||
|
|
||||||
|
# 端口已被监听:优先识别并终止前任残留的孤儿 Broker 子进程,以真正接管其
|
||||||
|
# 生命周期;否则视为不可托管的既有进程,复用(本进程不持有句柄)。
|
||||||
|
try:
|
||||||
|
prev_pid = int(worker_manager.cache.get(_broker_owner_key()) or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
prev_pid = None
|
||||||
|
if is_port_in_use(port, host):
|
||||||
|
if prev_pid and _pid_alive(prev_pid) and prev_pid != os.getpid():
|
||||||
|
logger.warning(
|
||||||
|
f"[mqtt-broker] 检出孤儿 Broker 子进程 PID {prev_pid},终止并接管重启"
|
||||||
|
)
|
||||||
|
_terminate_pid(prev_pid)
|
||||||
|
_wait_port_free(port, host)
|
||||||
|
if is_port_in_use(port, host):
|
||||||
|
logger.info(
|
||||||
|
f"MQTT Broker 端口 {port} 在终止孤儿后仍被占用,属不可托管进程,复用"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
logger.info(f"MQTT Broker 已在 {host}:{port} 运行,复用(不可托管)")
|
||||||
|
return None
|
||||||
|
|
||||||
|
proc = multiprocessing.Process(
|
||||||
|
target=_run_mqtt_broker, args=(host, port, username, password), daemon=False
|
||||||
|
)
|
||||||
|
proc.start()
|
||||||
|
app.state.mqtt_broker_process = proc
|
||||||
|
worker_manager.cache.set(_broker_owner_key(), str(proc.pid))
|
||||||
|
auth_note = f",认证用户 {username}" if username else "(匿名)"
|
||||||
|
logger.info(f"MQTT Broker 子进程已启动并登记托管 (PID: {proc.pid}){auth_note}")
|
||||||
|
for _ in range(50):
|
||||||
|
if is_port_in_use(port, host):
|
||||||
|
logger.info(f"MQTT Broker 已监听 {port} 端口")
|
||||||
|
return None
|
||||||
|
if not proc.is_alive():
|
||||||
|
raise RuntimeError("MQTT Broker 进程异常退出")
|
||||||
|
time.sleep(0.1)
|
||||||
|
raise RuntimeError("MQTT Broker 端口未就绪")
|
||||||
|
|
||||||
|
|
||||||
|
async def shutdown_mqtt_broker(app: FastAPI):
|
||||||
|
"""终止本进程托管的 MQTT Broker 子进程,并释放宿主级分布式排他锁、清理登记。"""
|
||||||
|
proc = getattr(app.state, "mqtt_broker_process", None)
|
||||||
|
if proc and proc.is_alive():
|
||||||
|
proc.terminate()
|
||||||
|
proc.join(timeout=5)
|
||||||
|
if proc.is_alive():
|
||||||
|
proc.kill()
|
||||||
|
logger.info("MQTT Broker 子进程已终止")
|
||||||
|
lock = getattr(app.state, "mqtt_broker_lock", None)
|
||||||
|
if lock is not None:
|
||||||
|
if proc: # 仅真正托管了子进程的进程才清理孤儿登记,避免误删他人登记
|
||||||
|
try:
|
||||||
|
lock._cache.delete(_broker_owner_key())
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.debug(f"清理 MQTT Broker 登记失败: {e}")
|
||||||
|
await lock.stop()
|
||||||
|
lock.release()
|
||||||
|
app.state.mqtt_broker_lock = None
|
||||||
|
app.state.mqtt_broker_lock_acquired = False
|
||||||
|
|
||||||
|
|
||||||
|
async def _start_mqtt(app: FastAPI, worker_manager: WorkerManager,
|
||||||
|
bridge_lock) -> MQTTBridge | None:
|
||||||
|
"""在已持有分布式锁的前提下构建并启动一个 MQTT Bridge 客户端。
|
||||||
|
|
||||||
|
该逻辑同时被 setup_mqtt(启动时)与 mqtt_bridge_supervisor(持有者自愈接管时)
|
||||||
|
复用。假定调用方已 acquire() 到 "mqtt_bridge" 分布式锁。
|
||||||
|
"""
|
||||||
|
env_config = config.get(env, {})
|
||||||
|
mqtt_config = env_config.get("mqtt", {})
|
||||||
|
host = mqtt_config.get("host", "localhost")
|
||||||
|
port = mqtt_config.get("port", 1883)
|
||||||
|
username = mqtt_config.get("username") or None
|
||||||
|
password = mqtt_config.get("password") or None
|
||||||
|
custom_topics = list(mqtt_config.get("custom_topics", []) or [])
|
||||||
|
custom_topics += list(getattr(app.state, "plugin_mqtt_topics", []) or [])
|
||||||
|
|
||||||
|
bridge = MQTTBridge(
|
||||||
|
worker_manager=worker_manager,
|
||||||
|
host=host,
|
||||||
|
port=port,
|
||||||
|
custom_topics=custom_topics,
|
||||||
|
username=username,
|
||||||
|
password=password,
|
||||||
|
)
|
||||||
|
app.state.mqtt_bridge = bridge
|
||||||
|
await bridge.start()
|
||||||
|
bridge_lock.start() # 后台续约租约,防止持有方宕机/长暂停导致锁被错误抢走
|
||||||
|
app.state.mqtt_bridge_lock = bridge_lock
|
||||||
|
# 把 Bridge 挂接给常驻路由注册表(动态主题 + 补推已到达的元信息)
|
||||||
|
registry = getattr(app.state, "route_registry", None)
|
||||||
|
if registry is not None:
|
||||||
|
registry.attach_bridge(bridge)
|
||||||
|
broker_mode = "内嵌 amqtt" if mqtt_config.get("use_embedded_broker", True) else "外部"
|
||||||
|
auth_note = f",认证用户 {username}" if username else "(匿名)"
|
||||||
|
logger.info(f"MQTT Bridge 已启动({broker_mode} Broker {host}:{port}{auth_note})")
|
||||||
|
return bridge
|
||||||
|
|
||||||
|
|
||||||
|
async def setup_mqtt(app: FastAPI, worker_manager: WorkerManager) -> MQTTBridge | None:
|
||||||
|
"""
|
||||||
|
启动 MQTT Bridge(纯客户端模式)。
|
||||||
|
|
||||||
|
从 config.toml 的环境段读取 host/port/enabled;若 enabled 为 false 则跳过。
|
||||||
|
Broker 来源由 use_embedded_broker 控制:
|
||||||
|
true (默认)= 内嵌 amqtt Broker(lifespan 第 3 步已启动子进程)
|
||||||
|
false = 外部 MQTT Broker(host/port 指向外部地址,网关不启动 Broker)
|
||||||
|
订阅主题 = 全局 [<env>.mqtt].custom_topics(可选兜底)
|
||||||
|
+ 各插件在其 config/workers/<type>/config.toml 中声明的 [[workers.mqtt_topics]]。
|
||||||
|
使用基于共享 cache 的分布式锁确保多进程/多网关中全局只有一个 Bridge 消费。
|
||||||
|
"""
|
||||||
|
env_config = config.get(env, {})
|
||||||
|
mqtt_config = env_config.get("mqtt", {})
|
||||||
|
if not mqtt_config.get("enabled", True):
|
||||||
|
logger.info("MQTT 已禁用")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# 排他锁:基于共享缓存(redis / 共享卷 diskcache),保证全局只有一个 Bridge 消费。
|
||||||
|
# 相比原文件锁,可跨多台主机收敛——多实例共享同一个外部 Broker 时,
|
||||||
|
# 避免一条 MQTT 请求被 N 台网关上各自消费、各回一份响应。
|
||||||
|
bridge_lock = DistributedLock(worker_manager.cache, "mqtt_bridge") # pyright: ignore[reportArgumentType]
|
||||||
|
if not bridge_lock.acquire():
|
||||||
|
logger.info(
|
||||||
|
"MQTT Bridge 已由其他网关实例持有(分布式锁),本网关跳过,"
|
||||||
|
"以规避共享 Broker 下单请求多响应"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return await _start_mqtt(app, worker_manager, bridge_lock)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.error(f"MQTT Bridge 启动失败: {e}")
|
||||||
|
await bridge_lock.stop()
|
||||||
|
bridge_lock.release()
|
||||||
|
app.state.mqtt_bridge = None
|
||||||
|
app.state.mqtt_bridge_lock = None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def mqtt_bridge_supervisor(app: FastAPI, worker_manager: WorkerManager):
|
||||||
|
"""MQTT Bridge 持有者崩溃自愈巡检(跨所有网关进程运行)。
|
||||||
|
|
||||||
|
与 inline 持有者同源的问题:仅靠 gunicorn 复活死掉 slot 无法 failover——新进程
|
||||||
|
可能在旧锁租约未过期时抢锁失败而照样跳过。本巡检在每个网关进程周期性运行:
|
||||||
|
|
||||||
|
- 本进程已持有一存活 Bridge → 正常托管,跳过。
|
||||||
|
- 否则尝试获得 "mqtt_bridge" 分布式锁(set_nx 保证全局唯一胜出,无分裂);
|
||||||
|
若锁仍被存活持有者把持 → 跳过;若空闲(原持有者租约过期=已死/失联)→ 接管。
|
||||||
|
- 接管时按需拉起内嵌 Broker(setup_mqtt_broker 幂等:端口被占则复用现有进程),
|
||||||
|
再重建 Bridge 客户端。
|
||||||
|
|
||||||
|
运行周期经 [<env>.mqtt].bridge_supervise_interval 配置,默认 15s。
|
||||||
|
"""
|
||||||
|
env_config = config.get(env, {})
|
||||||
|
mqtt_config = env_config.get("mqtt", {})
|
||||||
|
if not mqtt_config.get("enabled", True):
|
||||||
|
return # MQTT 禁用:无需巡检
|
||||||
|
interval = mqtt_config.get("bridge_supervise_interval", 15) or 15
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(interval)
|
||||||
|
try:
|
||||||
|
bridge = getattr(app.state, "mqtt_bridge", None)
|
||||||
|
# 本进程持有一存活 Bridge:正常托管
|
||||||
|
if bridge is not None and getattr(bridge, "is_running", True):
|
||||||
|
continue
|
||||||
|
|
||||||
|
newly_acquired = False
|
||||||
|
lock = getattr(app.state, "mqtt_bridge_lock", None)
|
||||||
|
if lock is None:
|
||||||
|
# 非持有者:仅当锁空闲(原持有者租约过期)时才能抢得
|
||||||
|
if worker_manager.cache is None:
|
||||||
|
continue
|
||||||
|
candidate = DistributedLock(worker_manager.cache, "mqtt_bridge")
|
||||||
|
if not candidate.acquire():
|
||||||
|
continue # 仍被存活持有者把持
|
||||||
|
candidate.start()
|
||||||
|
app.state.mqtt_bridge_lock = candidate
|
||||||
|
lock = candidate
|
||||||
|
newly_acquired = True
|
||||||
|
|
||||||
|
logger.warning("[mqtt-bridge] 检出持有者失效,接管重建 Broker+Bridge")
|
||||||
|
# 按需拉起内嵌 Broker(幂等;外部 Broker 时 setup_mqtt_broker 内直接返回)
|
||||||
|
try:
|
||||||
|
await setup_mqtt_broker(app, worker_manager)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.error(f"[mqtt-bridge] 接管时拉起 Broker 失败: {e}")
|
||||||
|
try:
|
||||||
|
await _start_mqtt(app, worker_manager, lock)
|
||||||
|
logger.info("[mqtt-bridge] 接管完成,Bridge 已重建")
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.error(f"[mqtt-bridge] 接管时重建 Bridge 失败: {e}")
|
||||||
|
app.state.mqtt_bridge = None
|
||||||
|
if newly_acquired:
|
||||||
|
await lock.stop()
|
||||||
|
lock.release()
|
||||||
|
app.state.mqtt_bridge_lock = None
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.error(f"MQTT Bridge 自愈巡检失败: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
async def shutdown_mqtt(app: FastAPI):
|
||||||
|
"""停止 MQTT Bridge 和 Broker 子进程(仅锁持有者释放锁)"""
|
||||||
|
bridge: MQTTBridge | None = getattr(app.state, "mqtt_bridge", None)
|
||||||
|
if bridge and bridge.is_running:
|
||||||
|
await bridge.stop()
|
||||||
|
# 停止续约并受保护释放分布式锁(仅本网关仍持有时删除)
|
||||||
|
bridge_lock = getattr(app.state, "mqtt_bridge_lock", None)
|
||||||
|
if bridge_lock is not None:
|
||||||
|
await bridge_lock.stop()
|
||||||
|
bridge_lock.release()
|
||||||
|
await shutdown_mqtt_broker(app)
|
||||||
|
|
||||||
|
|
||||||
|
async def cleanup_workers(app: FastAPI):
|
||||||
|
"""调用每个 inline worker 配置中定义的 cleanup 函数。
|
||||||
|
|
||||||
|
仅 inline worker 持有者进程执行:非持有者进程未初始化插件(未调用 create_workers,
|
||||||
|
on_init 未执行),cleanup/on_cleanup 无资源可清,且可能引用未创建的对象而误报错。
|
||||||
|
"""
|
||||||
|
# app.state.inline_owner_locked 恒为 bool(lifespan 4.2 设置),缺省按持有处理
|
||||||
|
if not getattr(app.state, "inline_owner_locked", True):
|
||||||
|
logger.info("本进程非 inline worker 持有者,跳过插件 cleanup")
|
||||||
|
return
|
||||||
|
workers_conf = _discover_plugins()
|
||||||
|
|
||||||
|
for wc in workers_conf:
|
||||||
|
if wc.get("load_mode", "inline") == "external":
|
||||||
|
continue
|
||||||
|
module_name = wc.get("module", "")
|
||||||
|
cleanup_name = wc.get("cleanup", "")
|
||||||
|
worker_type = wc.get("type", "")
|
||||||
|
|
||||||
|
if not module_name or not cleanup_name:
|
||||||
|
logger.warning(f"Worker 配置缺少 cleanup: {wc}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
module = _import_plugin_module(module_name)
|
||||||
|
cleanup = getattr(module, cleanup_name)
|
||||||
|
pid = os.getpid()
|
||||||
|
worker_id = f"worker/{worker_type}/worker-i-{pid}"
|
||||||
|
if inspect.iscoroutinefunction(cleanup):
|
||||||
|
await cleanup(worker_id)
|
||||||
|
else:
|
||||||
|
cleanup(worker_id)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"cleanup 调用失败 {module_name}: {e}")
|
||||||
|
|
||||||
@@ -0,0 +1,421 @@
|
|||||||
|
import re
|
||||||
|
import os
|
||||||
|
import atexit
|
||||||
|
import queue
|
||||||
|
import logging
|
||||||
|
import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Optional
|
||||||
|
from logging.handlers import RotatingFileHandler, QueueHandler, QueueListener
|
||||||
|
|
||||||
|
from chongming_config import get_config
|
||||||
|
|
||||||
|
|
||||||
|
config = get_config()
|
||||||
|
env = config["default"]["env"]
|
||||||
|
server_config = config[env]["server"]
|
||||||
|
log_config = config[env]["logging"]
|
||||||
|
|
||||||
|
|
||||||
|
class ColoredFormatter(logging.Formatter):
|
||||||
|
"""
|
||||||
|
为 HTTP 状态码添加颜色的日志格式化器
|
||||||
|
|
||||||
|
根据 HTTP 状态码的首位数字为日志消息中的状态码添加不同颜色:
|
||||||
|
- 2xx: 绿色(成功)
|
||||||
|
- 3xx: 青色(重定向)
|
||||||
|
- 4xx: 黄色(客户端错误)
|
||||||
|
- 5xx: 红色(服务器错误)
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
COLORS: 状态码首位数字到 ANSI 颜色代码的映射字典
|
||||||
|
RESET: ANSI 重置颜色代码
|
||||||
|
"""
|
||||||
|
|
||||||
|
COLORS = {
|
||||||
|
"2": "\033[32m", # 绿色 2xx
|
||||||
|
"3": "\033[36m", # 青色 3xx
|
||||||
|
"4": "\033[33m", # 黄色 4xx
|
||||||
|
"5": "\033[31m", # 红色 5xx
|
||||||
|
}
|
||||||
|
RESET = "\033[0m"
|
||||||
|
|
||||||
|
def format(self, record):
|
||||||
|
"""
|
||||||
|
格式化日志记录,为状态码添加颜色
|
||||||
|
|
||||||
|
如果日志记录包含 status_code 属性,则查找消息中的 "status=数字" 模式
|
||||||
|
并为其添加对应的颜色代码。格式化完成后恢复原始消息以避免影响其他 handler。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
record: 日志记录对象,应包含 msg 属性和可选的 status_code 属性
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: 格式化后的日志消息字符串,状态码部分可能包含 ANSI 颜色代码
|
||||||
|
"""
|
||||||
|
# 保存原始消息
|
||||||
|
original_msg = record.msg
|
||||||
|
# 如果日志记录包含 status_code 属性,则为其添加颜色
|
||||||
|
status_code = getattr(record, "status_code", None)
|
||||||
|
if status_code and isinstance(status_code, int):
|
||||||
|
first_digit = str(status_code)[0]
|
||||||
|
color = self.COLORS.get(first_digit, "")
|
||||||
|
colored_status = f"{color}{status_code}{self.RESET}"
|
||||||
|
# 替换消息中的 "status=数字" 为带颜色的版本
|
||||||
|
record.msg = re.sub(
|
||||||
|
rf"status={status_code}\b", f"status={colored_status}", original_msg
|
||||||
|
)
|
||||||
|
result = super().format(record)
|
||||||
|
# 恢复原始消息,避免影响其他 handler
|
||||||
|
record.msg = original_msg
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class DailyRotatingFileHandler(RotatingFileHandler):
|
||||||
|
"""
|
||||||
|
按自然日切分 + 大小轮转的文件处理器
|
||||||
|
|
||||||
|
日志文件名形如 {log_name}-2026-08-19.log:进程启动时按当天日期命名,
|
||||||
|
跨天后写入时自动切换到新日期文件,避免长时间运行的进程把跨天日志
|
||||||
|
全部堆积到启动当天的文件里。同时保留 RotatingFileHandler 的大小轮转
|
||||||
|
(单文件超限追加 .1/.2 后缀)。
|
||||||
|
|
||||||
|
历史文件(跨天文件与大小备份)按修改时间保留最近 backup_count 个,
|
||||||
|
超出部分在切换日期的瞬间清理。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
log_dir: str,
|
||||||
|
log_name: str,
|
||||||
|
max_bytes: int,
|
||||||
|
backup_count: int,
|
||||||
|
encoding: str = "utf-8",
|
||||||
|
):
|
||||||
|
self.log_dir = Path(log_dir)
|
||||||
|
self.log_name = log_name
|
||||||
|
self._current_date = datetime.date.today()
|
||||||
|
super().__init__(
|
||||||
|
str(self.log_dir / f"{log_name}-{self._current_date}.log"),
|
||||||
|
encoding=encoding,
|
||||||
|
maxBytes=max_bytes,
|
||||||
|
backupCount=backup_count,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _build_filename(self, day: datetime.date) -> str:
|
||||||
|
return str(self.log_dir / f"{self.log_name}-{day}.log")
|
||||||
|
|
||||||
|
def _cleanup_old_files(self) -> None:
|
||||||
|
"""保留最近 backup_count 个日志文件,删除更早的历史文件"""
|
||||||
|
# 按 (日期, 轮转序号) 排序,不依赖 mtime(同日备份可能同时落盘):
|
||||||
|
# 日期越大越新;同日内主文件(.log)最新,.N 备份按 N 递减更旧
|
||||||
|
pattern = re.compile(
|
||||||
|
rf"^{re.escape(self.log_name)}-(\d{{4}}-\d{{2}}-\d{{2}})\.log(?:\.(\d+))?$"
|
||||||
|
)
|
||||||
|
|
||||||
|
def sort_key(path: Path):
|
||||||
|
match = pattern.match(path.name)
|
||||||
|
if not match:
|
||||||
|
return (datetime.date.min, 0)
|
||||||
|
return (
|
||||||
|
datetime.date.fromisoformat(match.group(1)),
|
||||||
|
-int(match.group(2) or 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
files = sorted(
|
||||||
|
self.log_dir.glob(f"{self.log_name}-*.log*"),
|
||||||
|
key=sort_key,
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
for old in files[self.backupCount :]:
|
||||||
|
try:
|
||||||
|
old.unlink()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def emit(self, record):
|
||||||
|
# 跨天:关闭旧文件流,切换到新日期文件,并清理过期历史文件
|
||||||
|
today = datetime.date.today()
|
||||||
|
if today != self._current_date:
|
||||||
|
self._current_date = today
|
||||||
|
self.close()
|
||||||
|
self.baseFilename = self._build_filename(today)
|
||||||
|
self.stream = self._open()
|
||||||
|
self._cleanup_old_files()
|
||||||
|
super().emit(record)
|
||||||
|
|
||||||
|
|
||||||
|
class LoggerNameFilter(logging.Filter):
|
||||||
|
"""
|
||||||
|
按 log_name 路由日志记录的过滤器
|
||||||
|
|
||||||
|
只放行本 log_name 自身及其子 logger 的记录(如 chongming_worker.runtime
|
||||||
|
归入 chongming_worker),保证共享 QueueListener 下每个 handler 只处理
|
||||||
|
属于自己的记录:控制台每行只打印一次,各日志文件按名隔离。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, log_name: str):
|
||||||
|
super().__init__()
|
||||||
|
self._name = log_name
|
||||||
|
self._prefix = log_name + "."
|
||||||
|
|
||||||
|
def filter(self, record):
|
||||||
|
return record.name == self._name or record.name.startswith(self._prefix)
|
||||||
|
|
||||||
|
|
||||||
|
class ExcludeWatchFilesFilter(logging.Filter):
|
||||||
|
"""
|
||||||
|
日志过滤器,用于排除第三方库的日志输出
|
||||||
|
|
||||||
|
过滤掉来自 watchfiles、aiosqlite、apscheduler、sqlalchemy 等库的日志记录,
|
||||||
|
避免这些库的详细日志污染应用日志输出。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
"""
|
||||||
|
初始化过滤器,设置需要过滤的日志名称前缀列表
|
||||||
|
"""
|
||||||
|
super().__init__()
|
||||||
|
self._filter_starts_with_tulpe = (
|
||||||
|
"watchfiles",
|
||||||
|
"aiosqlite",
|
||||||
|
"apscheduler",
|
||||||
|
"sqlalchemy",
|
||||||
|
"sqlitedict",
|
||||||
|
"sqlalchemy.engine.Engine",
|
||||||
|
)
|
||||||
|
|
||||||
|
def filter(self, record):
|
||||||
|
"""
|
||||||
|
判断是否应该保留该日志记录
|
||||||
|
|
||||||
|
Args:
|
||||||
|
record: 日志记录对象,包含 name 属性表示日志来源
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bool: 如果日志名称不以过滤列表中的任何前缀开头则返回 True(保留),
|
||||||
|
否则返回 False(过滤掉)
|
||||||
|
"""
|
||||||
|
return not record.name.startswith(self._filter_starts_with_tulpe)
|
||||||
|
|
||||||
|
|
||||||
|
# ---- 异步日志队列(QueueHandler + QueueListener)----
|
||||||
|
# 主线程/事件循环只做"入队"(LogRecord 创建 + put,µs 级),文件/控制台 I/O
|
||||||
|
# 由后台监听线程执行,日志记录不再阻塞业务协程。
|
||||||
|
# 队列满时丢弃新日志(日志系统故障不影响业务);错误日志(>=400)不受采样影响。
|
||||||
|
|
||||||
|
_LOG_QUEUE_SIZE = 10000
|
||||||
|
|
||||||
|
|
||||||
|
class _AsyncQueueHandler(QueueHandler):
|
||||||
|
"""非阻塞入队 + 惰性格式化。
|
||||||
|
|
||||||
|
- 队列满时丢弃(而不是抛 queue.Full 打断业务)
|
||||||
|
- 不预格式化:% 样式消息的格式化推迟到监听线程(formatter 执行),
|
||||||
|
主线程只承担 LogRecord 创建与入队
|
||||||
|
- queue 经模块全局读取:gunicorn preload/fork 后自动指向子进程的新队列
|
||||||
|
"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def queue(self):
|
||||||
|
return _log_queue
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# queue 由 property 动态提供(fork 后自动指向新队列)。
|
||||||
|
# 只初始化 Handler 基类(level/filters 等),不走 QueueHandler.__init__
|
||||||
|
# 的 self.queue = queue 赋值(会与 property 冲突)
|
||||||
|
logging.Handler.__init__(self)
|
||||||
|
|
||||||
|
def enqueue(self, record):
|
||||||
|
try:
|
||||||
|
self.queue.put_nowait(record)
|
||||||
|
except queue.Full:
|
||||||
|
pass # 队列满:丢弃该日志
|
||||||
|
|
||||||
|
def prepare(self, record):
|
||||||
|
return record # 格式化留给监听线程的 formatter
|
||||||
|
|
||||||
|
|
||||||
|
class _RegistryListener(QueueListener):
|
||||||
|
"""handler 可动态注册的 QueueListener(每个 log_name 有独立 file handler)"""
|
||||||
|
|
||||||
|
def __init__(self, q: queue.Queue):
|
||||||
|
super().__init__(q)
|
||||||
|
self.handlers: list = [] # 可变列表,运行期可追加
|
||||||
|
|
||||||
|
def register(self, *handlers) -> None:
|
||||||
|
self.handlers.extend(handlers)
|
||||||
|
|
||||||
|
|
||||||
|
_log_queue: queue.Queue = queue.Queue(maxsize=_LOG_QUEUE_SIZE)
|
||||||
|
_listener: Optional[_RegistryListener] = None
|
||||||
|
# 全部真实 handler(文件/控制台),供 fork 后重建监听线程
|
||||||
|
_listener_handlers: List[logging.Handler] = []
|
||||||
|
|
||||||
|
|
||||||
|
def _start_listener() -> None:
|
||||||
|
global _listener
|
||||||
|
_listener = _RegistryListener(_log_queue)
|
||||||
|
_listener.register(*_listener_handlers)
|
||||||
|
_listener.start()
|
||||||
|
|
||||||
|
|
||||||
|
def _reopen_handler_stream(handler: logging.Handler) -> None:
|
||||||
|
"""fork 后在子进程里重新打开文件 handler 的流(控制台等非文件 handler 不动)。
|
||||||
|
|
||||||
|
子进程继承的是父进程的 BufferedWriter:内部锁/owner 状态在 fork 瞬间可能正被
|
||||||
|
父进程的监听线程占用(跨 fork 复制不可靠),子进程续写会**每条日志**都报
|
||||||
|
"reentrant call inside <_io.BufferedWriter>",文件日志整体失效;同时父子 fd
|
||||||
|
共享同一文件偏移,交错写还会互相踩。关闭继承流、在子进程里重新 open 即避开。
|
||||||
|
"""
|
||||||
|
open_stream = getattr(handler, "_open", None)
|
||||||
|
if not callable(open_stream):
|
||||||
|
return # 非文件 handler(控制台):继承的 stderr 是刻意共享的
|
||||||
|
try:
|
||||||
|
if handler.stream is not None:
|
||||||
|
handler.stream.close()
|
||||||
|
except Exception: # noqa: BLE001 - 继承流状态可能已损坏,关不掉就丢弃引用
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
handler.stream = open_stream()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
handler.stream = None
|
||||||
|
|
||||||
|
|
||||||
|
def _after_fork_child() -> None:
|
||||||
|
"""gunicorn/granian preload/fork 场景:线程不跨 fork,子进程重建队列与监听线程。
|
||||||
|
|
||||||
|
队列重建避免 fork 时锁状态不可用导致的死锁;文件 handler 的流也要在子进程里
|
||||||
|
重开(见 _reopen_handler_stream),否则子进程的文件日志会持续报 reentrant。
|
||||||
|
旧队列中的记录在 fork 瞬间丢失(启动窗口内可接受)。
|
||||||
|
"""
|
||||||
|
global _log_queue
|
||||||
|
_log_queue = queue.Queue(maxsize=_LOG_QUEUE_SIZE)
|
||||||
|
for handler in _listener_handlers:
|
||||||
|
_reopen_handler_stream(handler)
|
||||||
|
_start_listener()
|
||||||
|
|
||||||
|
|
||||||
|
if hasattr(os, "register_at_fork"):
|
||||||
|
os.register_at_fork(after_in_child=_after_fork_child)
|
||||||
|
|
||||||
|
|
||||||
|
def shutdown_logging() -> None:
|
||||||
|
"""停止监听线程并冲刷剩余日志(进程退出 / 应用关闭时调用),幂等。
|
||||||
|
|
||||||
|
QueueListener.stop() 停止后会将 _thread 置 None,重复调用需跳过;
|
||||||
|
幂等保证 atexit 与显式调用共存。
|
||||||
|
"""
|
||||||
|
global _listener
|
||||||
|
if _listener is not None:
|
||||||
|
if _listener._thread is not None:
|
||||||
|
_listener.stop() # flush=True:处理完队列中已有记录再停止
|
||||||
|
_listener = None
|
||||||
|
for h in _listener_handlers:
|
||||||
|
try:
|
||||||
|
h.close()
|
||||||
|
except Exception: # pragma: no cover - 关闭失败不影响退出
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
atexit.register(shutdown_logging)
|
||||||
|
|
||||||
|
|
||||||
|
_configured_loggers = set()
|
||||||
|
# 每个进程只打印一次日志配置信息(setup_logging 每个 log_name 都会调用一次)
|
||||||
|
_config_info_printed = False
|
||||||
|
|
||||||
|
|
||||||
|
def setup_logging(logger: logging.Logger, log_name: str = "app"):
|
||||||
|
"""
|
||||||
|
配置日志系统,包括文件和控制台输出
|
||||||
|
|
||||||
|
为指定的 logger 配置以下功能:
|
||||||
|
1. 设置日志级别和格式
|
||||||
|
2. 创建按自然日切分的轮转文件处理器(跨天自动切换文件,支持文件大小限制和备份数量控制)
|
||||||
|
3. 创建带颜色格式化的控制台处理器
|
||||||
|
4. 应用过滤器排除第三方库日志
|
||||||
|
|
||||||
|
该函数具有防重复配置机制,同一个 log_name 只会配置一次。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
logger: 需要配置的 logging.Logger 实例
|
||||||
|
log_name: 日志名称标识,用于区分不同的日志配置和生成日志文件名,默认为 "app"
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
无显式抛出异常,但可能因文件系统权限问题导致目录创建或文件写入失败
|
||||||
|
"""
|
||||||
|
if log_name in _configured_loggers:
|
||||||
|
return
|
||||||
|
_configured_loggers.add(log_name)
|
||||||
|
log_level = getattr(logging, log_config["level"], logging.INFO)
|
||||||
|
log_format = log_config["format"]
|
||||||
|
# 创建文件夹
|
||||||
|
log_file_path = Path(log_config["file"])
|
||||||
|
if log_file_path != Path("."):
|
||||||
|
log_file_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# 配置根日志记录器
|
||||||
|
logger.setLevel(log_level)
|
||||||
|
|
||||||
|
# 创建格式化器
|
||||||
|
plain_formatter = logging.Formatter(log_format)
|
||||||
|
color_formatter = ColoredFormatter(log_format)
|
||||||
|
|
||||||
|
# 创建按自然日切分的轮转文件处理器(跨天自动切文件,单文件超限再按大小轮转)
|
||||||
|
max_bytes = log_config.get("max_size", 10 * 1024 * 1024) # 默认 10MB
|
||||||
|
backup_count = log_config.get("backup_count", 5) # 默认保留 5 个历史文件
|
||||||
|
|
||||||
|
file_handler = DailyRotatingFileHandler(
|
||||||
|
log_config["file"],
|
||||||
|
log_name,
|
||||||
|
max_bytes=max_bytes,
|
||||||
|
backup_count=backup_count,
|
||||||
|
)
|
||||||
|
file_handler.setFormatter(plain_formatter)
|
||||||
|
file_handler.addFilter(ExcludeWatchFilesFilter())
|
||||||
|
file_handler.addFilter(LoggerNameFilter(log_name))
|
||||||
|
|
||||||
|
# 创建控制台处理器并添加过滤器
|
||||||
|
console_handler = logging.StreamHandler()
|
||||||
|
console_handler.setFormatter(color_formatter)
|
||||||
|
console_handler.addFilter(ExcludeWatchFilesFilter())
|
||||||
|
console_handler.addFilter(LoggerNameFilter(log_name))
|
||||||
|
|
||||||
|
# 真实 handler(文件/控制台)挂到后台监听线程;logger 上只挂队列 handler,
|
||||||
|
# 日志调用在主线程仅入队,I/O 由监听线程执行
|
||||||
|
_listener_handlers.extend([file_handler, console_handler])
|
||||||
|
if _listener is None:
|
||||||
|
_start_listener()
|
||||||
|
else:
|
||||||
|
_listener.register(file_handler, console_handler)
|
||||||
|
logger.addHandler(_AsyncQueueHandler())
|
||||||
|
|
||||||
|
global _config_info_printed
|
||||||
|
if not _config_info_printed:
|
||||||
|
_config_info_printed = True
|
||||||
|
print(
|
||||||
|
f"日志系统已配置 - 级别: {logging.getLevelName(log_level)}, 文件: {log_config['file']} "
|
||||||
|
f"(最大:{max_bytes / 1024 / 1024:.1f}MB, 备份:{backup_count})"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_logger(name: str) -> logging.Logger:
|
||||||
|
"""
|
||||||
|
获取配置好的日志记录器实例
|
||||||
|
|
||||||
|
根据名称获取或创建 logger 实例,并自动调用 setup_logging 进行配置。
|
||||||
|
如果该名称的 logger 已经配置过,则直接返回已配置的实例。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name: 日志记录器名称,通常使用模块名如 __name__
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
logging.Logger: 已配置完成的日志记录器实例,可直接用于记录日志
|
||||||
|
"""
|
||||||
|
logger = logging.getLogger(name)
|
||||||
|
setup_logging(logger, name)
|
||||||
|
return logger
|
||||||
@@ -0,0 +1,271 @@
|
|||||||
|
"""MQTT Bridge — 纯客户端模式,Broker 由子进程管理"""
|
||||||
|
import asyncio
|
||||||
|
import orjson
|
||||||
|
import re
|
||||||
|
from typing import Optional
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from amqtt.client import MQTTClient
|
||||||
|
from amqtt.mqtt.constants import QOS_1
|
||||||
|
from .logger import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("mqtt")
|
||||||
|
MQTT_TOPIC_PREFIX = "chongming"
|
||||||
|
MQTT_SUBSCRIBE_TOPIC = f"{MQTT_TOPIC_PREFIX}/+/+"
|
||||||
|
|
||||||
|
# 已按"非信封载荷"路由过告警的主题(每主题只告警一次,避免设备 5s 心跳刷屏)
|
||||||
|
_warned_legacy_topics: set = set()
|
||||||
|
|
||||||
|
|
||||||
|
def _compile_topic(topic: str) -> re.Pattern:
|
||||||
|
"""
|
||||||
|
将 MQTT 主题模板(支持 + / # 通配符)编译为正则表达式。
|
||||||
|
|
||||||
|
- ``+`` 匹配单级(编译为捕获组 ``([^/]+)``)
|
||||||
|
- ``#`` 匹配多级(编译为捕获组 ``(.*)``,须位于末尾)
|
||||||
|
捕获组按出现顺序对应,供 {1} {2}... 占位符替换使用。
|
||||||
|
"""
|
||||||
|
parts = topic.split("/")
|
||||||
|
regex = "^"
|
||||||
|
for i, part in enumerate(parts):
|
||||||
|
if i > 0:
|
||||||
|
regex += "/"
|
||||||
|
if part == "#":
|
||||||
|
regex += "(.*)"
|
||||||
|
elif part == "+":
|
||||||
|
regex += "([^/]+)"
|
||||||
|
else:
|
||||||
|
regex += re.escape(part)
|
||||||
|
regex += "$"
|
||||||
|
return re.compile(regex)
|
||||||
|
|
||||||
|
|
||||||
|
class MQTTBridge:
|
||||||
|
def __init__(self, worker_manager, host="localhost", port=1883, custom_topics=None,
|
||||||
|
username=None, password=None):
|
||||||
|
self._wm = worker_manager
|
||||||
|
self._host = host
|
||||||
|
self._port = port
|
||||||
|
self._username = username
|
||||||
|
self._password = password
|
||||||
|
self._client: Optional[MQTTClient] = None
|
||||||
|
self._lt: Optional[asyncio.Task] = None
|
||||||
|
self._running = False
|
||||||
|
# custom_topics: [{"topic","worker","subject"?,"response_topic"?}, ...]
|
||||||
|
# 混合模式双来源:
|
||||||
|
# _static_topics — inline 插件,构造时由网关从配置收集
|
||||||
|
# _dynamic_topics — external 插件,运行时由 RouteRegistry 推送
|
||||||
|
self._static_topics = custom_topics or []
|
||||||
|
self._dynamic_topics: list = []
|
||||||
|
self._compiled: list = []
|
||||||
|
self._rebuild_compiled()
|
||||||
|
self._subscribed_custom: set = set()
|
||||||
|
|
||||||
|
def _rebuild_compiled(self):
|
||||||
|
"""合并静态/动态主题并重编译匹配正则。"""
|
||||||
|
merged = self._static_topics + self._dynamic_topics
|
||||||
|
self._compiled = [
|
||||||
|
(ct, _compile_topic(ct.get("topic", "")))
|
||||||
|
for ct in merged if ct.get("topic")
|
||||||
|
]
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
if self._running:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
await self._sc()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"MQTT 不可用: {e}")
|
||||||
|
return
|
||||||
|
self._running = True
|
||||||
|
logger.info(f"MQTT Bridge 已连接 {self._host}:{self._port}")
|
||||||
|
|
||||||
|
async def stop(self):
|
||||||
|
if not self._running:
|
||||||
|
return
|
||||||
|
self._running = False
|
||||||
|
await self._sc_stop()
|
||||||
|
|
||||||
|
def _broker_uri(self) -> str:
|
||||||
|
"""构建 Broker URI;凭据经百分号编码(amqtt 连接时自动 unquote)。"""
|
||||||
|
if self._username:
|
||||||
|
user = quote(self._username, safe="")
|
||||||
|
pwd = quote(self._password or "", safe="")
|
||||||
|
return f"mqtt://{user}:{pwd}@{self._host}:{self._port}/"
|
||||||
|
return f"mqtt://{self._host}:{self._port}/"
|
||||||
|
|
||||||
|
async def _sc(self):
|
||||||
|
self._client = MQTTClient(config={"auto_reconnect": False})
|
||||||
|
await self._client.connect(self._broker_uri())
|
||||||
|
topics = [(MQTT_SUBSCRIBE_TOPIC, QOS_1)]
|
||||||
|
topics += [(ct["topic"], QOS_1) for ct, _ in self._compiled]
|
||||||
|
await self._client.subscribe(topics)
|
||||||
|
self._subscribed_custom = {ct["topic"] for ct, _ in self._compiled}
|
||||||
|
logger.info(f"已订阅 {MQTT_SUBSCRIBE_TOPIC}"
|
||||||
|
+ (f" 及 {len(self._compiled)} 个自定义主题" if self._compiled else ""))
|
||||||
|
self._lt = asyncio.create_task(self._loop())
|
||||||
|
|
||||||
|
async def update_dynamic_topics(self, custom_topics: list):
|
||||||
|
"""
|
||||||
|
动态更新 external 插件的自定义订阅主题(RouteRegistry 推送)。
|
||||||
|
|
||||||
|
与构造时的 inline 静态主题合并后增量重订阅(退订已移除主题、订阅新增主题),
|
||||||
|
运行中调用即生效。
|
||||||
|
"""
|
||||||
|
self._dynamic_topics = list(custom_topics or [])
|
||||||
|
self._rebuild_compiled()
|
||||||
|
new_topics = {ct["topic"] for ct, _ in self._compiled}
|
||||||
|
if not (self._running and self._client):
|
||||||
|
self._subscribed_custom = new_topics
|
||||||
|
return
|
||||||
|
removed = self._subscribed_custom - new_topics
|
||||||
|
added = new_topics - self._subscribed_custom
|
||||||
|
if removed:
|
||||||
|
await self._client.unsubscribe(list(removed))
|
||||||
|
if added:
|
||||||
|
await self._client.subscribe([(t, QOS_1) for t in added])
|
||||||
|
self._subscribed_custom = new_topics
|
||||||
|
if added or removed:
|
||||||
|
logger.info(
|
||||||
|
f"自定义订阅主题更新: +{len(added)} -{len(removed)}"
|
||||||
|
f"(当前 {len(new_topics)} 个)"
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _sc_stop(self):
|
||||||
|
# 先断开连接,让 deliver_message() 解锁
|
||||||
|
if self._client:
|
||||||
|
try:
|
||||||
|
await self._client.disconnect()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self._client = None
|
||||||
|
# 再取消消息循环
|
||||||
|
if self._lt and not self._lt.done():
|
||||||
|
self._lt.cancel()
|
||||||
|
try:
|
||||||
|
await self._lt
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def _loop(self):
|
||||||
|
while self._running:
|
||||||
|
try:
|
||||||
|
m = await self._client.deliver_message() # type: ignore
|
||||||
|
if m is None:
|
||||||
|
break
|
||||||
|
await self._handle(m)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
if self._running:
|
||||||
|
logger.error(f"消息异常: {e}")
|
||||||
|
break
|
||||||
|
|
||||||
|
async def _handle(self, m):
|
||||||
|
t: str = m.topic
|
||||||
|
p = m.data # type: ignore
|
||||||
|
try:
|
||||||
|
pl = orjson.loads(p)
|
||||||
|
except orjson.JSONDecodeError:
|
||||||
|
return
|
||||||
|
|
||||||
|
# 实例定向控制字段:与 HTTP 的 X-Chongming-Instance 头同义,指定请求发往的
|
||||||
|
# worker 进程实例。必须在下面归一化之前取出——归一化只保留 meta/data 两键,
|
||||||
|
# 之后控制字段就不在载荷里了。
|
||||||
|
target_instance: Optional[str] = None
|
||||||
|
if isinstance(pl, dict):
|
||||||
|
ti = pl.pop("target_instance", None)
|
||||||
|
if isinstance(ti, str) and ti.strip():
|
||||||
|
target_instance = ti.strip()
|
||||||
|
|
||||||
|
# 信封协议: {"meta": {...}, "data": {...}}
|
||||||
|
# 兼容非信封载荷(如第三方设备原始报文): 整包视为业务数据、meta 置空,
|
||||||
|
# 与 worker 运行时 runtime.py 的容错语义一致(method 默认 GET)——
|
||||||
|
# 若在此丢弃,盒子心跳等设备报文将永远到不了 Worker。
|
||||||
|
if not isinstance(pl, dict) or ("meta" not in pl and "data" not in pl):
|
||||||
|
data = pl if isinstance(pl, dict) else {}
|
||||||
|
pl = {"meta": {}, "data": data}
|
||||||
|
if t not in _warned_legacy_topics:
|
||||||
|
_warned_legacy_topics.add(t)
|
||||||
|
logger.warning(
|
||||||
|
"非信封格式载荷(缺少 meta/data)按原始报文路由 [%s]: "
|
||||||
|
"整包视为 data,建议设备侧改用 {'meta': {...}, 'data': {...}} 格式",
|
||||||
|
t,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
meta = pl.get("meta") or {}
|
||||||
|
data = pl.get("data") or {}
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
data = {}
|
||||||
|
pl = {"meta": meta, "data": data}
|
||||||
|
|
||||||
|
# 1. 优先匹配显式配置的自定义订阅主题(顺序即优先级)
|
||||||
|
for ct, compiled in self._compiled:
|
||||||
|
matched = compiled.match(t)
|
||||||
|
if matched:
|
||||||
|
await self._route_custom(ct, matched, pl, t, target_instance)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 2. 内置接口 chongming/{worker}/{operation}
|
||||||
|
parts = t.split("/")
|
||||||
|
if len(parts) < 3:
|
||||||
|
return
|
||||||
|
wt, op = parts[1], parts[2]
|
||||||
|
subj = f"{wt}.{op}"
|
||||||
|
rt = f"{MQTT_TOPIC_PREFIX}/{wt}/{op}/response"
|
||||||
|
await self._route(wt, subj, pl, rt, target_instance)
|
||||||
|
|
||||||
|
def _fill(self, template: str, matched: re.Match) -> str:
|
||||||
|
"""用匹配到的通配符值填充模板中的 {1} {2}... 占位符"""
|
||||||
|
if "{" not in template:
|
||||||
|
return template
|
||||||
|
for i, g in enumerate(matched.groups(), start=1):
|
||||||
|
template = template.replace("{" + str(i) + "}", g)
|
||||||
|
return template
|
||||||
|
|
||||||
|
async def _route_custom(self, ct: dict, matched: re.Match, pl: dict, topic: str,
|
||||||
|
target_instance: Optional[str] = None):
|
||||||
|
worker = ct.get("worker", "")
|
||||||
|
# subject 默认 "<worker>.<topic 模板最后一段>",与内置 <插件>.<操作> 约定一致
|
||||||
|
subject_tpl = ct.get("subject")
|
||||||
|
if not subject_tpl:
|
||||||
|
last = ct["topic"].split("/")[-1]
|
||||||
|
subject_tpl = f"{worker}.{last}" if last not in ("+", "#") else worker
|
||||||
|
subject = self._fill(subject_tpl, matched)
|
||||||
|
# response_topic 默认 "<实际匹配主题>/response"
|
||||||
|
rt_tpl = ct.get("response_topic")
|
||||||
|
rt = self._fill(rt_tpl, matched) if rt_tpl else f"{topic}/response"
|
||||||
|
await self._route(worker, subject, pl, rt, target_instance)
|
||||||
|
|
||||||
|
async def _route(self, wt: str, subj: str, pl: dict, rt: str,
|
||||||
|
target_instance: Optional[str] = None):
|
||||||
|
"""把一次 MQTT 请求转成 worker RPC。
|
||||||
|
|
||||||
|
target_instance 为实例定向目标(来自载荷顶层的 target_instance 控制字段,
|
||||||
|
已在 _handle 归一化前摘除);缺省时跨实例随机均衡,行为与以往一致。
|
||||||
|
"""
|
||||||
|
logger.info(f"MQTT -> {subj}")
|
||||||
|
meta = pl.get("meta") or {}
|
||||||
|
try:
|
||||||
|
r = await self._wm.call(worker_type=wt, subject=subj, payload=pl,
|
||||||
|
timeout=float(meta.get("timeout", 30)),
|
||||||
|
cacheable=bool(meta.get("cacheable", False)),
|
||||||
|
instance_id=target_instance or None)
|
||||||
|
if r is not None:
|
||||||
|
await self._pub(rt, r)
|
||||||
|
except RuntimeError as e:
|
||||||
|
await self._pub(rt, {"error": str(e)})
|
||||||
|
except Exception as e:
|
||||||
|
await self._pub(rt, {"error": str(e)})
|
||||||
|
|
||||||
|
async def _pub(self, topic, data):
|
||||||
|
try:
|
||||||
|
# data 均为 JSON 可序列化类型(worker 回复解析结果 / {"error": ...}),
|
||||||
|
# orjson.dumps 直接产出 UTF-8 字节,免去 str -> encode 两步
|
||||||
|
await self._client.publish(topic, orjson.dumps(data), QOS_1) # type: ignore
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"发布失败 [{topic}]: {e}")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_running(self):
|
||||||
|
return self._running
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
"""父进程死亡防护(Linux PR_SET_PDEATHSIG)——防"上游被 SIGKILL 后子进程成孤儿"。
|
||||||
|
|
||||||
|
进程树:onefile 引导进程 → granian master → N 个 worker(fork)+ 1 个 MQTT Broker
|
||||||
|
子进程。上游被 SIGKILL 时子进程会变孤儿:继续监听端口(SO_REUSEPORT)、继续在
|
||||||
|
服务发现里挂着,其它网关派发到这种"半死副本"每次要白吃一整轮超时(worker_manager
|
||||||
|
的 `超时 (>2s),尝试重试` 日志);SIGTERM/Ctrl-C 路径则由 granian 优雅停机覆盖。
|
||||||
|
|
||||||
|
PR_SET_PDEATHSIG 让内核在父进程死亡时(含 SIGKILL)给本进程发信号,走既有的
|
||||||
|
优雅停机摘除注册。注意三点(man prctl):
|
||||||
|
|
||||||
|
1. 该值**不跨 fork 继承**(内核为子进程清零,本机实测确认)——所以每个该"随父
|
||||||
|
而死"的进程都要自己设置:master 在 main.py 设一次(随引导进程/终端),
|
||||||
|
worker 在 lifespan 设一次(随 master),Broker 子进程随其宿主 worker;
|
||||||
|
2. 这里的"父"是**创建本进程的那个线程**,不是整个进程——granian 由主线程 fork
|
||||||
|
worker、worker 由事件循环所在线程拉起 Broker,语义都正确;
|
||||||
|
3. 设完必须立刻查 getppid(),父在设置前就已死的话信号不会再来(竞态),只能
|
||||||
|
自行退出。
|
||||||
|
|
||||||
|
仅 Linux 生效;`CHONGMING_NO_PARENT_GUARD=1` 可整体关闭(由外层编排负责清理时)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# 关闭开关:某些容器编排(PID 1 / cgroup 清理)自己负责回收时不需要本机制
|
||||||
|
GUARD_OFF_ENV = "CHONGMING_NO_PARENT_GUARD"
|
||||||
|
|
||||||
|
_PR_SET_PDEATHSIG = 1
|
||||||
|
_PR_GET_PDEATHSIG = 2
|
||||||
|
|
||||||
|
# 模块级缓存 libc 句柄(跨 fork 可安全复用;是否已设置不看缓存,看内核实际状态)
|
||||||
|
_libc = None
|
||||||
|
|
||||||
|
|
||||||
|
def _load_libc():
|
||||||
|
global _libc
|
||||||
|
if _libc is not None:
|
||||||
|
return _libc
|
||||||
|
import ctypes
|
||||||
|
|
||||||
|
for name in ("libc.so.6", None):
|
||||||
|
try:
|
||||||
|
_libc = ctypes.CDLL(name, use_errno=True)
|
||||||
|
return _libc
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _current_pdeathsig(libc) -> int | None:
|
||||||
|
"""读本进程当前的父死信号值(0 = 未设置)。"""
|
||||||
|
import ctypes
|
||||||
|
|
||||||
|
buf = ctypes.c_int(0)
|
||||||
|
if libc.prctl(_PR_GET_PDEATHSIG, ctypes.byref(buf)) != 0:
|
||||||
|
return None
|
||||||
|
return buf.value
|
||||||
|
|
||||||
|
|
||||||
|
def guard_parent_death(logger=None) -> bool:
|
||||||
|
"""设置父进程死亡信号为 SIGTERM(幂等)。返回是否生效。
|
||||||
|
|
||||||
|
生效后:父进程(含被 SIGKILL)死亡 → 内核给本进程发 SIGTERM → 触发既有
|
||||||
|
优雅停机(摘除路由/注册、关闭会话)。非 Linux、关闭开关打开、或 libc 不可用
|
||||||
|
时返回 False,不影响启动。
|
||||||
|
|
||||||
|
幂等性按**内核实际状态**判断(PR_GET_PDEATHSIG),不用模块级标志位——
|
||||||
|
master 先调用后 fork 出的 worker 会继承模块状态,用标志位会让 worker 误判为
|
||||||
|
"已设置"而直接跳过(PDEATHSIG 恰好在 fork 时被内核清零)。
|
||||||
|
"""
|
||||||
|
if os.environ.get(GUARD_OFF_ENV) == "1":
|
||||||
|
return False
|
||||||
|
if not sys.platform.startswith("linux"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
libc = _load_libc()
|
||||||
|
if libc is None:
|
||||||
|
if logger:
|
||||||
|
logger.warning("父进程死亡防护不可用:未能加载 libc")
|
||||||
|
return False
|
||||||
|
|
||||||
|
if _current_pdeathsig(libc) != int(signal.SIGTERM):
|
||||||
|
if libc.prctl(_PR_SET_PDEATHSIG, signal.SIGTERM) != 0:
|
||||||
|
import ctypes
|
||||||
|
|
||||||
|
if logger:
|
||||||
|
logger.warning("设置父进程死亡信号失败 (prctl errno=%s)", ctypes.get_errno())
|
||||||
|
return False
|
||||||
|
if logger:
|
||||||
|
logger.info("父进程死亡防护已启用(父进程退出时收 SIGTERM 走优雅停机)")
|
||||||
|
|
||||||
|
# 竞态守卫:父在 prctl 生效前已死 → 信号不会再来,自行退出(走 SIGTERM 停机路径)
|
||||||
|
if os.getppid() == 1:
|
||||||
|
if logger:
|
||||||
|
logger.warning("父进程已不存在(ppid=1),本进程自行退出")
|
||||||
|
os.kill(os.getpid(), signal.SIGTERM)
|
||||||
|
return True
|
||||||
@@ -0,0 +1,846 @@
|
|||||||
|
"""
|
||||||
|
动态路由生成器
|
||||||
|
|
||||||
|
根据配置文件中的 worker 路由定义,动态生成 FastAPI 端点。
|
||||||
|
为每个端点生成带有正确 Pydantic 类型注解的处理函数,确保 OpenAPI 文档
|
||||||
|
正确反映参数类型、响应结构,并注册到 components/schemas。
|
||||||
|
"""
|
||||||
|
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
|
||||||
|
from inspect import Parameter, Signature
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
|
||||||
|
import orjson
|
||||||
|
from fastapi import FastAPI, HTTPException, APIRouter, Query, Path as FastAPIPath, Request, Depends
|
||||||
|
from fastapi.responses import StreamingResponse, Response
|
||||||
|
from pydantic import create_model, Field, BaseModel
|
||||||
|
|
||||||
|
from chongming_worker.route_config import WORKER_CONFIG_DIR, load_worker_routes_from_dir
|
||||||
|
|
||||||
|
from .worker_manager import WorkerManager, StreamIdleTimeoutError
|
||||||
|
from .logger import get_logger
|
||||||
|
from .auth import store_user_to_request
|
||||||
|
from .call_stats import spawn, record_call
|
||||||
|
|
||||||
|
logger = get_logger("route_generator")
|
||||||
|
|
||||||
|
# 实例定向请求头:给定后该请求固定发往指定 worker 进程实例(同一实例下的多个副本
|
||||||
|
# 之间仍随机均衡),不参与跨实例的随机负载均衡。取值 = 实例身份(外部进程启动参数
|
||||||
|
# --instance / CHONGMING_WORKER_INSTANCE;inline 副本为网关自身实例 id)。
|
||||||
|
# 未携带该头时行为与以往完全一致(全类型随机均衡)。
|
||||||
|
INSTANCE_HEADER = "X-Chongming-Instance"
|
||||||
|
|
||||||
|
|
||||||
|
def _instance_from_request(request: Request) -> Optional[str]:
|
||||||
|
"""从请求头读取定向实例 id;空串/未携带视为不定向。"""
|
||||||
|
value = request.headers.get(INSTANCE_HEADER)
|
||||||
|
if value:
|
||||||
|
value = value.strip()
|
||||||
|
return value or None
|
||||||
|
|
||||||
|
|
||||||
|
# 简单类型映射
|
||||||
|
TYPE_MAP: Dict[str, Any] = {
|
||||||
|
"float": float,
|
||||||
|
"int": int,
|
||||||
|
"str": str,
|
||||||
|
"bool": bool,
|
||||||
|
"any": Any,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_python_type(type_str: str) -> Any:
|
||||||
|
"""解析配置中的类型字符串为 Python 类型对象"""
|
||||||
|
type_str = type_str.strip()
|
||||||
|
if type_str.startswith("list[") or type_str.startswith("List["):
|
||||||
|
inner = type_str[5:-1].strip()
|
||||||
|
inner_type = _parse_python_type(inner)
|
||||||
|
return List[inner_type] # type: ignore
|
||||||
|
if type_str.startswith("dict[") or type_str.startswith("Dict["):
|
||||||
|
inner = type_str[5:-1].strip()
|
||||||
|
parts = [p.strip() for p in inner.split(",", 1)]
|
||||||
|
if len(parts) == 2:
|
||||||
|
key_type = _parse_python_type(parts[0])
|
||||||
|
val_type = _parse_python_type(parts[1])
|
||||||
|
return Dict[key_type, val_type] # type: ignore
|
||||||
|
return Dict[str, Any] # type: ignore
|
||||||
|
return TYPE_MAP.get(type_str, str)
|
||||||
|
|
||||||
|
def generate_request_model(
|
||||||
|
param_spec: Dict[str, str], model_name: str = "AutoRequest"
|
||||||
|
) -> type[BaseModel]:
|
||||||
|
"""生成 Pydantic 请求体模型"""
|
||||||
|
fields: Dict[str, Any] = {}
|
||||||
|
for param_name, param_type_str in param_spec.items():
|
||||||
|
py_type = _parse_python_type(param_type_str)
|
||||||
|
fields[param_name] = (py_type, Field(...))
|
||||||
|
return create_model(model_name, **fields)
|
||||||
|
|
||||||
|
def generate_response_model(
|
||||||
|
response_spec: Dict[str, Any], model_name: str = "AutoResponse"
|
||||||
|
) -> Optional[type[BaseModel]]:
|
||||||
|
"""生成 Pydantic 响应模型"""
|
||||||
|
if not response_spec:
|
||||||
|
return None
|
||||||
|
fields: Dict[str, Any] = {}
|
||||||
|
for field_name, field_spec in response_spec.items():
|
||||||
|
if isinstance(field_spec, dict):
|
||||||
|
py_type = _parse_python_type(field_spec.get("type", "str"))
|
||||||
|
required = field_spec.get("required", True)
|
||||||
|
default_val = field_spec.get("default", ...)
|
||||||
|
if required:
|
||||||
|
fields[field_name] = (py_type, Field(...))
|
||||||
|
else:
|
||||||
|
fields[field_name] = (py_type, Field(default=default_val if default_val is not ... else None))
|
||||||
|
else:
|
||||||
|
py_type = _parse_python_type(str(field_spec))
|
||||||
|
fields[field_name] = (py_type, Field(...))
|
||||||
|
return create_model(model_name, **fields)
|
||||||
|
|
||||||
|
class RouteConfig:
|
||||||
|
"""单个路由配置模型"""
|
||||||
|
|
||||||
|
def __init__(self, data: Dict[str, Any]):
|
||||||
|
self.subject: str = data.get("subject", "")
|
||||||
|
self.path: str = data.get("path", "")
|
||||||
|
self.method: str = data.get("method", "GET").upper()
|
||||||
|
self.params: List[str] = data.get("params", [])
|
||||||
|
self.response_model: Dict[str, Any] = data.get("response_model", {})
|
||||||
|
self.summary: str = data.get("summary", "")
|
||||||
|
self.docstring: str = data.get("docstring", "")
|
||||||
|
self.timeout: float = data.get("timeout", 3.0)
|
||||||
|
self.cacheable: bool = data.get("cacheable", False)
|
||||||
|
self.auth_required: bool = data.get("auth_required", True)
|
||||||
|
self.internal: bool = data.get("internal", False)
|
||||||
|
# validate=false 时网关不解析/校验请求体,原样透传(校验职责下沉 worker,
|
||||||
|
# 失败经 reply_err 以 503 暴露);仅对 POST/PUT/PATCH 非流式端点生效
|
||||||
|
self.validate: bool = data.get("validate", True)
|
||||||
|
# 流式端点:以 SSE(text/event-stream)逐块输出
|
||||||
|
self.stream: bool = data.get("stream", False)
|
||||||
|
# 流式空闲超时(秒):连续未收到数据块的最大间隔(timeout 为流总寿命上限)
|
||||||
|
self.stream_idle_timeout: float = data.get("stream_idle_timeout", 10.0)
|
||||||
|
|
||||||
|
def parse_params(self) -> Dict[str, str]:
|
||||||
|
"""解析 params 列表为 {name: type} 字典"""
|
||||||
|
result: Dict[str, str] = {}
|
||||||
|
for param in self.params:
|
||||||
|
if ":" in param:
|
||||||
|
name, type_str = param.split(":", 1)
|
||||||
|
result[name.strip()] = type_str.strip()
|
||||||
|
else:
|
||||||
|
result[param.strip()] = "str"
|
||||||
|
return result
|
||||||
|
|
||||||
|
def extract_path_params(self) -> List[str]:
|
||||||
|
"""从路径中提取路径参数,如 /add/{a}/{b} -> ['a', 'b']"""
|
||||||
|
return [
|
||||||
|
seg[1:-1] for seg in self.path.split("/")
|
||||||
|
if seg.startswith("{") and seg.endswith("}")
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def register_route_group(
|
||||||
|
app: FastAPI,
|
||||||
|
worker_manager: WorkerManager,
|
||||||
|
worker_type: str,
|
||||||
|
routes: List[Dict[str, Any]],
|
||||||
|
prefix: str = "",
|
||||||
|
extra_schemas: Optional[Dict[str, type[BaseModel]]] = None,
|
||||||
|
raw_body_models: Optional[Dict[Tuple[str, str], str]] = None,
|
||||||
|
response_models: Optional[Dict[Tuple[str, str], str]] = None,
|
||||||
|
) -> List[Any]:
|
||||||
|
"""
|
||||||
|
为一个 worker_type 的一组路由描述符创建并挂载 APIRouter。
|
||||||
|
|
||||||
|
与 register_dynamic_routes(盘扫路径)共用同一注册逻辑,
|
||||||
|
process 模式下 RouteRegistry 也调用本函数做动态挂载。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
app: FastAPI 应用
|
||||||
|
worker_manager: WorkerManager 实例
|
||||||
|
worker_type: worker 类型(OpenAPI 分组标签)
|
||||||
|
routes: 路由描述符列表(TOML 兼容格式,与 load_worker_routes_from_dir 输出一致)
|
||||||
|
prefix: 全局路由前缀(如 /api/v1)
|
||||||
|
extra_schemas: 额外 schema 收集 dict(GET 请求模型注入 OpenAPI 用)
|
||||||
|
raw_body_models: (path, method) -> 请求模型名 的收集 dict
|
||||||
|
(validate=false 端点:请求体文档经 requestBody $ref 注入,见 _inject_schemas_to_openapi)
|
||||||
|
response_models: (path, method) -> 响应模型名 的收集 dict
|
||||||
|
(端点注册不传 response_model 以砍掉注册期模型反射;响应文档经
|
||||||
|
responses[200].content $ref 手动注入,见 _inject_schemas_to_openapi)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List[Any]: 本次新增到 app.router.routes 的 APIRoute 对象列表
|
||||||
|
(调用方可记录引用,用于动态移除)
|
||||||
|
"""
|
||||||
|
router = APIRouter(prefix=prefix)
|
||||||
|
for route_conf in routes:
|
||||||
|
_register_single_route(router, worker_manager, worker_type, route_conf, extra_schemas, raw_body_models, response_models)
|
||||||
|
|
||||||
|
# raw_body_models / response_models 记录的 (path, method) 为端点声明路径(不含 prefix),
|
||||||
|
# 而 OpenAPI paths 键为最终挂载路径(prefix + path),这里统一补前缀
|
||||||
|
if prefix:
|
||||||
|
for key in list((raw_body_models or {}).keys()):
|
||||||
|
path, method = key
|
||||||
|
raw_body_models[(prefix + path, method)] = raw_body_models.pop(key)
|
||||||
|
for key in list((response_models or {}).keys()):
|
||||||
|
path, method = key
|
||||||
|
response_models[(prefix + path, method)] = response_models.pop(key)
|
||||||
|
|
||||||
|
before = set(id(r) for r in app.router.routes)
|
||||||
|
app.include_router(router)
|
||||||
|
added = [r for r in app.router.routes if id(r) not in before]
|
||||||
|
logger.info(
|
||||||
|
f"已挂载 worker '{worker_type}' 的路由组: {len(routes)} 个描述符 -> "
|
||||||
|
f"{len(added)} 条 HTTP 路由 (prefix={prefix or '/'})"
|
||||||
|
)
|
||||||
|
return added
|
||||||
|
|
||||||
|
|
||||||
|
def register_dynamic_routes(
|
||||||
|
app: FastAPI,
|
||||||
|
worker_manager: WorkerManager,
|
||||||
|
prefix: str = "",
|
||||||
|
worker_types: Optional[List[str]] = None,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
根据 config/workers/ 目录结构注册所有动态路由。
|
||||||
|
|
||||||
|
worker_types: 白名单(混合模式下只注册 inline 插件;
|
||||||
|
None = 全部注册,兼容纯 inline 部署)。
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
workers_base = Path(WORKER_CONFIG_DIR)
|
||||||
|
if not workers_base.is_dir():
|
||||||
|
logger.info(f"Worker 配置目录不存在: {WORKER_CONFIG_DIR}")
|
||||||
|
return
|
||||||
|
|
||||||
|
extra_schemas: Dict[str, type[BaseModel]] = {}
|
||||||
|
# validate=false 端点的 (path, method) -> 请求模型名(OpenAPI requestBody 注入用)
|
||||||
|
raw_body_models: Dict[Tuple[str, str], str] = {}
|
||||||
|
# 非流式端点的 (path, method) -> 响应模型名(OpenAPI responses[200] $ref 注入用)
|
||||||
|
response_models: Dict[Tuple[str, str], str] = {}
|
||||||
|
|
||||||
|
# 扫描所有 worker 子目录(每个子目录名 = worker_type)
|
||||||
|
for worker_dir in sorted(workers_base.iterdir()):
|
||||||
|
if not worker_dir.is_dir():
|
||||||
|
continue
|
||||||
|
worker_type = worker_dir.name
|
||||||
|
if worker_types is not None and worker_type not in worker_types:
|
||||||
|
continue
|
||||||
|
routes = load_worker_routes_from_dir(worker_type)
|
||||||
|
if not routes:
|
||||||
|
continue
|
||||||
|
|
||||||
|
register_route_group(app, worker_manager, worker_type, routes, prefix, extra_schemas, raw_body_models, response_models)
|
||||||
|
|
||||||
|
# 将额外 schema 注册到 OpenAPI 的 components/schemas
|
||||||
|
if extra_schemas:
|
||||||
|
_inject_schemas_to_openapi(app, lambda: extra_schemas, lambda: raw_body_models, lambda: response_models)
|
||||||
|
|
||||||
|
logger.info(f"动态路由注册完成,worker 配置目录: {WORKER_CONFIG_DIR}")
|
||||||
|
|
||||||
|
|
||||||
|
def _inject_schemas_to_openapi(app: FastAPI, schemas_provider, raw_body_provider=None, response_provider=None):
|
||||||
|
"""
|
||||||
|
将自定义 Pydantic 模型注册到 FastAPI 的 OpenAPI components/schemas 中。
|
||||||
|
|
||||||
|
通过重写 app.openapi() 方法注入,因为 FastAPI 在生命周期内只生成一次
|
||||||
|
OpenAPI schema,之后会被缓存。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
schemas_provider: 无参 callable,返回 {name: BaseModel} 字典。
|
||||||
|
盘扫路径传静态 dict;process 模式传 RouteRegistry 的动态 dict,
|
||||||
|
使路由表变化后重新生成的 OpenAPI 能反映最新 schema。
|
||||||
|
raw_body_provider: 可选无参 callable,返回 {(path, method): model_name} 字典。
|
||||||
|
validate=false 端点未声明 body 参数(FastAPI 不会自动生成 requestBody),
|
||||||
|
这里按 (path, method) 补注入 requestBody $ref,保持 OpenAPI 请求体文档完整。
|
||||||
|
response_provider: 可选无参 callable,返回 {(path, method): model_name} 字典。
|
||||||
|
端点注册未传 response_model(见 _register_single_route:砍掉注册期模型反射),
|
||||||
|
这里按 (path, method) 补注入 responses[200] 的 $ref,保持响应文档完整。
|
||||||
|
"""
|
||||||
|
original_openapi = app.openapi
|
||||||
|
|
||||||
|
def patched_openapi() -> dict[str, Any]:
|
||||||
|
if app.openapi_schema is not None:
|
||||||
|
return app.openapi_schema
|
||||||
|
schema = original_openapi()
|
||||||
|
if schema is None:
|
||||||
|
return {"openapi": "3.1.0", "info": {"title": "Chongming", "version": "1.0.0"}, "paths": {}}
|
||||||
|
# 注入我们的 schema
|
||||||
|
schemas_dict = schema.setdefault("components", {}).setdefault("schemas", {})
|
||||||
|
for name, model in schemas_provider().items():
|
||||||
|
if name not in schemas_dict:
|
||||||
|
schemas_dict[name] = model.model_json_schema()
|
||||||
|
# validate=false 端点:补注入 requestBody(请求体文档,模型已注册到 components)
|
||||||
|
if raw_body_provider is not None:
|
||||||
|
for (path, method), model_name in raw_body_provider().items():
|
||||||
|
op = schema.get("paths", {}).get(path, {}).get(method.lower())
|
||||||
|
if op is not None:
|
||||||
|
op["requestBody"] = {
|
||||||
|
"required": True,
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {"$ref": f"#/components/schemas/{model_name}"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
# 响应模型:补注入 responses[200] 的 $ref(保留 description,仅加 content)
|
||||||
|
if response_provider is not None:
|
||||||
|
for (path, method), model_name in response_provider().items():
|
||||||
|
op = schema.get("paths", {}).get(path, {}).get(method.lower())
|
||||||
|
if op is not None:
|
||||||
|
op.setdefault("responses", {}).setdefault("200", {})
|
||||||
|
op["responses"]["200"].setdefault(
|
||||||
|
"content", {}
|
||||||
|
).setdefault("application/json", {})["schema"] = {
|
||||||
|
"$ref": f"#/components/schemas/{model_name}"
|
||||||
|
}
|
||||||
|
app.openapi_schema = schema
|
||||||
|
return schema
|
||||||
|
|
||||||
|
app.openapi = patched_openapi
|
||||||
|
|
||||||
|
|
||||||
|
def _spawn_call_record(
|
||||||
|
request: Request,
|
||||||
|
worker_type: str,
|
||||||
|
subject: str,
|
||||||
|
rc: RouteConfig,
|
||||||
|
status: int,
|
||||||
|
duration_ms: float,
|
||||||
|
req_bytes: int = 0,
|
||||||
|
resp_bytes: int = 0,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
以后台任务记录一次调用(统计 + 审计),非关键后置操作。
|
||||||
|
|
||||||
|
通过 spawn() 以 fire-and-forget 方式执行,端点立即返回,
|
||||||
|
不增加接口响应延迟;后台失败由 record_call 内部兜底。
|
||||||
|
"""
|
||||||
|
entry = {
|
||||||
|
"ts": time.time(),
|
||||||
|
"worker_type": worker_type,
|
||||||
|
"subject": subject,
|
||||||
|
"method": rc.method,
|
||||||
|
"path": request.url.path,
|
||||||
|
"user": getattr(request.state, "user", None),
|
||||||
|
"client": request.client.host if request.client else None,
|
||||||
|
"status": status,
|
||||||
|
"duration_ms": round(duration_ms, 3),
|
||||||
|
"req_bytes": req_bytes,
|
||||||
|
"resp_bytes": resp_bytes,
|
||||||
|
}
|
||||||
|
spawn(record_call(entry))
|
||||||
|
|
||||||
|
|
||||||
|
def _make_body_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, RequestModel: type[BaseModel]):
|
||||||
|
"""创建用于 Body 请求(POST/PUT/PATCH)的端点函数"""
|
||||||
|
path = rc.path
|
||||||
|
subject = rc.subject
|
||||||
|
timeout = rc.timeout
|
||||||
|
http_method = rc.method
|
||||||
|
cacheable = rc.cacheable
|
||||||
|
|
||||||
|
async def endpoint(
|
||||||
|
request: Request,
|
||||||
|
body: RequestModel, # type: ignore
|
||||||
|
) -> Any:
|
||||||
|
start: Optional[float] = None
|
||||||
|
payload_bytes = b""
|
||||||
|
try:
|
||||||
|
# 信封协议: {"meta": {...}, "data": {...}}
|
||||||
|
meta = {"method": http_method}
|
||||||
|
user_info = getattr(request.state, "user", None)
|
||||||
|
if user_info:
|
||||||
|
meta["user_info"] = user_info
|
||||||
|
payload = {"meta": meta, "data": body.model_dump()}
|
||||||
|
payload_bytes = orjson.dumps(payload)
|
||||||
|
start = time.perf_counter()
|
||||||
|
result = await worker_manager.call_raw(
|
||||||
|
worker_type=worker_type,
|
||||||
|
subject=subject,
|
||||||
|
payload=payload_bytes,
|
||||||
|
timeout=timeout,
|
||||||
|
cacheable=cacheable,
|
||||||
|
instance_id=_instance_from_request(request),
|
||||||
|
)
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 200, duration_ms,
|
||||||
|
req_bytes=len(payload_bytes), resp_bytes=len(result),
|
||||||
|
)
|
||||||
|
# 回复为 worker 原始字节,零解析直接转发(response_model 仅用于 OpenAPI 文档)
|
||||||
|
return Response(content=result, media_type="application/json")
|
||||||
|
except RuntimeError as e:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 503, duration_ms,
|
||||||
|
req_bytes=len(payload_bytes),
|
||||||
|
)
|
||||||
|
logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e)
|
||||||
|
raise HTTPException(status_code=503, detail=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 500, duration_ms,
|
||||||
|
req_bytes=len(payload_bytes),
|
||||||
|
)
|
||||||
|
logger.error(f"路由 {path} 调用异常: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||||
|
|
||||||
|
return endpoint
|
||||||
|
|
||||||
|
|
||||||
|
def _make_raw_body_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str):
|
||||||
|
"""
|
||||||
|
创建 validate=false 的 body 端点:请求体原样透传(网关零解析/校验)。
|
||||||
|
|
||||||
|
信封结构不变({"meta": ..., "data": <原始 JSON 值>}),通过字节拼接构造,
|
||||||
|
网关既不解析 body 也不转义;参数校验/类型转换职责下沉到 worker
|
||||||
|
(失败经 reply_err 通道以 503 暴露)。空/空白 body 兜底为 {}。
|
||||||
|
请求模型(params TOML 生成)仅用于 OpenAPI 文档(extra_schemas + requestBody 注入)。
|
||||||
|
"""
|
||||||
|
path = rc.path
|
||||||
|
subject = rc.subject
|
||||||
|
timeout = rc.timeout
|
||||||
|
http_method = rc.method
|
||||||
|
cacheable = rc.cacheable
|
||||||
|
|
||||||
|
async def endpoint(request: Request) -> Any:
|
||||||
|
start: Optional[float] = None
|
||||||
|
payload = b""
|
||||||
|
try:
|
||||||
|
meta = {"method": http_method}
|
||||||
|
user_info = getattr(request.state, "user", None)
|
||||||
|
if user_info:
|
||||||
|
meta["user_info"] = user_info
|
||||||
|
raw = await request.body()
|
||||||
|
data_part = raw.strip() or b"{}"
|
||||||
|
payload = b'{"meta":' + orjson.dumps(meta) + b',"data":' + data_part + b'}'
|
||||||
|
start = time.perf_counter()
|
||||||
|
result = await worker_manager.call_raw(
|
||||||
|
worker_type=worker_type,
|
||||||
|
subject=subject,
|
||||||
|
payload=payload,
|
||||||
|
timeout=timeout,
|
||||||
|
cacheable=cacheable,
|
||||||
|
instance_id=_instance_from_request(request),
|
||||||
|
)
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 200, duration_ms,
|
||||||
|
req_bytes=len(payload), resp_bytes=len(result),
|
||||||
|
)
|
||||||
|
return Response(content=result, media_type="application/json")
|
||||||
|
except RuntimeError as e:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 503, duration_ms,
|
||||||
|
req_bytes=len(payload),
|
||||||
|
)
|
||||||
|
logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e)
|
||||||
|
raise HTTPException(status_code=503, detail=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 500, duration_ms,
|
||||||
|
req_bytes=len(payload),
|
||||||
|
)
|
||||||
|
logger.error(f"路由 {path} 调用异常: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||||
|
|
||||||
|
return endpoint
|
||||||
|
|
||||||
|
|
||||||
|
def _make_query_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, path_params: List[str], query_params: Dict[str, str]):
|
||||||
|
"""
|
||||||
|
创建用于 Query 请求(GET/DELETE)的端点函数。
|
||||||
|
|
||||||
|
通过设置函数 __signature__ 和 __annotations__ 为每个参数赋予正确的 Pydantic 类型,
|
||||||
|
使 FastAPI 能在 OpenAPI schema 中生成正确的 type / format 信息。
|
||||||
|
请求模型 schema 通过 extra_schemas 机制单独注入到 components/schemas。
|
||||||
|
"""
|
||||||
|
path = rc.path
|
||||||
|
subject = rc.subject
|
||||||
|
timeout = rc.timeout
|
||||||
|
http_method = rc.method
|
||||||
|
cacheable = rc.cacheable
|
||||||
|
|
||||||
|
# 合并路径参数和查询参数
|
||||||
|
param_spec_combined: Dict[str, str] = {}
|
||||||
|
param_spec_combined.update({k: query_params.get(k, "str") for k in path_params if k in query_params})
|
||||||
|
param_spec_combined.update(query_params)
|
||||||
|
|
||||||
|
# 无参数情况
|
||||||
|
if not param_spec_combined:
|
||||||
|
async def _simple_endpoint(request: Request) -> Any:
|
||||||
|
# 信封协议: {"meta": {...}, "data": {}}
|
||||||
|
meta = {"method": http_method}
|
||||||
|
user_info = getattr(request.state, "user", None)
|
||||||
|
if user_info:
|
||||||
|
meta["user_info"] = user_info
|
||||||
|
payload_bytes = orjson.dumps({"meta": meta, "data": {}})
|
||||||
|
start = time.perf_counter()
|
||||||
|
try:
|
||||||
|
result = await worker_manager.call_raw(
|
||||||
|
worker_type=worker_type,
|
||||||
|
subject=subject,
|
||||||
|
payload=payload_bytes,
|
||||||
|
timeout=timeout,
|
||||||
|
cacheable=cacheable,
|
||||||
|
instance_id=_instance_from_request(request),
|
||||||
|
)
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 200, duration_ms,
|
||||||
|
req_bytes=len(payload_bytes), resp_bytes=len(result),
|
||||||
|
)
|
||||||
|
return Response(content=result, media_type="application/json")
|
||||||
|
except RuntimeError as e:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 503, duration_ms,
|
||||||
|
req_bytes=len(payload_bytes),
|
||||||
|
)
|
||||||
|
logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e)
|
||||||
|
raise HTTPException(status_code=503, detail=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 500, duration_ms,
|
||||||
|
req_bytes=len(payload_bytes),
|
||||||
|
)
|
||||||
|
logger.error(f"路由 {path} 调用异常: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||||
|
return _simple_endpoint
|
||||||
|
|
||||||
|
# 构造参数名列表(用于从 kwargs 中提取)
|
||||||
|
param_names = list(param_spec_combined.keys())
|
||||||
|
|
||||||
|
async def endpoint(request: Request, **kwargs: Any) -> Any:
|
||||||
|
# 信封协议: {"meta": {...}, "data": {...}}
|
||||||
|
meta = {"method": http_method}
|
||||||
|
user_info = getattr(request.state, "user", None)
|
||||||
|
if user_info:
|
||||||
|
meta["user_info"] = user_info
|
||||||
|
payload_bytes = orjson.dumps({"meta": meta, "data": {k: kwargs[k] for k in param_names}})
|
||||||
|
start = time.perf_counter()
|
||||||
|
try:
|
||||||
|
result = await worker_manager.call_raw(
|
||||||
|
worker_type=worker_type,
|
||||||
|
subject=subject,
|
||||||
|
payload=payload_bytes,
|
||||||
|
timeout=timeout,
|
||||||
|
cacheable=cacheable,
|
||||||
|
instance_id=_instance_from_request(request),
|
||||||
|
)
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 200, duration_ms,
|
||||||
|
req_bytes=len(payload_bytes), resp_bytes=len(result),
|
||||||
|
)
|
||||||
|
return Response(content=result, media_type="application/json")
|
||||||
|
except RuntimeError as e:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 503, duration_ms,
|
||||||
|
req_bytes=len(payload_bytes),
|
||||||
|
)
|
||||||
|
logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e)
|
||||||
|
raise HTTPException(status_code=503, detail=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 500, duration_ms,
|
||||||
|
req_bytes=len(payload_bytes),
|
||||||
|
)
|
||||||
|
logger.error(f"路由 {path} 调用异常: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||||
|
|
||||||
|
# 构建带类型注解的 Parameter 列表
|
||||||
|
params: List[Parameter] = [
|
||||||
|
Parameter("request", Parameter.POSITIONAL_OR_KEYWORD, annotation=Request)
|
||||||
|
]
|
||||||
|
annotations: Dict[str, Any] = {}
|
||||||
|
for name, type_str in param_spec_combined.items():
|
||||||
|
py_type = _parse_python_type(type_str)
|
||||||
|
annotations[name] = py_type
|
||||||
|
if name in path_params:
|
||||||
|
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=FastAPIPath(...), annotation=py_type))
|
||||||
|
else:
|
||||||
|
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=Query(...), annotation=py_type))
|
||||||
|
|
||||||
|
# 设置 __annotations__(FastAPI 通过 get_type_hints 读取)
|
||||||
|
endpoint.__annotations__ = annotations
|
||||||
|
# 设置 __signature__(FastAPI 优先于此获取参数列表)
|
||||||
|
endpoint.__signature__ = Signature(params, return_annotation=Dict[str, Any]) # type: ignore
|
||||||
|
|
||||||
|
return endpoint
|
||||||
|
|
||||||
|
|
||||||
|
# 流式端点响应头:禁止缓存;通知反向代理(nginx)关闭缓冲保证 SSE 即时性
|
||||||
|
_STREAM_HEADERS = {
|
||||||
|
"Cache-Control": "no-cache",
|
||||||
|
"X-Accel-Buffering": "no",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _make_stream_endpoint(
|
||||||
|
worker_manager: WorkerManager,
|
||||||
|
rc: RouteConfig,
|
||||||
|
worker_type: str,
|
||||||
|
path_params: List[str] = (),
|
||||||
|
query_params: Optional[Dict[str, str]] = None,
|
||||||
|
RequestModel: Optional[type[BaseModel]] = None,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
创建流式端点(TOML 中 stream = true):以 SSE(text/event-stream)逐块输出。
|
||||||
|
|
||||||
|
请求参数形态与普通端点一致(GET 走 query/path 参数,POST/PUT/PATCH 走 body 模型)。
|
||||||
|
错误语义:
|
||||||
|
- 初始错误(无可用 worker 等)-> HTTP 503
|
||||||
|
- 流中错误(Worker 返回 {"error": ...} 收尾块 / 空闲超时)-> SSE error 事件
|
||||||
|
(此时 HTTP 200 已提交,无法改状态码)
|
||||||
|
不校验响应模型:流式输出是逐块 JSON dict。
|
||||||
|
"""
|
||||||
|
path = rc.path
|
||||||
|
subject = rc.subject
|
||||||
|
http_method = rc.method
|
||||||
|
timeout = rc.timeout
|
||||||
|
idle_timeout = rc.stream_idle_timeout
|
||||||
|
param_names = list((query_params or {}).keys())
|
||||||
|
|
||||||
|
def build_payload(request: Request, body=None, kwargs=None) -> Dict[str, Any]:
|
||||||
|
"""构造信封协议 payload(与普通端点一致)"""
|
||||||
|
meta = {"method": http_method}
|
||||||
|
user_info = getattr(request.state, "user", None)
|
||||||
|
if user_info:
|
||||||
|
meta["user_info"] = user_info
|
||||||
|
if body is not None:
|
||||||
|
data = body.model_dump()
|
||||||
|
else:
|
||||||
|
data = {k: kwargs[k] for k in param_names}
|
||||||
|
return {"meta": meta, "data": data}
|
||||||
|
|
||||||
|
def _sse_frame(data: Dict[str, Any]) -> str:
|
||||||
|
return f"data: {orjson.dumps(data).decode('utf-8')}\n\n"
|
||||||
|
|
||||||
|
async def _sse_body(gen, start: float, request: Request, req_bytes: int) -> AsyncGenerator[str, None]:
|
||||||
|
"""SSE 分帧;流中异常转为 SSE error 事件(HTTP 200 已提交)。
|
||||||
|
|
||||||
|
流结束(正常/空闲超时/中断/客户端断开)后在 finally 中后台记录
|
||||||
|
调用统计与审计,时长覆盖整个流。
|
||||||
|
"""
|
||||||
|
status = 200
|
||||||
|
try:
|
||||||
|
async for chunk in gen:
|
||||||
|
yield f"data: {chunk.decode('utf-8')}\n\n"
|
||||||
|
except StreamIdleTimeoutError as e:
|
||||||
|
status = 504
|
||||||
|
logger.warning("路由 %s 流式空闲超时: %s", path, e)
|
||||||
|
yield _sse_frame({"error": str(e)})
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
status = 499 # 客户端断开,流未完成
|
||||||
|
raise
|
||||||
|
except GeneratorExit:
|
||||||
|
status = 499 # 服务端主动关闭流(如连接回收),流未完成
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
status = 500
|
||||||
|
logger.error("路由 %s 流式中断: %s", path, e)
|
||||||
|
yield _sse_frame({"error": "内部服务器错误"})
|
||||||
|
finally:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, status, duration_ms,
|
||||||
|
req_bytes=req_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _start_stream(request: Request, payload: Dict[str, Any]) -> StreamingResponse:
|
||||||
|
req_bytes = len(orjson.dumps(payload))
|
||||||
|
start = time.perf_counter()
|
||||||
|
try:
|
||||||
|
gen = await worker_manager.call_stream(
|
||||||
|
worker_type=worker_type,
|
||||||
|
subject=subject,
|
||||||
|
payload=payload,
|
||||||
|
timeout=timeout,
|
||||||
|
idle_timeout=idle_timeout,
|
||||||
|
instance_id=_instance_from_request(request),
|
||||||
|
)
|
||||||
|
except RuntimeError as e:
|
||||||
|
duration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
_spawn_call_record(
|
||||||
|
request, worker_type, subject, rc, 503, duration_ms,
|
||||||
|
req_bytes=req_bytes,
|
||||||
|
)
|
||||||
|
logger.warning("路由 %s [%s] 流式调用失败: %s", path, worker_type, e)
|
||||||
|
raise HTTPException(status_code=503, detail=str(e))
|
||||||
|
return StreamingResponse(
|
||||||
|
_sse_body(gen, start, request, req_bytes),
|
||||||
|
media_type="text/event-stream",
|
||||||
|
headers=_STREAM_HEADERS,
|
||||||
|
)
|
||||||
|
|
||||||
|
if RequestModel is not None:
|
||||||
|
async def endpoint(request: Request, body: RequestModel) -> StreamingResponse: # type: ignore
|
||||||
|
return await _start_stream(request, build_payload(request, body=body))
|
||||||
|
return endpoint
|
||||||
|
|
||||||
|
# GET/DELETE 形态
|
||||||
|
if not param_names:
|
||||||
|
async def _simple_endpoint(request: Request) -> StreamingResponse:
|
||||||
|
return await _start_stream(request, build_payload(request, kwargs=None))
|
||||||
|
return _simple_endpoint
|
||||||
|
|
||||||
|
async def endpoint(request: Request, **kwargs: Any) -> StreamingResponse:
|
||||||
|
return await _start_stream(request, build_payload(request, kwargs=kwargs))
|
||||||
|
|
||||||
|
# 附加类型注解与签名(与 _make_query_endpoint 一致),保证 OpenAPI 参数文档正确
|
||||||
|
params: List[Parameter] = [
|
||||||
|
Parameter("request", Parameter.POSITIONAL_OR_KEYWORD, annotation=Request)
|
||||||
|
]
|
||||||
|
annotations: Dict[str, Any] = {}
|
||||||
|
for name, type_str in (query_params or {}).items():
|
||||||
|
py_type = _parse_python_type(type_str)
|
||||||
|
annotations[name] = py_type
|
||||||
|
if name in path_params:
|
||||||
|
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=FastAPIPath(...), annotation=py_type))
|
||||||
|
else:
|
||||||
|
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=Query(...), annotation=py_type))
|
||||||
|
|
||||||
|
endpoint.__annotations__ = annotations
|
||||||
|
endpoint.__signature__ = Signature(params, return_annotation=StreamingResponse) # type: ignore
|
||||||
|
|
||||||
|
return endpoint
|
||||||
|
|
||||||
|
|
||||||
|
def _register_single_route(
|
||||||
|
router: APIRouter,
|
||||||
|
worker_manager: WorkerManager,
|
||||||
|
worker_type: str,
|
||||||
|
route_conf: Dict[str, Any],
|
||||||
|
extra_schemas: Optional[Dict[str, type[BaseModel]]] = None,
|
||||||
|
raw_body_models: Optional[Dict[Tuple[str, str], str]] = None,
|
||||||
|
response_models: Optional[Dict[Tuple[str, str], str]] = None,
|
||||||
|
):
|
||||||
|
"""注册单个路由,生成正确的函数签名以便 OpenAPI 正确生成文档"""
|
||||||
|
rc = RouteConfig(route_conf)
|
||||||
|
|
||||||
|
# MQTT 端点:不注册 HTTP 路由,仅通过 MQTT Bridge 调用
|
||||||
|
if rc.method == "MQTT":
|
||||||
|
logger.info(
|
||||||
|
f"MQTT 端点: subject={rc.subject} (仅 MQTT,不注册 HTTP 路由)"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
# internal 端点:不注册 HTTP 路由,仅保留 Zenoh RPC 内部调用能力
|
||||||
|
if rc.internal:
|
||||||
|
logger.info(
|
||||||
|
f"跳过 internal 路由注册: {rc.method} {rc.path} -> worker {worker_type}.{rc.subject}"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
param_spec = rc.parse_params()
|
||||||
|
path_params = rc.extract_path_params()
|
||||||
|
query_params = {k: v for k, v in param_spec.items() if k not in path_params}
|
||||||
|
|
||||||
|
# 生成干净的模型名称(替换路径中的 / 和 {} 为 _)
|
||||||
|
# 模型名称包含 HTTP method 以区分同名路径的不同调用类型
|
||||||
|
clean_path = rc.path.replace("/", "_").replace("{", "").replace("}", "").strip("_")
|
||||||
|
if not clean_path:
|
||||||
|
clean_path = "root"
|
||||||
|
method_tag = rc.method.upper()
|
||||||
|
response_model_name = f"{worker_type}_{clean_path}_{method_tag}_Response"
|
||||||
|
request_model_name = f"{worker_type}_{clean_path}_{method_tag}_Request"
|
||||||
|
|
||||||
|
# 生成响应模型
|
||||||
|
if rc.stream:
|
||||||
|
# 流式端点:逐块 SSE 输出,不校验响应模型(OpenAPI 不声明响应 schema)
|
||||||
|
ResponseModel = None
|
||||||
|
if rc.method in ("POST", "PUT", "PATCH"):
|
||||||
|
# body 形态:RequestModel 传给端点做请求体校验
|
||||||
|
RequestModel = generate_request_model(
|
||||||
|
param_spec, request_model_name
|
||||||
|
)
|
||||||
|
endpoint = _make_stream_endpoint(
|
||||||
|
worker_manager, rc, worker_type, path_params, query_params, RequestModel
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# GET/DELETE 形态:query/path 参数走签名注解;
|
||||||
|
# RequestModel 仅注册到 extra_schemas 供 OpenAPI 展示(与普通 GET 一致)
|
||||||
|
if param_spec:
|
||||||
|
RequestModel = generate_request_model(
|
||||||
|
param_spec, request_model_name
|
||||||
|
)
|
||||||
|
if extra_schemas is not None and request_model_name not in extra_schemas:
|
||||||
|
extra_schemas[request_model_name] = RequestModel
|
||||||
|
endpoint = _make_stream_endpoint(
|
||||||
|
worker_manager, rc, worker_type, path_params, query_params, None
|
||||||
|
)
|
||||||
|
elif rc.method in ("POST", "PUT", "PATCH"):
|
||||||
|
ResponseModel = generate_response_model(
|
||||||
|
rc.response_model, response_model_name
|
||||||
|
)
|
||||||
|
# 响应模型仅注册到 components/schemas + responses[200] $ref 注入,
|
||||||
|
# 不传给 add_api_route:端点返回 Response 对象本就走透传(零运行时校验),
|
||||||
|
# 省掉 FastAPI 注册期对 response_model 的反射解析(external 插件频繁
|
||||||
|
# 挂载/摘除时每次重建都会重复这一开销)
|
||||||
|
if ResponseModel is not None:
|
||||||
|
if extra_schemas is not None and response_model_name not in extra_schemas:
|
||||||
|
extra_schemas[response_model_name] = ResponseModel
|
||||||
|
if response_models is not None:
|
||||||
|
response_models[(rc.path, rc.method)] = response_model_name
|
||||||
|
RequestModel = generate_request_model(
|
||||||
|
param_spec, request_model_name
|
||||||
|
)
|
||||||
|
if rc.validate:
|
||||||
|
endpoint = _make_body_endpoint(worker_manager, rc, worker_type, RequestModel)
|
||||||
|
else:
|
||||||
|
# validate=false:请求体原样透传(网关零解析/校验,校验下沉 worker)。
|
||||||
|
# RequestModel 仅用于 OpenAPI 请求体文档:
|
||||||
|
# 经 extra_schemas 注册 components/schemas + requestBody $ref 注入
|
||||||
|
if extra_schemas is not None and request_model_name not in extra_schemas:
|
||||||
|
extra_schemas[request_model_name] = RequestModel
|
||||||
|
if raw_body_models is not None:
|
||||||
|
raw_body_models[(rc.path, rc.method)] = request_model_name
|
||||||
|
endpoint = _make_raw_body_endpoint(worker_manager, rc, worker_type)
|
||||||
|
else:
|
||||||
|
ResponseModel = generate_response_model(
|
||||||
|
rc.response_model, response_model_name
|
||||||
|
)
|
||||||
|
# 同上:响应模型走 extra_schemas + responses $ref 注入,不传 add_api_route
|
||||||
|
if ResponseModel is not None:
|
||||||
|
if extra_schemas is not None and response_model_name not in extra_schemas:
|
||||||
|
extra_schemas[response_model_name] = ResponseModel
|
||||||
|
if response_models is not None:
|
||||||
|
response_models[(rc.path, rc.method)] = response_model_name
|
||||||
|
# 对 GET 端点也生成 Request 模型并注册到 extra_schemas
|
||||||
|
if param_spec:
|
||||||
|
RequestModel = generate_request_model(
|
||||||
|
param_spec, request_model_name
|
||||||
|
)
|
||||||
|
if extra_schemas is not None and request_model_name not in extra_schemas:
|
||||||
|
extra_schemas[request_model_name] = RequestModel
|
||||||
|
endpoint = _make_query_endpoint(worker_manager, rc, worker_type, path_params, query_params)
|
||||||
|
|
||||||
|
dependencies = []
|
||||||
|
if rc.auth_required:
|
||||||
|
dependencies.append(Depends(store_user_to_request))
|
||||||
|
|
||||||
|
# 注册路由 - 使用 worker_type 作为 OpenAPI 分组标签
|
||||||
|
# 每个 worker 类型自动成为一个独立的分组,显示在 Swagger UI 中
|
||||||
|
# 不传 response_model:端点返回 Response 对象走透传,运行时本就零校验;
|
||||||
|
# 响应文档由 response_models 收集 + _inject_schemas_to_openapi 补 $ref
|
||||||
|
router.add_api_route(
|
||||||
|
path=rc.path,
|
||||||
|
endpoint=endpoint,
|
||||||
|
methods=[rc.method],
|
||||||
|
summary=rc.summary,
|
||||||
|
description=rc.docstring,
|
||||||
|
tags=[worker_type],
|
||||||
|
include_in_schema=True,
|
||||||
|
dependencies=dependencies,
|
||||||
|
)
|
||||||
|
logger.info(f"已注册动态路由: {rc.method} {rc.path} -> worker {worker_type}.{rc.subject} | tags=[{worker_type}]")
|
||||||
@@ -0,0 +1,271 @@
|
|||||||
|
"""
|
||||||
|
动态路由注册表(process 模式)
|
||||||
|
|
||||||
|
网关订阅 `chongming/routes/**` 命名空间,独立 worker 进程上线时 put 路由描述符、
|
||||||
|
下线时 delete,网关据此动态挂载/摘除 FastAPI 路由并同步 MQTT 自定义订阅主题。
|
||||||
|
|
||||||
|
协议(描述符 schema_version=1):
|
||||||
|
chongming/routes/<type>/__meta__ 插件级描述: {"schema_version":1,"mqtt_topics":[...]}
|
||||||
|
chongming/routes/<type>/<route_id> 端点描述(TOML 兼容 dict,与 RouteConfig 对位):
|
||||||
|
{"schema_version":1, "type","subject","method","path","params",
|
||||||
|
"response_model","summary","timeout","cacheable","auth_required","internal"}
|
||||||
|
|
||||||
|
特性:
|
||||||
|
- SampleKind.PUT -> upsert;DELETE -> remove
|
||||||
|
- 同一 route_id 内容相同(hash)视为多副本重复发布,忽略;不同 -> 告警并保留先到者
|
||||||
|
- type 级 2 秒防抖:worker 重启风暴合并为一次路由重建
|
||||||
|
- 迟到一致性:worker 侧周期性重发布描述符(RoutePublisher 由调用方周期调用),
|
||||||
|
网关后启动或样本丢失时,路由在一个发布周期内自动补齐
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import orjson
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
import zenoh
|
||||||
|
|
||||||
|
from chongming_worker.routes_publisher import META_KEY, ROUTES_PREFIX, SCHEMA_VERSION
|
||||||
|
|
||||||
|
from .logger import get_logger
|
||||||
|
from .route_generator import register_route_group, _inject_schemas_to_openapi
|
||||||
|
|
||||||
|
logger = get_logger("route_registry")
|
||||||
|
|
||||||
|
# worker 重启风暴防抖窗口(秒)
|
||||||
|
DEBOUNCE_SECONDS = 2.0
|
||||||
|
|
||||||
|
|
||||||
|
def _descriptor_hash(descriptor: Dict[str, Any]) -> str:
|
||||||
|
"""描述符内容 hash(多副本幂等去重用)"""
|
||||||
|
payload = orjson.dumps(descriptor, option=orjson.OPT_SORT_KEYS)
|
||||||
|
return hashlib.sha1(payload).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
class RouteRegistry:
|
||||||
|
"""process 模式下的网关侧动态路由注册表。"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
app,
|
||||||
|
worker_manager,
|
||||||
|
prefix: str = "",
|
||||||
|
):
|
||||||
|
self._app = app
|
||||||
|
self._wm = worker_manager
|
||||||
|
self._prefix = prefix
|
||||||
|
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||||
|
|
||||||
|
# worker_type -> route_id -> descriptor
|
||||||
|
self._groups: Dict[str, Dict[str, Dict[str, Any]]] = {}
|
||||||
|
# worker_type -> 插件级描述(mqtt_topics 等)
|
||||||
|
self._meta: Dict[str, Dict[str, Any]] = {}
|
||||||
|
# worker_type -> 已挂载的 APIRoute 对象(用于动态摘除)
|
||||||
|
self._route_objs: Dict[str, List[Any]] = {}
|
||||||
|
# OpenAPI 额外 schema(GET 请求模型),随路由表变化
|
||||||
|
self._schemas: Dict[str, Any] = {}
|
||||||
|
# validate=false 端点的 (path, method) -> 请求模型名(OpenAPI requestBody 注入用)
|
||||||
|
self._raw_body_models: Dict[Tuple[str, str], str] = {}
|
||||||
|
# 非流式端点的 (path, method) -> 响应模型名(OpenAPI responses[200] $ref 注入用)
|
||||||
|
self._response_models: Dict[Tuple[str, str], str] = {}
|
||||||
|
# worker_type -> 防抖任务
|
||||||
|
self._debounce: Dict[str, asyncio.Task] = {}
|
||||||
|
|
||||||
|
self._subscriber: Optional[zenoh.Subscriber] = None
|
||||||
|
self._mqtt_bridge = None
|
||||||
|
# 网关进程内加载(inline)的插件类型:忽略其描述符,防止与盘扫注册重复挂载
|
||||||
|
self._inline_types: set = set()
|
||||||
|
|
||||||
|
# 将动态 schema 注入 OpenAPI(provider 实时读取本注册表)
|
||||||
|
_inject_schemas_to_openapi(
|
||||||
|
app,
|
||||||
|
lambda: self._schemas,
|
||||||
|
lambda: self._raw_body_models,
|
||||||
|
lambda: self._response_models,
|
||||||
|
)
|
||||||
|
|
||||||
|
def set_inline_types(self, types: set):
|
||||||
|
"""设置 inline 插件类型集合(混合模式防双注册)。"""
|
||||||
|
self._inline_types = set(types)
|
||||||
|
|
||||||
|
async def start(self, session: zenoh.Session):
|
||||||
|
"""订阅 chongming/routes/**,开始接收路由描述符。"""
|
||||||
|
self._loop = asyncio.get_running_loop()
|
||||||
|
self._subscriber = session.declare_subscriber(
|
||||||
|
f"{ROUTES_PREFIX}/**", self._on_sample
|
||||||
|
)
|
||||||
|
logger.info(f"路由注册表已就绪: 订阅 {ROUTES_PREFIX}/**")
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""停止订阅(shutdown 时调用)。"""
|
||||||
|
if self._subscriber is not None:
|
||||||
|
self._subscriber.undeclare()
|
||||||
|
self._subscriber = None
|
||||||
|
for task in self._debounce.values():
|
||||||
|
task.cancel()
|
||||||
|
|
||||||
|
def attach_bridge(self, mqtt_bridge):
|
||||||
|
"""挂接 MQTT Bridge:后续路由表变化会推送 external 插件的动态订阅主题。"""
|
||||||
|
self._mqtt_bridge = mqtt_bridge
|
||||||
|
# Bridge 晚于路由到达时,立即补推当前已收集的主题
|
||||||
|
topics = self._collect_mqtt_topics()
|
||||||
|
if topics:
|
||||||
|
self._loop.create_task(mqtt_bridge.update_dynamic_topics(topics))
|
||||||
|
|
||||||
|
@property
|
||||||
|
def route_types(self) -> List[str]:
|
||||||
|
"""当前已注册路由的 worker 类型列表(观测用)。"""
|
||||||
|
return sorted(self._groups.keys())
|
||||||
|
|
||||||
|
def ready_summary(self) -> str:
|
||||||
|
"""路由就绪状态摘要(日志/观测用)。"""
|
||||||
|
parts = []
|
||||||
|
for wtype in sorted(self._groups):
|
||||||
|
http = sum(
|
||||||
|
1 for d in self._groups[wtype].values()
|
||||||
|
if d.get("method") not in ("MQTT",) and not d.get("internal")
|
||||||
|
)
|
||||||
|
parts.append(f"{wtype}={len(self._groups[wtype])} 端点/{http} HTTP")
|
||||||
|
return ", ".join(parts) if parts else "(空)"
|
||||||
|
|
||||||
|
# ---- zenoh 回调线程 -> 事件循环 ----
|
||||||
|
|
||||||
|
def _on_sample(self, sample: zenoh.Sample):
|
||||||
|
"""zenoh 后台线程回调:转交事件循环处理。"""
|
||||||
|
if self._loop is None:
|
||||||
|
return
|
||||||
|
key = str(sample.key_expr)
|
||||||
|
kind = sample.kind
|
||||||
|
payload = bytes(sample.payload) if sample.payload else b""
|
||||||
|
asyncio.run_coroutine_threadsafe(
|
||||||
|
self._handle_sample(key, kind, payload), self._loop
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _handle_sample(self, key: str, kind: zenoh.SampleKind, payload: bytes):
|
||||||
|
"""解析样本并更新内存路由表(在事件循环内执行)。"""
|
||||||
|
parts = key.split("/")
|
||||||
|
if len(parts) < 4:
|
||||||
|
return
|
||||||
|
wtype, route_id = parts[2], parts[3]
|
||||||
|
|
||||||
|
if wtype in self._inline_types:
|
||||||
|
# 该插件由网关进程内加载(盘扫注册路由),忽略外部描述符防双注册
|
||||||
|
return
|
||||||
|
|
||||||
|
if kind == zenoh.SampleKind.DELETE:
|
||||||
|
self._remove(wtype, route_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
descriptor = orjson.loads(payload)
|
||||||
|
except orjson.JSONDecodeError:
|
||||||
|
logger.warning("丢弃非法路由描述符 [%s]: JSON 解析失败", key)
|
||||||
|
return
|
||||||
|
if not isinstance(descriptor, dict):
|
||||||
|
logger.warning("丢弃非法路由描述符 [%s]: 非 dict", key)
|
||||||
|
return
|
||||||
|
|
||||||
|
if route_id == META_KEY:
|
||||||
|
self._upsert_meta(wtype, descriptor)
|
||||||
|
else:
|
||||||
|
self._upsert_route(wtype, route_id, descriptor)
|
||||||
|
|
||||||
|
def _upsert_meta(self, wtype: str, descriptor: Dict[str, Any]):
|
||||||
|
old = self._meta.get(wtype)
|
||||||
|
new_hash = _descriptor_hash(descriptor)
|
||||||
|
if old is not None and _descriptor_hash(old) == new_hash:
|
||||||
|
return
|
||||||
|
self._meta[wtype] = descriptor
|
||||||
|
self._schedule_rebuild(wtype)
|
||||||
|
|
||||||
|
def _upsert_route(self, wtype: str, route_id: str, descriptor: Dict[str, Any]):
|
||||||
|
group = self._groups.setdefault(wtype, {})
|
||||||
|
existing = group.get(route_id)
|
||||||
|
if existing is not None:
|
||||||
|
if _descriptor_hash(existing) == _descriptor_hash(descriptor):
|
||||||
|
# 多副本重复发布同一描述符,幂等忽略
|
||||||
|
return
|
||||||
|
logger.warning(
|
||||||
|
"路由描述符冲突 [%s/%s]: 已存在版本保留,忽略新版本",
|
||||||
|
wtype, route_id,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
# 补全描述符字段(发布端缺省时的兜底)
|
||||||
|
descriptor.setdefault("type", wtype)
|
||||||
|
descriptor.setdefault("schema_version", SCHEMA_VERSION)
|
||||||
|
group[route_id] = descriptor
|
||||||
|
logger.info(
|
||||||
|
"收到路由注册: %s %s %s (subject=%s)",
|
||||||
|
descriptor.get("method", "?"),
|
||||||
|
descriptor.get("path", "?"),
|
||||||
|
f"[{wtype}]",
|
||||||
|
descriptor.get("subject", "?"),
|
||||||
|
)
|
||||||
|
self._schedule_rebuild(wtype)
|
||||||
|
|
||||||
|
def _remove(self, wtype: str, route_id: str):
|
||||||
|
if route_id == META_KEY:
|
||||||
|
if self._meta.pop(wtype, None) is not None:
|
||||||
|
self._schedule_rebuild(wtype)
|
||||||
|
return
|
||||||
|
group = self._groups.get(wtype)
|
||||||
|
if group and group.pop(route_id, None) is not None:
|
||||||
|
logger.info("收到路由摘除: [%s] %s", wtype, route_id)
|
||||||
|
self._schedule_rebuild(wtype)
|
||||||
|
|
||||||
|
def _schedule_rebuild(self, wtype: str):
|
||||||
|
"""type 级防抖:合并短时间内的多次变更。"""
|
||||||
|
prev = self._debounce.get(wtype)
|
||||||
|
if prev is not None and not prev.done():
|
||||||
|
prev.cancel()
|
||||||
|
self._debounce[wtype] = asyncio.create_task(self._debounced_rebuild(wtype))
|
||||||
|
|
||||||
|
async def _debounced_rebuild(self, wtype: str):
|
||||||
|
try:
|
||||||
|
await asyncio.sleep(DEBOUNCE_SECONDS)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
return
|
||||||
|
self._debounce.pop(wtype, None)
|
||||||
|
await self._rebuild(wtype)
|
||||||
|
|
||||||
|
async def _rebuild(self, wtype: str):
|
||||||
|
"""整组重建该 worker_type 的 HTTP 路由 + 同步 MQTT 自定义主题。"""
|
||||||
|
routes = list(self._groups.get(wtype, {}).values())
|
||||||
|
routes.sort(key=lambda r: (r.get("path", ""), r.get("method", "")))
|
||||||
|
|
||||||
|
# 1. 摘除旧路由(按对象引用从 router.routes 过滤)
|
||||||
|
for r in self._route_objs.pop(wtype, []):
|
||||||
|
if r in self._app.router.routes:
|
||||||
|
self._app.router.routes.remove(r)
|
||||||
|
|
||||||
|
# 2. 挂载新路由(MQTT / internal 端点不注册 HTTP,但保留在路由表中)
|
||||||
|
http_routes = [
|
||||||
|
r for r in routes
|
||||||
|
if r.get("method") not in ("MQTT",) and not r.get("internal")
|
||||||
|
]
|
||||||
|
if http_routes:
|
||||||
|
self._route_objs[wtype] = register_route_group(
|
||||||
|
self._app, self._wm, wtype, http_routes, self._prefix,
|
||||||
|
self._schemas, self._raw_body_models, self._response_models,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info("[%s] 无 HTTP 路由可挂载(%d 个描述符,全部为 MQTT/internal)",
|
||||||
|
wtype, len(routes))
|
||||||
|
|
||||||
|
# 路由表变化后 OpenAPI 缓存失效,下一次访问时重建
|
||||||
|
self._app.openapi_schema = None
|
||||||
|
|
||||||
|
# 3. 同步 external 插件的 MQTT 动态订阅主题
|
||||||
|
if self._mqtt_bridge is not None:
|
||||||
|
await self._mqtt_bridge.update_dynamic_topics(self._collect_mqtt_topics())
|
||||||
|
|
||||||
|
def _collect_mqtt_topics(self) -> List[Dict[str, Any]]:
|
||||||
|
"""汇总所有插件 __meta__ 声明的自定义订阅主题。"""
|
||||||
|
topics: List[Dict[str, Any]] = []
|
||||||
|
for wtype in sorted(self._meta):
|
||||||
|
meta = self._meta[wtype]
|
||||||
|
for t in meta.get("mqtt_topics") or []:
|
||||||
|
if isinstance(t, dict) and t.get("topic"):
|
||||||
|
topic = dict(t)
|
||||||
|
topic.setdefault("worker", wtype)
|
||||||
|
topics.append(topic)
|
||||||
|
return topics
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
from fastapi import HTTPException
|
||||||
|
from fastapi.responses import Response
|
||||||
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
from sqlite_vfs.core import SQLiteVFS
|
||||||
|
|
||||||
|
|
||||||
|
class SVFSStaticFiles(StaticFiles):
|
||||||
|
"""
|
||||||
|
基于 SQLite VFS 的静态文件服务类
|
||||||
|
|
||||||
|
该类继承自 FastAPI 的 StaticFiles,使用 SQLite 虚拟文件系统来存储和提供静态文件。
|
||||||
|
支持单页应用(SPA)的路由回退机制,当请求的文件不存在时,会自动返回 index.html。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, directory, vfs_db_path="static.svfs"):
|
||||||
|
"""
|
||||||
|
初始化 SVFSStaticFiles 实例
|
||||||
|
|
||||||
|
参数:
|
||||||
|
directory (str): 静态文件的根目录路径
|
||||||
|
vfs_db_path (str): SQLite VFS 数据库文件路径,默认为 "static.svfs"
|
||||||
|
|
||||||
|
返回值:
|
||||||
|
无
|
||||||
|
"""
|
||||||
|
vfs_db_path = vfs_db_path
|
||||||
|
self._vfs = SQLiteVFS(vfs_db_path, compress=False)
|
||||||
|
super().__init__(directory=directory)
|
||||||
|
|
||||||
|
async def get_response(self, path: str, scope):
|
||||||
|
"""
|
||||||
|
根据请求路径获取对应的静态文件响应
|
||||||
|
|
||||||
|
该方法处理静态文件的请求逻辑,包括:
|
||||||
|
1. 路径标准化处理
|
||||||
|
2. 文件存在性检查
|
||||||
|
3. SPA 路由回退支持
|
||||||
|
4. 目录索引处理
|
||||||
|
5. MIME 类型自动识别
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path (str): 请求的文件路径
|
||||||
|
scope: ASGI 作用域对象,包含请求上下文信息
|
||||||
|
|
||||||
|
返回值:
|
||||||
|
Response: FastAPI 响应对象,包含文件内容和正确的 Content-Type
|
||||||
|
|
||||||
|
异常:
|
||||||
|
HTTPException: 当文件不存在且无法回退到 index.html 时,抛出 404 错误
|
||||||
|
"""
|
||||||
|
# 统一路径格式:以 / 开头,并将 Windows 反斜杠转为正斜杠
|
||||||
|
normalized_path = "/" + path.replace("\\", "/")
|
||||||
|
|
||||||
|
# 获取文件信息
|
||||||
|
file_info = self._vfs.get_file_info(normalized_path)
|
||||||
|
|
||||||
|
# 处理文件不存在的情况,尝试 SPA 回退机制
|
||||||
|
if file_info is None:
|
||||||
|
# SPA 回退:如果不是静态资源扩展名,尝试返回 index.html
|
||||||
|
if not any(
|
||||||
|
normalized_path.endswith(ext)
|
||||||
|
for ext in [
|
||||||
|
".js",
|
||||||
|
".css",
|
||||||
|
".png",
|
||||||
|
".jpg",
|
||||||
|
".ico",
|
||||||
|
".svg",
|
||||||
|
".webp",
|
||||||
|
".json",
|
||||||
|
]
|
||||||
|
):
|
||||||
|
index_info = self._vfs.get_file_info("/index.html")
|
||||||
|
if index_info:
|
||||||
|
content = self._vfs.read_file("/index.html")
|
||||||
|
return Response(content=content, media_type="text/html")
|
||||||
|
raise HTTPException(status_code=404, detail="File not found")
|
||||||
|
|
||||||
|
# 处理目录请求,返回目录下的 index.html
|
||||||
|
if file_info["is_directory"]:
|
||||||
|
index_info = self._vfs.get_file_info(f"{normalized_path}/index.html")
|
||||||
|
if index_info:
|
||||||
|
content = self._vfs.read_file(f"{normalized_path}/index.html")
|
||||||
|
return Response(content=content, media_type="text/html")
|
||||||
|
raise HTTPException(status_code=404, detail="Directory index not found")
|
||||||
|
|
||||||
|
# 读取文件内容并确定 MIME 类型
|
||||||
|
content = self._vfs.read_file(normalized_path)
|
||||||
|
content_type = file_info.get("content_type") or "application/octet-stream"
|
||||||
|
# 简单根据扩展名补充 MIME 类型(可选)
|
||||||
|
if normalized_path.endswith(".js"):
|
||||||
|
content_type = "application/javascript"
|
||||||
|
elif normalized_path.endswith(".css"):
|
||||||
|
content_type = "text/css"
|
||||||
|
elif normalized_path.endswith(".html"):
|
||||||
|
content_type = "text/html"
|
||||||
|
elif normalized_path.endswith(".ico"):
|
||||||
|
content_type = "image/x-icon"
|
||||||
|
|
||||||
|
return Response(content=content, media_type=content_type)
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
|||||||
|
from .stats import router as stats_router
|
||||||
|
from .worker_admin import router as worker_admin_router
|
||||||
|
from .auth import router as auth_router
|
||||||
|
|
||||||
|
__all__ = ["stats_router", "worker_admin_router", "auth_router"]
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
"""内置登录端点 —— auth 域的基础设施部分(管理平面可用性底线)。
|
||||||
|
|
||||||
|
拆分自 plugins.auth:登录 / token 签发不再依赖 auth 插件是否加载,
|
||||||
|
auth 插件缩容到 0 或全部副本故障时,登录与 admin API 仍可用;
|
||||||
|
用户管理功能(注册 / 资料 / 改密)仍由 auth 插件提供。
|
||||||
|
|
||||||
|
共享契约(Users 模型 / claims / 失败文案)来自 chongming_auth 包,
|
||||||
|
路径与响应 shape 与拆分前完全一致(前端零改动)。
|
||||||
|
"""
|
||||||
|
import time
|
||||||
|
|
||||||
|
from chongming_auth import (
|
||||||
|
DEFAULT_TOKEN_EXPIRES_SECONDS,
|
||||||
|
LoginPostRequest,
|
||||||
|
LoginPostResponse,
|
||||||
|
issue_token,
|
||||||
|
touch_last_login,
|
||||||
|
verify_login,
|
||||||
|
)
|
||||||
|
from chongming_config import get_config
|
||||||
|
from fastapi import APIRouter, Request
|
||||||
|
|
||||||
|
config = get_config()
|
||||||
|
|
||||||
|
router = APIRouter(prefix=config["default"]["prefix"], tags=["auth"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/auth/login",
|
||||||
|
response_model=LoginPostResponse,
|
||||||
|
summary="登录",
|
||||||
|
description="根据用户名和密码进行登录,登录成功返回token,获取请求需带式token进行请求",
|
||||||
|
)
|
||||||
|
async def login(body: LoginPostRequest, request: Request) -> LoginPostResponse:
|
||||||
|
db = request.app.state.auth_db
|
||||||
|
user, message = await verify_login(db, body.username, body.password)
|
||||||
|
if user is None:
|
||||||
|
return LoginPostResponse(
|
||||||
|
result="fail", token="", message=message, timestamp=time.time()
|
||||||
|
)
|
||||||
|
|
||||||
|
await touch_last_login(db, user)
|
||||||
|
|
||||||
|
security = get_config()["security"]
|
||||||
|
token = issue_token(
|
||||||
|
user,
|
||||||
|
security["secret_key"],
|
||||||
|
security["algorithm"],
|
||||||
|
expires_in_seconds=DEFAULT_TOKEN_EXPIRES_SECONDS,
|
||||||
|
)
|
||||||
|
return LoginPostResponse(
|
||||||
|
result="success", token=token, message="登录成功", timestamp=time.time()
|
||||||
|
)
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# src/chongming/app/routes/stats.py
|
||||||
|
"""
|
||||||
|
网关级调用统计查看端点
|
||||||
|
|
||||||
|
数据来源为 core.call_stats 的内存统计(进程内视角):
|
||||||
|
动态路由每次调用在后台任务中更新计数/延迟聚合/最近调用缓冲,
|
||||||
|
此处提供只读快照,便于观测接口调用量、错误分布与延迟。
|
||||||
|
"""
|
||||||
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
from chongming_config import get_config
|
||||||
|
|
||||||
|
from ..core import call_stats
|
||||||
|
|
||||||
|
config = get_config()
|
||||||
|
|
||||||
|
router = APIRouter(prefix=config["default"]["prefix"], tags=["stats"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/stats", summary="调用统计")
|
||||||
|
async def get_call_stats():
|
||||||
|
"""返回内存调用统计快照:总量、per worker/subject/status 计数、延迟聚合、最近调用"""
|
||||||
|
return call_stats.summary()
|
||||||
@@ -0,0 +1,329 @@
|
|||||||
|
"""
|
||||||
|
Worker 管理端点(插件 / 单副本重启)
|
||||||
|
|
||||||
|
提供插件实例盘点与"精细化重启单个 worker 副本"的能力:
|
||||||
|
- 经 Zenoh 下发管理指令到目标的 admin Queryable(worker/<type>/<id>/__admin__)
|
||||||
|
- 仅超级管理员(JWT payload.is_superuser)可访问
|
||||||
|
|
||||||
|
协议说明(见 chongming_worker.runtime 单副本管理协议):
|
||||||
|
- 每个 worker 副本在 worker/<type>/<worker_id>/__admin__ 注册内部 Queryable
|
||||||
|
- {"data": {"cmd": "restart"}} → 副本平滑退出并以新 worker_id 重建
|
||||||
|
- {"data": {"cmd": "stop"}} → 副本平滑退出且不自动重建
|
||||||
|
"""
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
import pathlib
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
from chongming_config import get_config
|
||||||
|
|
||||||
|
from ..core.auth import jwt_auth_dependency
|
||||||
|
from ..core.logger import get_logger
|
||||||
|
|
||||||
|
config = get_config()
|
||||||
|
|
||||||
|
logger = get_logger("worker_admin")
|
||||||
|
|
||||||
|
router = APIRouter(prefix=config["default"]["prefix"], tags=["worker_admin"])
|
||||||
|
|
||||||
|
|
||||||
|
def require_admin(
|
||||||
|
payload: dict = Depends(jwt_auth_dependency),
|
||||||
|
) -> dict:
|
||||||
|
"""仅超级管理员可操作(重启属高风险操作,普通登录用户无权限)。"""
|
||||||
|
if not payload.get("is_superuser"):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=403,
|
||||||
|
detail="需要超级管理员权限 (is_superuser)",
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _get_manager(request: Request):
|
||||||
|
"""从应用状态获取 WorkerManager;未初始化时(lifespan 外)报 503。"""
|
||||||
|
manager = getattr(request.app.state, "worker_manager", None)
|
||||||
|
if manager is None:
|
||||||
|
raise HTTPException(status_code=503, detail="WorkerManager 尚未就绪")
|
||||||
|
return manager
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/admin/workers", summary="盘点 Worker 实例")
|
||||||
|
async def list_workers(
|
||||||
|
request: Request,
|
||||||
|
_admin: dict = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""返回全部插件实例(worker_type / worker_id / mode / alive),供前端渲染与选择。"""
|
||||||
|
manager = _get_manager(request)
|
||||||
|
return {"workers": manager.list_workers(), "success": True}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/admin/workers/restart", summary="重启单个 Worker 副本")
|
||||||
|
async def restart_worker(
|
||||||
|
request: Request,
|
||||||
|
worker_type: str,
|
||||||
|
worker_id: str,
|
||||||
|
_admin: dict = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""精细化重启单个 worker 副本:经 Zenoh 下发 restart,副本以新 worker_id 重建。
|
||||||
|
|
||||||
|
- worker_type: 插件类型(如 example)
|
||||||
|
- worker_id: 副本 id(如 worker-0-1-12345),由 GET /admin/workers 提供
|
||||||
|
"""
|
||||||
|
manager = _get_manager(request)
|
||||||
|
try:
|
||||||
|
result = await manager.restart_worker(worker_type, worker_id)
|
||||||
|
except RuntimeError as e:
|
||||||
|
logger.warning(
|
||||||
|
"重启 Worker 副本失败: %s/%s 用户=%s 错误=%s",
|
||||||
|
worker_type, worker_id,
|
||||||
|
_admin.get("username", _admin.get("sub", "-")),
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=502,
|
||||||
|
detail=f"重启失败: {e}",
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"重启 Worker 副本: %s/%s 用户=%s 结果=%s",
|
||||||
|
worker_type, worker_id,
|
||||||
|
_admin.get("username", _admin.get("sub", "-")),
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
return {"success": True, "result": result}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/admin/workers/{worker_type}/restart", summary="重启整个插件(全部副本)")
|
||||||
|
async def restart_all(
|
||||||
|
request: Request,
|
||||||
|
worker_type: str,
|
||||||
|
_admin: dict = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""重启某插件的全部副本(逐个下发 restart,各自以新 id 重建)。
|
||||||
|
|
||||||
|
注意:网关进程内加载的 inline 插件副本经本接口重启,仅重建 worker 协程,
|
||||||
|
不会卸载插件;external 插件同理逐个副本重启。
|
||||||
|
"""
|
||||||
|
manager = _get_manager(request)
|
||||||
|
candidates = [
|
||||||
|
w for w in manager.list_workers()
|
||||||
|
if w["worker_type"] == worker_type
|
||||||
|
]
|
||||||
|
if not candidates:
|
||||||
|
raise HTTPException(status_code=404, detail=f"无类型为 {worker_type} 的 worker")
|
||||||
|
results = []
|
||||||
|
failed = []
|
||||||
|
for w in candidates:
|
||||||
|
try:
|
||||||
|
res = await manager.restart_worker(worker_type, w["worker_id"])
|
||||||
|
results.append({"worker_id": w["worker_id"], "ok": True, "result": res})
|
||||||
|
except Exception as e: # noqa: BLE001 - 批量重启不因单个失败中断
|
||||||
|
failed.append({"worker_id": w["worker_id"], "ok": False, "error": str(e)})
|
||||||
|
results.append({"worker_id": w["worker_id"], "ok": False, "error": str(e)})
|
||||||
|
logger.info(
|
||||||
|
"重启插件全部副本: %s (成功 %d/%d) 用户=%s",
|
||||||
|
worker_type, len(results) - len(failed), len(results),
|
||||||
|
_admin.get("username", _admin.get("sub", "-")),
|
||||||
|
)
|
||||||
|
return {"success": not failed, "total": len(results), "failed": len(failed), "results": results}
|
||||||
|
|
||||||
|
|
||||||
|
class ScaleBody(BaseModel):
|
||||||
|
"""副本数量调整请求体:目标副本数。"""
|
||||||
|
count: int = Field(..., ge=0, description="目标副本数(>=0)")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/admin/workers/{worker_type}/scale", summary="调整同类 worker 副本数量(扩缩容)")
|
||||||
|
async def scale_workers(
|
||||||
|
request: Request,
|
||||||
|
worker_type: str,
|
||||||
|
body: ScaleBody,
|
||||||
|
_admin: dict = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""把某类型的副本数调整为 body.count 并持久化到配置。
|
||||||
|
|
||||||
|
- 扩容新增副本;缩容随机关闭副本,并**优雅排空**(摘除 liveliness/业务
|
||||||
|
Queryable 隔绝新请求,等待正在处理的在途请求完成后再关闭)。
|
||||||
|
- inline 直接调整本进程 Worker 池;external 经 Zenoh 下发 rescale 到存活副本,
|
||||||
|
由其进程内池扩缩并写回自身配置。
|
||||||
|
"""
|
||||||
|
manager = _get_manager(request)
|
||||||
|
try:
|
||||||
|
result = await manager.scale_workers(worker_type, body.count)
|
||||||
|
except Exception as e: # noqa: BLE001 - 无副本/失败映射为 4xx
|
||||||
|
logger.warning(
|
||||||
|
"调整副本数量失败: %s -> %d 用户=%s 错误=%s",
|
||||||
|
worker_type, body.count,
|
||||||
|
_admin.get("username", _admin.get("sub", "-")),
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
raise HTTPException(status_code=404, detail=str(e))
|
||||||
|
logger.info(
|
||||||
|
"调整副本数量: %s -> %d 用户=%s 结果=%s",
|
||||||
|
worker_type, body.count,
|
||||||
|
_admin.get("username", _admin.get("sub", "-")),
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
return {"success": True, "result": result}
|
||||||
|
|
||||||
|
|
||||||
|
class ScaleInstanceBody(BaseModel):
|
||||||
|
"""逐实例精细扩缩请求体:目标实例 + 该实例的目标副本数。"""
|
||||||
|
instance_id: str = Field(..., description="外部进程/实例身份(list_instances 返回)")
|
||||||
|
count: int = Field(..., ge=0, description="该实例的目标副本数(绝对数,>=0)")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/admin/workers/{worker_type}/instances",
|
||||||
|
summary="按实例分组列出某类型 worker(精细扩缩寻址)",
|
||||||
|
)
|
||||||
|
async def list_worker_instances(
|
||||||
|
request: Request,
|
||||||
|
worker_type: str,
|
||||||
|
_admin: dict = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""把某类型的在线副本按「外部进程实例」分组返回。
|
||||||
|
|
||||||
|
每个实例 = 一个可独立承载副本的执行单元(inline=本网关进程/host,
|
||||||
|
external=独立进程/容器),含当前副本数与可选中的 worker_id。
|
||||||
|
"""
|
||||||
|
manager = _get_manager(request)
|
||||||
|
try:
|
||||||
|
instances = await manager.list_instances(worker_type)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.warning("列出实例失败: %s 错误=%s", worker_type, e)
|
||||||
|
raise HTTPException(status_code=404, detail=str(e))
|
||||||
|
return {"success": True, "result": instances}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/admin/workers/{worker_type}/scale_instance",
|
||||||
|
summary="把某实例精确扩缩到指定副本数(精细扩缩容)",
|
||||||
|
)
|
||||||
|
async def scale_worker_instance(
|
||||||
|
request: Request,
|
||||||
|
worker_type: str,
|
||||||
|
body: ScaleInstanceBody,
|
||||||
|
_admin: dict = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""把指定外部进程/**实例**精确调整到 body.count 个副本(绝对数),不影响其它实例。
|
||||||
|
|
||||||
|
经该实例任一副本下发 rescale{scale, instance_id, absolute:true},由其进程内
|
||||||
|
WorkerPool 直接扩缩并记入逐实例覆盖表;重启后仍保持该绝对数。
|
||||||
|
"""
|
||||||
|
manager = _get_manager(request)
|
||||||
|
try:
|
||||||
|
result = await manager.scale_instance(
|
||||||
|
worker_type, body.instance_id, body.count
|
||||||
|
)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.warning(
|
||||||
|
"逐实例扩缩失败: %s instance=%s -> %d 用户=%s 错误=%s",
|
||||||
|
worker_type, body.instance_id, body.count,
|
||||||
|
_admin.get("username", _admin.get("sub", "-")),
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
raise HTTPException(status_code=404, detail=str(e))
|
||||||
|
logger.info(
|
||||||
|
"逐实例扩缩: %s instance=%s -> %d 用户=%s 结果=%s",
|
||||||
|
worker_type, body.instance_id, body.count,
|
||||||
|
_admin.get("username", _admin.get("sub", "-")),
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
return {"success": True, "result": result}
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigUpdateBody(BaseModel):
|
||||||
|
"""配置更新请求体:写目标插件 config.toml 的 [workers.config] 段。"""
|
||||||
|
updates: dict = Field(..., description="待写入 [workers.config] 的键值(路由/加载控制键除外)")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/admin/workers/{worker_type}/config", summary="读取插件现有运行时配置")
|
||||||
|
async def read_worker_config(
|
||||||
|
request: Request,
|
||||||
|
worker_type: str,
|
||||||
|
_admin: dict = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""读取某插件 config.toml 的 [workers.config] 现有值(供前端展示)。
|
||||||
|
|
||||||
|
经 Zenoh 向该类任一存活副本下发 get_config,由其访问自身缓存(或回退 toml 种子)
|
||||||
|
返回 —— 网关不直接接触 external worker 的缓存后端。无存活副本 / 旧版运行时
|
||||||
|
不支持 get_config 时,回退回直读 config/workers/<type>/config.toml 的 [workers].config。
|
||||||
|
路由配置(add.toml 等)不在此范围。
|
||||||
|
|
||||||
|
响应中的 source/mode 供前端如实标注展示值的来路(避免把回退读到的 toml 值当成缓存值):
|
||||||
|
- source="cache":值来自运行时缓存;source="toml":缓存缺失或副本未响应,值取自文件;
|
||||||
|
- mode:该类插件的配置优先来源模式(toml 模式以文件为准、cache 模式以缓存为准)。
|
||||||
|
|
||||||
|
- worker_type: 插件类型(如 example)
|
||||||
|
"""
|
||||||
|
manager = _get_manager(request)
|
||||||
|
try:
|
||||||
|
info = await manager.get_worker_config(worker_type)
|
||||||
|
cfg = info.get("config")
|
||||||
|
if isinstance(cfg, dict) and info.get("status") in ("ok", None):
|
||||||
|
return {
|
||||||
|
"worker_type": worker_type,
|
||||||
|
"worker_id": info.get("worker_id"),
|
||||||
|
"source": info.get("source", "cache"),
|
||||||
|
"mode": info.get("mode") or manager.config_mode,
|
||||||
|
"config": cfg,
|
||||||
|
"success": True,
|
||||||
|
}
|
||||||
|
logger.debug("worker get_config 未返回有效配置(%s),回退读 toml", info)
|
||||||
|
except Exception as e: # noqa: BLE001 - 副本未响应等一律回退读 toml
|
||||||
|
logger.debug("worker get_config 查询失败,回退读 toml: %s", e)
|
||||||
|
|
||||||
|
path = pathlib.Path("config/workers") / worker_type / "config.toml"
|
||||||
|
if not path.exists():
|
||||||
|
raise HTTPException(status_code=404, detail=f"插件 {worker_type} 的配置文件不存在: {path}")
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
data = tomllib.load(f)
|
||||||
|
workers = data.get("workers") or {}
|
||||||
|
cfg = workers.get("config")
|
||||||
|
return {
|
||||||
|
"worker_type": worker_type,
|
||||||
|
"config": cfg if isinstance(cfg, dict) else {},
|
||||||
|
"source": "toml",
|
||||||
|
"mode": manager.config_mode,
|
||||||
|
"success": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/admin/workers/{worker_type}/config", summary="更新插件运行时配置(下发至 worker)")
|
||||||
|
async def update_worker_config(
|
||||||
|
request: Request,
|
||||||
|
worker_type: str,
|
||||||
|
body: ConfigUpdateBody,
|
||||||
|
_admin: dict = Depends(require_admin),
|
||||||
|
):
|
||||||
|
"""更新某插件的运行时配置 `[workers.config]`(路由配置 add.toml 等除外)。
|
||||||
|
|
||||||
|
流程:网关经 Zenoh 下发 config_set 给该类型全部副本 → 由 worker 触发写回各自
|
||||||
|
配置文件(仅 index0 落盘避免同文件并发写)→ 写成功则以新配置重启所有对应副本;
|
||||||
|
任一失败则返回失败明细(error 信息)。
|
||||||
|
|
||||||
|
- worker_type: 插件类型(如 example)
|
||||||
|
- body.updates: 待更新的键值,如 {"data_dir": "./data2"}
|
||||||
|
"""
|
||||||
|
manager = _get_manager(request)
|
||||||
|
if not body.updates:
|
||||||
|
raise HTTPException(status_code=422, detail="updates 不能为空")
|
||||||
|
try:
|
||||||
|
result = await manager.update_worker_config(worker_type, body.updates)
|
||||||
|
except Exception as e: # noqa: BLE001 - 无副本等运行时错误映射为 4xx
|
||||||
|
logger.warning(
|
||||||
|
"更新插件配置失败: %s 用户=%s 错误=%s",
|
||||||
|
worker_type,
|
||||||
|
_admin.get("username", _admin.get("sub", "-")),
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
raise HTTPException(status_code=404, detail=str(e))
|
||||||
|
logger.info(
|
||||||
|
"更新插件配置: %s\n updates=%s\n 结果=成功 %d/%d (失败: %s) 用户=%s",
|
||||||
|
worker_type,
|
||||||
|
body.updates,
|
||||||
|
result["total"] - result["failed"], result["total"],
|
||||||
|
result["errors"],
|
||||||
|
_admin.get("username", _admin.get("sub", "-")),
|
||||||
|
)
|
||||||
|
return result
|
||||||
Reference in New Issue
Block a user