diff --git a/config.toml b/config.toml index 1fe40dc..315180f 100644 --- a/config.toml +++ b/config.toml @@ -101,8 +101,17 @@ use_embedded_broker = false config_path = "./config/workers/config.toml" [pyinstaller_build_import] +# 打包收集清单（PyInstaller --hidden-import / Nuitka --include-module）。 +# 网关本体依赖随 chongming-gateway 包发布，打包时能沿 `from chongming.app import app` +# 静态跟到；本清单保留两类： +# - 动态导入、静态分析看不到的（granian 内部子模块、amqtt 插件、 +# sqlite_vfs.contrib、chongming_worker 子模块等） +# - public 骨架自身（server / utils）与服务器可选实现（uvloop 等） +# 内置插件专属依赖（网关本体不 import，如 openai / bytedance）在根目录 +# plugins_deps.toml 按插件分节维护，打包时由 build 自动并入本清单 base = [ - "utils", # public 骨架 + "httptools", + "utils", # public 骨架 "utils.launch", "utils.config", "zenoh", @@ -113,17 +122,17 @@ base = [ "chongming_request", "chongming_auth", "server", - "module_bank", # 模块化打包工具 + "module_bank", # 模块化打包工具 "sqlite_vfs", "sqlite_vfs.core", "sqlite_vfs.contrib", "sqlite_vfs.contrib.fastapi_static", - "fastapi", # web 应用依赖 + "fastapi", # web 应用依赖 "fastapi.staticfiles", "fastapi.middleware.cors", "fastapi.middleware.gzip", - "uvicorn", - "granian", # 备选服务器（APP_SERVER=granian，见 public/main.py），含 Rust 内核 + # "uvicorn", + "granian", # 备选服务器（APP_SERVER=granian，见 public/main.py），含 Rust 内核 "granian._compat", "granian._futures", "granian._granian", @@ -149,7 +158,6 @@ base = [ "chongming_cache.backends.sqlite_", "orjson", "python_multipart", - "volcenginesdkarkruntime", "chongming_jwt", "chongming_database", "chongming_config", @@ -164,22 +172,32 @@ base = [ "aiofiles.os", "sqlmodel", "amqtt", - "bytedance", - "bytedance.volcengine_aicc_sdk", - "bytedance.volcengine_aicc_sdk.secure_http_client", + # amqtt 内嵌 Broker 的插件：amqtt 经 entry points 动态加载，静态分析看不到 + # （Nuitka 尤其如此；缺了会在启动时告警 "Plugin import failed: amqtt.plugins...."） + "amqtt.plugins.authentication", + "amqtt.plugins.logging_amqtt", + "amqtt.plugins.sys.broker", + "amqtt.plugins.topic_checking", "aiofiles", - "openai", ] linux = [ - "gunicorn.glogging", - "gunicorn.app", - "gunicorn.app.wsgiapp", + # "gunicorn.glogging", + # "gunicorn.app", + # "gunicorn.app.wsgiapp", + "uvloop", ] windows = [] [production.module_system] +# 网关本体（chongming-gateway 包）不再打进 .mbank：打包时 PyInstaller/Nuitka 从 +# site-packages 静态收集（public/server.py 里 `from chongming.app import app`）， +# 改网关走正常依赖升级（uv lock --upgrade-package chongming-gateway）而不是换 .mbank。 +# 模块银行只剩内置插件一项：插件仍是加密交付 + 可只换 plugins.mbank 现场更新。 path = [ - ["resources/app.mbank", "src/chongming/app", "app"], - ["resources/plugins.mbank", "src/plugins", "plugins"], + [ + "resources/plugins.mbank", + "src/plugins", + "plugins", + ], ] type = "module-bank" diff --git a/plugins_deps.toml b/plugins_deps.toml new file mode 100644 index 0000000..099c3ad --- /dev/null +++ b/plugins_deps.toml @@ -0,0 +1,40 @@ +# 内置插件专属依赖清单（打包收集用） +# +# 分工规则： +# - 网关本体依赖 → chongming-gateway 包自己的 pyproject.toml（消费者自动获得； +# 打包沿 `from chongming.app import app` 静态收集，不必在别处重复声明） +# - public 骨架自身 + 动态导入（静态分析看不到，如 granian 子模块）→ 根 config.toml +# [pyinstaller_build_import] +# - 内置插件（src/plugins//）专属、网关本体不 import 的依赖 → 本文件， +# 按插件分节声明 +# +# `uv run build` 打包时读取本文件，把各节 imports 并入 PyInstaller --hidden-import / +# Nuitka --include-module（inline 插件在 mbank 内 import，静态分析看不到，必须显式声明）。 +# 需 chongming-cli ≥ 0.3.4 的 build 才读取；更早版本会忽略本文件（插件依赖只在 +# config.toml 里找），升级前不要从那里删依赖。 +# 插件下线、或改为 external 进程插件（依赖随 workers//pyproject.toml 自包含）后， +# 删掉对应节即可，网关构建配置不受影响。 +# +# 注意：本文件只影响"打进产物 exe 的收集清单"；venv 安装清单仍在根 pyproject.toml +# （dev 运行 inline 插件与打包共用同一 venv），两处需同步维护。 +# +# 结构（节名 = src/plugins/ 下的插件名；顶层 imports 为多个插件共用）： +# imports = ["xxx"] +# [插件名] +# imports = ["pkg", "pkg.submodule"] + +# 多个插件共用的插件专属依赖 +# httpx: aibox_sophon_adapter / auth_oauth / ip_sound_column 的 HTTP 调用 +imports = [ + "httpx", +] + +[doubao] +# 豆包视觉理解：openai SDK + 火山方舟 SDK + AICC 安全通信通道（bytedance 工具包） +imports = [ + "openai", + "volcenginesdkarkruntime", + "bytedance", + "bytedance.volcengine_aicc_sdk", + "bytedance.volcengine_aicc_sdk.secure_http_client", +] diff --git a/public/config.toml b/public/config.toml index c932cce..94abbd6 100644 --- a/public/config.toml +++ b/public/config.toml @@ -84,8 +84,9 @@ use_embedded_broker = true # 详细参数参考: https://github.com/Yakifo/amqtt [production.module_system] +# 运行期模块银行加载清单（frozen 产物用）：网关本体是普通依赖包（chongming-gateway， +# 已由 PyInstaller/Nuitka 打进产物），只有内置插件经模块银行加载。 path = [ - "resources/app.mbank", "resources/plugins.mbank", ] type = "module-bank" diff --git a/public/main.py b/public/main.py index ee16cd1..1c8861b 100644 --- a/public/main.py +++ b/public/main.py @@ -18,22 +18,22 @@ import os import platform import sys from multiprocessing import freeze_support -from pathlib import Path -import server # noqa: F401 # 预加载：master 内完成模块银行装载，fork 出的 worker 共享 +import server # noqa: F401 # 预加载：master 内完成插件模块银行装载，fork 出的 worker 共享 import uvloop # noqa: F401 # 显式导入保证打包时收集 uvloop（uvicorn/granian loop 依赖） +from chongming import RELOAD_PATHS # 热重载监听目录（网关包内定义） +from chongming.app.core.logger import get_logger +from chongming.app.core.parent_death import guard_parent_death from utils.launch import launch from uvicorn.workers import UvicornWorker use_shell = platform.system() == "Windows" # 未设置 APP_SERVER 时的平台默认（保持现状） -DEFAULT_SERVER = "uvicorn" if use_shell else "gunicorn" +# DEFAULT_SERVER = "uvicorn" if use_shell else "gunicorn" +DEFAULT_SERVER = "granian" SUPPORTED_SERVERS = ("gunicorn", "uvicorn", "granian") -# 热重载（reload=true）监听的源码目录，避免 granian 默认扫 cwd 时被 logs/ 写日志触发 -RELOAD_PATHS = [Path("src/chongming"), Path("src/plugins")] - class UvloopUvicornWorker(UvicornWorker): CONFIG_KWARGS = {"loop": "uvloop"} @@ -134,8 +134,6 @@ def app_run(app_config: dict, default_config: dict): Returns: None: 此函数不返回值，直接启动服务器并阻塞运行 """ - from app.core.logger import get_logger # type: ignore - logger = get_logger("app") params = _server_params(app_config) @@ -182,6 +180,10 @@ def main(): Returns: None: 此函数不返回值，调用launch函数启动应用 """ + # 父进程死亡防护（master 侧）：引导进程（onefile 引导/终端 shell）死亡时 + # ——含被 SIGKILL——内核给本进程发 SIGTERM，granian 走优雅停机收掉全部 worker； + # worker 侧在 app 的 lifespan 里各自设置（PR_SET_PDEATHSIG 不跨 fork 继承）。 + guard_parent_death(get_logger("app")) freeze_support() launch(app_run, "production") diff --git a/public/server.py b/public/server.py index 6e586ec..6d43907 100644 --- a/public/server.py +++ b/public/server.py @@ -3,18 +3,17 @@ import sys from utils.launch import init_module_bank from utils.config import ConfigLoader -# 加载配置文件并初始化模块银行 +# 加载配置文件并初始化模块银行（当前只剩内置插件 plugins.mbank） config = ConfigLoader("config.toml") init_module_bank(config) -# 从数据库动态导入 app 模块 -# 该模块存储在 SQLite 数据库中，通过自定义导入机制加载 +# 网关本体是普通依赖包（chongming-gateway），直接导入即可——打包时 +# PyInstaller/Nuitka 也沿这行静态收集网关及其依赖。 +# 插件（plugins 包）由上面的模块银行提供，在 lifespan 启动时被网关按配置加载。 try: - import app # type: ignore - - app_instance = app.app + from chongming.app import app as app_instance except ImportError as e: - print(f"无法从数据库导入 app 模块: {e}") + print(f"无法导入网关（chongming.app）: {e}") sys.exit(1) # 暴露 ASGI 应用实例，供 uvicorn 服务器使用 diff --git a/pyproject.toml b/pyproject.toml index 4f44031..999a871 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,36 +1,40 @@ -[build-system] -build-backend = "hatchling.build" -requires = ["hatchling"] - [project] +# 应用/打包项目：本仓负责宿主骨架（public/）、内置插件集成（src/plugins）、构建配置与 +# 交付产物。网关本体是独立包 chongming-gateway（源码在 utils/gateway，独立仓库； +# 发布到私有源后按版本依赖），本文件只声明宿主自己需要的依赖。 dependencies = [ - "chongming-jwt", - "chongming-request", - "eclipse-zenoh", - "fastapi[uvicorn]", - "gunicorn", - "granian[reload,uvloop]>=2.8.4", + # —— 网关本体（私有源发布版；本地联调见下方 [tool.uv.sources] 注释）—— + "chongming-gateway", + # —— public 骨架（exe 入口）自己的依赖 —— + # "granian[reload,uvloop]>=2.8.4", + # "gunicorn", + # "httptools", "module-bank", + # "uvicorn", + "uvloop", + # —— 共用 venv：内置插件 + 测试所需（dev 运行 inline 插件与打包共用同一 venv）—— + # 注：注释项由 chongming-gateway 传递引入，保留注释便于查看来源，需要时再显式声明 + "aiofiles", + # "amqtt", + "bytedance", + # "chongming-auth", + # "chongming-cache", + # "chongming-jwt", + "chongming-mqtt", + "chongming-password", + "chongming-request", + "chongming-stream-manager", + # "chongming-worker", + # "eclipse-zenoh", + # "fastapi", "openai", "python-multipart", - "sqlite-vfs", + # "sqlite-vfs", "tomli", "typing-extensions", - "uvicorn", "volcengine-python-sdk[ark]", - "chongming-password", - "chongming-worker", - "amqtt", - "aiofiles", - "chongming-stream-manager", - "uvloop", - "chongming-mqtt", - "httptools", - "bytedance", - "chongming-cache", - "chongming-auth", ] -description = "基于 modules-bank 的 fastapi 打包脚手架" +description = "基于 modules-bank 的 fastapi 打包脚手架（宿主项目：骨架 + 插件集成 + 构建）" name = "chongming-zenoh" readme = "README.md" requires-python = ">=3.12" @@ -38,25 +42,30 @@ version = "0.1.0" [dependency-groups] dev = [ - "chongming-cli", - "greenlet>=3.5.6", - "nuitka", - "pyarmor", - "pyinstaller", - "ruff", + "chongming-cli", + "greenlet>=3.5.6", + # onefile extra：带 zstandard，否则 onefile 产物不做压缩（Nuitka 自身推荐写法） + "nuitka[onefile]", + # Nuitka standalone/onefile 打包在 Linux 必需（链接完成后才检查，缺了会白跑一整轮编译） + "patchelf; sys_platform == 'linux'", + "pyarmor", + "pyinstaller", + "ruff", ] -[tool.hatch.build.targets.wheel] -packages = ["src/chongming"] +[tool.uv] +# 宿主项目自身不再出包（网关搬到 chongming-gateway 后本仓无 Python 包）， +# uv sync 只装依赖与 dev 工具，不再尝试构建/安装本仓 editable。 +package = false + +# 本地联调网关时取消注释：直接用 utils/gateway 源码（editable），改完无需发版即可被 +# 本仓（含打包构建）使用；联调结束恢复按发布版依赖，并用 uv lock --upgrade-package +# chongming-gateway 刷新版本。 +# [tool.uv.sources] +# chongming-gateway = { path = "utils/gateway", editable = true } [[tool.uv.index]] url = "http://124.71.68.6:3000/api/packages/chongming/pypi/simple/" [[tool.uv.index]] url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" - -[project.scripts] -gunicorn = "chongming:gunicorn_serve" -granian = "chongming:granian_serve" -serve = "chongming:serve" -worker_serve = "chongming_worker.process:worker_serve" diff --git a/src/chongming/__init__.py b/src/chongming/__init__.py deleted file mode 100644 index a95f3b8..0000000 --- a/src/chongming/__init__.py +++ /dev/null @@ -1,147 +0,0 @@ -"""chongming 网关启动入口（uvicorn / granian / gunicorn）。 - -config.toml 中 `[.server]` 的启动参数保持现状（host / port / workers / -reload / access_log / timeout_keep_alive），由 `_to_uvicorn`、`_to_granian` 在 -启动时转换为对应服务器的关键字参数，切换服务器不必改配置格式。 -""" - -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -import uvicorn -from chongming_config import Server, load_config - -from .app.core.logger import get_logger - -# 开发期热重载（reload=true）监听的源码目录（granian 用；uvicorn 走其默认扫描） -RELOAD_PATHS = [Path("src/chongming"), Path("src/plugins")] - - -def _server_config_view(server_config: Server) -> Mapping[str, Any]: - """[.server] 的宽松视图：实际 TOML 可能带 Server 之外的键 - （如 timeout_keep_alive，chongming_config.Server 尚未声明）。""" - return server_config - - -def _to_uvicorn(server_config: Server) -> dict: - """把 [.server] 的现状参数转为 uvicorn.run 的关键字参数。 - - uvicorn 的参数名与配置项一致，这里只做缺省兜底。 - """ - cfg = _server_config_view(server_config) - converted: dict = { - "host": cfg.get("host", "0.0.0.0"), - "port": cfg.get("port", 8000), - "reload": cfg.get("reload", False), - } - for key in ("workers", "access_log", "timeout_keep_alive"): - if key in cfg: - converted[key] = cfg[key] - return converted - - -def _to_granian(server_config: Server) -> dict: - """把 [.server] 的现状参数转为 granian.Granian 的关键字参数。 - - 仅两处改名：host → address、access_log → log_access； - timeout_keep_alive 在 granian 无对应项（HTTP/1 keep-alive 只有开关、无空闲 - 超时配置），忽略。 - """ - cfg = _server_config_view(server_config) - converted: dict = { - "address": cfg.get("host", "0.0.0.0"), - "port": cfg.get("port", 8000), - "interface": "asgi", - } - for key in ("reload", "workers"): - if key in cfg: - converted[key] = cfg[key] - if "access_log" in cfg: - converted["log_access"] = cfg["access_log"] - return converted - - -def serve(): - """ - 使用 uvicorn 启动服务 - - 从配置文件中加载服务器配置和日志配置，然后使用 uvicorn 启动应用服务。 - 该函数适用于开发环境或单进程部署场景。 - - Returns: - None: 该函数不返回值，会阻塞运行直到服务停止 - """ - config = load_config() - env = config["default"]["env"] - server_config = config[env]["server"] - log_config = config[env]["logging"] - logger = get_logger("app") - logger.info(f"启动服务: {server_config}") - logger.info(f"启动日志: {log_config}") - uvicorn.run("chongming.app:app", **_to_uvicorn(server_config)) - - -def granian_serve(): - """ - 使用 granian 启动服务 - - 从配置文件中加载服务器配置和日志配置，然后使用 granian 启动应用服务。 - 该函数适用于开发环境或单进程部署场景。 - - Returns: - None: 该函数不返回值，会阻塞运行直到服务停止 - """ - # 延迟导入：未选用 granian 的部署环境不必安装它 - import granian - - config = load_config() - env = config["default"]["env"] - server_config = config[env]["server"] - log_config = config[env]["logging"] - logger = get_logger("app") - logger.info(f"启动服务: {server_config}") - logger.info(f"启动日志: {log_config}") - granian.Granian( - target="chongming.app:app", - **_to_granian(server_config), - reload_paths=RELOAD_PATHS, - ).serve() - - -def gunicorn_serve(): - """ - 使用 gunicorn 启动服务 - - 通过 gunicorn 作为进程管理器启动应用，使用 UvicornWorker 作为 worker 类型， - 支持多进程部署，适用于生产环境。 - - 从配置文件中读取服务器配置，动态设置 gunicorn 的 worker 数量、绑定地址和端口等参数。 - - Returns: - None: 该函数不返回值，会阻塞运行直到服务停止 - """ - import sys - - from gunicorn.app.wsgiapp import run - - config = load_config() - env = config["default"]["env"] - server_config = config[env]["server"] - - # 构建 gunicorn 命令行参数并启动服务 - sys.argv = [ - "gunicorn", - "chongming.app:app", - "-k", - "uvicorn.workers.UvicornWorker", - "-w", - f"{server_config.get('workers', 4)}", - "-b", - f"{server_config.get('host', '0.0.0.0')}:{server_config.get('port', 8000)}", - ] - run() - - -if __name__ == "__main__": - serve() diff --git a/src/chongming/app/__init__.py b/src/chongming/app/__init__.py deleted file mode 100644 index f7201a4..0000000 --- a/src/chongming/app/__init__.py +++ /dev/null @@ -1,342 +0,0 @@ -""" -chongming FastAPI 应用入口 - -重构为配置驱动的微服务网关： -- WorkerManager: 统一管理 Zenoh worker 存活检测、负载均衡、超时重试、缓存 -- RouteGenerator: 根据配置自动生成 FastAPI 端点 -- 插件工厂函数动态加载 -""" - -import time -import asyncio -import itertools -from contextlib import asynccontextmanager - -from fastapi import FastAPI, Request -from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse -from fastapi.middleware.gzip import GZipMiddleware -from fastapi.middleware.cors import CORSMiddleware - -from chongming_config import get_config -from starlette.types import ASGIApp, Message, Receive, Scope, Send -from .core.logger import get_logger -from sqlite_vfs.contrib.fastapi_static import SVFSStaticFiles -from .core import lifespan_setup, call_stats -from .routes import stats_router, worker_admin_router, auth_router - - -config = get_config() -logger = get_logger("app") - -try: - env = config["default"]["env"] -except KeyError: # pragma: no cover - raise ValueError("配置不存在") - -@asynccontextmanager -async def lifespan(app: FastAPI): -# ---- 启动阶段 ---- - # 1. 本地存储（替代 Nanio S3，省 ~172 MB） - lifespan_setup.setup_storage(app) - - # 1.2 内置 auth 库（users 表 + admin 种子）—— auth 域的基础设施部分： - # 登录/签发内置在网关（管理平面底线），不依赖 auth 插件是否加载； - # 与 auth 插件（用户管理）共用同一张 users 表 - await lifespan_setup.setup_auth_store(app) - - # 2. Zenoh - session = lifespan_setup.setup_zenoh(app) - - # 3. WorkerManager - manager = lifespan_setup.setup_worker_manager(app, session) - - # 3.1 内嵌 amqtt Broker（宿主级分布式锁；需在 WorkerManager 就绪后提供共享 cache） - await lifespan_setup.setup_mqtt_broker(app, manager) - - # 4.1 全局副本数协调：把当前网关登记进在线列表，供按在线网关分摊 quota - manager.register_gateway() - - # 4.2 宿主级 inline worker 领导选举（基于共享 cache 的分布式锁，key 按网关隔离）： - # gunicorn 多 worker 下每实例仅一个进程托管 inline worker（避免 N×count 冗余）， - # 其余进程只做 HTTP 并经 Zenoh liveliness 发现调用；多实例各选各的 leader。 - is_inline_owner = manager.acquire_inline_owner() - app.state.inline_owner_locked = is_inline_owner - - # 5. 插件加载（混合模式：inline 进程内加载；external 由独立进程运行， - # 其路由经 Zenoh 动态注册——进程编排交给 Docker/K8s，网关不拉起进程） - worker_tasks, custom_routers, inline_types, external_types = ( - await lifespan_setup.load_workers( - app, session, manager, is_inline_owner=is_inline_owner - ) - ) - - # 5.1 动态路由注册表常驻启动：订阅 chongming/routes/**，external 插件的 - # 路由经 Zenoh 注册/摘除，无需网关侧保留插件配置（external_types 仅观测） - await lifespan_setup.setup_route_registry( - app, session, manager, inline_types=inline_types - ) - - # 5.5 契约校验（仅 inline 插件；external 插件在自己进程内校验），失败则启动终止 - lifespan_setup.validate_contracts(app) - - # 6. 注册路由（inline 插件的 TOML 动态路由 + 插件自定义路由） - await lifespan_setup.register_routes( - app, manager, custom_routers, worker_types=inline_types - ) - - # 7. 启动 MQTT Bridge（内嵌 Broker + Worker 桥接） - await lifespan_setup.setup_mqtt(app, manager) - - # 7.1 启动 MQTT Bridge 持有者自愈巡检（跨所有网关进程，崩溃时自动接管重建） - app.state.mqtt_bridge_supervise_task = asyncio.create_task( - lifespan_setup.mqtt_bridge_supervisor(app, manager) - ) - - # 8. 短暂等待，确保 worker 状态同步 - await asyncio.sleep(0.5) - - # 打印 worker 就绪状态 - worker_status = { - wtype: len(workers) - for wtype, workers in manager._workers_cache.items() - } - logger.info(f"Worker 就绪状态: {worker_status}") - - # ---- 应用运行 ---- - yield - - # ---- 关闭阶段 ---- - logger.info("正在关闭应用...") - - # 1. 停止 MQTT Bridge 持有者自愈巡检，先于 Bridge 关停，避免关停期间再次接管 - _sup = getattr(app.state, "mqtt_bridge_supervise_task", None) - if _sup: - _sup.cancel() - try: - await _sup - except (asyncio.CancelledError, Exception): - pass - - # 1.0 停止 MQTT Bridge - await lifespan_setup.shutdown_mqtt(app) - - # 1.5 停止路由注册表订阅（常驻，随应用生命周期启停） - registry = getattr(app.state, "route_registry", None) - if registry is not None: - registry.stop() - - # 2. 清理 inline workers（cleanup_workers 内部按 load_mode 过滤） - await lifespan_setup.cleanup_workers(app) - - # 2.2 关闭内置 auth 库 - auth_db = getattr(app.state, "auth_db", None) - if auth_db is not None: - await auth_db.dispose() - - # 2.1 释放 inline worker 所有权分布式锁（仅持锁进程，避免误删其他进程获得的新锁） - if getattr(app.state, "inline_owner_locked", False): - await manager.release_inline_owner() - - # 3. 取消刷新任务 - refresh_task = getattr(app.state, "refresh_task", None) - if refresh_task: - refresh_task.cancel() - try: - await refresh_task - except asyncio.CancelledError: - pass - - # 4. 取消所有 worker 任务（含 self-heal 接管后拉起的任务） - all_worker_tasks = list(worker_tasks) + manager.extra_worker_tasks() - for task in all_worker_tasks: - task.cancel() - if all_worker_tasks: - await asyncio.gather(*all_worker_tasks, return_exceptions=True) - - # 4.5 取消后台统计/审计任务 - await call_stats.shutdown() - - # 4.6 全局副本数协调：关闭时把当前网关从在线列表移除（其余网关将重新分摊） - manager.unregister_gateway() - - # 5. 关闭 Zenoh 会话 - session.close() - logger.info("Zenoh 会话已关闭") - - logger.info("应用已关闭") - - -cors = config[env].get("cors", {}) -app = FastAPI( - title=config["default"]["app"]["name"], - version=config["default"]["app"]["version"], - description=config["default"]["app"]["description"], - docs_url="/docs" if config[env]["debug"] else None, - redoc_url="/redoc" if config[env]["debug"] else None, - openapi_url="/openapi.json" if config[env]["debug"] else None, - lifespan=lifespan, -) - -app.add_middleware(CORSMiddleware, **cors) -# GZip 压缩响应（<500B 不压缩；starlette 默认排除 text/event-stream， -# 保证 SSE 流式端点即时性；>=128KB 的响应在独立线程压缩，不阻塞事件循环） -app.add_middleware(GZipMiddleware) - - -# access 日志采样率（logging.access_sample_rate，0.1 = 只记录 10% 的成功请求； -# 错误 >=400 始终全量记录，保证可追溯；1.0/缺省 = 全量） -ACCESS_SAMPLE_RATE = float(config[env].get("logging", {}).get("access_sample_rate", 1.0)) -# 成功请求采样计数器（每进程独立；错误请求不消耗计数） -_access_log_counter = itertools.count() - - -def _should_log_access(status_code: int, sample_rate: float = ACCESS_SAMPLE_RATE) -> bool: - """错误全量；成功按采样率（采样为确定性计数，无随机数开销）""" - if status_code >= 400 or sample_rate >= 1.0: - return True - denom = max(1, int(round(1.0 / sample_rate))) - return next(_access_log_counter) % denom == 0 - - -def _log_access(scope: Scope, status_code: int, start: float) -> None: - """按采样规则写一条访问日志（成功按 access_sample_rate，错误 >=400 全量）""" - if not _should_log_access(status_code): - return - client = scope.get("client") - logger.info( - "%s %s status=%s duration=%.2fms client=%s", - scope.get("method", "-"), scope.get("path", "-"), status_code, - (time.perf_counter() - start) * 1000, client[0] if client else "-", - extra={"status_code": status_code}, - ) - - -class AccessLogMiddleware: - """ - HTTP访问日志中间件（纯 ASGI 实现），记录每个请求的详细信息。 - - 记录的信息包括： - - 请求方法和路径 - - 响应状态码 - - 请求处理时长（毫秒，按慢路径：从进入中间件到响应头产出） - - 客户端IP地址 - - 高吞吐优化： - - 采样：成功请求按 access_sample_rate 记录，错误（>=400）全量 - - 异步：% 参数化消息 + 队列日志（QueueHandler），格式化与文件 I/O - 在后台监听线程执行，主线程仅入队 - - 不用 BaseHTTPMiddleware（@app.middleware("http")）：它会为每请求建 anyio - task group 并把响应体经内存流过一手，压测实测高并发下吃掉 30~50% 吞吐； - 这里只包一层 send 截获 response.start，无缓冲、无额外任务 - """ - - def __init__(self, app: ASGIApp) -> None: - self.app = app - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] != "http": - await self.app(scope, receive, send) - return - - start = time.perf_counter() - logged = False - - async def send_wrapper(message: Message) -> None: - nonlocal logged - # 响应头产出即记录（流式响应因此只计到首字节，不计整段流） - if message["type"] == "http.response.start" and not logged: - logged = True - _log_access(scope, message["status"], start) - await send(message) - - try: - await self.app(scope, receive, send_wrapper) - except Exception: - # 未产出响应就异常（外层 500）：补一条全量错误日志后原样抛出 - if not logged: - _log_access(scope, 500, start) - raise - - -app.add_middleware(AccessLogMiddleware) - - -@app.get("/", summary="根路径") -async def root(): - """ - 根路径端点，返回应用的基本信息。 - - 返回: - dict: 包含应用名称、版本、文档地址和健康检查地址的字典 - """ - return { - "message": f"欢迎使用 {config['default']['app']['name']}", - "version": config["default"]["app"]["version"], - "docs": "/docs" if config[env]["debug"] else None, - "health": f"{config['default']['prefix']}/health", - } - - -@app.exception_handler(RequestValidationError) -async def validation_exception_handler(request: Request, exc: RequestValidationError): - """ - 请求验证异常处理器，记录详细的验证错误信息和请求体。 - 帮助快速定位 Pydantic 模型与前端/设备上报数据的不匹配问题。 - """ - body = None - try: - body = await request.body() - body_str = body.decode("utf-8")[:2000] if body else "" - except Exception: - body_str = "" - - errors = exc.errors() - error_summary = [ - f"{' -> '.join(str(loc) for loc in e['loc'])}: {e['msg']}" - for e in errors - ] - logger.error( - f"请求验证失败 [{request.method} {request.url.path}]:\n" - f" 错误: {'; '.join(error_summary)}\n" - f" 请求体: {body_str}" - ) - return JSONResponse( - status_code=422, - content={"detail": errors}, - ) - - -@app.exception_handler(Exception) -async def global_exception_handler(_: Request, exc: Exception): - """ - 全局异常处理器，捕获并处理应用中未捕获的异常。 - - 在调试模式下返回详细的异常信息，在生产模式下返回通用错误提示。 - - 参数: - _: FastAPI请求对象（未使用） - exc: 捕获到的异常对象 - - 返回: - JSONResponse: 包含错误信息的JSON响应，状态码为500 - """ - logger.error(f"全局异常: {exc}") - return JSONResponse( - status_code=500, - content={ - "error": "内部服务器错误", - "detail": str(exc) if config[env]["debug"] else "请查看服务器日志", - }, - ) - - -# 静态文件服务（可选） -if vfs_db_path := config[env].get("file_system", {}).get("path", None): - logger.info("静态文件已存在，将使用静态文件服务") - static_files_handler = SVFSStaticFiles(vfs_db_path=vfs_db_path) - app.mount("/static", static_files_handler, name="static") - -app.include_router(stats_router) -app.include_router(worker_admin_router) -app.include_router(auth_router) diff --git a/src/chongming/app/core/__init__.py b/src/chongming/app/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/chongming/app/core/auth.py b/src/chongming/app/core/auth.py deleted file mode 100644 index 2907ec2..0000000 --- a/src/chongming/app/core/auth.py +++ /dev/null @@ -1,141 +0,0 @@ -import hashlib -import os -import time -from collections import OrderedDict - -from fastapi import Depends, HTTPException, status, Request -from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer -from chongming_jwt import JWT -from chongming_cache import Cache -from chongming_config import get_config -import orjson - - -security_scheme = HTTPBearer(auto_error=False) - -# JWT 解析共享缓存：与 WorkerManager 共用 [cache] 配置段。 -# 默认 sqlite（本机）；横向拓展时 config.toml 切 backend = "redis"， -# 多网关实例共享同一 Redis 即可全局只解密一次，代码零改动。 -# 注意：缓存存的是"已验证结果"，信任模型与 worker 回复缓存一致 -# （缓存基础设施可信；Redis 请配置密码并置于内网）。 -_jwt_cache = Cache.from_config(get_config()["cache"]) -# 条目 TTL（秒）：控制存储占用与密钥轮换后的自然清理； -# 安全性由命中时的 exp 复核保证，TTL 不承担过期判断 -_JWT_CACHE_TTL = 60 - -# ── 一级：进程内 JWT 结果缓存（LRU + TTL）──────────────────────────── -# 鉴权依赖是同步依赖（def），每次请求都要走一遍共享缓存——backend=redis 时 -# 即"每请求一次同步网络往返 + FastAPI 线程池 hop"。同一 token 在 TTL 内被 -# 反复使用是常态，故在进程内加一层 L1：命中直接返回，不碰网络。 -# 语义与 L2 完全一致：key 含密钥/算法指纹；命中仍复核 exp（过期=拒绝而非放行）； -# 条目按 TTL 惰性清理并限长。CHONGMING_JWT_L1=0 可关闭（便于 A/B 与排障）。 -_JWT_L1_ENABLED = os.environ.get("CHONGMING_JWT_L1", "1") != "0" -_JWT_L1_MAX = 1024 -# 存 orjson 字节而非 dict：命中时反序列化出新副本，保证"调用方修改返回值 -# 不污染缓存"这一 L2 既有契约（L2 同样是 JSON 往返后返回新对象） -_jwt_l1: "OrderedDict[str, tuple[bytes, float, float]]" = OrderedDict() # key → (payload_json, exp, 写入时刻) - - -def _l1_get(key: str): - """命中返回 (payload_json: bytes, exp)；条目超过共享缓存 TTL（运行时可调，便于测试/运维）视为过期""" - entry = _jwt_l1.get(key) - if entry is None: - return None - _payload_json, _exp, ts = entry - if time.time() - ts >= _JWT_CACHE_TTL: - _jwt_l1.pop(key, None) - return None - _jwt_l1.move_to_end(key) - return entry - - -def _l1_set(key: str, payload_json: bytes, exp: float) -> None: - _jwt_l1[key] = (payload_json, exp, time.time()) - _jwt_l1.move_to_end(key) - while len(_jwt_l1) > _JWT_L1_MAX: - _jwt_l1.popitem(last=False) - - -def _l1_sweep(now: float) -> None: - """惰性清理：超 TTL 的整条丢弃（token 永不重复时防堆积），只在表较大时调用""" - stale = [k for k, v in _jwt_l1.items() if now - v[2] >= _JWT_CACHE_TTL] - for k in stale: - _jwt_l1.pop(k, None) - - -def _cache_key(token: str, secret: str, algorithm: str) -> str: - """缓存 key：token 内容 + 密钥/算法指纹，不同密钥/算法不串用""" - return "jwt:" + hashlib.sha256(f"{secret}:{algorithm}:{token}".encode()).hexdigest() - - -def _cached_decode(token: str, secret: str, algorithm: str) -> tuple: - """JWT 解密结果缓存（(payload, exp)）：一级进程内 LRU，二级共享缓存。 - - - 一级命中且未过期 -> 直接返回（零网络、零密码学）；过期 -> 直接拒绝 - - 一级未命中 -> 查二级共享缓存；命中且未过期 -> 回填一级后返回 - - 二级也未命中 -> 完整解码（RSA/ECDSA 验证）后写二级 + 一级（TTL=60s/30s） - - decode 失败（无效 token）不缓存，每次都完整验证后拒绝 - """ - key = _cache_key(token, secret, algorithm) - now = time.time() - if _JWT_L1_ENABLED: - hit = _l1_get(key) - if hit is not None: - payload_json, exp, _ts = hit - if not exp or exp >= now: - # 反序列化出新副本：调用方修改不影响缓存条目（与 L2 契约一致） - return orjson.loads(payload_json), exp - raise ValueError("Token 已过期") - if len(_jwt_l1) > _JWT_L1_MAX // 2: # 仅大表时惰性清理，避免常态 O(n) - _l1_sweep(now) - - entry = _jwt_cache.get(key) - if entry is not None: - payload = entry["payload"] - exp = entry.get("exp", 0) - if not exp or exp >= time.time(): - if _JWT_L1_ENABLED: - _l1_set(key, orjson.dumps(payload), exp) - return payload, exp - raise ValueError("Token 已过期") - jwt = JWT(secret, algorithm) - payload_str = jwt.decode(token) - payload = orjson.loads(payload_str.encode("utf-8")) - exp = payload.get("exp", 0) - _jwt_cache.set(key, {"payload": payload, "exp": exp}, expire=_JWT_CACHE_TTL) - if _JWT_L1_ENABLED: - _l1_set(key, orjson.dumps(payload), exp) - return payload, exp - - -def jwt_auth_dependency( - credentials: HTTPAuthorizationCredentials = Depends(security_scheme) -) -> dict: - """JWT 校验依赖，返回解码后的 payload，校验失败抛出 401""" - config = get_config() - secret = config["security"]["secret_key"] - algorithm = config["security"]["algorithm"] - - if not credentials: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Missing authorization header", - headers={"WWW-Authenticate": "Bearer"}, - ) - token = credentials.credentials - try: - payload, _ = _cached_decode(token, secret, algorithm) - # 缓存后端返回的是独立对象（JSON 反序列化后为新副本），调用方修改不影响缓存 - return payload - except Exception as e: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail=f"Invalid token: {str(e)}", - headers={"WWW-Authenticate": "Bearer"}, - ) - -async def store_user_to_request( - request: Request, - user: dict = Depends(jwt_auth_dependency) -) -> None: - request.state.user = user diff --git a/src/chongming/app/core/call_stats.py b/src/chongming/app/core/call_stats.py deleted file mode 100644 index 1a64a6e..0000000 --- a/src/chongming/app/core/call_stats.py +++ /dev/null @@ -1,170 +0,0 @@ -""" -调用统计与审计日志（后台任务执行） - -route_generator 端点在 call_raw 返回后仅做非关键后置操作，全部经 -spawn() 以 fire-and-forget 后台任务执行，端点立即返回，不增加接口 -响应延迟、不占用请求协程资源： - 1. 更新内存调用统计（per worker_type/subject/status 计数 + 延迟聚合 + 最近调用缓冲） - 2. 写审计日志（logs/audit-YYYY-MM-DD.log，复用 logger 按日切分 + 大小轮转设施） - -record_call 内部全兜底：统计或审计任一步失败仅记 warning，绝不影响响应。 -统计为进程内视角（gunicorn 多 worker 下各进程独立聚合）。 - -后台任务随应用生命周期管理：lifespan 关闭阶段调用 shutdown() 统一取消。 -""" -import asyncio -import datetime -import orjson -import random -import time -from collections import deque -from typing import Any, Coroutine, Dict - -from chongming_config import get_config - -from .logger import get_logger - -logger = get_logger("call_stats") -# 审计日志独立文件（audit-YYYY-MM-DD.log），与业务日志分离便于检索 -audit_logger = get_logger("audit") - -# 最近调用环形缓冲上限 -MAX_RECENT = 1000 - -# 审计日志采样率：与 access 日志同源配置（access_sample_rate，0.1 = 只写 10% 的成功请求； -# 错误 status >= 400 始终全量）。压测实测全量审计写盘 8.8MB/4.3 万行，是高吞吐下的隐藏 I/O 大头。 -_audit_config = get_config() -_audit_env = _audit_config["default"]["env"] -_AUDIT_SAMPLE_RATE = float( - _audit_config[_audit_env].get("logging", {}).get("access_sample_rate", 1.0) -) - - -def _should_record_audit(status: int) -> bool: - """审计日志采样判定（与 access 日志同模式）""" - if status >= 400 or _AUDIT_SAMPLE_RATE >= 1.0: - return True - denom = max(1, int(round(1.0 / _AUDIT_SAMPLE_RATE))) - return random.randrange(denom) == 0 - -# 后台任务集合（fire-and-forget，shutdown 时统一取消） -_tasks: set[asyncio.Task] = set() - - -def spawn(coro: Coroutine) -> asyncio.Task: - """以 fire-and-forget 方式创建后台任务并纳入跟踪（shutdown 时取消）""" - task = asyncio.create_task(coro) - _tasks.add(task) - task.add_done_callback(_tasks.discard) - return task - - -async def shutdown() -> None: - """取消所有后台统计/审计任务（应用关闭时调用）""" - tasks = list(_tasks) - if not tasks: - return - for task in tasks: - task.cancel() - await asyncio.gather(*tasks, return_exceptions=True) - - -class CallStats: - """内存调用统计：计数 + 延迟聚合 + 最近调用环形缓冲。 - - asyncio 单线程事件循环内访问，无需加锁。 - """ - - def __init__(self, max_recent: int = MAX_RECENT): - self._counts: Dict[tuple, int] = {} - self._latency: Dict[str, Dict[str, float]] = {} - self._recent: deque = deque(maxlen=max_recent) - - def record(self, entry: Dict[str, Any]) -> None: - key = (entry["worker_type"], entry["subject"], entry["status"]) - self._counts[key] = self._counts.get(key, 0) + 1 - - wt = entry["worker_type"] - lat = self._latency.setdefault(wt, {"total": 0.0, "count": 0, "max": 0.0}) - dur = entry.get("duration_ms", 0.0) - lat["total"] += dur - lat["count"] += 1 - if dur > lat["max"]: - lat["max"] = dur - - self._recent.append(entry) - - def summary(self) -> Dict[str, Any]: - """聚合快照：总量、per worker/subject/status 计数、延迟、最近调用""" - calls = [ - {"worker_type": wt, "subject": subj, "status": status, "count": count} - for (wt, subj, status), count in sorted(self._counts.items()) - ] - latency = [ - { - "worker_type": wt, - "count": lat["count"], - "avg_ms": round(lat["total"] / lat["count"], 2) if lat["count"] else 0.0, - "max_ms": round(lat["max"], 2), - } - for wt, lat in sorted(self._latency.items()) - ] - return { - "total_calls": sum(self._counts.values()), - "calls": calls, - "latency": latency, - "recent": list(self._recent)[-50:], - } - - -_stats = CallStats() - - -def summary() -> Dict[str, Any]: - """全局统计快照（供 GET /stats 端点使用）""" - return _stats.summary() - - -def _format_audit(entry: Dict[str, Any]) -> str: - """审计日志单行 key=value 格式，便于 grep 检索""" - ts = entry.get("ts") or time.time() - ts_str = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] - user = entry.get("user") - if isinstance(user, dict): - # JWT payload（auth.py store_user_to_request 注入），orjson 默认紧凑单行输出 - user_str = orjson.dumps(user).decode() - elif isinstance(user, str): - user_str = user - else: - user_str = "-" - parts = [ - "AUDIT", - f"ts={ts_str}", - f"worker={entry.get('worker_type', '-')}", - f"subject={entry.get('subject', '-')}", - f"method={entry.get('method', '-')}", - f"path={entry.get('path', '-')}", - f"user={user_str}", - f"client={entry.get('client') or '-'}", - f"status={entry.get('status', '-')}", - f"duration_ms={entry.get('duration_ms', 0):.3f}", - f"req_bytes={entry.get('req_bytes', 0)}", - f"resp_bytes={entry.get('resp_bytes', 0)}", - ] - return " ".join(parts) - - -async def record_call(entry: Dict[str, Any]) -> None: - """后台执行体：更新内存统计 + 写审计日志。 - - 整体兜底：任一步失败仅记 warning，绝不向端点冒泡。 - """ - try: - _stats.record(entry) - except Exception as e: # pragma: no cover - 防御性兜底 - logger.warning("调用统计记录失败: %s", e) - try: - if _should_record_audit(entry.get("status", 200)): - audit_logger.info(_format_audit(entry)) - except Exception as e: # pragma: no cover - 防御性兜底 - logger.warning("审计日志写入失败: %s", e) diff --git a/src/chongming/app/core/contracts.py b/src/chongming/app/core/contracts.py deleted file mode 100644 index 8714938..0000000 --- a/src/chongming/app/core/contracts.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -启动契约校验（网关 inline 模式） - -校验逻辑的唯一实现在 chongming_worker.contracts（网关与 worker 进程共享）， -本模块只做网关侧的编排：对 inline 插件批量校验、日志汇总、硬失败终止启动。 - -external 插件在自己进程内做同样校验（validate_worker_contracts），网关侧跳过。 -""" -from typing import Any, Dict, List - -from chongming_worker import get_all_rpc_methods -from chongming_worker.contracts import check_plugin, load_plugin_models -from chongming_worker.route_config import load_worker_routes_from_dir - -from .logger import get_logger - -logger = get_logger("contracts") - - -def validate_contracts(plugins: List[Dict[str, Any]]) -> None: - """ - 对每个发现的 inline 插件执行三方一致性校验。 - - 挂载于 lifespan 的 load_workers 与 register_routes 之间，此时插件模块已导入 - （@rpc_method 注册表就绪）且 TOML 路由尚未注册到 FastAPI。 - - Args: - plugins: _discover_plugins() 返回的 inline 插件配置列表 - - Raises: - RuntimeError: 存在硬失败项（HTTP 路由缺失 @rpc_method 注册）时 - """ - registry = get_all_rpc_methods() - hard_errors: List[str] = [] - warnings: List[str] = [] - - for wc in plugins: - worker_type = wc.get("type", "") - if not worker_type: - continue - - toml_routes = load_worker_routes_from_dir(worker_type) - registered = registry.get(worker_type, {}) - model_names = load_plugin_models(worker_type) - hard, warns = check_plugin(worker_type, toml_routes, registered, model_names) - hard_errors.extend(hard) - warnings.extend(warns) - - # 输出结果 - if warnings: - logger.warning("契约校验发现 %d 个警告:", len(warnings)) - for w in warnings: - logger.warning(" ⚠ %s", w) - - if hard_errors: - logger.error("契约校验失败，发现 %d 个硬性错误:", len(hard_errors)) - for e in hard_errors: - logger.error(" ✗ %s", e) - raise RuntimeError( - "启动契约校验失败: " - + "; ".join(hard_errors) - ) - - logger.info("契约校验通过（TOML 路由 ↔ @rpc_method 注册表一致）") diff --git a/src/chongming/app/core/lifespan_setup.py b/src/chongming/app/core/lifespan_setup.py deleted file mode 100644 index 40fc38f..0000000 --- a/src/chongming/app/core/lifespan_setup.py +++ /dev/null @@ -1,888 +0,0 @@ -import os -import inspect -import time -import signal -import tomllib -import glob -import multiprocessing -import asyncio -import socket -import sys - -from chongming_cache import Cache, DistributedLock -from fastapi import FastAPI -import zenoh - -from chongming_config import get_config, get_workers_config -from ..core.logger import get_logger -from ..core.route_generator import register_dynamic_routes -from ..core.worker_manager import WorkerManager -from ..core.mqtt_bridge import MQTTBridge - -config = get_config() -logger = get_logger("app") - -# 挂载框架/插件父 logger 的 handler：logger 模块只为具名 logger 挂 handler， -# 子 logger（chongming_worker.runtime、plugins.doubao 等）经 propagate 到父级才可见 -get_logger("chongming_worker") -get_logger("plugins") - -try: - env = config["default"]["env"] -except KeyError: # pragma: no cover - raise ValueError("配置不存在") - -def is_port_in_use(port: int, host: str = "127.0.0.1") -> bool: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(0.5) - return s.connect_ex((host, port)) == 0 - - -def _broker_lock_key() -> str: - """内嵌 Broker 排他锁 key。内嵌 Broker 绑定本机端口，故按宿主隔离 - （多实例共享同一 cache/redis 时，各宿主仍各起各的 Broker）。""" - return f"mqtt_broker:{socket.gethostname()}" - - -def _broker_owner_key() -> str: - """记录内嵌 Broker 子进程实际 PID 的缓存 key（按宿主隔离）。 - - 持有者在启动 Broker 子进程后登记其 PID；前任持有者崩溃时，接管的进程据此 - 识别并终止孤儿 Broker 子进程，从而真正掌握其生命周期。""" - return f"mqtt_broker_owner:{socket.gethostname()}" - - -def _pid_alive(pid) -> bool: - try: - os.kill(int(pid), 0) - return True - except (ProcessLookupError, OSError, ValueError, TypeError): - return False - - -def _terminate_pid(pid): - """尽力终止进程：先 SIGTERM，宽限后仍存活则 SIGKILL。""" - try: - os.kill(int(pid), signal.SIGTERM) - except (ProcessLookupError, OSError, ValueError, TypeError): - return - for _ in range(20): - if not _pid_alive(pid): - return - time.sleep(0.1) - try: - os.kill(int(pid), signal.SIGKILL) - except (ProcessLookupError, OSError, ValueError, TypeError): - pass - - -def _wait_port_free(port: int, host: str = "127.0.0.1", timeout: float = 3.0): - end = time.time() + timeout - while time.time() < end and is_port_in_use(port, host): - time.sleep(0.1) - - -def _run_mqtt_broker(host: str, port: int, username: str | None = None, - password: str | None = None): - """子进程入口：启动 amqtt Broker（配置了账号密码时启用认证）""" - import asyncio as _asyncio - from amqtt.broker import Broker - - # amqtt 在 broker 进程内无 handler，WARNING 会经 lastResort 直接刷到 stderr： - # - sys_interval=0（禁用 $SYS）被 amqtt 当 falsy 报警 - # - 客户端连上却不发 CONNECT 时每次刷 2 行 "No data from"/"Failed to initialize..." - # 统一压到 ERROR：认证失败本身是 DEBUG 级不受影响，真实故障仍会输出 - import logging as _logging - - _logging.getLogger("amqtt").setLevel(_logging.ERROR) - - # ---- monkey-patch 1: amqtt StreamWriterAdapter.close() 忽略断开的传输层 ---- - from amqtt.adapters import StreamWriterAdapter as _SWA - - async def _patched_close(self): - if self.is_closed: - return - self.is_closed = True - try: - await self._writer.drain() - except (OSError, ConnectionError): - pass - try: - if self._writer.can_write_eof(): - self._writer.write_eof() - except OSError: - pass # 传输层已断开 (Errno 107) - try: - self._writer.close() - except OSError: - pass - with __import__("contextlib").suppress(AttributeError): - await self._writer.wait_closed() - - _SWA.close = _patched_close - - # ---- monkey-patch 2: 协议版本不匹配时不打 traceback（仅 warning） ---- - import amqtt.errors as _amqtt_err - from amqtt.utils import format_client_message as _fmt_client, gen_client_id as _gen_cid - from amqtt.broker import Broker as _Broker - from amqtt.mqtt.protocol.broker_handler import BrokerProtocolHandler as _BPH - - async def _patched_init_client(self, reader, writer, remote_address, remote_port): - """与原始方法逻辑一致，但 MQTTError 使用 warning 而非 exception""" - try: - handler, client_session = await _BPH.init_from_connect( - reader, writer, self.plugins_manager - ) - except _amqtt_err.AMQTTError as exc: - self.logger.warning( - "[MQTT-3.1.0-1] %s: Can't read first packet as CONNECT: %s", - _fmt_client(address=remote_address, port=remote_port), - exc, - ) - raise - except _amqtt_err.MQTTError as exc: - self.logger.warning( - "%s (client 可能使用 MQTT 5.0，amqtt 仅支持 3.1.1)", - str(exc), - ) - try: - await writer.close() - except OSError: - pass - raise - except _amqtt_err.NoDataError as exc: - self.logger.warning( - "No data from %s: %s", - _fmt_client(address=remote_address, port=remote_port), - exc, - ) - raise _amqtt_err.AMQTTError(exc) from exc - - if client_session.clean_session: - if client_session.client_id is not None and client_session.client_id != "": - await self._delete_session(client_session.client_id) - else: - client_session.client_id = _gen_cid() - - return handler, client_session - - _Broker._initialize_client_session = _patched_init_client - - async def _serve(): - # 认证插件：配置了账号密码 → FileAuthPlugin（argon2 密码文件，仅此账号可连）； - # 未配置 → 匿名（默认行为） - auth_plugins = {} - passwd_file = None - if username and password: - import tempfile - from pwdlib import PasswordHash - - try: - fd, passwd_file = tempfile.mkstemp(prefix="chongming_mqtt_", suffix=".passwd") - with os.fdopen(fd, "w") as f: - f.write(f"{username}:{PasswordHash.recommended().hash(password)}\n") - auth_plugins = { - "amqtt.plugins.authentication.FileAuthPlugin": { - "password_file": passwd_file, - }, - } - except Exception as e: - logger.warning(f"MQTT 认证初始化失败，回退匿名模式: {e}") - if passwd_file: - os.unlink(passwd_file) - passwd_file = None - else: - auth_plugins = { - "amqtt.plugins.authentication.AnonymousAuthPlugin": {"allow_anonymous": True}, - } - - cfg = { - "listeners": { - "default": { - "type": "tcp", "bind": f"{host}:{port}", "max_connections": 500, - }, - }, - "timeout_disconnect_delay": 2, - "plugins": { - **auth_plugins, - "amqtt.plugins.sys.broker.BrokerSysPlugin": {"sys_interval": 0}, - }, - } - broker = Broker(cfg) - try: - await broker.start() - except Exception: - # 端口已被其他 worker 的 Broker 占用，静默退出 - return - finally: - # 密码文件仅在启动时读取，启动完成后即可删除 - if passwd_file: - try: - os.unlink(passwd_file) - except OSError: - pass - auth_note = f" (认证用户: {username})" if username else " (匿名)" - logger.info("MQTT Broker 已启动 @ %s:%s%s", host, port, auth_note) - try: - while True: - await _asyncio.sleep(1) - except (KeyboardInterrupt, _asyncio.CancelledError): - pass - finally: - await broker.shutdown() - - try: - _asyncio.run(_serve()) - except KeyboardInterrupt: - pass - - -def setup_storage(app: FastAPI): - """初始化本地文件存储（替代 Nanio S3，省 ~172 MB）""" - upload_root = os.path.join(os.getcwd(), "data", "uploads") - os.makedirs(upload_root, exist_ok=True) - app.state.upload_root = upload_root - logger.info(f"本地存储已就绪: {upload_root}") - - -async def setup_auth_store(app: FastAPI) -> None: - """初始化内置 auth 库（users 表 + admin 种子）—— auth 域的基础设施部分。 - - 拆分自 plugins.auth：登录/签发内置在网关（管理平面底线），auth 插件 - 缩容到 0 或全部副本故障时登录与 admin API 仍可用。库位置与 auth 插件 - 读同一份配置（config/workers/auth/config.toml 的 [workers.config]： - data_dir / database），指向同一张 users 表；插件配置缺失（如彻底移除 - auth 插件）时回退默认 ./data/auth.db。 - """ - from chongming_auth import open_database, seed_admin - - workers_conf = {} - auth_conf_path = "config/workers/auth/config.toml" - if os.path.isfile(auth_conf_path): - try: - with open(auth_conf_path, "rb") as f: - workers_conf = tomllib.load(f).get("workers", {}).get("config", {}) - except Exception as e: # noqa: BLE001 - 配置损坏不阻断启动，走默认路径 - logger.warning(f"无法读取 auth 插件配置 {auth_conf_path}: {e}（使用默认 data_dir）") - - db = await open_database(workers_conf) - created = await seed_admin(db) - app.state.auth_db = db - logger.info(f"内置 auth 库已就绪 (admin 种子: {'新建' if created else '已存在'})") - - -def setup_zenoh(app: FastAPI) -> zenoh.Session: - """建立 Zenoh 会话并存入 app.state。""" - zenoh_config = config.get("zenoh", {}) - - zconfig = zenoh.Config() - zconfig.from_file(zenoh_config.get("json5_path", "config/zenoh_config.json5")) - - try: - session = zenoh.open(zconfig) - logger.info("Zenoh 会话已建立 (TCP 自连接模式)") - except Exception as e: - logger.error(f"Zenoh 会话建立失败: {e}") - raise - - app.state.zenoh_session = session - return session - -def setup_worker_manager(app: FastAPI, session: zenoh.Session) -> WorkerManager: - """创建 WorkerManager，启动定期刷新任务，并存入 app.state。""" - cache = Cache.from_config(config["cache"]) - launch_worker_count = str( - config["cache"].get("launch_worker_count", "toml") - ).lower() - manager = WorkerManager(session, cache, launch_worker_count=launch_worker_count) - app.state.worker_manager = manager - - refresh_task = asyncio.create_task(manager.refresh_workers()) - app.state.refresh_task = refresh_task - return manager - -async def setup_route_registry( - app: FastAPI, - session: zenoh.Session, - manager: WorkerManager, - inline_types: list[str] | None = None, -): - """ - 启动动态路由注册表（常驻订阅 chongming/routes/**）。 - - 网关启动即订阅，不依赖网关侧是否登记 external 插件配置：external 插件 - 以独立进程运行（由 Docker/K8s 等编排拉起，网关不负责进程管理），其路由 - 描述符经 Zenoh 动态注册；inline 插件仍在网关进程内加载，两者同时存在 - （混合模式）。盘扫发现的 external 插件条目仅用于启动日志观测。 - - inline_types: 网关进程内加载的插件类型，注册表忽略其描述符以防重复挂载。 - - 注意：external 插件的自定义 APIRouter（如 multipart 上传端点）无法由 - 网关发现（网关不 import 该插件），此类端点需保持 inline 或等 P2 专门通道。 - """ - from ..core.route_registry import RouteRegistry - - prefix = config["default"]["prefix"] - registry = RouteRegistry(app, manager, prefix=prefix) - if inline_types: - registry.set_inline_types(set(inline_types)) - await registry.start(session) - app.state.route_registry = registry - logger.info( - f"路由注册表已启动（prefix={prefix}, 外部插件路由动态注册, " - f"inline 插件: {sorted(inline_types or []) or '无'}）" - ) - return registry - - -def _discover_plugins() -> list[dict]: - """ - 发现所有插件配置。 - - 优先从各插件的独立配置文件 config/workers//config.toml 的 [workers] 节读取， - 兼容旧的 config/workers/config.toml 全局配置。 - - Returns: - [{"type": "doubao", "module": "plugins.doubao", "factory": "create_workers", - "cleanup": "cleanup", "count": 3, "config": {...}}, ...] - """ - plugins = [] - - # 1. 自动扫描 config/workers/*/config.toml - for path in sorted(glob.glob("config/workers/*/config.toml")): - plugin_type = os.path.basename(os.path.dirname(path)) - try: - with open(path, "rb") as f: - plugin_conf = tomllib.load(f) - except Exception as e: - logger.warning("无法读取插件配置 %s: %s", path, e) - continue - - workers_section = plugin_conf.get("workers", {}) - if not workers_section: - continue - - plugin_type = workers_section.get("type", plugin_type) - # 插件运行模式（按插件粒度，混合共存）: - # "inline"（默认） = 网关进程内加载 - # "external" = 独立进程运行（Docker/K8s 编排），经 Zenoh 注册路由 - load_mode = workers_section.get("load_mode", "inline") - # 插件声明的 MQTT Bridge 额外订阅主题（[[workers.mqtt_topics]]） - # 网关级路由配置，不传给插件运行时；worker 目标即本插件类型。 - mqtt_topics = [] - for t in workers_section.get("mqtt_topics", []) or []: - if not isinstance(t, dict) or not t.get("topic"): - continue - t = dict(t) - t.setdefault("worker", plugin_type) - mqtt_topics.append(t) - - # 合并运行时 config: - # 1. plugin_conf 顶层 (worker, worker.registration 等 gen_models 用) - # 2. [workers] 节的注册字段展平 (type, module, factory, cleanup, count) - # — 其中 type 被 chongming_worker.runtime.create_workers 用于匹配 RPC 方法 - # 3. [workers.config] 展平到顶层 (api_key, data_dir, heartbeat 等) - workers_flat = {k: v for k, v in workers_section.items() - if k not in ("config", "mqtt_topics", "load_mode")} - runtime_config = { - **plugin_conf, - **workers_flat, - **workers_section.get("config", {}), - } - - plugins.append({ - "type": plugin_type, - "module": workers_section.get("module", f"plugins.{plugin_type}"), - "factory": workers_section.get("factory", "create_workers"), - "cleanup": workers_section.get("cleanup", "cleanup"), - "count": workers_section.get("count", 1), - "load_mode": load_mode, - "mqtt_topics": mqtt_topics, - "config": runtime_config, - # [workers.config] 展平前的原始段：供 load_workers 按「配置优先来源模式」 - # 引导/覆盖共享缓存，使 inline 插件初始化即拿到缓存值（cache 模式） - "config_section": dict(workers_section.get("config") or {}), - }) - - # 2. 兼容旧写法: config/workers/config.toml（仅加载尚未发现的插件） - old_config_path = config.get("plugins_worker", {}).get("config_path", "config/workers/config.toml") - if os.path.isfile(old_config_path): - existing_types = {p["type"] for p in plugins} - old_conf = get_workers_config(old_config_path).get("workers", []) - for wc in old_conf: - wtype = wc.get("type", "") - if wtype and wtype not in existing_types: - plugins.append({ - "type": wtype, - "module": wc.get("module", ""), - "factory": wc.get("factory", ""), - "cleanup": wc.get("cleanup", ""), - "count": wc.get("count", 1), - "config": wc.get("config", {}), - }) - logger.info("从旧配置加载插件: %s (config/workers/config.toml)", wtype) - - return plugins - - -async def load_workers(app: FastAPI, session: zenoh.Session, manager: WorkerManager, - is_inline_owner: bool = True): - """ - 加载 inline 插件，启动对应的 worker 任务（混合模式）。 - - 自动从 config/workers//config.toml 发现插件；load_mode="external" - 的插件不在网关内加载（由独立进程运行，其路由经常驻 RouteRegistry 动态 - 注册，无需网关侧保留插件配置，盘扫发现的 external 条目仅用于观测）。 - - 同时自动发现插件模块中暴露的 APIRouter（约定：模块顶层名为 `router` 的 - APIRouter 实例），收集后供 register_custom_routes() 统一注册。 - - is_inline_owner: 本进程是否为宿主级 inline worker 持有者。gunicorn 多 worker - 下仅持有者进程真正拉起 inline worker（避免 N×count 冗余副本）；非持有者进程 - 仍 import 插件模块以便注册自定义路由/契约校验，但不 create_workers，worker 经 - Zenoh liveliness 被跨进程发现。单进程（uvicorn serve）时为 True，行为不变。 - - 返回: (worker_tasks, custom_routers, inline_types, external_types) - """ - workers_conf = _discover_plugins() - inline_plugins = [wc for wc in workers_conf if wc.get("load_mode", "inline") != "external"] - external_plugins = [wc for wc in workers_conf if wc.get("load_mode", "inline") == "external"] - inline_types = sorted({wc["type"] for wc in inline_plugins}) - external_types = sorted({wc["type"] for wc in external_plugins}) - logger.info( - f"发现 {len(workers_conf)} 个 worker 插件: " - f"inline={inline_types or '无'}, external={external_types or '无'} " - f"(本进程 inline 持有者: {is_inline_owner})" - ) - - for wc in external_plugins: - logger.info( - "插件 %s 为 external 模式，跳过进程内加载（由独立进程经 Zenoh 注册路由）", - wc["type"], - ) - - # 汇总 inline 插件声明的 MQTT Bridge 额外订阅主题，供 setup_mqtt 使用 - # （external 插件的主题由 RouteRegistry 经 __meta__ 动态推送） - app.state.plugin_mqtt_topics = [ - topic - for wc in inline_plugins - for topic in (wc.get("mqtt_topics") or []) - ] - - worker_tasks = [] - custom_routers = [] - for wc in inline_plugins: - module_name = wc["module"] - factory_name = wc["factory"] - # count 以 chongming-cache 为单一事实来源：cache 有值时用 cache 的全局总量， - # 否则用配置值引导并写回 cache（多网关共享同一 cache 时自动对齐）。 - count = wc["count"] - worker_type = wc["type"] - # [workers.config] 单一事实来源：按配置优先来源模式对齐（cache 模式取缓存值、 - # toml 模式以文件为准并覆盖缓存），必须在注册 starter/创建副本之前完成—— - # 插件实例（on_init 重建客户端等）用的就是这份配置。 - worker_config = manager.align_inline_config( - worker_type, wc["config"], wc.get("config_section") - ) - - if not module_name or not factory_name: - logger.warning(f"Worker 配置不完整: {wc}") - continue - - logger.debug("加载插件 %s: config/workers/%s/config.toml (count=%d)", - worker_type, worker_type, count) - - try: - module = __import__(module_name, fromlist=[factory_name]) - - # 自动发现插件的自定义 FastAPI 路由（约定：模块暴露 `router` 属性） - plugin_router = getattr(module, "router", None) - if plugin_router is not None: - from fastapi import APIRouter - if isinstance(plugin_router, APIRouter): - custom_routers.append(plugin_router) - logger.info(f"发现插件自定义路由: {module_name}.router") - - # 非持有者进程：不立即拉起 inline worker，但仍在下方（build starter 后） - # 注册自愈启动器；worker 经 liveliness 被本进程发现调用，@rpc_method 与 - # 自定义路由/契约校验依赖的模块 import 已在上文完成不依赖拉起。 - factory = getattr(module, factory_name) - - # 定义"拉起本类型 worker"的协程，供持有者崩溃后的自愈接管复用； - # 用默认参数捕获循环变量，避免闭包迟到绑定。 - def _make_starter(session=session, config=worker_config, cnt=count, - mgr=manager, fac=factory): - async def starter(): - if inspect.iscoroutinefunction(fac): - return await fac(session, config, cnt, worker_manager=mgr) - return fac(session, config, cnt, worker_manager=mgr) - return starter - - starter = _make_starter() - manager.register_worker_starter(worker_type, starter) - - # 非持有者进程：不拉起 inline worker（启动器已注册，持有者崩溃时由 refresh 自愈接管） - if not is_inline_owner: - logger.info( - "本进程非 %s 的 inline 持有者，跳过创建副本（liveliness 发现即可调用）", - worker_type, - ) - continue - - tasks = await starter() - worker_tasks.extend(tasks) - logger.info(f"已加载 Worker 插件: {module_name}.{factory_name} (副本数: {count})") - except Exception as e: - import traceback - logger.error( - f"加载 Worker 插件失败 {module_name}: {e}\n" - f" 配置: type={worker_type}, count={count}\n" - f" 堆栈:\n{traceback.format_exc()}" - ) - - app.state.worker_tasks = worker_tasks - app.state.inline_types = inline_types - app.state.external_types = external_types - return worker_tasks, custom_routers, inline_types, external_types - -async def register_routes( - app: FastAPI, - manager: WorkerManager, - custom_routers: list | None = None, - worker_types: list[str] | None = None, -): - """注册动态路由和插件自定义路由。 - - worker_types: 仅注册这些插件类型的 TOML 动态路由（混合模式下只注册 - inline 插件；external 插件路由由 RouteRegistry 动态挂载）。 - """ - prefix = config["default"]["prefix"] - register_dynamic_routes(app, manager, prefix=prefix, worker_types=worker_types) - # 注册插件自定义路由（如 FORM-DATA 文件上传等 TOML 无法描述的端点） - # 统一加上全局 prefix（如 /api/v1），与 TOML 动态路由保持一致 - if custom_routers: - for r in custom_routers: - app.include_router(r, prefix=prefix) - logger.info(f"已注册 {len(custom_routers)} 个插件自定义路由（prefix={prefix}）") - - -def validate_contracts(app: FastAPI | None = None): - """ - 契约校验（TOML 路由 ↔ @rpc_method 注册表 ↔ 生成模型），仅覆盖 inline 插件。 - - 在 load_workers 之后、register_routes 之前调用：此时插件模块已导入 - （注册表就绪）。HTTP 路由缺失对应 @rpc_method 时抛 RuntimeError 终止启动， - 其余不一致以警告日志汇总。 - - external 插件在自己进程内做同样校验（validate_worker_contracts），网关侧跳过。 - """ - from ..core.contracts import validate_contracts as _validate - - plugins = _discover_plugins() - inline_plugins = [p for p in plugins if p.get("load_mode", "inline") != "external"] - _validate(inline_plugins) - -async def setup_mqtt_broker(app: FastAPI, worker_manager: WorkerManager): - """ - 启动内嵌 amqtt Broker 子进程（宿主级分布式锁确保多 worker/多实例仅本宿主启动一个）。 - - 由配置 [.mqtt].use_embedded_broker 控制： - true （默认）→ 网关启动 amqtt Broker 子进程，host/port 为监听地址 - false → 使用外部 MQTT Broker，网关仅运行 Bridge 客户端，跳过本步 - 配置了 username/password 时 Broker 启用认证，否则保持匿名。 - - 排他锁基于 chongming-cache 的 DistributedLock，key 按宿主隔离（内嵌 Broker 绑定 - 本机端口）。持有者在缓存登记其 Broker 子进程 PID；前任持有者崩溃/被强杀而留下 - 孤儿 Broker 子进程时，接管的进程借此 PID 识别并终止之，再启动自己托管的子进程， - 从而真正掌握 Broker 生命周期（可关停、可接管），而非留下不可管控的残留下端点。 - """ - env_config = config.get(env, {}) - mqtt_config = env_config.get("mqtt", {}) - if not mqtt_config.get("enabled", True): - logger.info("MQTT 已禁用") - return None - if not mqtt_config.get("use_embedded_broker", True): - logger.info( - f"use_embedded_broker=false：使用外部 MQTT Broker " - f"({mqtt_config.get('host', 'localhost')}:{mqtt_config.get('port', 1883)})，" - f"跳过内嵌 amqtt Broker 启动" - ) - return None - if worker_manager.cache is None: - logger.warning( - "MQTT 内嵌 Broker 启用但未配置共享 cache（[cache]），无法获取排他锁；" - "请配置 redis 或共享卷 sqlite，或改用外部 Broker" - ) - return None - - host = mqtt_config.get("host", "localhost") - port = mqtt_config.get("port", 1883) - username = mqtt_config.get("username") or None - password = mqtt_config.get("password") or None - - lock = DistributedLock(worker_manager.cache, _broker_lock_key()) - if not lock.acquire(): - # 非持有者：等待持有者把 Broker 拉起并监听端口 - app.state.mqtt_broker_lock_acquired = False - app.state.mqtt_broker_lock = None - for _ in range(50): - if is_port_in_use(port, host): - logger.info(f"MQTT Broker 已在 {host}:{port} 运行（由其他进程托管）") - return None - time.sleep(0.1) - logger.warning(f"等待内嵌 MQTT Broker 端口 {port} 就绪超时") - return None - - # 本进程赢得宿主锁：持有租约续期，取得 Broker 托管权 - lock.start() - app.state.mqtt_broker_lock = lock - app.state.mqtt_broker_lock_acquired = True - - # 端口已被监听：优先识别并终止前任残留的孤儿 Broker 子进程，以真正接管其 - # 生命周期；否则视为不可托管的既有进程，复用（本进程不持有句柄）。 - try: - prev_pid = int(worker_manager.cache.get(_broker_owner_key()) or 0) - except (TypeError, ValueError): - prev_pid = None - if is_port_in_use(port, host): - if prev_pid and _pid_alive(prev_pid) and prev_pid != os.getpid(): - logger.warning( - f"[mqtt-broker] 检出孤儿 Broker 子进程 PID {prev_pid}，终止并接管重启" - ) - _terminate_pid(prev_pid) - _wait_port_free(port, host) - if is_port_in_use(port, host): - logger.info( - f"MQTT Broker 端口 {port} 在终止孤儿后仍被占用，属不可托管进程，复用" - ) - return None - else: - logger.info(f"MQTT Broker 已在 {host}:{port} 运行，复用（不可托管）") - return None - - proc = multiprocessing.Process( - target=_run_mqtt_broker, args=(host, port, username, password), daemon=False - ) - proc.start() - app.state.mqtt_broker_process = proc - worker_manager.cache.set(_broker_owner_key(), str(proc.pid)) - auth_note = f"，认证用户 {username}" if username else "（匿名）" - logger.info(f"MQTT Broker 子进程已启动并登记托管 (PID: {proc.pid}){auth_note}") - for _ in range(50): - if is_port_in_use(port, host): - logger.info(f"MQTT Broker 已监听 {port} 端口") - return None - if not proc.is_alive(): - raise RuntimeError("MQTT Broker 进程异常退出") - time.sleep(0.1) - raise RuntimeError("MQTT Broker 端口未就绪") - - -async def shutdown_mqtt_broker(app: FastAPI): - """终止本进程托管的 MQTT Broker 子进程，并释放宿主级分布式排他锁、清理登记。""" - proc = getattr(app.state, "mqtt_broker_process", None) - if proc and proc.is_alive(): - proc.terminate() - proc.join(timeout=5) - if proc.is_alive(): - proc.kill() - logger.info("MQTT Broker 子进程已终止") - lock = getattr(app.state, "mqtt_broker_lock", None) - if lock is not None: - if proc: # 仅真正托管了子进程的进程才清理孤儿登记，避免误删他人登记 - try: - lock._cache.delete(_broker_owner_key()) - except Exception as e: # noqa: BLE001 - logger.debug(f"清理 MQTT Broker 登记失败: {e}") - await lock.stop() - lock.release() - app.state.mqtt_broker_lock = None - app.state.mqtt_broker_lock_acquired = False - - -async def _start_mqtt(app: FastAPI, worker_manager: WorkerManager, - bridge_lock) -> MQTTBridge | None: - """在已持有分布式锁的前提下构建并启动一个 MQTT Bridge 客户端。 - - 该逻辑同时被 setup_mqtt（启动时）与 mqtt_bridge_supervisor（持有者自愈接管时） - 复用。假定调用方已 acquire() 到 "mqtt_bridge" 分布式锁。 - """ - env_config = config.get(env, {}) - mqtt_config = env_config.get("mqtt", {}) - host = mqtt_config.get("host", "localhost") - port = mqtt_config.get("port", 1883) - username = mqtt_config.get("username") or None - password = mqtt_config.get("password") or None - custom_topics = list(mqtt_config.get("custom_topics", []) or []) - custom_topics += list(getattr(app.state, "plugin_mqtt_topics", []) or []) - - bridge = MQTTBridge( - worker_manager=worker_manager, - host=host, - port=port, - custom_topics=custom_topics, - username=username, - password=password, - ) - app.state.mqtt_bridge = bridge - await bridge.start() - bridge_lock.start() # 后台续约租约，防止持有方宕机/长暂停导致锁被错误抢走 - app.state.mqtt_bridge_lock = bridge_lock - # 把 Bridge 挂接给常驻路由注册表（动态主题 + 补推已到达的元信息） - registry = getattr(app.state, "route_registry", None) - if registry is not None: - registry.attach_bridge(bridge) - broker_mode = "内嵌 amqtt" if mqtt_config.get("use_embedded_broker", True) else "外部" - auth_note = f"，认证用户 {username}" if username else "（匿名）" - logger.info(f"MQTT Bridge 已启动（{broker_mode} Broker {host}:{port}{auth_note}）") - return bridge - - -async def setup_mqtt(app: FastAPI, worker_manager: WorkerManager) -> MQTTBridge | None: - """ - 启动 MQTT Bridge（纯客户端模式）。 - - 从 config.toml 的环境段读取 host/port/enabled；若 enabled 为 false 则跳过。 - Broker 来源由 use_embedded_broker 控制： - true （默认）= 内嵌 amqtt Broker（lifespan 第 3 步已启动子进程） - false = 外部 MQTT Broker（host/port 指向外部地址，网关不启动 Broker） - 订阅主题 = 全局 [.mqtt].custom_topics（可选兜底） - + 各插件在其 config/workers//config.toml 中声明的 [[workers.mqtt_topics]]。 - 使用基于共享 cache 的分布式锁确保多进程/多网关中全局只有一个 Bridge 消费。 - """ - env_config = config.get(env, {}) - mqtt_config = env_config.get("mqtt", {}) - if not mqtt_config.get("enabled", True): - logger.info("MQTT 已禁用") - return None - - # 排他锁：基于共享缓存（redis / 共享卷 diskcache），保证全局只有一个 Bridge 消费。 - # 相比原文件锁，可跨多台主机收敛——多实例共享同一个外部 Broker 时， - # 避免一条 MQTT 请求被 N 台网关上各自消费、各回一份响应。 - bridge_lock = DistributedLock(worker_manager.cache, "mqtt_bridge") # pyright: ignore[reportArgumentType] - if not bridge_lock.acquire(): - logger.info( - "MQTT Bridge 已由其他网关实例持有（分布式锁），本网关跳过，" - "以规避共享 Broker 下单请求多响应" - ) - return None - try: - return await _start_mqtt(app, worker_manager, bridge_lock) - except Exception as e: # noqa: BLE001 - logger.error(f"MQTT Bridge 启动失败: {e}") - await bridge_lock.stop() - bridge_lock.release() - app.state.mqtt_bridge = None - app.state.mqtt_bridge_lock = None - return None - - -async def mqtt_bridge_supervisor(app: FastAPI, worker_manager: WorkerManager): - """MQTT Bridge 持有者崩溃自愈巡检（跨所有网关进程运行）。 - - 与 inline 持有者同源的问题：仅靠 gunicorn 复活死掉 slot 无法 failover——新进程 - 可能在旧锁租约未过期时抢锁失败而照样跳过。本巡检在每个网关进程周期性运行： - - - 本进程已持有一存活 Bridge → 正常托管，跳过。 - - 否则尝试获得 "mqtt_bridge" 分布式锁（set_nx 保证全局唯一胜出，无分裂）； - 若锁仍被存活持有者把持 → 跳过；若空闲（原持有者租约过期=已死/失联）→ 接管。 - - 接管时按需拉起内嵌 Broker（setup_mqtt_broker 幂等：端口被占则复用现有进程）， - 再重建 Bridge 客户端。 - - 运行周期经 [.mqtt].bridge_supervise_interval 配置，默认 15s。 - """ - env_config = config.get(env, {}) - mqtt_config = env_config.get("mqtt", {}) - if not mqtt_config.get("enabled", True): - return # MQTT 禁用：无需巡检 - interval = mqtt_config.get("bridge_supervise_interval", 15) or 15 - while True: - await asyncio.sleep(interval) - try: - bridge = getattr(app.state, "mqtt_bridge", None) - # 本进程持有一存活 Bridge：正常托管 - if bridge is not None and getattr(bridge, "is_running", True): - continue - - newly_acquired = False - lock = getattr(app.state, "mqtt_bridge_lock", None) - if lock is None: - # 非持有者：仅当锁空闲（原持有者租约过期）时才能抢得 - if worker_manager.cache is None: - continue - candidate = DistributedLock(worker_manager.cache, "mqtt_bridge") - if not candidate.acquire(): - continue # 仍被存活持有者把持 - candidate.start() - app.state.mqtt_bridge_lock = candidate - lock = candidate - newly_acquired = True - - logger.warning("[mqtt-bridge] 检出持有者失效，接管重建 Broker+Bridge") - # 按需拉起内嵌 Broker（幂等；外部 Broker 时 setup_mqtt_broker 内直接返回） - try: - await setup_mqtt_broker(app, worker_manager) - except Exception as e: # noqa: BLE001 - logger.error(f"[mqtt-bridge] 接管时拉起 Broker 失败: {e}") - try: - await _start_mqtt(app, worker_manager, lock) - logger.info("[mqtt-bridge] 接管完成，Bridge 已重建") - except Exception as e: # noqa: BLE001 - logger.error(f"[mqtt-bridge] 接管时重建 Bridge 失败: {e}") - app.state.mqtt_bridge = None - if newly_acquired: - await lock.stop() - lock.release() - app.state.mqtt_bridge_lock = None - except Exception as e: # noqa: BLE001 - logger.error(f"MQTT Bridge 自愈巡检失败: {e}") - - -async def shutdown_mqtt(app: FastAPI): - """停止 MQTT Bridge 和 Broker 子进程（仅锁持有者释放锁）""" - bridge: MQTTBridge | None = getattr(app.state, "mqtt_bridge", None) - if bridge and bridge.is_running: - await bridge.stop() - # 停止续约并受保护释放分布式锁（仅本网关仍持有时删除） - bridge_lock = getattr(app.state, "mqtt_bridge_lock", None) - if bridge_lock is not None: - await bridge_lock.stop() - bridge_lock.release() - await shutdown_mqtt_broker(app) - - -async def cleanup_workers(app: FastAPI): - """调用每个 inline worker 配置中定义的 cleanup 函数。 - - 仅 inline worker 持有者进程执行：非持有者进程未初始化插件（未调用 create_workers， - on_init 未执行），cleanup/on_cleanup 无资源可清，且可能引用未创建的对象而误报错。 - """ - # app.state.inline_owner_locked 恒为 bool（lifespan 4.2 设置），缺省按持有处理 - if not getattr(app.state, "inline_owner_locked", True): - logger.info("本进程非 inline worker 持有者，跳过插件 cleanup") - return - workers_conf = _discover_plugins() - - for wc in workers_conf: - if wc.get("load_mode", "inline") == "external": - continue - module_name = wc.get("module", "") - cleanup_name = wc.get("cleanup", "") - worker_type = wc.get("type", "") - - if not module_name or not cleanup_name: - logger.warning(f"Worker 配置缺少 cleanup: {wc}") - continue - - try: - module = __import__(module_name, fromlist=[cleanup_name]) - cleanup = getattr(module, cleanup_name) - pid = os.getpid() - worker_id = f"worker/{worker_type}/worker-i-{pid}" - if inspect.iscoroutinefunction(cleanup): - await cleanup(worker_id) - else: - cleanup(worker_id) - except Exception as e: - logger.error(f"cleanup 调用失败 {module_name}: {e}") - \ No newline at end of file diff --git a/src/chongming/app/core/logger.py b/src/chongming/app/core/logger.py deleted file mode 100644 index 92688b0..0000000 --- a/src/chongming/app/core/logger.py +++ /dev/null @@ -1,396 +0,0 @@ -import re -import os -import atexit -import queue -import logging -import datetime -from pathlib import Path -from typing import List, Optional -from logging.handlers import RotatingFileHandler, QueueHandler, QueueListener - -from chongming_config import get_config - - -config = get_config() -env = config["default"]["env"] -server_config = config[env]["server"] -log_config = config[env]["logging"] - - -class ColoredFormatter(logging.Formatter): - """ - 为 HTTP 状态码添加颜色的日志格式化器 - - 根据 HTTP 状态码的首位数字为日志消息中的状态码添加不同颜色： - - 2xx: 绿色（成功） - - 3xx: 青色（重定向） - - 4xx: 黄色（客户端错误） - - 5xx: 红色（服务器错误） - - Attributes: - COLORS: 状态码首位数字到 ANSI 颜色代码的映射字典 - RESET: ANSI 重置颜色代码 - """ - - COLORS = { - "2": "\033[32m", # 绿色 2xx - "3": "\033[36m", # 青色 3xx - "4": "\033[33m", # 黄色 4xx - "5": "\033[31m", # 红色 5xx - } - RESET = "\033[0m" - - def format(self, record): - """ - 格式化日志记录，为状态码添加颜色 - - 如果日志记录包含 status_code 属性，则查找消息中的 "status=数字" 模式 - 并为其添加对应的颜色代码。格式化完成后恢复原始消息以避免影响其他 handler。 - - Args: - record: 日志记录对象，应包含 msg 属性和可选的 status_code 属性 - - Returns: - str: 格式化后的日志消息字符串，状态码部分可能包含 ANSI 颜色代码 - """ - # 保存原始消息 - original_msg = record.msg - # 如果日志记录包含 status_code 属性，则为其添加颜色 - status_code = getattr(record, "status_code", None) - if status_code and isinstance(status_code, int): - first_digit = str(status_code)[0] - color = self.COLORS.get(first_digit, "") - colored_status = f"{color}{status_code}{self.RESET}" - # 替换消息中的 "status=数字" 为带颜色的版本 - record.msg = re.sub( - rf"status={status_code}\b", f"status={colored_status}", original_msg - ) - result = super().format(record) - # 恢复原始消息，避免影响其他 handler - record.msg = original_msg - return result - - -class DailyRotatingFileHandler(RotatingFileHandler): - """ - 按自然日切分 + 大小轮转的文件处理器 - - 日志文件名形如 {log_name}-2026-08-19.log：进程启动时按当天日期命名， - 跨天后写入时自动切换到新日期文件，避免长时间运行的进程把跨天日志 - 全部堆积到启动当天的文件里。同时保留 RotatingFileHandler 的大小轮转 - （单文件超限追加 .1/.2 后缀）。 - - 历史文件（跨天文件与大小备份）按修改时间保留最近 backup_count 个， - 超出部分在切换日期的瞬间清理。 - """ - - def __init__( - self, - log_dir: str, - log_name: str, - max_bytes: int, - backup_count: int, - encoding: str = "utf-8", - ): - self.log_dir = Path(log_dir) - self.log_name = log_name - self._current_date = datetime.date.today() - super().__init__( - str(self.log_dir / f"{log_name}-{self._current_date}.log"), - encoding=encoding, - maxBytes=max_bytes, - backupCount=backup_count, - ) - - def _build_filename(self, day: datetime.date) -> str: - return str(self.log_dir / f"{self.log_name}-{day}.log") - - def _cleanup_old_files(self) -> None: - """保留最近 backup_count 个日志文件，删除更早的历史文件""" - # 按 (日期, 轮转序号) 排序，不依赖 mtime（同日备份可能同时落盘）： - # 日期越大越新；同日内主文件(.log)最新，.N 备份按 N 递减更旧 - pattern = re.compile( - rf"^{re.escape(self.log_name)}-(\d{{4}}-\d{{2}}-\d{{2}})\.log(?:\.(\d+))?$" - ) - - def sort_key(path: Path): - match = pattern.match(path.name) - if not match: - return (datetime.date.min, 0) - return ( - datetime.date.fromisoformat(match.group(1)), - -int(match.group(2) or 0), - ) - - files = sorted( - self.log_dir.glob(f"{self.log_name}-*.log*"), - key=sort_key, - reverse=True, - ) - for old in files[self.backupCount :]: - try: - old.unlink() - except OSError: - pass - - def emit(self, record): - # 跨天：关闭旧文件流，切换到新日期文件，并清理过期历史文件 - today = datetime.date.today() - if today != self._current_date: - self._current_date = today - self.close() - self.baseFilename = self._build_filename(today) - self.stream = self._open() - self._cleanup_old_files() - super().emit(record) - - -class LoggerNameFilter(logging.Filter): - """ - 按 log_name 路由日志记录的过滤器 - - 只放行本 log_name 自身及其子 logger 的记录（如 chongming_worker.runtime - 归入 chongming_worker），保证共享 QueueListener 下每个 handler 只处理 - 属于自己的记录：控制台每行只打印一次，各日志文件按名隔离。 - """ - - def __init__(self, log_name: str): - super().__init__() - self._name = log_name - self._prefix = log_name + "." - - def filter(self, record): - return record.name == self._name or record.name.startswith(self._prefix) - - -class ExcludeWatchFilesFilter(logging.Filter): - """ - 日志过滤器，用于排除第三方库的日志输出 - - 过滤掉来自 watchfiles、aiosqlite、apscheduler、sqlalchemy 等库的日志记录， - 避免这些库的详细日志污染应用日志输出。 - """ - - def __init__(self): - """ - 初始化过滤器，设置需要过滤的日志名称前缀列表 - """ - super().__init__() - self._filter_starts_with_tulpe = ( - "watchfiles", - "aiosqlite", - "apscheduler", - "sqlalchemy", - "sqlitedict", - "sqlalchemy.engine.Engine", - ) - - def filter(self, record): - """ - 判断是否应该保留该日志记录 - - Args: - record: 日志记录对象，包含 name 属性表示日志来源 - - Returns: - bool: 如果日志名称不以过滤列表中的任何前缀开头则返回 True（保留）， - 否则返回 False（过滤掉） - """ - return not record.name.startswith(self._filter_starts_with_tulpe) - - -# ---- 异步日志队列（QueueHandler + QueueListener）---- -# 主线程/事件循环只做"入队"（LogRecord 创建 + put，µs 级），文件/控制台 I/O -# 由后台监听线程执行，日志记录不再阻塞业务协程。 -# 队列满时丢弃新日志（日志系统故障不影响业务）；错误日志（>=400）不受采样影响。 - -_LOG_QUEUE_SIZE = 10000 - - -class _AsyncQueueHandler(QueueHandler): - """非阻塞入队 + 惰性格式化。 - - - 队列满时丢弃（而不是抛 queue.Full 打断业务） - - 不预格式化：% 样式消息的格式化推迟到监听线程（formatter 执行）， - 主线程只承担 LogRecord 创建与入队 - - queue 经模块全局读取：gunicorn preload/fork 后自动指向子进程的新队列 - """ - - @property - def queue(self): - return _log_queue - - def __init__(self): - # queue 由 property 动态提供（fork 后自动指向新队列）。 - # 只初始化 Handler 基类（level/filters 等），不走 QueueHandler.__init__ - # 的 self.queue = queue 赋值（会与 property 冲突） - logging.Handler.__init__(self) - - def enqueue(self, record): - try: - self.queue.put_nowait(record) - except queue.Full: - pass # 队列满：丢弃该日志 - - def prepare(self, record): - return record # 格式化留给监听线程的 formatter - - -class _RegistryListener(QueueListener): - """handler 可动态注册的 QueueListener（每个 log_name 有独立 file handler）""" - - def __init__(self, q: queue.Queue): - super().__init__(q) - self.handlers: list = [] # 可变列表，运行期可追加 - - def register(self, *handlers) -> None: - self.handlers.extend(handlers) - - -_log_queue: queue.Queue = queue.Queue(maxsize=_LOG_QUEUE_SIZE) -_listener: Optional[_RegistryListener] = None -# 全部真实 handler（文件/控制台），供 fork 后重建监听线程 -_listener_handlers: List[logging.Handler] = [] - - -def _start_listener() -> None: - global _listener - _listener = _RegistryListener(_log_queue) - _listener.register(*_listener_handlers) - _listener.start() - - -def _after_fork_child() -> None: - """gunicorn preload/fork 场景：线程不跨 fork，子进程重建队列与监听线程。 - - 队列重建避免 fork 时锁状态不可用导致的死锁；旧队列中的记录在 fork - 瞬间丢失（启动窗口内可接受）。 - """ - global _log_queue - _log_queue = queue.Queue(maxsize=_LOG_QUEUE_SIZE) - _start_listener() - - -if hasattr(os, "register_at_fork"): - os.register_at_fork(after_in_child=_after_fork_child) - - -def shutdown_logging() -> None: - """停止监听线程并冲刷剩余日志（进程退出 / 应用关闭时调用），幂等。 - - QueueListener.stop() 停止后会将 _thread 置 None，重复调用需跳过； - 幂等保证 atexit 与显式调用共存。 - """ - global _listener - if _listener is not None: - if _listener._thread is not None: - _listener.stop() # flush=True：处理完队列中已有记录再停止 - _listener = None - for h in _listener_handlers: - try: - h.close() - except Exception: # pragma: no cover - 关闭失败不影响退出 - pass - - -atexit.register(shutdown_logging) - - -_configured_loggers = set() -# 每个进程只打印一次日志配置信息（setup_logging 每个 log_name 都会调用一次） -_config_info_printed = False - - -def setup_logging(logger: logging.Logger, log_name: str = "app"): - """ - 配置日志系统，包括文件和控制台输出 - - 为指定的 logger 配置以下功能： - 1. 设置日志级别和格式 - 2. 创建按自然日切分的轮转文件处理器（跨天自动切换文件，支持文件大小限制和备份数量控制） - 3. 创建带颜色格式化的控制台处理器 - 4. 应用过滤器排除第三方库日志 - - 该函数具有防重复配置机制，同一个 log_name 只会配置一次。 - - Args: - logger: 需要配置的 logging.Logger 实例 - log_name: 日志名称标识，用于区分不同的日志配置和生成日志文件名，默认为 "app" - - Returns: - None - - Raises: - 无显式抛出异常，但可能因文件系统权限问题导致目录创建或文件写入失败 - """ - if log_name in _configured_loggers: - return - _configured_loggers.add(log_name) - log_level = getattr(logging, log_config["level"], logging.INFO) - log_format = log_config["format"] - # 创建文件夹 - log_file_path = Path(log_config["file"]) - if log_file_path != Path("."): - log_file_path.mkdir(parents=True, exist_ok=True) - - # 配置根日志记录器 - logger.setLevel(log_level) - - # 创建格式化器 - plain_formatter = logging.Formatter(log_format) - color_formatter = ColoredFormatter(log_format) - - # 创建按自然日切分的轮转文件处理器（跨天自动切文件，单文件超限再按大小轮转） - max_bytes = log_config.get("max_size", 10 * 1024 * 1024) # 默认 10MB - backup_count = log_config.get("backup_count", 5) # 默认保留 5 个历史文件 - - file_handler = DailyRotatingFileHandler( - log_config["file"], - log_name, - max_bytes=max_bytes, - backup_count=backup_count, - ) - file_handler.setFormatter(plain_formatter) - file_handler.addFilter(ExcludeWatchFilesFilter()) - file_handler.addFilter(LoggerNameFilter(log_name)) - - # 创建控制台处理器并添加过滤器 - console_handler = logging.StreamHandler() - console_handler.setFormatter(color_formatter) - console_handler.addFilter(ExcludeWatchFilesFilter()) - console_handler.addFilter(LoggerNameFilter(log_name)) - - # 真实 handler（文件/控制台）挂到后台监听线程；logger 上只挂队列 handler， - # 日志调用在主线程仅入队，I/O 由监听线程执行 - _listener_handlers.extend([file_handler, console_handler]) - if _listener is None: - _start_listener() - else: - _listener.register(file_handler, console_handler) - logger.addHandler(_AsyncQueueHandler()) - - global _config_info_printed - if not _config_info_printed: - _config_info_printed = True - print( - f"日志系统已配置 - 级别: {logging.getLevelName(log_level)}, 文件: {log_config['file']} " - f"(最大：{max_bytes / 1024 / 1024:.1f}MB, 备份：{backup_count})" - ) - - -def get_logger(name: str) -> logging.Logger: - """ - 获取配置好的日志记录器实例 - - 根据名称获取或创建 logger 实例，并自动调用 setup_logging 进行配置。 - 如果该名称的 logger 已经配置过，则直接返回已配置的实例。 - - Args: - name: 日志记录器名称，通常使用模块名如 __name__ - - Returns: - logging.Logger: 已配置完成的日志记录器实例，可直接用于记录日志 - """ - logger = logging.getLogger(name) - setup_logging(logger, name) - return logger diff --git a/src/chongming/app/core/mqtt_bridge.py b/src/chongming/app/core/mqtt_bridge.py deleted file mode 100644 index 25d1f03..0000000 --- a/src/chongming/app/core/mqtt_bridge.py +++ /dev/null @@ -1,271 +0,0 @@ -"""MQTT Bridge — 纯客户端模式，Broker 由子进程管理""" -import asyncio -import orjson -import re -from typing import Optional -from urllib.parse import quote - -from amqtt.client import MQTTClient -from amqtt.mqtt.constants import QOS_1 -from .logger import get_logger - -logger = get_logger("mqtt") -MQTT_TOPIC_PREFIX = "chongming" -MQTT_SUBSCRIBE_TOPIC = f"{MQTT_TOPIC_PREFIX}/+/+" - -# 已按"非信封载荷"路由过告警的主题（每主题只告警一次，避免设备 5s 心跳刷屏） -_warned_legacy_topics: set = set() - - -def _compile_topic(topic: str) -> re.Pattern: - """ - 将 MQTT 主题模板（支持 + / # 通配符）编译为正则表达式。 - - - ``+`` 匹配单级（编译为捕获组 ``([^/]+)``） - - ``#`` 匹配多级（编译为捕获组 ``(.*)``，须位于末尾） - 捕获组按出现顺序对应，供 {1} {2}... 占位符替换使用。 - """ - parts = topic.split("/") - regex = "^" - for i, part in enumerate(parts): - if i > 0: - regex += "/" - if part == "#": - regex += "(.*)" - elif part == "+": - regex += "([^/]+)" - else: - regex += re.escape(part) - regex += "$" - return re.compile(regex) - - -class MQTTBridge: - def __init__(self, worker_manager, host="localhost", port=1883, custom_topics=None, - username=None, password=None): - self._wm = worker_manager - self._host = host - self._port = port - self._username = username - self._password = password - self._client: Optional[MQTTClient] = None - self._lt: Optional[asyncio.Task] = None - self._running = False - # custom_topics: [{"topic","worker","subject"?,"response_topic"?}, ...] - # 混合模式双来源： - # _static_topics — inline 插件，构造时由网关从配置收集 - # _dynamic_topics — external 插件，运行时由 RouteRegistry 推送 - self._static_topics = custom_topics or [] - self._dynamic_topics: list = [] - self._compiled: list = [] - self._rebuild_compiled() - self._subscribed_custom: set = set() - - def _rebuild_compiled(self): - """合并静态/动态主题并重编译匹配正则。""" - merged = self._static_topics + self._dynamic_topics - self._compiled = [ - (ct, _compile_topic(ct.get("topic", ""))) - for ct in merged if ct.get("topic") - ] - - async def start(self): - if self._running: - return - try: - await self._sc() - except Exception as e: - logger.warning(f"MQTT 不可用: {e}") - return - self._running = True - logger.info(f"MQTT Bridge 已连接 {self._host}:{self._port}") - - async def stop(self): - if not self._running: - return - self._running = False - await self._sc_stop() - - def _broker_uri(self) -> str: - """构建 Broker URI；凭据经百分号编码（amqtt 连接时自动 unquote）。""" - if self._username: - user = quote(self._username, safe="") - pwd = quote(self._password or "", safe="") - return f"mqtt://{user}:{pwd}@{self._host}:{self._port}/" - return f"mqtt://{self._host}:{self._port}/" - - async def _sc(self): - self._client = MQTTClient(config={"auto_reconnect": False}) - await self._client.connect(self._broker_uri()) - topics = [(MQTT_SUBSCRIBE_TOPIC, QOS_1)] - topics += [(ct["topic"], QOS_1) for ct, _ in self._compiled] - await self._client.subscribe(topics) - self._subscribed_custom = {ct["topic"] for ct, _ in self._compiled} - logger.info(f"已订阅 {MQTT_SUBSCRIBE_TOPIC}" - + (f" 及 {len(self._compiled)} 个自定义主题" if self._compiled else "")) - self._lt = asyncio.create_task(self._loop()) - - async def update_dynamic_topics(self, custom_topics: list): - """ - 动态更新 external 插件的自定义订阅主题（RouteRegistry 推送）。 - - 与构造时的 inline 静态主题合并后增量重订阅（退订已移除主题、订阅新增主题）， - 运行中调用即生效。 - """ - self._dynamic_topics = list(custom_topics or []) - self._rebuild_compiled() - new_topics = {ct["topic"] for ct, _ in self._compiled} - if not (self._running and self._client): - self._subscribed_custom = new_topics - return - removed = self._subscribed_custom - new_topics - added = new_topics - self._subscribed_custom - if removed: - await self._client.unsubscribe(list(removed)) - if added: - await self._client.subscribe([(t, QOS_1) for t in added]) - self._subscribed_custom = new_topics - if added or removed: - logger.info( - f"自定义订阅主题更新: +{len(added)} -{len(removed)}" - f"（当前 {len(new_topics)} 个）" - ) - - async def _sc_stop(self): - # 先断开连接，让 deliver_message() 解锁 - if self._client: - try: - await self._client.disconnect() - except Exception: - pass - self._client = None - # 再取消消息循环 - if self._lt and not self._lt.done(): - self._lt.cancel() - try: - await self._lt - except asyncio.CancelledError: - pass - - async def _loop(self): - while self._running: - try: - m = await self._client.deliver_message() # type: ignore - if m is None: - break - await self._handle(m) - except asyncio.CancelledError: - break - except Exception as e: - if self._running: - logger.error(f"消息异常: {e}") - break - - async def _handle(self, m): - t: str = m.topic - p = m.data # type: ignore - try: - pl = orjson.loads(p) - except orjson.JSONDecodeError: - return - - # 实例定向控制字段：与 HTTP 的 X-Chongming-Instance 头同义，指定请求发往的 - # worker 进程实例。必须在下面归一化之前取出——归一化只保留 meta/data 两键， - # 之后控制字段就不在载荷里了。 - target_instance: Optional[str] = None - if isinstance(pl, dict): - ti = pl.pop("target_instance", None) - if isinstance(ti, str) and ti.strip(): - target_instance = ti.strip() - - # 信封协议: {"meta": {...}, "data": {...}} - # 兼容非信封载荷（如第三方设备原始报文）: 整包视为业务数据、meta 置空， - # 与 worker 运行时 runtime.py 的容错语义一致（method 默认 GET）—— - # 若在此丢弃，盒子心跳等设备报文将永远到不了 Worker。 - if not isinstance(pl, dict) or ("meta" not in pl and "data" not in pl): - data = pl if isinstance(pl, dict) else {} - pl = {"meta": {}, "data": data} - if t not in _warned_legacy_topics: - _warned_legacy_topics.add(t) - logger.warning( - "非信封格式载荷（缺少 meta/data）按原始报文路由 [%s]: " - "整包视为 data，建议设备侧改用 {'meta': {...}, 'data': {...}} 格式", - t, - ) - else: - meta = pl.get("meta") or {} - data = pl.get("data") or {} - if not isinstance(data, dict): - data = {} - pl = {"meta": meta, "data": data} - - # 1. 优先匹配显式配置的自定义订阅主题（顺序即优先级） - for ct, compiled in self._compiled: - matched = compiled.match(t) - if matched: - await self._route_custom(ct, matched, pl, t, target_instance) - return - - # 2. 内置接口 chongming/{worker}/{operation} - parts = t.split("/") - if len(parts) < 3: - return - wt, op = parts[1], parts[2] - subj = f"{wt}.{op}" - rt = f"{MQTT_TOPIC_PREFIX}/{wt}/{op}/response" - await self._route(wt, subj, pl, rt, target_instance) - - def _fill(self, template: str, matched: re.Match) -> str: - """用匹配到的通配符值填充模板中的 {1} {2}... 占位符""" - if "{" not in template: - return template - for i, g in enumerate(matched.groups(), start=1): - template = template.replace("{" + str(i) + "}", g) - return template - - async def _route_custom(self, ct: dict, matched: re.Match, pl: dict, topic: str, - target_instance: Optional[str] = None): - worker = ct.get("worker", "") - # subject 默认 "."，与内置 <插件>.<操作> 约定一致 - subject_tpl = ct.get("subject") - if not subject_tpl: - last = ct["topic"].split("/")[-1] - subject_tpl = f"{worker}.{last}" if last not in ("+", "#") else worker - subject = self._fill(subject_tpl, matched) - # response_topic 默认 "<实际匹配主题>/response" - rt_tpl = ct.get("response_topic") - rt = self._fill(rt_tpl, matched) if rt_tpl else f"{topic}/response" - await self._route(worker, subject, pl, rt, target_instance) - - async def _route(self, wt: str, subj: str, pl: dict, rt: str, - target_instance: Optional[str] = None): - """把一次 MQTT 请求转成 worker RPC。 - - target_instance 为实例定向目标（来自载荷顶层的 target_instance 控制字段， - 已在 _handle 归一化前摘除）；缺省时跨实例随机均衡，行为与以往一致。 - """ - logger.info(f"MQTT -> {subj}") - meta = pl.get("meta") or {} - try: - r = await self._wm.call(worker_type=wt, subject=subj, payload=pl, - timeout=float(meta.get("timeout", 30)), - cacheable=bool(meta.get("cacheable", False)), - instance_id=target_instance or None) - if r is not None: - await self._pub(rt, r) - except RuntimeError as e: - await self._pub(rt, {"error": str(e)}) - except Exception as e: - await self._pub(rt, {"error": str(e)}) - - async def _pub(self, topic, data): - try: - # data 均为 JSON 可序列化类型（worker 回复解析结果 / {"error": ...}）， - # orjson.dumps 直接产出 UTF-8 字节，免去 str -> encode 两步 - await self._client.publish(topic, orjson.dumps(data), QOS_1) # type: ignore - except Exception as e: - logger.error(f"发布失败 [{topic}]: {e}") - - @property - def is_running(self): - return self._running diff --git a/src/chongming/app/core/route_generator.py b/src/chongming/app/core/route_generator.py deleted file mode 100644 index e9b2236..0000000 --- a/src/chongming/app/core/route_generator.py +++ /dev/null @@ -1,846 +0,0 @@ -""" -动态路由生成器 - -根据配置文件中的 worker 路由定义，动态生成 FastAPI 端点。 -为每个端点生成带有正确 Pydantic 类型注解的处理函数，确保 OpenAPI 文档 -正确反映参数类型、响应结构，并注册到 components/schemas。 -""" -from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple -from inspect import Parameter, Signature -import asyncio -import time - -import orjson -from fastapi import FastAPI, HTTPException, APIRouter, Query, Path as FastAPIPath, Request, Depends -from fastapi.responses import StreamingResponse, Response -from pydantic import create_model, Field, BaseModel - -from chongming_worker.route_config import WORKER_CONFIG_DIR, load_worker_routes_from_dir - -from .worker_manager import WorkerManager, StreamIdleTimeoutError -from .logger import get_logger -from .auth import store_user_to_request -from .call_stats import spawn, record_call - -logger = get_logger("route_generator") - -# 实例定向请求头：给定后该请求固定发往指定 worker 进程实例（同一实例下的多个副本 -# 之间仍随机均衡），不参与跨实例的随机负载均衡。取值 = 实例身份（外部进程启动参数 -# --instance / CHONGMING_WORKER_INSTANCE；inline 副本为网关自身实例 id）。 -# 未携带该头时行为与以往完全一致（全类型随机均衡）。 -INSTANCE_HEADER = "X-Chongming-Instance" - - -def _instance_from_request(request: Request) -> Optional[str]: - """从请求头读取定向实例 id；空串/未携带视为不定向。""" - value = request.headers.get(INSTANCE_HEADER) - if value: - value = value.strip() - return value or None - - -# 简单类型映射 -TYPE_MAP: Dict[str, Any] = { - "float": float, - "int": int, - "str": str, - "bool": bool, - "any": Any, -} - - -def _parse_python_type(type_str: str) -> Any: - """解析配置中的类型字符串为 Python 类型对象""" - type_str = type_str.strip() - if type_str.startswith("list[") or type_str.startswith("List["): - inner = type_str[5:-1].strip() - inner_type = _parse_python_type(inner) - return List[inner_type] # type: ignore - if type_str.startswith("dict[") or type_str.startswith("Dict["): - inner = type_str[5:-1].strip() - parts = [p.strip() for p in inner.split(",", 1)] - if len(parts) == 2: - key_type = _parse_python_type(parts[0]) - val_type = _parse_python_type(parts[1]) - return Dict[key_type, val_type] # type: ignore - return Dict[str, Any] # type: ignore - return TYPE_MAP.get(type_str, str) - -def generate_request_model( - param_spec: Dict[str, str], model_name: str = "AutoRequest" -) -> type[BaseModel]: - """生成 Pydantic 请求体模型""" - fields: Dict[str, Any] = {} - for param_name, param_type_str in param_spec.items(): - py_type = _parse_python_type(param_type_str) - fields[param_name] = (py_type, Field(...)) - return create_model(model_name, **fields) - -def generate_response_model( - response_spec: Dict[str, Any], model_name: str = "AutoResponse" -) -> Optional[type[BaseModel]]: - """生成 Pydantic 响应模型""" - if not response_spec: - return None - fields: Dict[str, Any] = {} - for field_name, field_spec in response_spec.items(): - if isinstance(field_spec, dict): - py_type = _parse_python_type(field_spec.get("type", "str")) - required = field_spec.get("required", True) - default_val = field_spec.get("default", ...) - if required: - fields[field_name] = (py_type, Field(...)) - else: - fields[field_name] = (py_type, Field(default=default_val if default_val is not ... else None)) - else: - py_type = _parse_python_type(str(field_spec)) - fields[field_name] = (py_type, Field(...)) - return create_model(model_name, **fields) - -class RouteConfig: - """单个路由配置模型""" - - def __init__(self, data: Dict[str, Any]): - self.subject: str = data.get("subject", "") - self.path: str = data.get("path", "") - self.method: str = data.get("method", "GET").upper() - self.params: List[str] = data.get("params", []) - self.response_model: Dict[str, Any] = data.get("response_model", {}) - self.summary: str = data.get("summary", "") - self.docstring: str = data.get("docstring", "") - self.timeout: float = data.get("timeout", 3.0) - self.cacheable: bool = data.get("cacheable", False) - self.auth_required: bool = data.get("auth_required", True) - self.internal: bool = data.get("internal", False) - # validate=false 时网关不解析/校验请求体，原样透传（校验职责下沉 worker， - # 失败经 reply_err 以 503 暴露）；仅对 POST/PUT/PATCH 非流式端点生效 - self.validate: bool = data.get("validate", True) - # 流式端点：以 SSE（text/event-stream）逐块输出 - self.stream: bool = data.get("stream", False) - # 流式空闲超时（秒）：连续未收到数据块的最大间隔（timeout 为流总寿命上限） - self.stream_idle_timeout: float = data.get("stream_idle_timeout", 10.0) - - def parse_params(self) -> Dict[str, str]: - """解析 params 列表为 {name: type} 字典""" - result: Dict[str, str] = {} - for param in self.params: - if ":" in param: - name, type_str = param.split(":", 1) - result[name.strip()] = type_str.strip() - else: - result[param.strip()] = "str" - return result - - def extract_path_params(self) -> List[str]: - """从路径中提取路径参数，如 /add/{a}/{b} -> ['a', 'b']""" - return [ - seg[1:-1] for seg in self.path.split("/") - if seg.startswith("{") and seg.endswith("}") - ] - - -def register_route_group( - app: FastAPI, - worker_manager: WorkerManager, - worker_type: str, - routes: List[Dict[str, Any]], - prefix: str = "", - extra_schemas: Optional[Dict[str, type[BaseModel]]] = None, - raw_body_models: Optional[Dict[Tuple[str, str], str]] = None, - response_models: Optional[Dict[Tuple[str, str], str]] = None, -) -> List[Any]: - """ - 为一个 worker_type 的一组路由描述符创建并挂载 APIRouter。 - - 与 register_dynamic_routes（盘扫路径）共用同一注册逻辑， - process 模式下 RouteRegistry 也调用本函数做动态挂载。 - - Args: - app: FastAPI 应用 - worker_manager: WorkerManager 实例 - worker_type: worker 类型（OpenAPI 分组标签） - routes: 路由描述符列表（TOML 兼容格式，与 load_worker_routes_from_dir 输出一致） - prefix: 全局路由前缀（如 /api/v1） - extra_schemas: 额外 schema 收集 dict（GET 请求模型注入 OpenAPI 用） - raw_body_models: (path, method) -> 请求模型名 的收集 dict - （validate=false 端点：请求体文档经 requestBody $ref 注入，见 _inject_schemas_to_openapi） - response_models: (path, method) -> 响应模型名 的收集 dict - （端点注册不传 response_model 以砍掉注册期模型反射；响应文档经 - responses[200].content $ref 手动注入，见 _inject_schemas_to_openapi） - - Returns: - List[Any]: 本次新增到 app.router.routes 的 APIRoute 对象列表 - （调用方可记录引用，用于动态移除） - """ - router = APIRouter(prefix=prefix) - for route_conf in routes: - _register_single_route(router, worker_manager, worker_type, route_conf, extra_schemas, raw_body_models, response_models) - - # raw_body_models / response_models 记录的 (path, method) 为端点声明路径（不含 prefix）， - # 而 OpenAPI paths 键为最终挂载路径（prefix + path），这里统一补前缀 - if prefix: - for key in list((raw_body_models or {}).keys()): - path, method = key - raw_body_models[(prefix + path, method)] = raw_body_models.pop(key) - for key in list((response_models or {}).keys()): - path, method = key - response_models[(prefix + path, method)] = response_models.pop(key) - - before = set(id(r) for r in app.router.routes) - app.include_router(router) - added = [r for r in app.router.routes if id(r) not in before] - logger.info( - f"已挂载 worker '{worker_type}' 的路由组: {len(routes)} 个描述符 -> " - f"{len(added)} 条 HTTP 路由 (prefix={prefix or '/'})" - ) - return added - - -def register_dynamic_routes( - app: FastAPI, - worker_manager: WorkerManager, - prefix: str = "", - worker_types: Optional[List[str]] = None, -): - """ - 根据 config/workers/ 目录结构注册所有动态路由。 - - worker_types: 白名单（混合模式下只注册 inline 插件； - None = 全部注册，兼容纯 inline 部署）。 - """ - from pathlib import Path - - workers_base = Path(WORKER_CONFIG_DIR) - if not workers_base.is_dir(): - logger.info(f"Worker 配置目录不存在: {WORKER_CONFIG_DIR}") - return - - extra_schemas: Dict[str, type[BaseModel]] = {} - # validate=false 端点的 (path, method) -> 请求模型名（OpenAPI requestBody 注入用） - raw_body_models: Dict[Tuple[str, str], str] = {} - # 非流式端点的 (path, method) -> 响应模型名（OpenAPI responses[200] $ref 注入用） - response_models: Dict[Tuple[str, str], str] = {} - - # 扫描所有 worker 子目录（每个子目录名 = worker_type） - for worker_dir in sorted(workers_base.iterdir()): - if not worker_dir.is_dir(): - continue - worker_type = worker_dir.name - if worker_types is not None and worker_type not in worker_types: - continue - routes = load_worker_routes_from_dir(worker_type) - if not routes: - continue - - register_route_group(app, worker_manager, worker_type, routes, prefix, extra_schemas, raw_body_models, response_models) - - # 将额外 schema 注册到 OpenAPI 的 components/schemas - if extra_schemas: - _inject_schemas_to_openapi(app, lambda: extra_schemas, lambda: raw_body_models, lambda: response_models) - - logger.info(f"动态路由注册完成，worker 配置目录: {WORKER_CONFIG_DIR}") - - -def _inject_schemas_to_openapi(app: FastAPI, schemas_provider, raw_body_provider=None, response_provider=None): - """ - 将自定义 Pydantic 模型注册到 FastAPI 的 OpenAPI components/schemas 中。 - - 通过重写 app.openapi() 方法注入，因为 FastAPI 在生命周期内只生成一次 - OpenAPI schema，之后会被缓存。 - - Args: - schemas_provider: 无参 callable，返回 {name: BaseModel} 字典。 - 盘扫路径传静态 dict；process 模式传 RouteRegistry 的动态 dict， - 使路由表变化后重新生成的 OpenAPI 能反映最新 schema。 - raw_body_provider: 可选无参 callable，返回 {(path, method): model_name} 字典。 - validate=false 端点未声明 body 参数（FastAPI 不会自动生成 requestBody）， - 这里按 (path, method) 补注入 requestBody $ref，保持 OpenAPI 请求体文档完整。 - response_provider: 可选无参 callable，返回 {(path, method): model_name} 字典。 - 端点注册未传 response_model（见 _register_single_route：砍掉注册期模型反射）， - 这里按 (path, method) 补注入 responses[200] 的 $ref，保持响应文档完整。 - """ - original_openapi = app.openapi - - def patched_openapi() -> dict[str, Any]: - if app.openapi_schema is not None: - return app.openapi_schema - schema = original_openapi() - if schema is None: - return {"openapi": "3.1.0", "info": {"title": "Chongming", "version": "1.0.0"}, "paths": {}} - # 注入我们的 schema - schemas_dict = schema.setdefault("components", {}).setdefault("schemas", {}) - for name, model in schemas_provider().items(): - if name not in schemas_dict: - schemas_dict[name] = model.model_json_schema() - # validate=false 端点：补注入 requestBody（请求体文档，模型已注册到 components） - if raw_body_provider is not None: - for (path, method), model_name in raw_body_provider().items(): - op = schema.get("paths", {}).get(path, {}).get(method.lower()) - if op is not None: - op["requestBody"] = { - "required": True, - "content": { - "application/json": { - "schema": {"$ref": f"#/components/schemas/{model_name}"} - } - }, - } - # 响应模型：补注入 responses[200] 的 $ref（保留 description，仅加 content） - if response_provider is not None: - for (path, method), model_name in response_provider().items(): - op = schema.get("paths", {}).get(path, {}).get(method.lower()) - if op is not None: - op.setdefault("responses", {}).setdefault("200", {}) - op["responses"]["200"].setdefault( - "content", {} - ).setdefault("application/json", {})["schema"] = { - "$ref": f"#/components/schemas/{model_name}" - } - app.openapi_schema = schema - return schema - - app.openapi = patched_openapi - - -def _spawn_call_record( - request: Request, - worker_type: str, - subject: str, - rc: RouteConfig, - status: int, - duration_ms: float, - req_bytes: int = 0, - resp_bytes: int = 0, -) -> None: - """ - 以后台任务记录一次调用（统计 + 审计），非关键后置操作。 - - 通过 spawn() 以 fire-and-forget 方式执行，端点立即返回， - 不增加接口响应延迟；后台失败由 record_call 内部兜底。 - """ - entry = { - "ts": time.time(), - "worker_type": worker_type, - "subject": subject, - "method": rc.method, - "path": request.url.path, - "user": getattr(request.state, "user", None), - "client": request.client.host if request.client else None, - "status": status, - "duration_ms": round(duration_ms, 3), - "req_bytes": req_bytes, - "resp_bytes": resp_bytes, - } - spawn(record_call(entry)) - - -def _make_body_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, RequestModel: type[BaseModel]): - """创建用于 Body 请求（POST/PUT/PATCH）的端点函数""" - path = rc.path - subject = rc.subject - timeout = rc.timeout - http_method = rc.method - cacheable = rc.cacheable - - async def endpoint( - request: Request, - body: RequestModel, # type: ignore - ) -> Any: - start: Optional[float] = None - payload_bytes = b"" - try: - # 信封协议: {"meta": {...}, "data": {...}} - meta = {"method": http_method} - user_info = getattr(request.state, "user", None) - if user_info: - meta["user_info"] = user_info - payload = {"meta": meta, "data": body.model_dump()} - payload_bytes = orjson.dumps(payload) - start = time.perf_counter() - result = await worker_manager.call_raw( - worker_type=worker_type, - subject=subject, - payload=payload_bytes, - timeout=timeout, - cacheable=cacheable, - instance_id=_instance_from_request(request), - ) - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, 200, duration_ms, - req_bytes=len(payload_bytes), resp_bytes=len(result), - ) - # 回复为 worker 原始字节，零解析直接转发（response_model 仅用于 OpenAPI 文档） - return Response(content=result, media_type="application/json") - except RuntimeError as e: - duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0 - _spawn_call_record( - request, worker_type, subject, rc, 503, duration_ms, - req_bytes=len(payload_bytes), - ) - logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e) - raise HTTPException(status_code=503, detail=str(e)) - except Exception as e: - duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0 - _spawn_call_record( - request, worker_type, subject, rc, 500, duration_ms, - req_bytes=len(payload_bytes), - ) - logger.error(f"路由 {path} 调用异常: {e}") - raise HTTPException(status_code=500, detail="内部服务器错误") - - return endpoint - - -def _make_raw_body_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str): - """ - 创建 validate=false 的 body 端点：请求体原样透传（网关零解析/校验）。 - - 信封结构不变（{"meta": ..., "data": <原始 JSON 值>}），通过字节拼接构造， - 网关既不解析 body 也不转义；参数校验/类型转换职责下沉到 worker - （失败经 reply_err 通道以 503 暴露）。空/空白 body 兜底为 {}。 - 请求模型（params TOML 生成）仅用于 OpenAPI 文档（extra_schemas + requestBody 注入）。 - """ - path = rc.path - subject = rc.subject - timeout = rc.timeout - http_method = rc.method - cacheable = rc.cacheable - - async def endpoint(request: Request) -> Any: - start: Optional[float] = None - payload = b"" - try: - meta = {"method": http_method} - user_info = getattr(request.state, "user", None) - if user_info: - meta["user_info"] = user_info - raw = await request.body() - data_part = raw.strip() or b"{}" - payload = b'{"meta":' + orjson.dumps(meta) + b',"data":' + data_part + b'}' - start = time.perf_counter() - result = await worker_manager.call_raw( - worker_type=worker_type, - subject=subject, - payload=payload, - timeout=timeout, - cacheable=cacheable, - instance_id=_instance_from_request(request), - ) - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, 200, duration_ms, - req_bytes=len(payload), resp_bytes=len(result), - ) - return Response(content=result, media_type="application/json") - except RuntimeError as e: - duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0 - _spawn_call_record( - request, worker_type, subject, rc, 503, duration_ms, - req_bytes=len(payload), - ) - logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e) - raise HTTPException(status_code=503, detail=str(e)) - except Exception as e: - duration_ms = (time.perf_counter() - start) * 1000 if start else 0.0 - _spawn_call_record( - request, worker_type, subject, rc, 500, duration_ms, - req_bytes=len(payload), - ) - logger.error(f"路由 {path} 调用异常: {e}") - raise HTTPException(status_code=500, detail="内部服务器错误") - - return endpoint - - -def _make_query_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, path_params: List[str], query_params: Dict[str, str]): - """ - 创建用于 Query 请求（GET/DELETE）的端点函数。 - - 通过设置函数 __signature__ 和 __annotations__ 为每个参数赋予正确的 Pydantic 类型， - 使 FastAPI 能在 OpenAPI schema 中生成正确的 type / format 信息。 - 请求模型 schema 通过 extra_schemas 机制单独注入到 components/schemas。 - """ - path = rc.path - subject = rc.subject - timeout = rc.timeout - http_method = rc.method - cacheable = rc.cacheable - - # 合并路径参数和查询参数 - param_spec_combined: Dict[str, str] = {} - param_spec_combined.update({k: query_params.get(k, "str") for k in path_params if k in query_params}) - param_spec_combined.update(query_params) - - # 无参数情况 - if not param_spec_combined: - async def _simple_endpoint(request: Request) -> Any: - # 信封协议: {"meta": {...}, "data": {}} - meta = {"method": http_method} - user_info = getattr(request.state, "user", None) - if user_info: - meta["user_info"] = user_info - payload_bytes = orjson.dumps({"meta": meta, "data": {}}) - start = time.perf_counter() - try: - result = await worker_manager.call_raw( - worker_type=worker_type, - subject=subject, - payload=payload_bytes, - timeout=timeout, - cacheable=cacheable, - instance_id=_instance_from_request(request), - ) - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, 200, duration_ms, - req_bytes=len(payload_bytes), resp_bytes=len(result), - ) - return Response(content=result, media_type="application/json") - except RuntimeError as e: - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, 503, duration_ms, - req_bytes=len(payload_bytes), - ) - logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e) - raise HTTPException(status_code=503, detail=str(e)) - except Exception as e: - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, 500, duration_ms, - req_bytes=len(payload_bytes), - ) - logger.error(f"路由 {path} 调用异常: {e}") - raise HTTPException(status_code=500, detail="内部服务器错误") - return _simple_endpoint - - # 构造参数名列表（用于从 kwargs 中提取） - param_names = list(param_spec_combined.keys()) - - async def endpoint(request: Request, **kwargs: Any) -> Any: - # 信封协议: {"meta": {...}, "data": {...}} - meta = {"method": http_method} - user_info = getattr(request.state, "user", None) - if user_info: - meta["user_info"] = user_info - payload_bytes = orjson.dumps({"meta": meta, "data": {k: kwargs[k] for k in param_names}}) - start = time.perf_counter() - try: - result = await worker_manager.call_raw( - worker_type=worker_type, - subject=subject, - payload=payload_bytes, - timeout=timeout, - cacheable=cacheable, - instance_id=_instance_from_request(request), - ) - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, 200, duration_ms, - req_bytes=len(payload_bytes), resp_bytes=len(result), - ) - return Response(content=result, media_type="application/json") - except RuntimeError as e: - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, 503, duration_ms, - req_bytes=len(payload_bytes), - ) - logger.warning("路由 %s [%s] 调用失败: %s", path, worker_type, e) - raise HTTPException(status_code=503, detail=str(e)) - except Exception as e: - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, 500, duration_ms, - req_bytes=len(payload_bytes), - ) - logger.error(f"路由 {path} 调用异常: {e}") - raise HTTPException(status_code=500, detail="内部服务器错误") - - # 构建带类型注解的 Parameter 列表 - params: List[Parameter] = [ - Parameter("request", Parameter.POSITIONAL_OR_KEYWORD, annotation=Request) - ] - annotations: Dict[str, Any] = {} - for name, type_str in param_spec_combined.items(): - py_type = _parse_python_type(type_str) - annotations[name] = py_type - if name in path_params: - params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=FastAPIPath(...), annotation=py_type)) - else: - params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=Query(...), annotation=py_type)) - - # 设置 __annotations__（FastAPI 通过 get_type_hints 读取） - endpoint.__annotations__ = annotations - # 设置 __signature__（FastAPI 优先于此获取参数列表） - endpoint.__signature__ = Signature(params, return_annotation=Dict[str, Any]) # type: ignore - - return endpoint - - -# 流式端点响应头：禁止缓存；通知反向代理（nginx）关闭缓冲保证 SSE 即时性 -_STREAM_HEADERS = { - "Cache-Control": "no-cache", - "X-Accel-Buffering": "no", -} - - -def _make_stream_endpoint( - worker_manager: WorkerManager, - rc: RouteConfig, - worker_type: str, - path_params: List[str] = (), - query_params: Optional[Dict[str, str]] = None, - RequestModel: Optional[type[BaseModel]] = None, -): - """ - 创建流式端点（TOML 中 stream = true）：以 SSE（text/event-stream）逐块输出。 - - 请求参数形态与普通端点一致（GET 走 query/path 参数，POST/PUT/PATCH 走 body 模型）。 - 错误语义: - - 初始错误（无可用 worker 等）-> HTTP 503 - - 流中错误（Worker 返回 {"error": ...} 收尾块 / 空闲超时）-> SSE error 事件 - （此时 HTTP 200 已提交，无法改状态码） - 不校验响应模型：流式输出是逐块 JSON dict。 - """ - path = rc.path - subject = rc.subject - http_method = rc.method - timeout = rc.timeout - idle_timeout = rc.stream_idle_timeout - param_names = list((query_params or {}).keys()) - - def build_payload(request: Request, body=None, kwargs=None) -> Dict[str, Any]: - """构造信封协议 payload（与普通端点一致）""" - meta = {"method": http_method} - user_info = getattr(request.state, "user", None) - if user_info: - meta["user_info"] = user_info - if body is not None: - data = body.model_dump() - else: - data = {k: kwargs[k] for k in param_names} - return {"meta": meta, "data": data} - - def _sse_frame(data: Dict[str, Any]) -> str: - return f"data: {orjson.dumps(data).decode('utf-8')}\n\n" - - async def _sse_body(gen, start: float, request: Request, req_bytes: int) -> AsyncGenerator[str, None]: - """SSE 分帧；流中异常转为 SSE error 事件（HTTP 200 已提交）。 - - 流结束（正常/空闲超时/中断/客户端断开）后在 finally 中后台记录 - 调用统计与审计，时长覆盖整个流。 - """ - status = 200 - try: - async for chunk in gen: - yield f"data: {chunk.decode('utf-8')}\n\n" - except StreamIdleTimeoutError as e: - status = 504 - logger.warning("路由 %s 流式空闲超时: %s", path, e) - yield _sse_frame({"error": str(e)}) - except asyncio.CancelledError: - status = 499 # 客户端断开，流未完成 - raise - except GeneratorExit: - status = 499 # 服务端主动关闭流（如连接回收），流未完成 - raise - except Exception as e: - status = 500 - logger.error("路由 %s 流式中断: %s", path, e) - yield _sse_frame({"error": "内部服务器错误"}) - finally: - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, status, duration_ms, - req_bytes=req_bytes, - ) - - async def _start_stream(request: Request, payload: Dict[str, Any]) -> StreamingResponse: - req_bytes = len(orjson.dumps(payload)) - start = time.perf_counter() - try: - gen = await worker_manager.call_stream( - worker_type=worker_type, - subject=subject, - payload=payload, - timeout=timeout, - idle_timeout=idle_timeout, - instance_id=_instance_from_request(request), - ) - except RuntimeError as e: - duration_ms = (time.perf_counter() - start) * 1000 - _spawn_call_record( - request, worker_type, subject, rc, 503, duration_ms, - req_bytes=req_bytes, - ) - logger.warning("路由 %s [%s] 流式调用失败: %s", path, worker_type, e) - raise HTTPException(status_code=503, detail=str(e)) - return StreamingResponse( - _sse_body(gen, start, request, req_bytes), - media_type="text/event-stream", - headers=_STREAM_HEADERS, - ) - - if RequestModel is not None: - async def endpoint(request: Request, body: RequestModel) -> StreamingResponse: # type: ignore - return await _start_stream(request, build_payload(request, body=body)) - return endpoint - - # GET/DELETE 形态 - if not param_names: - async def _simple_endpoint(request: Request) -> StreamingResponse: - return await _start_stream(request, build_payload(request, kwargs=None)) - return _simple_endpoint - - async def endpoint(request: Request, **kwargs: Any) -> StreamingResponse: - return await _start_stream(request, build_payload(request, kwargs=kwargs)) - - # 附加类型注解与签名（与 _make_query_endpoint 一致），保证 OpenAPI 参数文档正确 - params: List[Parameter] = [ - Parameter("request", Parameter.POSITIONAL_OR_KEYWORD, annotation=Request) - ] - annotations: Dict[str, Any] = {} - for name, type_str in (query_params or {}).items(): - py_type = _parse_python_type(type_str) - annotations[name] = py_type - if name in path_params: - params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=FastAPIPath(...), annotation=py_type)) - else: - params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=Query(...), annotation=py_type)) - - endpoint.__annotations__ = annotations - endpoint.__signature__ = Signature(params, return_annotation=StreamingResponse) # type: ignore - - return endpoint - - -def _register_single_route( - router: APIRouter, - worker_manager: WorkerManager, - worker_type: str, - route_conf: Dict[str, Any], - extra_schemas: Optional[Dict[str, type[BaseModel]]] = None, - raw_body_models: Optional[Dict[Tuple[str, str], str]] = None, - response_models: Optional[Dict[Tuple[str, str], str]] = None, -): - """注册单个路由，生成正确的函数签名以便 OpenAPI 正确生成文档""" - rc = RouteConfig(route_conf) - - # MQTT 端点：不注册 HTTP 路由，仅通过 MQTT Bridge 调用 - if rc.method == "MQTT": - logger.info( - f"MQTT 端点: subject={rc.subject} (仅 MQTT，不注册 HTTP 路由)" - ) - return - - # internal 端点：不注册 HTTP 路由，仅保留 Zenoh RPC 内部调用能力 - if rc.internal: - logger.info( - f"跳过 internal 路由注册: {rc.method} {rc.path} -> worker {worker_type}.{rc.subject}" - ) - return - - param_spec = rc.parse_params() - path_params = rc.extract_path_params() - query_params = {k: v for k, v in param_spec.items() if k not in path_params} - - # 生成干净的模型名称（替换路径中的 / 和 {} 为 _） - # 模型名称包含 HTTP method 以区分同名路径的不同调用类型 - clean_path = rc.path.replace("/", "_").replace("{", "").replace("}", "").strip("_") - if not clean_path: - clean_path = "root" - method_tag = rc.method.upper() - response_model_name = f"{worker_type}_{clean_path}_{method_tag}_Response" - request_model_name = f"{worker_type}_{clean_path}_{method_tag}_Request" - - # 生成响应模型 - if rc.stream: - # 流式端点：逐块 SSE 输出，不校验响应模型（OpenAPI 不声明响应 schema） - ResponseModel = None - if rc.method in ("POST", "PUT", "PATCH"): - # body 形态：RequestModel 传给端点做请求体校验 - RequestModel = generate_request_model( - param_spec, request_model_name - ) - endpoint = _make_stream_endpoint( - worker_manager, rc, worker_type, path_params, query_params, RequestModel - ) - else: - # GET/DELETE 形态：query/path 参数走签名注解； - # RequestModel 仅注册到 extra_schemas 供 OpenAPI 展示（与普通 GET 一致） - if param_spec: - RequestModel = generate_request_model( - param_spec, request_model_name - ) - if extra_schemas is not None and request_model_name not in extra_schemas: - extra_schemas[request_model_name] = RequestModel - endpoint = _make_stream_endpoint( - worker_manager, rc, worker_type, path_params, query_params, None - ) - elif rc.method in ("POST", "PUT", "PATCH"): - ResponseModel = generate_response_model( - rc.response_model, response_model_name - ) - # 响应模型仅注册到 components/schemas + responses[200] $ref 注入， - # 不传给 add_api_route：端点返回 Response 对象本就走透传（零运行时校验）， - # 省掉 FastAPI 注册期对 response_model 的反射解析（external 插件频繁 - # 挂载/摘除时每次重建都会重复这一开销） - if ResponseModel is not None: - if extra_schemas is not None and response_model_name not in extra_schemas: - extra_schemas[response_model_name] = ResponseModel - if response_models is not None: - response_models[(rc.path, rc.method)] = response_model_name - RequestModel = generate_request_model( - param_spec, request_model_name - ) - if rc.validate: - endpoint = _make_body_endpoint(worker_manager, rc, worker_type, RequestModel) - else: - # validate=false：请求体原样透传（网关零解析/校验，校验下沉 worker）。 - # RequestModel 仅用于 OpenAPI 请求体文档： - # 经 extra_schemas 注册 components/schemas + requestBody $ref 注入 - if extra_schemas is not None and request_model_name not in extra_schemas: - extra_schemas[request_model_name] = RequestModel - if raw_body_models is not None: - raw_body_models[(rc.path, rc.method)] = request_model_name - endpoint = _make_raw_body_endpoint(worker_manager, rc, worker_type) - else: - ResponseModel = generate_response_model( - rc.response_model, response_model_name - ) - # 同上：响应模型走 extra_schemas + responses $ref 注入，不传 add_api_route - if ResponseModel is not None: - if extra_schemas is not None and response_model_name not in extra_schemas: - extra_schemas[response_model_name] = ResponseModel - if response_models is not None: - response_models[(rc.path, rc.method)] = response_model_name - # 对 GET 端点也生成 Request 模型并注册到 extra_schemas - if param_spec: - RequestModel = generate_request_model( - param_spec, request_model_name - ) - if extra_schemas is not None and request_model_name not in extra_schemas: - extra_schemas[request_model_name] = RequestModel - endpoint = _make_query_endpoint(worker_manager, rc, worker_type, path_params, query_params) - - dependencies = [] - if rc.auth_required: - dependencies.append(Depends(store_user_to_request)) - - # 注册路由 - 使用 worker_type 作为 OpenAPI 分组标签 - # 每个 worker 类型自动成为一个独立的分组，显示在 Swagger UI 中 - # 不传 response_model：端点返回 Response 对象走透传，运行时本就零校验； - # 响应文档由 response_models 收集 + _inject_schemas_to_openapi 补 $ref - router.add_api_route( - path=rc.path, - endpoint=endpoint, - methods=[rc.method], - summary=rc.summary, - description=rc.docstring, - tags=[worker_type], - include_in_schema=True, - dependencies=dependencies, - ) - logger.info(f"已注册动态路由: {rc.method} {rc.path} -> worker {worker_type}.{rc.subject} | tags=[{worker_type}]") diff --git a/src/chongming/app/core/route_registry.py b/src/chongming/app/core/route_registry.py deleted file mode 100644 index 7ef3553..0000000 --- a/src/chongming/app/core/route_registry.py +++ /dev/null @@ -1,271 +0,0 @@ -""" -动态路由注册表（process 模式） - -网关订阅 `chongming/routes/**` 命名空间，独立 worker 进程上线时 put 路由描述符、 -下线时 delete，网关据此动态挂载/摘除 FastAPI 路由并同步 MQTT 自定义订阅主题。 - -协议（描述符 schema_version=1）: - chongming/routes//__meta__ 插件级描述: {"schema_version":1,"mqtt_topics":[...]} - chongming/routes// 端点描述（TOML 兼容 dict，与 RouteConfig 对位）: - {"schema_version":1, "type","subject","method","path","params", - "response_model","summary","timeout","cacheable","auth_required","internal"} - -特性: - - SampleKind.PUT -> upsert；DELETE -> remove - - 同一 route_id 内容相同（hash）视为多副本重复发布，忽略；不同 -> 告警并保留先到者 - - type 级 2 秒防抖：worker 重启风暴合并为一次路由重建 - - 迟到一致性：worker 侧周期性重发布描述符（RoutePublisher 由调用方周期调用）， - 网关后启动或样本丢失时，路由在一个发布周期内自动补齐 -""" -import asyncio -import hashlib -import orjson -from typing import Any, Dict, List, Optional, Tuple - -import zenoh - -from chongming_worker.routes_publisher import META_KEY, ROUTES_PREFIX, SCHEMA_VERSION - -from .logger import get_logger -from .route_generator import register_route_group, _inject_schemas_to_openapi - -logger = get_logger("route_registry") - -# worker 重启风暴防抖窗口（秒） -DEBOUNCE_SECONDS = 2.0 - - -def _descriptor_hash(descriptor: Dict[str, Any]) -> str: - """描述符内容 hash（多副本幂等去重用）""" - payload = orjson.dumps(descriptor, option=orjson.OPT_SORT_KEYS) - return hashlib.sha1(payload).hexdigest() - - -class RouteRegistry: - """process 模式下的网关侧动态路由注册表。""" - - def __init__( - self, - app, - worker_manager, - prefix: str = "", - ): - self._app = app - self._wm = worker_manager - self._prefix = prefix - self._loop: Optional[asyncio.AbstractEventLoop] = None - - # worker_type -> route_id -> descriptor - self._groups: Dict[str, Dict[str, Dict[str, Any]]] = {} - # worker_type -> 插件级描述（mqtt_topics 等） - self._meta: Dict[str, Dict[str, Any]] = {} - # worker_type -> 已挂载的 APIRoute 对象（用于动态摘除） - self._route_objs: Dict[str, List[Any]] = {} - # OpenAPI 额外 schema（GET 请求模型），随路由表变化 - self._schemas: Dict[str, Any] = {} - # validate=false 端点的 (path, method) -> 请求模型名（OpenAPI requestBody 注入用） - self._raw_body_models: Dict[Tuple[str, str], str] = {} - # 非流式端点的 (path, method) -> 响应模型名（OpenAPI responses[200] $ref 注入用） - self._response_models: Dict[Tuple[str, str], str] = {} - # worker_type -> 防抖任务 - self._debounce: Dict[str, asyncio.Task] = {} - - self._subscriber: Optional[zenoh.Subscriber] = None - self._mqtt_bridge = None - # 网关进程内加载（inline）的插件类型：忽略其描述符，防止与盘扫注册重复挂载 - self._inline_types: set = set() - - # 将动态 schema 注入 OpenAPI（provider 实时读取本注册表） - _inject_schemas_to_openapi( - app, - lambda: self._schemas, - lambda: self._raw_body_models, - lambda: self._response_models, - ) - - def set_inline_types(self, types: set): - """设置 inline 插件类型集合（混合模式防双注册）。""" - self._inline_types = set(types) - - async def start(self, session: zenoh.Session): - """订阅 chongming/routes/**，开始接收路由描述符。""" - self._loop = asyncio.get_running_loop() - self._subscriber = session.declare_subscriber( - f"{ROUTES_PREFIX}/**", self._on_sample - ) - logger.info(f"路由注册表已就绪: 订阅 {ROUTES_PREFIX}/**") - - def stop(self): - """停止订阅（shutdown 时调用）。""" - if self._subscriber is not None: - self._subscriber.undeclare() - self._subscriber = None - for task in self._debounce.values(): - task.cancel() - - def attach_bridge(self, mqtt_bridge): - """挂接 MQTT Bridge：后续路由表变化会推送 external 插件的动态订阅主题。""" - self._mqtt_bridge = mqtt_bridge - # Bridge 晚于路由到达时，立即补推当前已收集的主题 - topics = self._collect_mqtt_topics() - if topics: - self._loop.create_task(mqtt_bridge.update_dynamic_topics(topics)) - - @property - def route_types(self) -> List[str]: - """当前已注册路由的 worker 类型列表（观测用）。""" - return sorted(self._groups.keys()) - - def ready_summary(self) -> str: - """路由就绪状态摘要（日志/观测用）。""" - parts = [] - for wtype in sorted(self._groups): - http = sum( - 1 for d in self._groups[wtype].values() - if d.get("method") not in ("MQTT",) and not d.get("internal") - ) - parts.append(f"{wtype}={len(self._groups[wtype])} 端点/{http} HTTP") - return ", ".join(parts) if parts else "(空)" - - # ---- zenoh 回调线程 -> 事件循环 ---- - - def _on_sample(self, sample: zenoh.Sample): - """zenoh 后台线程回调：转交事件循环处理。""" - if self._loop is None: - return - key = str(sample.key_expr) - kind = sample.kind - payload = bytes(sample.payload) if sample.payload else b"" - asyncio.run_coroutine_threadsafe( - self._handle_sample(key, kind, payload), self._loop - ) - - async def _handle_sample(self, key: str, kind: zenoh.SampleKind, payload: bytes): - """解析样本并更新内存路由表（在事件循环内执行）。""" - parts = key.split("/") - if len(parts) < 4: - return - wtype, route_id = parts[2], parts[3] - - if wtype in self._inline_types: - # 该插件由网关进程内加载（盘扫注册路由），忽略外部描述符防双注册 - return - - if kind == zenoh.SampleKind.DELETE: - self._remove(wtype, route_id) - return - - try: - descriptor = orjson.loads(payload) - except orjson.JSONDecodeError: - logger.warning("丢弃非法路由描述符 [%s]: JSON 解析失败", key) - return - if not isinstance(descriptor, dict): - logger.warning("丢弃非法路由描述符 [%s]: 非 dict", key) - return - - if route_id == META_KEY: - self._upsert_meta(wtype, descriptor) - else: - self._upsert_route(wtype, route_id, descriptor) - - def _upsert_meta(self, wtype: str, descriptor: Dict[str, Any]): - old = self._meta.get(wtype) - new_hash = _descriptor_hash(descriptor) - if old is not None and _descriptor_hash(old) == new_hash: - return - self._meta[wtype] = descriptor - self._schedule_rebuild(wtype) - - def _upsert_route(self, wtype: str, route_id: str, descriptor: Dict[str, Any]): - group = self._groups.setdefault(wtype, {}) - existing = group.get(route_id) - if existing is not None: - if _descriptor_hash(existing) == _descriptor_hash(descriptor): - # 多副本重复发布同一描述符，幂等忽略 - return - logger.warning( - "路由描述符冲突 [%s/%s]: 已存在版本保留，忽略新版本", - wtype, route_id, - ) - return - # 补全描述符字段（发布端缺省时的兜底） - descriptor.setdefault("type", wtype) - descriptor.setdefault("schema_version", SCHEMA_VERSION) - group[route_id] = descriptor - logger.info( - "收到路由注册: %s %s %s (subject=%s)", - descriptor.get("method", "?"), - descriptor.get("path", "?"), - f"[{wtype}]", - descriptor.get("subject", "?"), - ) - self._schedule_rebuild(wtype) - - def _remove(self, wtype: str, route_id: str): - if route_id == META_KEY: - if self._meta.pop(wtype, None) is not None: - self._schedule_rebuild(wtype) - return - group = self._groups.get(wtype) - if group and group.pop(route_id, None) is not None: - logger.info("收到路由摘除: [%s] %s", wtype, route_id) - self._schedule_rebuild(wtype) - - def _schedule_rebuild(self, wtype: str): - """type 级防抖：合并短时间内的多次变更。""" - prev = self._debounce.get(wtype) - if prev is not None and not prev.done(): - prev.cancel() - self._debounce[wtype] = asyncio.create_task(self._debounced_rebuild(wtype)) - - async def _debounced_rebuild(self, wtype: str): - try: - await asyncio.sleep(DEBOUNCE_SECONDS) - except asyncio.CancelledError: - return - self._debounce.pop(wtype, None) - await self._rebuild(wtype) - - async def _rebuild(self, wtype: str): - """整组重建该 worker_type 的 HTTP 路由 + 同步 MQTT 自定义主题。""" - routes = list(self._groups.get(wtype, {}).values()) - routes.sort(key=lambda r: (r.get("path", ""), r.get("method", ""))) - - # 1. 摘除旧路由（按对象引用从 router.routes 过滤） - for r in self._route_objs.pop(wtype, []): - if r in self._app.router.routes: - self._app.router.routes.remove(r) - - # 2. 挂载新路由（MQTT / internal 端点不注册 HTTP，但保留在路由表中） - http_routes = [ - r for r in routes - if r.get("method") not in ("MQTT",) and not r.get("internal") - ] - if http_routes: - self._route_objs[wtype] = register_route_group( - self._app, self._wm, wtype, http_routes, self._prefix, - self._schemas, self._raw_body_models, self._response_models, - ) - else: - logger.info("[%s] 无 HTTP 路由可挂载（%d 个描述符，全部为 MQTT/internal）", - wtype, len(routes)) - - # 路由表变化后 OpenAPI 缓存失效，下一次访问时重建 - self._app.openapi_schema = None - - # 3. 同步 external 插件的 MQTT 动态订阅主题 - if self._mqtt_bridge is not None: - await self._mqtt_bridge.update_dynamic_topics(self._collect_mqtt_topics()) - - def _collect_mqtt_topics(self) -> List[Dict[str, Any]]: - """汇总所有插件 __meta__ 声明的自定义订阅主题。""" - topics: List[Dict[str, Any]] = [] - for wtype in sorted(self._meta): - meta = self._meta[wtype] - for t in meta.get("mqtt_topics") or []: - if isinstance(t, dict) and t.get("topic"): - topic = dict(t) - topic.setdefault("worker", wtype) - topics.append(topic) - return topics diff --git a/src/chongming/app/core/static_files.py b/src/chongming/app/core/static_files.py deleted file mode 100644 index 0880cbe..0000000 --- a/src/chongming/app/core/static_files.py +++ /dev/null @@ -1,100 +0,0 @@ -from fastapi import HTTPException -from fastapi.responses import Response -from fastapi.staticfiles import StaticFiles -from sqlite_vfs.core import SQLiteVFS - - -class SVFSStaticFiles(StaticFiles): - """ - 基于 SQLite VFS 的静态文件服务类 - - 该类继承自 FastAPI 的 StaticFiles，使用 SQLite 虚拟文件系统来存储和提供静态文件。 - 支持单页应用（SPA）的路由回退机制，当请求的文件不存在时，会自动返回 index.html。 - """ - - def __init__(self, directory, vfs_db_path="static.svfs"): - """ - 初始化 SVFSStaticFiles 实例 - - 参数: - directory (str): 静态文件的根目录路径 - vfs_db_path (str): SQLite VFS 数据库文件路径，默认为 "static.svfs" - - 返回值: - 无 - """ - vfs_db_path = vfs_db_path - self._vfs = SQLiteVFS(vfs_db_path, compress=False) - super().__init__(directory=directory) - - async def get_response(self, path: str, scope): - """ - 根据请求路径获取对应的静态文件响应 - - 该方法处理静态文件的请求逻辑，包括： - 1. 路径标准化处理 - 2. 文件存在性检查 - 3. SPA 路由回退支持 - 4. 目录索引处理 - 5. MIME 类型自动识别 - - 参数: - path (str): 请求的文件路径 - scope: ASGI 作用域对象，包含请求上下文信息 - - 返回值: - Response: FastAPI 响应对象，包含文件内容和正确的 Content-Type - - 异常: - HTTPException: 当文件不存在且无法回退到 index.html 时，抛出 404 错误 - """ - # 统一路径格式：以 / 开头，并将 Windows 反斜杠转为正斜杠 - normalized_path = "/" + path.replace("\\", "/") - - # 获取文件信息 - file_info = self._vfs.get_file_info(normalized_path) - - # 处理文件不存在的情况，尝试 SPA 回退机制 - if file_info is None: - # SPA 回退：如果不是静态资源扩展名，尝试返回 index.html - if not any( - normalized_path.endswith(ext) - for ext in [ - ".js", - ".css", - ".png", - ".jpg", - ".ico", - ".svg", - ".webp", - ".json", - ] - ): - index_info = self._vfs.get_file_info("/index.html") - if index_info: - content = self._vfs.read_file("/index.html") - return Response(content=content, media_type="text/html") - raise HTTPException(status_code=404, detail="File not found") - - # 处理目录请求，返回目录下的 index.html - if file_info["is_directory"]: - index_info = self._vfs.get_file_info(f"{normalized_path}/index.html") - if index_info: - content = self._vfs.read_file(f"{normalized_path}/index.html") - return Response(content=content, media_type="text/html") - raise HTTPException(status_code=404, detail="Directory index not found") - - # 读取文件内容并确定 MIME 类型 - content = self._vfs.read_file(normalized_path) - content_type = file_info.get("content_type") or "application/octet-stream" - # 简单根据扩展名补充 MIME 类型（可选） - if normalized_path.endswith(".js"): - content_type = "application/javascript" - elif normalized_path.endswith(".css"): - content_type = "text/css" - elif normalized_path.endswith(".html"): - content_type = "text/html" - elif normalized_path.endswith(".ico"): - content_type = "image/x-icon" - - return Response(content=content, media_type=content_type) diff --git a/src/chongming/app/core/worker_manager.py b/src/chongming/app/core/worker_manager.py deleted file mode 100644 index eb316a7..0000000 --- a/src/chongming/app/core/worker_manager.py +++ /dev/null @@ -1,1405 +0,0 @@ -import asyncio -import concurrent.futures -import hashlib -import os -import random -import socket -import time -import orjson -from collections import OrderedDict -from typing import Any, AsyncGenerator, Awaitable, Callable, Dict, List, Optional -import zenoh -from chongming_cache import Cache, DistributedLock -from .logger import get_logger - -logger = get_logger("worker_manager") - -# ── 插件运行时配置（chongming-cache 单一事实来源）── -# 键与 worker SDK（chongming_worker.runtime._configs_cache_key）一致，全局唯一一份 -# [workers.config]；读写规则见 WorkerManager.align_inline_config。 -_CONFIGS_KEY = "worker:cm:configs:{type}" - -# ── 全局副本数（chongming-cache 单一事实来源）── -# 多网关共用同一 cache（单实例=本地 diskcache，多实例=共享卷/redis）， -# 「count」语义 = 该类 worker 的**全局目标总量**，各网关按在线数量分摊。 -_SCALE_GATEWAYS_KEY = "worker:cm:gateways" # 在线网关列表：[[id, 最近心跳 ts], ...]（JSON） - -# 在线网关租约（秒）：网关心跳每 refresh_interval(15s) 更新一次自身 ts，超过 LEASE 未刷新 -# 视为离线的死网关，在下一次读写时被清理。容忍多次漏跳，且 crash/kill -9 不残留。 -_GATEWAY_LEASE = 60 - -_SENTINEL = object() # 哨兵值，用于标记"No reply received" - -# Zenoh RPC 专用线程池：隔离于 asyncio.to_thread 的全局默认池 -# （默认池上限 min(32, cpu+4)，高并发下会被其他阻塞调用挤占导致排队）。 -# zenoh session.get 是同步阻塞调用，每个进行中的查询占用一个线程； -# wait_for 超时后线程仍阻塞在 zenoh 内部直到其自身超时返回，故池上限即并发上限。 -# 按业务并发量调整（256 ≈ 256 个并发 Zenoh 查询）；线程 daemon=True，进程退出不等待。 -_ZENOH_EXECUTOR = concurrent.futures.ThreadPoolExecutor( - max_workers=256, thread_name_prefix="zenoh-rpc" -) - - -class StreamIdleTimeoutError(TimeoutError): - """流式调用空闲超时：在 idle_timeout 秒内未收到任何数据块""" - - -def _next_reply(iterator): - """线程安全地获取迭代器的下一个元素。 - - 用 for 循环 + 哨兵代替直接 next()，避免 StopIteration 在 - asyncio.to_thread 中传播的问题（与 call() 的 _get_first 同模式）。 - 迭代耗尽时返回 _SENTINEL。 - """ - for item in iterator: - return item - return _SENTINEL - - -class _MemLRU: - """极简进程内一级缓存：LRU + TTL。 - - 仅度热路径（cacheable RPC 命中）/JWT 等高频只读复用。单事件循环内读写、 - OrderedDict 实现，O(1) 无锁；作为 diskcache/redis 二级共享缓存的进程级第一级， - 命中时完全绕过网络/磁盘与 Zenoh，降低热点请求延迟与后端压力。 - maxsize 达到上限时淘汰最久未用（LRU）条目；TTL 削防陈旧数据长期驻留。 - """ - - def __init__(self, maxsize: int = 2048, ttl: float = 60.0): - self._d: "OrderedDict[str, tuple]" = OrderedDict() - self._maxsize = maxsize - self._ttl = ttl - - def get(self, key: str): - item = self._d.get(key) - if item is None: - return None - val, expire = item - if expire < time.monotonic(): - self._d.pop(key, None) - return None - self._d.move_to_end(key) # 更新为最近使用 - return val - - def set(self, key: str, val) -> None: - self._d[key] = (val, time.monotonic() + self._ttl) - self._d.move_to_end(key) - if len(self._d) > self._maxsize: - self._d.popitem(last=False) - - -# ── 进程内直调（inline 快路径）───────────────────────────────────────────── -# inline 副本由本进程托管（与网关同事件循环），业务请求却仍要绕 -# "zenoh 查询 → router → queryable 回调 → 跨线程回本 loop"。分派逻辑已统一到 -# chongming_worker.dispatch，故此处可直接 await 同一个 dispatcher，省掉整圈往返。 -# 语义不变：信封/错误通道/超时/重试/换副本/实例定向/in-flight 与 drain 全部沿用原逻辑； -# 非本地副本（external 插件、w>1 时非托管进程）与旧版 worker 包自动回退 zenoh。 -_RPC_LOCAL = os.environ.get("CHONGMING_RPC_LOCAL", "1") != "0" -try: # 旧版 chongming-worker（<0.5）无 dispatch 模块：保持纯 zenoh 行为 - from chongming_worker.dispatch import get_dispatcher as _get_local_dispatcher -except Exception: # noqa: BLE001 - _get_local_dispatcher = None - - -def _local_dispatcher_for(key_expr: str) -> Optional[Any]: - """'worker//' → 本进程托管该副本的 dispatcher；非本地/不可用返回 None""" - if not _RPC_LOCAL or _get_local_dispatcher is None: - return None - parts = key_expr.split("/") - if len(parts) != 3 or parts[0] != "worker": - return None - return _get_local_dispatcher(parts[1], parts[2]) - - -def _err_text(payload: Any) -> str: - """把错误载荷归一为文本（兼容 zenoh ZBytes 与直调的 bytes）""" - if payload is None: - return "unknown error" - if isinstance(payload, (bytes, bytearray)): - return bytes(payload).decode("utf-8", "replace") or "unknown error" - return str(payload) or "unknown error" - - -class WorkerManager: - - """ - 统一 Worker 管理器 - - 封装 Zenoh 会话，维护每个 worker 类型的存活节点列表， - 提供负载均衡、超时重试、缓存等能力的统一 RPC 调用入口。 - - 支持两种 worker 注册方式： - 1. 分布式：通过 Zenoh liveliness 自动发现 - 2. 本地：通过 register_worker() 直接注册（用于同进程 worker） - """ - - def __init__(self, session: zenoh.Session, cache: Optional[Cache] = None, - launch_worker_count: str = "toml"): - self.session = session - self.cache = cache - # RPC 缓存进程内一级（LRU+TTL），兜底并加速 cacheable 命中； - # 二级为共享 cache（diskcache/redis），I/O 移出事件循环走 executor。 - self._rpc_mem_cache = _MemLRU() - # 启动时全局副本数的优先来源："toml"=用 toml 的 count 覆盖 cache(经典模式)； - # "cache"=以 cache 为单一事实来源。对应 config.toml [cache].launch_worker_count。 - self._launch_worker_count = launch_worker_count - self._workers_cache: Dict[str, List[str]] = {} # type -> list of worker key_exprs - self._local_workers: Dict[str, List[str]] = {} # type -> list of locally registered workers - # ── 实例级定向（把请求固定到某个进程实例而非随机负载均衡）── - # 副本经 liveliness 只暴露 worker//，实例身份取自副本 admin status - # 的 instance_id（Python/Rust 运行时均返回），故定向索引按需查询并缓存： - # _instance_index[type][instance_id] = [worker//, ...] - # _instance_unknown[type] = [已查询但身份不可得的副本 key, ...] - # 只缓存已查询过的副本；每次按需刷新时整表重建并对新副本补查询、对离线副本剪枝。 - self._instance_index: Dict[str, Dict[str, List[str]]] = {} - self._instance_unknown: Dict[str, List[str]] = {} - self._instance_locks: Dict[str, asyncio.Lock] = {} - self._lock = asyncio.Lock() - self._refresh_interval = 15 # seconds - # 当前网关心跳标识：优先环境变量（多实例需唯一，尤其同宿主多网关须显式指定）， - # 否则回退 hostname（稳定，重启不变——避免用 PID 导致每次重启标识变化、旧条目 - # 残留撑大在线列表，进而让配额计算失真把副本全排空）。用于全局副本数按在线网关分摊。 - self._gateway_id: str = ( - os.environ.get("CHONGMING_GATEWAY_ID") or socket.gethostname() - ) - # inline worker 领导选举持有的分布式锁及持有标记（acquire_inline_owner 时设置） - self._inline_owner_lock: Optional[DistributedLock] = None - self._inline_owner: bool = False - # inline worker 自愈接管：类型 -> 启动协程（load_workers 注册）；_started_extra - # 记录接管后拉起的任务，供关停时取消。 - self._worker_starters: Dict[str, Callable[[], Awaitable[List[Any]]]] = {} - self._started_extra: List[Any] = [] - - @property - def config_mode(self) -> str: - """本网关 inline 插件的配置优先来源模式。 - - "toml"（默认）= 以 config/workers//config.toml 为准，启动时覆盖缓存； - "cache" = 以 chongming-cache 为准（跨重启保持），toml 仅作引导。 - 对应 config.toml 的 [cache].launch_worker_count，与 worker 侧 - chongming_worker.runtime._resolve_mode 取值一致。 - """ - return "cache" if str(self._launch_worker_count).lower() == "cache" else "toml" - - def align_inline_config( - self, - worker_type: str, - worker_config: Dict[str, Any], - config_section: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: - """inline 插件初始化前，按配置优先来源模式对齐其 [workers.config]（网关侧兜底）。 - - worker SDK 的 Plugin.create_workers 会做同样的事（权威实现，见 - chongming_worker.runtime.resolve_startup_config）；网关侧再兜一层，是为了让 - 网关（随本仓库发布）与 worker 包版本解耦：worker 包仍是旧版时，旧版 on_init - 只拿到 config/workers//config.toml 展平出来的配置，会出现“前端显示缓存值、 - 实际跑 toml”。规则与 SDK 一致： - - - "cache"：缓存里有该段就用它覆盖传入配置（toml 仅作缓存缺失时的引导并写回， - 之后以缓存为准、跨重启保持）； - - "toml" ：以 config.toml 为准，把该段覆盖写入缓存（前端读缓存 → 随之显示文件值）； - - 未配置缓存后端（cache 为 None）时保持传入配置不动。 - - Args: - worker_type: 插件类型（如 aibox_sophon_adapter） - worker_config: _discover_plugins 展平出来的运行时配置 - config_section: 该插件 config.toml 的 [workers.config] 原始段（引导/覆盖缓存用） - """ - if self.cache is None: - return worker_config - key = _CONFIGS_KEY.format(type=worker_type) - section = config_section if isinstance(config_section, dict) else None - if self.config_mode == "cache": - cached = self.cache.get(key) - if isinstance(cached, dict): - logger.info( - "%s [workers.config] 取自缓存（cache 模式，%d 键）", - worker_type, len(cached), - ) - return {**worker_config, **cached} - if section: - self.cache[key] = dict(section) # 缓存缺失：用 toml 引导并写回 - logger.info("%s [workers.config] 缓存缺失，已用 config.toml 引导", worker_type) - return worker_config - if section is not None: - self.cache[key] = dict(section) # 以文件为准：覆盖缓存，前端随之显示文件值 - logger.info("%s [workers.config] 以 config.toml 为准并覆盖缓存（toml 模式）", worker_type) - return worker_config - - # ── 全局副本数协调（chongming-cache 单一事实来源）──────────── - # count/quota 的读写与分摊已收归 worker 层（chongming_worker.runtime）： - # 网关不再直写这些缓存键，仅维护「在线网关列表」（register/heartbeat/unregister）。 - - def _read_gateways_raw(self) -> List[List]: - """读取在线网关原始列表 [[id, ts], ...]。""" - if self.cache is None: - return [] - return list(self.cache.get(_SCALE_GATEWAYS_KEY) or []) - - def _write_gateways(self, gws: List[List]) -> None: - """写回在线网关列表（去重，按 id 稳定）。""" - if self.cache is None: - return - dedup = {} - for gw_id, ts in gws: - dedup.setdefault(gw_id, ts) # 保序去重 - self.cache[_SCALE_GATEWAYS_KEY] = [[gw_id, ts] for gw_id, ts in dedup.items()] - - def _prune_gateways(self, gws: List[List]) -> List[List]: - """剔除超过租约未心跳的死网关，写回清理结果。 - - 兼容旧版残留：旧格式为纯字符串 id 列表（无心跳 ts），视为失效，仅保留 - [[id, ts], ...] 两元组且 ts 在租约内的在线网关。 - """ - deadline = time.time() - _GATEWAY_LEASE - alive: List[List] = [] - for entry in gws: - if not isinstance(entry, (list, tuple)) or len(entry) != 2: - continue # 旧格式/畸形条目视为失效，一并清理 - gw_id, ts = entry - if isinstance(ts, (int, float)) and isinstance(gw_id, str) and ts >= deadline: - alive.append([gw_id, ts]) - if len(alive) != len(gws): - self._write_gateways(alive) - return alive - - def _gateways(self) -> List[str]: - """读取在线网关 id 列表（先按心跳租约清理僵尸条目，杜绝死网关残留）。""" - gws = self._prune_gateways(self._read_gateways_raw()) - return [gw_id for gw_id, _ in gws] - - def register_gateway(self) -> None: - """把当前网关登记进在线列表（带当前心跳时间戳），参与全局分摊。启动时调用一次。""" - if self.cache is None: - return - gws = self._prune_gateways(self._read_gateways_raw()) - gws = [[gw_id, ts] for gw_id, ts in gws if gw_id != self._gateway_id] - gws.append([self._gateway_id, time.time()]) - self._write_gateways(gws) - - def heartbeat_gateway(self) -> None: - """周期性地刷新自身心跳 ts，保持在线；同时清理死网关。由 refresh_workers 循环调用。""" - if self.cache is None: - return - gws = self._prune_gateways(self._read_gateways_raw()) - gws = [[gw_id, ts] for gw_id, ts in gws if gw_id != self._gateway_id] - gws.append([self._gateway_id, time.time()]) - self._write_gateways(gws) - - def unregister_gateway(self) -> None: - """关闭时从在线列表移除当前网关（并清理死网关）。""" - if self.cache is None: - return - gws = self._prune_gateways(self._read_gateways_raw()) - gws = [[gw_id, ts] for gw_id, ts in gws if gw_id != self._gateway_id] - self._write_gateways(gws) - - # ── inline worker 领导选举（gunicorn 多 worker 兼容）──────────────── - # gunicorn `-w N` 会 fork N 个网关进程，各自跑 lifespan 并拉起 inline worker， - # 副本数变成 N×count（副本数量超预期）。用基于共享 cache 的分布式锁做领导选举： - # 每个网关（gateway_id 隔离的实例/宿主）只有一个进程托管其 inline worker，其余 - # 进程只做 HTTP、经 Zenoh liveliness 照常发现调用。 - # 与 MQTT Bridge 同一把锁模式：经共享 cache 后端（redis/共享卷 diskcache）可跨 - # 机器收敛；key 含 gateway_id，多实例各选各的 leader、互不抢占，不破坏分摊语义。 - - def acquire_inline_owner(self) -> bool: - """尝试成为本网关 inline worker 持有者（cache 支撑的分布式锁）。 - - 成功（全局唯一持有者）返回 True，并在后台续约租约；持锁进程崩溃后租约过期， - gunicorn 复活的进程可重新获得（failover）。未配置共享 cache 时单进程视为持有者。 - """ - if self.cache is None: - self._inline_owner = True # 无共享 cache：单进程即持有者 - return True - lock = DistributedLock(self.cache, f"inline_owner:{self._gateway_id}") - if not lock.acquire(): - self._inline_owner = False - return False - lock.start() - self._inline_owner_lock = lock - self._inline_owner = True - logger.info( - "[LOCK] inline owner: 本进程获得持有权 (gw=%s)", self._gateway_id - ) - return True - - async def release_inline_owner(self) -> None: - """释放 inline worker 持有权分布式锁（受保护：仅仍由本进程持有时才删除）。""" - lock = self._inline_owner_lock - if lock is not None: - await lock.stop() - lock.release() - self._inline_owner_lock = None - self._inline_owner = False - - def is_inline_owner(self) -> bool: - """返回本进程是否持有 inline worker 所有权（用于管理/诊断）。""" - return bool(self._inline_owner) - - def register_worker_starter(self, worker_type: str, starter) -> None: - """为某 inline 类型注册一个"拉起 worker"的协程（load_workers 调用）， - 供持有者崩溃后的自愈接管使用。starter: async () -> List[Task] - """ - self._worker_starters[worker_type] = starter - - def extra_worker_tasks(self) -> List[Any]: - """返回因自愈接管而拉起的 worker 任务（供网关关停时取消）。""" - return list(self._started_extra) - - async def _maybe_heal_inline(self) -> None: - """持有者进程崩溃后的自愈接管。 - - 若本进程非持有者，却存在「已配置 inline 类型但没有无线活副本」的情况， - 说明原持有者已失效：尝试经分布式锁抢得领导权成为新持有者，并拉起这些 worker。 - 分布式锁保证同一时刻只有一个进程胜出，因此不会出现双写/分裂（split-brain）。 - - 背景：gunicorn 仅复活死掉的 worker 槽位不足以 failover——槽位重启可能在旧锁 - 租约尚未过期时发生，新进程抢锁失败而照样跳过 create_workers。本函数随 refresh - 每周期（~15s）重试，租约过期后即可接管，形成兜底。 - """ - # 已是持有者：正常托管，不走接管路径（避免与常规池任务重复 init） - if self._inline_owner or not self._worker_starters: - return - missing = [ - wtype - for wtype in self._worker_starters - if not self._local_workers.get(wtype) - and not self._workers_cache.get(wtype) - ] - if not missing: - return - # 仍需保证同一时刻仅一个进程成为新持有者 - if not self.acquire_inline_owner(): - return - logger.warning( - "[heal] 检出持有者进程失效，本进程接管并拉起 inline workers: %s", missing - ) - for wtype in missing: - try: - tasks = await self._worker_starters[wtype]() - self._started_extra.extend(tasks or []) - logger.info("[heal] 已拉起 %s (tasks=%d)", wtype, len(tasks or [])) - except Exception as e: # noqa: BLE001 - logger.error("[heal] 拉起 %s 失败: %s", wtype, e) - - def register_worker(self, worker_type: str, worker_id: str): - """ - 直接注册一个本地 worker（同进程内使用） - 这样无需等待 liveliness 发现，立即可用。 - - 同步执行（仅修改内存字典），确保调用后立即可被 get_worker 发现。 - """ - key_expr = f"worker/{worker_type}/{worker_id}" - # 同步直接写入 _local_workers，无需异步锁 - # 因为 _local_workers 和 _workers_cache 在调用 call() 时通过 await get_worker() 读取 - self._local_workers.setdefault(worker_type, []).append(key_expr) - logger.debug(f"本地 worker 已注册: {key_expr}") - - def unregister_worker(self, worker_type: str, worker_id: str): - """取消注册一个本地 worker""" - key_expr = f"worker/{worker_type}/{worker_id}" - if worker_type in self._local_workers: - try: - self._local_workers[worker_type].remove(key_expr) - except ValueError: - pass - # 同步从存活缓存剔除：已关闭/被 drain 的 inline 副本若仍留在 _workers_cache， - # 会在下次 liveliness 刷新（15s）前被 list_workers 误判为 external 并残留显示。 - cache = self._workers_cache.get(worker_type) - if cache and key_expr in cache: - self._workers_cache[worker_type] = [k for k in cache if k != key_expr] - - async def refresh_workers(self): - """定期刷新存活 worker 列表（合并 Zenoh liveliness + 本地注册）""" - while True: - try: - # 1. 从 Zenoh liveliness 获取分布式 workers - replies = self.session.liveliness().get("worker/*/*") - new_cache: Dict[str, List[str]] = {} - for reply in replies: - if reply.ok: - key = str(reply.ok.key_expr) # e.g. "worker/example/worker-0" - parts = key.split("/") - if len(parts) >= 3: - worker_type = parts[1] - new_cache.setdefault(worker_type, []).append(key) - - # 2. 合并本地注册的 workers - for wtype, workers in self._local_workers.items(): - existing = set(new_cache.get(wtype, [])) - for w in workers: - if w not in existing: - new_cache.setdefault(wtype, []).append(w) - - async with self._lock: - self._workers_cache = new_cache - logger.debug(f"Worker 存活列表更新: {self._workers_cache}") - # 3. 网关心跳：刷新自身在线时间戳并清理死网关，保持在线列表准确 - try: - self.heartbeat_gateway() - except Exception as e: # noqa: BLE001 - logger.error(f"网关心跳失败: {e}") - # 4. inline 持有者崩溃自愈接管（见 _maybe_heal_inline） - try: - await self._maybe_heal_inline() - except Exception as e: # noqa: BLE001 - logger.error(f"inline 自愈检查失败: {e}") - except Exception as e: - logger.error(f"刷新 worker 列表失败: {e}") - await asyncio.sleep(self._refresh_interval) - - def _all_workers(self, worker_type: str) -> List[str]: - """该类型当前全部候选副本（liveliness 列表优先，启动初期回退本地注册表）。""" - workers = list(self._workers_cache.get(worker_type, [])) - if not workers: - workers = list(self._local_workers.get(worker_type, [])) - return workers - - async def _query_instance_id(self, worker_type: str, worker_key: str) -> Optional[str]: - """问单个副本要它的实例身份（admin status 的 instance_id）。 - - 复用单副本管理协议 worker///__admin__；查询失败或副本运行旧版 - 运行时（回复无 instance_id）返回 None，由调用方归入 unknown 组。 - """ - wid = worker_key.split("/")[2] if "/" in worker_key else worker_key - try: - reply = await self._send_admin_cmd(worker_type, wid, "query", timeout=2.0) - except Exception as e: # noqa: BLE001 - 单副本查询失败不影响其它副本 - logger.warning("[instance] %s/%s 身份查询失败: %s", worker_type, wid, e) - return None - iid = reply.get("instance_id") if isinstance(reply, dict) else None - return str(iid) if iid else None - - async def _refresh_instance_map(self, worker_type: str) -> Dict[str, List[str]]: - """重建某类型的「实例 → 副本」索引，并按需补查询新副本。 - - 只对从未查询过的副本发一次 admin status（本地注册的 inline 副本直接返回 - 网关自身实例，零查询）；已查询过的副本沿用缓存，因此常规路径是纯内存查询， - 仅新上线副本会引入一次额外 RPC。离线副本在重建时被剪枝。 - """ - workers = self._all_workers(worker_type) - known = self._instance_index.get(worker_type, {}) - unknown = set(self._instance_unknown.get(worker_type, [])) - # 反查已有归属：副本 -> 实例（副本 key 唯一，一个副本只属一个实例） - owner = {w: iid for iid, ws in known.items() for w in ws} - - index: Dict[str, List[str]] = {} - still_unknown: List[str] = [] - queried = owner.keys() | unknown - for w in workers: - if w in queried: - iid = owner.get(w) - else: - # 本地注册（inline）副本天然属于本网关实例，无需查询 - iid = self._gateway_id if w in self._local_workers.get(worker_type, []) else None - if iid is None: - iid = await self._query_instance_id(worker_type, w) - if iid is None: - still_unknown.append(w) - if iid: - index.setdefault(iid, []).append(w) - - self._instance_index[worker_type] = index - self._instance_unknown[worker_type] = still_unknown - return index - - async def _instance_workers(self, worker_type: str, instance_id: str) -> List[str]: - """取某实例下该类型的在线副本 key 列表（无则空表）。""" - index = self._instance_index.get(worker_type) - if index is None: - # 同类型并发请求只让一个跑刷新，其余等结果，避免重复查询同一批副本 - lock = self._instance_locks.setdefault(worker_type, asyncio.Lock()) - async with lock: - index = self._instance_index.get(worker_type) - if index is None: - index = await self._refresh_instance_map(worker_type) - return index.get(instance_id, []) - - async def get_worker( - self, worker_type: str, instance_id: Optional[str] = None - ) -> Optional[str]: - """获取一个可用的 worker 地址。 - - instance_id 为 None 时按简单随机负载均衡挑选（默认，行为不变）； - 指定实例时只在该实例的在线副本中挑选，把请求固定到那个进程实例。 - 实例无在线副本（或该实例在旧版运行时下无法识别身份）返回 None。 - - 读路径无锁：_workers_cache 由 refresh_workers 整表原子替换， - _local_workers 由同步方法修改，单事件循环内读取安全。 - """ - if instance_id: - return await self._pick_from_instance(worker_type, instance_id) - workers = self._all_workers(worker_type) - return random.choice(workers) if workers else None - - async def _pick_from_instance( - self, worker_type: str, instance_id: str - ) -> Optional[str]: - """在指定实例的在线副本中随机挑一个（定向路径，永不跨实例回退）。 - - 定向的语义就是"必须打在同一个进程实例上"，因此实例离线时返回 None 让 - 调用方报错，而不是悄悄换成别的实例——后者会让粘滞状态类的插件跑在错误 - 副本上，产生难查的数据错乱。 - """ - candidates = await self._instance_workers(worker_type, instance_id) - if candidates: - return random.choice(candidates) - logger.warning( - "[instance] 目标实例 %s 下无 %s 的在线副本（已识别实例: %s；%d 个副本读不出身份）", - instance_id, worker_type, sorted(self._instance_index.get(worker_type, {})), - len(self._instance_unknown.get(worker_type, [])), - ) - return None - - async def _pick_other_worker( - self, worker_type: str, exclude: List[str], - instance_id: Optional[str] = None, - ) -> Optional[str]: - """ - 选择一个不同于已尝试列表的 worker。 - - Args: - worker_type: Worker 类型 - exclude: 已尝试（需排除）的 worker key_expr 列表 - instance_id: 定向实例；给定时只在同实例内换副本，绝不跨实例 - - Returns: - 可用的 worker key_expr，若无其他可用则返回 None - """ - if instance_id: - candidates = [ - w for w in await self._instance_workers(worker_type, instance_id) - if w not in exclude - ] - return random.choice(candidates) if candidates else None - workers = self._workers_cache.get(worker_type, []) - candidates = [w for w in workers if w not in exclude] - if candidates: - return random.choice(candidates) - # 也检查本地注册表 - local = self._local_workers.get(worker_type, []) - local_candidates = [w for w in local if w not in exclude] - if local_candidates: - return random.choice(local_candidates) - return None - - def is_success_response(self, result: dict) -> bool: - """根据业务约定判断响应是否成功。 - 假设成功响应没有 'error' 字段，或 'code' 为 0。 - """ - if "error" in result and result["error"]: - return False - if "code" in result and result["code"] != 0: - return False - return True - - async def _no_worker_message( - self, worker_type: str, instance_id: Optional[str] = None - ) -> str: - """ - 无可用 Worker 时的统一诊断文案。 - - 供 call() / call_stream() 共用，保证两类调用错误信息一致；定向（instance_id） - 调用额外列出当前已识别实例，便于区分"实例名写错"与"实例离线"。 - """ - all_known = list(self._workers_cache.keys()) - local_known = list(self._local_workers.keys()) - all_types = sorted(set(all_known + local_known)) - msg = f"类型 '{worker_type}' 没有可用 Worker。" - if instance_id: - msg = ( - f"类型 '{worker_type}' 的实例 '{instance_id}' 没有在线副本（定向调用不" - f"跨实例回退）。已识别实例: " - f"{sorted(self._instance_index.get(worker_type, {})) or '(无)'}。" - f"请检查: 1) 该实例 id 是否写错（取值见 GET " - f"/admin/workers/{worker_type}/instances 或副本启动时的 --instance）" - f" 2) 该实例进程是否在线 3) 该实例的 worker 是否运行旧版运行时" - f"（不支持实例身份上报）" - ) - else: - msg += ( - f"当前已知的 Worker 类型: {all_types if all_types else '(无)'}。" - f"请检查: 1) config/workers/config.toml 是否配置了该类型 " - f"2) 插件初始化是否成功（查看启动日志中的 '加载 Worker 插件失败' 错误）" - f"3) Worker 进程是否正常运行" - ) - return msg - - async def call_stream( - self, - worker_type: str, - subject: str, - payload: Dict[str, Any], - timeout: float = 300.0, - idle_timeout: float = 10.0, - instance_id: Optional[str] = None, - ) -> AsyncGenerator[bytes, None]: - """ - 流式调用：消费 Worker 的多回复流，逐块产出 JSON 字节。 - - 与 call() 的区别: - - 不合并回复（ConsolidationMode.NONE），逐块 yield，实现流式传输 - - 不重试（流一旦开始输出无法安全重放，中途失败只能重建） - - 超时模型: timeout = 流总寿命上限（zenoh query deadline，硬上限， - 流式端点须配大如 300s）；idle_timeout = 连续无数据块的最大间隔 - - 错误约定: 块内含 "error" 键（Worker 收尾错误块）或 reply.err - （zenoh 层错误）均为流的终止信号，转发后结束 - - Args: - worker_type: Worker 类型标识（如 "example"） - subject: 调用的主题（如 "stream_chat"） - payload: 请求参数（信封格式，由路由层构造） - timeout: 流总寿命上限（秒），需大于业务最长生成时长 - idle_timeout: 空闲超时（秒），超时抛 StreamIdleTimeoutError - instance_id: 定向进程实例（None=随机负载均衡）；流本身不可重试， - 故实例离线时直接报错，不会中途改派到别的实例 - - Yields: - bytes: 每个回复块的 JSON 字节 - - Raises: - RuntimeError: 无可用 worker（含定向实例无在线副本） - StreamIdleTimeoutError: 空闲超时 - """ - # 急切解析 worker：无可用时立即抛错，让路由层映射 HTTP 503 - # （异步生成器函数体在首次迭代才执行，故这里用普通 async 函数返回内部生成器） - worker_expr = await self.get_worker(worker_type, instance_id=instance_id) - if not worker_expr: - raise RuntimeError(await self._no_worker_message(worker_type, instance_id)) - - full_key = f"{worker_expr}/{subject}" - - # 进程内直调（inline 快路径）：与 zenoh 路径同语义（逐块产出、错误块终止、无重试） - local = _local_dispatcher_for(worker_expr) - if local is not None and local.has_subject(subject): - async def _local_stream(): - agen = local.invoke_stream(subject, orjson.dumps(payload)) - try: - while True: - try: - data = await asyncio.wait_for(agen.__anext__(), timeout=idle_timeout) - except StopAsyncIteration: - break - except asyncio.TimeoutError: - raise StreamIdleTimeoutError( - f"Worker {worker_expr} 流式空闲超时 ({idle_timeout}s 无数据)" - ) from None - if not data: - continue - decoded = orjson.loads(data) - if isinstance(decoded, dict) and decoded.get("error"): - # Worker 错误收尾块：转发后终止流 - yield data - return - yield data - finally: - # 消费方提前退出（SSE 断连/网关放弃）：停流并取消在途派发 - await agen.aclose() - - return _local_stream() - - reply_iter = self.session.get( - full_key, - payload=orjson.dumps(payload), - consolidation=zenoh.ConsolidationMode.NONE, - timeout=timeout, - ) - - async def _stream(): - # 逐块消费；zenoh reply 迭代器是同步阻塞的，必须放线程执行， - # 避免阻塞事件循环（与 call() 的 _get_first 同模式）。 - # 走专用 _ZENOH_EXECUTOR 池，不与全局默认池的其它阻塞调用争抢 - while True: - try: - loop = asyncio.get_running_loop() - reply = await asyncio.wait_for( - loop.run_in_executor(_ZENOH_EXECUTOR, _next_reply, reply_iter), - timeout=idle_timeout, - ) - except asyncio.TimeoutError: - raise StreamIdleTimeoutError( - f"Worker {worker_expr} 流式空闲超时 ({idle_timeout}s 无数据)" - ) from None - - if reply is _SENTINEL: - break - - if reply.ok: # type: ignore - data = bytes(reply.ok.payload) if reply.ok.payload else b"" # type: ignore - if not data: - continue - decoded = orjson.loads(data) - if isinstance(decoded, dict) and decoded.get("error"): - # Worker 错误收尾块：转发后终止流 - yield data - return - yield data - else: - # zenoh 层错误回复（如 queryable 异常）：转发后终止流 - error_payload = reply.err.payload # type: ignore - error_msg = str(error_payload) if error_payload else "unknown error" - logger.warning( - f"Worker {worker_expr} 流式返回错误: {error_msg}" - ) - yield orjson.dumps( - {"error": f"Worker {worker_expr} 返回错误: {error_msg}"} - ) - return - - return _stream() - - # ── 共享缓存二级 I/O（executor 化，不阻塞事件循环）──────────── - # 进程内一级（_rpc_mem_cache）在事件循环内 O(1) 直读；二级 diskcache/redis - # 的 get/set 是同步 I/O，统一丢到 _ZENOH_EXECUTOR 线程执行，避免高并发下 - # 磁盘/网络 I/O 卡死事件循环。值恒为 bytes，选 None 作为未命中哨兵安全。 - - async def _cache_get(self, key: str) -> Optional[bytes]: - if self.cache is None: - return None - loop = asyncio.get_running_loop() - return await loop.run_in_executor(_ZENOH_EXECUTOR, self._sync_cache_get, key) - - def _sync_cache_get(self, key: str) -> Optional[bytes]: - try: - return self.cache.get(key) if self.cache is not None else None - except Exception: - return None - - async def _cache_set(self, key: str, value: bytes) -> None: - if self.cache is None: - return - loop = asyncio.get_running_loop() - await loop.run_in_executor(_ZENOH_EXECUTOR, self._sync_cache_set, key, value) - - def _sync_cache_set(self, key: str, value: bytes) -> None: - try: - if self.cache is not None: - self.cache[key] = value - except Exception: - pass - - async def call( - self, - worker_type: str, - subject: str, - payload: Dict[str, Any], - timeout: float = 3.0, - retries: int = 1, - cacheable: bool = False, - instance_id: Optional[str] = None, - ) -> Any: - """ - 统一调用入口（dict 语义，MQTT Bridge 等调用方使用） - - Args: - worker_type: Worker 类型标识（如 "example"） - subject: 调用的主题（如 "calc.add"） - payload: 请求参数（信封格式，由路由层构造） - timeout: 单次调用超时（秒） - retries: 失败重试次数 - cacheable: 是否允许缓存结果（若为 False 则跳过缓存读写） - instance_id: 定向进程实例；给定时请求固定发往该实例（不参与随机 - 负载均衡，重试也只在该实例内换副本），None=默认随机均衡 - - Returns: - Dict[str, Any]: 调用结果（worker 回复解析后的 dict） - """ - # cacheable=False 时不计算 cache_key，省一次排序序列化 - cache_key = None - if cacheable: - # 定向调用的缓存键含实例名：定向通常用于读取该实例的本地状态 - # （本地库/设备连接），共用缓存会拿到其它实例的结果 - cache_key = ( - f"{worker_type}:{subject}" - f":{orjson.dumps(payload, option=orjson.OPT_SORT_KEYS).decode()}" - ) - if instance_id: - cache_key = f"{instance_id}@{cache_key}" - reply_bytes = await self._call_impl( - worker_type, subject, orjson.dumps(payload), - timeout=timeout, retries=retries, cacheable=cacheable, cache_key=cache_key, - instance_id=instance_id, - ) - return orjson.loads(reply_bytes) if reply_bytes else {} - - async def call_raw( - self, - worker_type: str, - subject: str, - payload: bytes, - timeout: float = 3.0, - retries: int = 1, - cacheable: bool = False, - instance_id: Optional[str] = None, - ) -> bytes: - """ - 统一调用入口（bytes 语义，HTTP 端点透传用）：返回 worker 回复的原始字节。 - - 与 call() 的区别: - - payload 为预序列化的请求字节（普通信封或字节拼接信封），网关不再二次序列化 - - 回复不做 JSON 解析，直接返回原始字节（配合 Response(content=bytes) 零开销转发） - - cacheable 时仅在缓存写入前解析一次做成功检查（业务错误不缓存）， - 缓存与返回均保留原始字节 - - Args: - worker_type: Worker 类型标识（如 "example"） - subject: 调用的主题（如 "calc.add"） - payload: 预序列化的请求字节（信封 JSON） - timeout: 单次调用超时（秒） - retries: 失败重试次数 - cacheable: 是否允许缓存结果（缓存原始字节） - instance_id: 定向进程实例（见 call()）；None=默认随机均衡 - - Returns: - bytes: worker 回复的原始字节（空回复兜底 b"{}"，与 call() 的 {} 语义一致） - """ - # cacheable=False 时不计算 cache_key（当前唯一用途），省一次 sha256 - cache_key = None - if cacheable: - cache_key = f"{worker_type}:{subject}:{hashlib.sha256(payload).hexdigest()}" - if instance_id: - cache_key = f"{instance_id}@{cache_key}" - return await self._call_impl( - worker_type, subject, payload, - timeout=timeout, retries=retries, cacheable=cacheable, cache_key=cache_key, - instance_id=instance_id, - ) - - async def _call_impl( - self, - worker_type: str, - subject: str, - payload_bytes: bytes, - timeout: float = 3.0, - retries: int = 1, - cacheable: bool = False, - cache_key: Optional[str] = None, - instance_id: Optional[str] = None, - ) -> bytes: - """ - 调用实现：缓存 > 选副本（随机均衡/实例定向）> 超时重试，返回回复的原始字节。 - - call() / call_raw() 共用本实现，仅入参出参形态不同（dict ↔ bytes）。 - instance_id 给定时选副本与重试都被约束在该实例内（定向语义，不跨实例回退）。 - """ - # 1. 缓存检查（仅在 cacheable=True 时检查） - if cacheable and cache_key is not None: - # 一级：进程内内存 LRU（事件循环内 O(1)，零 I/O，命中完全绕过 Zenoh） - mem_val = self._rpc_mem_cache.get(cache_key) - if mem_val is not None: - logger.debug(f"命中内存缓存: {cache_key}") - return mem_val - # 二级：共享缓存（diskcache/redis），I/O 移到 executor，不阻塞事件循环 - if self.cache is not None: - disk_val = await self._cache_get(cache_key) - if disk_val is not None: - self._rpc_mem_cache.set(cache_key, disk_val) - logger.debug(f"命中共享缓存: {cache_key}") - return disk_val - - # 2. 获取可用 worker（instance_id 给定时固定到该实例，否则随机均衡） - worker_expr = await self.get_worker(worker_type, instance_id=instance_id) - if not worker_expr: - raise RuntimeError(await self._no_worker_message(worker_type, instance_id)) - - full_key = f"{worker_expr}/{subject}" - - last_error = None - tried_workers: List[str] = [] - - async def _switch_worker(why: str) -> bool: - """换一个副本重试；返回是否值得继续（定向调用只在同实例内换副本）。 - - 换完更新闭包外层的 worker_expr / full_key。定向实例内已无未试副本时 - 返回 False —— 继续循环只会重复打同一个坏副本，白等一整轮超时。 - """ - nonlocal worker_expr, full_key, last_error - new_worker = await self._pick_other_worker( - worker_type, tried_workers, instance_id=instance_id - ) - if new_worker: - if new_worker != worker_expr: - logger.info(f"{why}切换 Worker: {worker_expr} -> {new_worker}") - worker_expr = new_worker - full_key = f"{worker_expr}/{subject}" - return True - # 没有任何未试过的副本可换：继续循环只会重复打同一个副本（若它已死， - # 白等一整轮超时；若它已回业务错误，结果必然相同），直接结束重试。 - # 保留具体失败原因（last_error 已由调用方按场景写入），便于定位。 - scope = f"实例 '{instance_id}' 内" if instance_id else "" - last_error = f"{scope}已无其它未试副本可换（{why}）：{last_error}" - return False - - for attempt in range(retries + 1): - # 记录已尝试的 worker - if worker_expr not in tried_workers: - tried_workers.append(worker_expr) - - try: - local = _local_dispatcher_for(worker_expr) - if local is not None and local.has_subject(subject): - # 进程内直调（inline 快路径）：省 zenoh 往返，语义与下面 zenoh 路径一致 - logger.debug(f"[local-rpc] {worker_expr}/{subject} 进程内直调") - local_res = await asyncio.wait_for( - local.invoke_unary(subject, payload_bytes), timeout=timeout - ) - reply_ok, reply_payload = local_res.ok, local_res.payload - else: - reply_iter = self.session.get( - full_key, - payload=payload_bytes, - consolidation=zenoh.ConsolidationMode.NONE, - timeout=timeout, - ) - # 使用辅助函数获取第一个回复，避免 StopIteration 在 asyncio.to_thread 中传播的问题 - def _get_first(iterator): - """线程安全地获取迭代器的第一个元素""" - for item in iterator: - return item - return _SENTINEL - - # 走专用 _ZENOH_EXECUTOR 池（隔离于全局默认池，避免高并发下排队） - loop = asyncio.get_running_loop() - first_reply = await asyncio.wait_for( - loop.run_in_executor(_ZENOH_EXECUTOR, _get_first, reply_iter), - timeout=timeout, - ) - if first_reply is _SENTINEL: - logger.warning( - f"Worker {worker_expr} 无回复，尝试重试 {attempt+1}/{retries+1}" - ) - last_error = f"Worker {worker_expr} 无回复" - if not await _switch_worker("无回复"): - break - continue - - reply = first_reply - if reply.ok: # type: ignore - reply_ok = True - reply_payload = bytes(reply.ok.payload) if reply.ok.payload else b"{}" - else: - reply_ok = False - reply_payload = reply.err.payload # type: ignore - - if reply_ok: - reply_bytes = bytes(reply_payload) if reply_payload else b"{}" - # 缓存写入（仅在 cacheable=True 时）：解析一次做成功检查， - # 但缓存与返回均保留原始字节（网关侧零二次序列化）。 - # 先写内存一级，再异步写共享二级（executor，不阻塞响应返回） - if cacheable and cache_key is not None and reply_bytes: - parsed = orjson.loads(reply_bytes) - if self.is_success_response(parsed): - self._rpc_mem_cache.set(cache_key, reply_bytes) - await self._cache_set(cache_key, reply_bytes) - return reply_bytes - else: - error_msg = _err_text(reply_payload) - logger.warning(f"Worker 返回错误: {error_msg} (尝试 {attempt+1}/{retries+1})") - last_error = f"Worker {worker_expr} 返回错误: {error_msg}" - # 错误回复（含 Zenoh 超时）也换一个 worker 重试 - if not await _switch_worker("错误后"): - break - continue # 重试 - except asyncio.TimeoutError: - logger.warning( - f"Worker {worker_expr} 超时 (>{timeout}s)，尝试重试 {attempt+1}/{retries+1}" - ) - last_error = f"Worker {worker_expr} 超时 (>{timeout}s)" - # 超时后换一个 worker 重试，避免打到仍在处理中的同一个 worker - if not await _switch_worker("超时后"): - break - continue - except Exception as e: - logger.error(f"调用 worker 异常: {e}") - raise - - raise RuntimeError( - f"Worker '{worker_type}' 调用失败 (已尝试 {len(tried_workers)} 个 Worker, " - f"实际尝试 {attempt + 1} 次/上限 {retries + 1} 次, timeout={timeout}s): {last_error}" - ) - - # ------------------------------------------------------------------ - # 管理能力：盘点 worker 实例 + 精细化控制单个副本（admin Queryable） - # - # 单副本协议（worker 框架声明）：worker///__admin__ - # POST {"data": {"cmd": "restart"}} 平滑退出并以新 worker_id 重建 - # POST {"data": {"cmd": "stop"}} 平滑退出且不自动重建 - # （无 cmd / GET） 返回副本实时状态 - # ------------------------------------------------------------------ - - def list_workers(self) -> List[Dict[str, Any]]: - """ - 盘点当前全部 worker 实例。 - - 合并 Zenoh liveliness 与本地注册表： - - 仅出现在本地注册表 / 也出现在 liveliness 的 = inline（网关进程内加载） - - 仅出现在 liveliness 的 = external（独立进程） - 每个实例含 worker_type / worker_id / mode / alive。 - - Returns: - List[{"worker_type", "worker_id", "mode"}] - """ - result: List[Dict[str, Any]] = [] - # 本地注册（inline）：type -> [key]，key 形如 "worker//" - local_keys: Dict[str, List[str]] = { - t: list(v) for t, v in self._local_workers.items() - } - for wtype, keys in local_keys.items(): - for key in keys: - parts = key.split("/") - wid = parts[2] if len(parts) >= 3 else key - result.append({ - "worker_type": wtype, - "worker_id": wid, - "mode": "inline", - "alive": True, - }) - # liveliness（inline 亦会出现，去重；其余为 external） - known = {(r["worker_type"], r["worker_id"]) for r in result} - for wtype, keys in self._workers_cache.items(): - for key in keys: - parts = key.split("/") - if len(parts) < 3: - continue - wid = parts[2] - if (wtype, wid) in known: - continue - result.append({ - "worker_type": wtype, - "worker_id": wid, - "mode": "external", - "alive": True, - }) - result.sort(key=lambda w: (w["worker_type"], w["worker_id"])) - return result - - async def _send_admin_cmd( - self, - worker_type: str, - worker_id: str, - cmd: str, - timeout: float = 5.0, - data_extra: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: - """向单个副本的 admin Queryable 下发指令并等待回复。 - - Args: - worker_type: Worker 类型 - worker_id: 副本 id（如 worker-0-1-12345） - cmd: "restart" / "stop" / "config_set" - timeout: 单次下发超时（秒） - data_extra: 追加到 data 字段的附加载荷（如 config_set 的 config） - - Returns: - worker 侧回复（dict） - - Raises: - RuntimeError: 无可用副本 / 下发失败 / 超时 - """ - admin_key = f"worker/{worker_type}/{worker_id}/__admin__" - data = {"cmd": cmd, **(data_extra or {})} - payload = orjson.dumps({"meta": {"method": "POST"}, "data": data}) - reply_iter = self.session.get( - admin_key, - payload=payload, - consolidation=zenoh.ConsolidationMode.NONE, - timeout=timeout, - ) - loop = asyncio.get_running_loop() - reply = await asyncio.wait_for( - loop.run_in_executor(_ZENOH_EXECUTOR, _next_reply, reply_iter), - timeout=timeout, - ) - if reply is _SENTINEL: - raise RuntimeError( - f"副本 {worker_type}/{worker_id} 未响应管理指令（可能不在线，" - "或该 Worker 运行旧版运行时未声明 __admin__ 管理能力）" - ) - if reply.ok: # type: ignore - data = bytes(reply.ok.payload) if reply.ok.payload else b"{}" # type: ignore - return orjson.loads(data) - error_payload = reply.err.payload # type: ignore - error_msg = str(error_payload) if error_payload else "unknown error" - raise RuntimeError(f"副本 {worker_type}/{worker_id} 管理指令失败: {error_msg}") - - async def restart_worker( - self, - worker_type: str, - worker_id: str, - timeout: float = 5.0, - ) -> Dict[str, Any]: - """精细化重启单个 worker 副本（经 Zenoh 下发 restart，副本以新 id 重建）。""" - return await self._send_admin_cmd(worker_type, worker_id, "restart", timeout) - - async def stop_worker( - self, - worker_type: str, - worker_id: str, - timeout: float = 5.0, - ) -> Dict[str, Any]: - """精细化停止单个 worker 副本（经 Zenoh 下发 stop，副本退出且不自动重建）。""" - return await self._send_admin_cmd(worker_type, worker_id, "stop", timeout) - - async def scale_workers( - self, - worker_type: str, - count: int, - timeout: float = 5.0, - ) -> Dict[str, Any]: - """把某类型的副本数调整为 count（扩缩容）并持久化到配置。 - - 下发逻辑（与 inline/external 的统一管理协议一致）： - - inline：类型在本进程内有 WorkerPool，只下发本网关进程内的池（进程内扩缩， - 无需跨进程广播，副本按其所在网关分摊配额）； - - external：类型由独立进程/容器运行，向该类型**全部存活**副本经 Zenoh 广播 - rescale 指令，由各进程内 WorkerPool 按「全局 count ÷ 存活实例数」重新分摊 - 并各写回自身配置（避免只缩/扩其中一个实例造成不一致）。 - 全局缩放同时清除各实例的「逐实例绝对副本数」覆盖，回到对称分摊——因此它是 - 「一键复位对称」入口；逐实例精细缩放走 scale_instance（见后）。 - - 缩容走优雅排空：摘除 liveliness/业务 Queryable 隔绝新请求，等正在处理的 - 在途请求完成后再关闭副本（避免中断业务）。 - - Args: - worker_type: 插件类型 - count: 目标副本总数（全局总量，>= 0） - timeout: 下发超时（秒） - - Returns: - {"worker_type", "target"(全局总量), "mode", "sent_to", "reply"} - - Raises: - RuntimeError: count 非法 / 类型无可用副本 - """ - count = max(0, int(count)) - # 「count=全局总量」收归 worker 层：网关不再直写 count/quota 缓存、不再直调进程池。 - # 统一经 rescale 下发全局 target，由 worker 读取在线网关分摊、写缓存并扩缩本进程副本； - # inline/external 口径一致（inline 副本在本进程，经 Zenoh 直达其 __admin__）。 - inline = False - try: - from chongming_worker.runtime import get_process_pool - inline = get_process_pool(worker_type) is not None - except Exception: # noqa: BLE001 - 运行时缺依赖视为非 inline - inline = False - candidates = [ - w for w in self.list_workers() - if w["worker_type"] == worker_type and w.get("alive") - ] - if not candidates: - raise RuntimeError(f"类型 '{worker_type}' 没有可用副本，无法调整数量") - - # 目标选择： - # - inline：只下发本网关进程内的池（任意一个 inline 候选都会落到本地池，幂等）； - # - external：同一类型可能有多个独立进程/实例，必须广播到全部存活实例，由各自按 - # 「全局 count ÷ 存活实例数」重新分摊，避免只缩/扩其中一个进程造成不一致。 - if inline: - inline_cands = [c for c in candidates if c.get("mode") == "inline"] or candidates - targets = [inline_cands[0]] - else: - targets = candidates - replies: Dict[str, Any] = {} - for cand in targets: - try: - replies[cand["worker_id"]] = await self._send_admin_cmd( - worker_type, cand["worker_id"], "rescale", - timeout=timeout, data_extra={"scale": count}, - ) - except Exception as e: # noqa: BLE001 - 单个实例失败不影响其它实例 - logger.warning("[scale] %s 下发失败 %s: %s", worker_type, cand["worker_id"], e) - if not replies: - raise RuntimeError(f"类型 '{worker_type}' 没有副本响应 rescale，无法调整数量") - logger.info( - "[scale] %s 全局目标=%d 下发 %d 个实例（%s）", - worker_type, count, len(replies), "inline" if inline else "external", - ) - return { - "worker_type": worker_type, - "target": count, - "mode": "inline" if inline else "external", - "sent_to": list(replies.keys()), - "reply": replies, - } - - async def list_instances( - self, - worker_type: str, - timeout: float = 3.0, - ) -> List[Dict[str, Any]]: - """按「外部进程实例」分组盘点某类型的落地实体。 - - 对类型下每个存活副本发轻量 status 查询，从回复取 instance_id 归组； - 一个副本组 = 一个能独立承载若干副本的执行单元（inline=本网关进程/网关 host， - external=独立进程/容器）。副本数=组内在线副本个数，worker_ids 供精细缩放寻址。 - """ - workers = [ - w for w in self.list_workers() - if w["worker_type"] == worker_type and w.get("alive") - ] - groups: Dict[str, Dict[str, Any]] = {} - for w in workers: - try: - reply = await self._send_admin_cmd( - worker_type, w["worker_id"], "query", timeout=timeout - ) - iid = reply.get("instance_id") or "unknown" - except Exception as e: # noqa: BLE001 - 单副本查询失败不阻塞整体 - logger.warning("[instances] %s/%s 查询失败: %s", worker_type, w["worker_id"], e) - iid = "unknown" - g = groups.setdefault(iid, { - "worker_type": worker_type, - "instance_id": iid, - "mode": w.get("mode", "external"), - "replicas": 0, - "worker_ids": [], - }) - g["replicas"] += 1 - g["worker_ids"].append(w["worker_id"]) - groups.pop("unknown", None) # 拿不到身份的一律归入 unknown，避免误当独立实例 - return list(groups.values()) - - async def scale_instance( - self, - worker_type: str, - instance_id: str, - count: int, - timeout: float = 5.0, - ) -> Dict[str, Any]: - """把某个外部进程/**实例**精确调到 count 个副本（绝对数，不影响其它实例）。 - - 经该实例任一副本的 admin Queryable 下发 `rescale{scale:count, instance_id, - absolute:true}`，由其进程内 WorkerPool 直接 set_target(count) 并写入 instcounts - 覆盖表（进程重启后仍生效）；目标其它实例的指令被 worker 安全忽略。 - 返回与原 /scale 结构兼容：{worker_type, target, mode, sent_to, reply}。 - """ - count = max(0, int(count)) - insts = await self.list_instances(worker_type) - inst = next((i for i in insts if i["instance_id"] == instance_id), None) - if inst is None or not inst["worker_ids"]: - raise RuntimeError( - f"类型 '{worker_type}' 下没有在线副本归属实例 '{instance_id}'，无法调整数量" - ) - wid = inst["worker_ids"][0] - reply = await self._send_admin_cmd( - worker_type, wid, "rescale", - timeout=timeout, - data_extra={"scale": count, "instance_id": instance_id, "absolute": True}, - ) - logger.info( - "[scale_instance] %s 实例=%s 目标=%d → 下发 %s", worker_type, instance_id, count, wid, - ) - return { - "worker_type": worker_type, - "instance_id": instance_id, - "target": count, - "mode": inst.get("mode", "external"), - "sent_to": wid, - "reply": reply, - } - - async def get_worker_config( - self, - worker_type: str, - timeout: float = 5.0, - ) -> Dict[str, Any]: - """经 Zenoh 向该类型任一存活副本查询其 [workers.config]。 - - 由 worker 访问自身缓存（或回退 toml 种子）返回；网关不直接接触外部 worker 的缓存后端。 - - Returns: - {"status", "config": {...}, "source": "cache"|"toml", "mode": "toml"|"cache", - "worker_id", ...} - - Raises: - RuntimeError: 无存活副本 / 副本未响应（可能运行旧版运行时无 get_config 能力） - """ - candidate = next( - ( - w for w in self.list_workers() - if w["worker_type"] == worker_type and w.get("alive") - ), - None, - ) - if candidate is None: - raise RuntimeError(f"类型 '{worker_type}' 没有在线副本，无法读取其配置") - reply = await self._send_admin_cmd( - worker_type, candidate["worker_id"], "get_config", timeout=timeout - ) - return { - "worker_type": worker_type, - "worker_id": candidate["worker_id"], - "status": reply.get("status"), - "config": reply.get("config"), - "source": reply.get("source", "toml"), - # 旧版运行时回复无 mode：按本网关 inline 模式兜底（external 副本以自身 - # [workers.cache].launch_worker_count 为准，新运行时会在回复里带回） - "mode": reply.get("mode") or self.config_mode, - } - - async def update_worker_config( - self, - worker_type: str, - updates: Dict[str, Any], - timeout: float = 5.0, - ) -> Dict[str, Any]: - """经 Zenoh 下发 config_set 指令，让目标类型的全部副本修改配置并自我重启。 - - worker 侧行为（见 chongming_worker.runtime）： - - 仅 index0 副本把 updates 写入共享缓存的 [workers.config]（key - worker:cm:configs:{type}，全局唯一，路由/保留键会被拒绝）；不再回写 config.toml。 - - 所有副本合并 updates 后以新配置重启（新 worker_id）。 - - 本方法只负责"下发 + 汇总"，写操作由 worker 访问其自身缓存触发执行，故进程插件 - 位于不同路径/宿主机时也能把配置持久化到各自的共享缓存后端。 - - Args: - worker_type: 插件类型 - updates: 待写入 [workers.config] 的键值（{"key": value}） - timeout: 单副本下发超时（秒） - - Returns: - { - "total": 下发副本数, - "failed": 失败数, "ok": bool, - "results": [{worker_id, ok, reply|error}], - "errors": [错误信息列表] - } - - Raises: - RuntimeError: 该类型无任何存活副本 - """ - candidates = [ - w for w in self.list_workers() - if w["worker_type"] == worker_type - ] - if not candidates: - raise RuntimeError(f"类型 '{worker_type}' 没有可用 Worker，无法更新其配置") - - results: List[Dict[str, Any]] = [] - errors: List[str] = [] - for w in candidates: - try: - reply = await self._send_admin_cmd( - worker_type, w["worker_id"], "config_set", - timeout=timeout, data_extra={"config": updates}, - ) - ok = reply.get("status") == "accepted" - if not ok: - errors.append( - f"{w['worker_id']}: {reply.get('error', '未知错误')}" - ) - results.append({ - "worker_id": w["worker_id"], "ok": ok, "reply": reply, - }) - except Exception as e: # noqa: BLE001 - 单副本失败不中断其他副本 - errors.append(f"{w['worker_id']}: {e}") - results.append({ - "worker_id": w["worker_id"], "ok": False, "error": str(e), - }) - - return { - "total": len(results), - "failed": len(errors), - "ok": not errors, - "results": results, - "errors": errors, - } diff --git a/src/chongming/app/routes/__init__.py b/src/chongming/app/routes/__init__.py deleted file mode 100644 index 194527a..0000000 --- a/src/chongming/app/routes/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .stats import router as stats_router -from .worker_admin import router as worker_admin_router -from .auth import router as auth_router - -__all__ = ["stats_router", "worker_admin_router", "auth_router"] \ No newline at end of file diff --git a/src/chongming/app/routes/auth.py b/src/chongming/app/routes/auth.py deleted file mode 100644 index f6681cb..0000000 --- a/src/chongming/app/routes/auth.py +++ /dev/null @@ -1,53 +0,0 @@ -"""内置登录端点 —— auth 域的基础设施部分（管理平面可用性底线）。 - -拆分自 plugins.auth：登录 / token 签发不再依赖 auth 插件是否加载， -auth 插件缩容到 0 或全部副本故障时，登录与 admin API 仍可用； -用户管理功能（注册 / 资料 / 改密）仍由 auth 插件提供。 - -共享契约（Users 模型 / claims / 失败文案）来自 chongming_auth 包， -路径与响应 shape 与拆分前完全一致（前端零改动）。 -""" -import time - -from chongming_auth import ( - DEFAULT_TOKEN_EXPIRES_SECONDS, - LoginPostRequest, - LoginPostResponse, - issue_token, - touch_last_login, - verify_login, -) -from chongming_config import get_config -from fastapi import APIRouter, Request - -config = get_config() - -router = APIRouter(prefix=config["default"]["prefix"], tags=["auth"]) - - -@router.post( - "/auth/login", - response_model=LoginPostResponse, - summary="登录", - description="根据用户名和密码进行登录，登录成功返回token，获取请求需带式token进行请求", -) -async def login(body: LoginPostRequest, request: Request) -> LoginPostResponse: - db = request.app.state.auth_db - user, message = await verify_login(db, body.username, body.password) - if user is None: - return LoginPostResponse( - result="fail", token="", message=message, timestamp=time.time() - ) - - await touch_last_login(db, user) - - security = get_config()["security"] - token = issue_token( - user, - security["secret_key"], - security["algorithm"], - expires_in_seconds=DEFAULT_TOKEN_EXPIRES_SECONDS, - ) - return LoginPostResponse( - result="success", token=token, message="登录成功", timestamp=time.time() - ) diff --git a/src/chongming/app/routes/stats.py b/src/chongming/app/routes/stats.py deleted file mode 100644 index 637ff42..0000000 --- a/src/chongming/app/routes/stats.py +++ /dev/null @@ -1,23 +0,0 @@ -# src/chongming/app/routes/stats.py -""" -网关级调用统计查看端点 - -数据来源为 core.call_stats 的内存统计（进程内视角）： -动态路由每次调用在后台任务中更新计数/延迟聚合/最近调用缓冲， -此处提供只读快照，便于观测接口调用量、错误分布与延迟。 -""" -from fastapi import APIRouter - -from chongming_config import get_config - -from ..core import call_stats - -config = get_config() - -router = APIRouter(prefix=config["default"]["prefix"], tags=["stats"]) - - -@router.get("/stats", summary="调用统计") -async def get_call_stats(): - """返回内存调用统计快照：总量、per worker/subject/status 计数、延迟聚合、最近调用""" - return call_stats.summary() diff --git a/src/chongming/app/routes/worker_admin.py b/src/chongming/app/routes/worker_admin.py deleted file mode 100644 index 6fb7c55..0000000 --- a/src/chongming/app/routes/worker_admin.py +++ /dev/null @@ -1,329 +0,0 @@ -""" -Worker 管理端点（插件 / 单副本重启） - -提供插件实例盘点与"精细化重启单个 worker 副本"的能力： - - 经 Zenoh 下发管理指令到目标的 admin Queryable（worker///__admin__） - - 仅超级管理员（JWT payload.is_superuser）可访问 - -协议说明（见 chongming_worker.runtime 单副本管理协议）： - - 每个 worker 副本在 worker///__admin__ 注册内部 Queryable - - {"data": {"cmd": "restart"}} → 副本平滑退出并以新 worker_id 重建 - - {"data": {"cmd": "stop"}} → 副本平滑退出且不自动重建 -""" -from fastapi import APIRouter, Depends, HTTPException, Request -from pydantic import BaseModel, Field -import pathlib -import tomllib - -from chongming_config import get_config - -from ..core.auth import jwt_auth_dependency -from ..core.logger import get_logger - -config = get_config() - -logger = get_logger("worker_admin") - -router = APIRouter(prefix=config["default"]["prefix"], tags=["worker_admin"]) - - -def require_admin( - payload: dict = Depends(jwt_auth_dependency), -) -> dict: - """仅超级管理员可操作（重启属高风险操作，普通登录用户无权限）。""" - if not payload.get("is_superuser"): - raise HTTPException( - status_code=403, - detail="需要超级管理员权限 (is_superuser)", - ) - return payload - - -def _get_manager(request: Request): - """从应用状态获取 WorkerManager；未初始化时（lifespan 外）报 503。""" - manager = getattr(request.app.state, "worker_manager", None) - if manager is None: - raise HTTPException(status_code=503, detail="WorkerManager 尚未就绪") - return manager - - -@router.get("/admin/workers", summary="盘点 Worker 实例") -async def list_workers( - request: Request, - _admin: dict = Depends(require_admin), -): - """返回全部插件实例（worker_type / worker_id / mode / alive），供前端渲染与选择。""" - manager = _get_manager(request) - return {"workers": manager.list_workers(), "success": True} - - -@router.post("/admin/workers/restart", summary="重启单个 Worker 副本") -async def restart_worker( - request: Request, - worker_type: str, - worker_id: str, - _admin: dict = Depends(require_admin), -): - """精细化重启单个 worker 副本：经 Zenoh 下发 restart，副本以新 worker_id 重建。 - - - worker_type: 插件类型（如 example） - - worker_id: 副本 id（如 worker-0-1-12345），由 GET /admin/workers 提供 - """ - manager = _get_manager(request) - try: - result = await manager.restart_worker(worker_type, worker_id) - except RuntimeError as e: - logger.warning( - "重启 Worker 副本失败: %s/%s 用户=%s 错误=%s", - worker_type, worker_id, - _admin.get("username", _admin.get("sub", "-")), - e, - ) - raise HTTPException( - status_code=502, - detail=f"重启失败: {e}", - ) - logger.info( - "重启 Worker 副本: %s/%s 用户=%s 结果=%s", - worker_type, worker_id, - _admin.get("username", _admin.get("sub", "-")), - result, - ) - return {"success": True, "result": result} - - -@router.post("/admin/workers/{worker_type}/restart", summary="重启整个插件（全部副本）") -async def restart_all( - request: Request, - worker_type: str, - _admin: dict = Depends(require_admin), -): - """重启某插件的全部副本（逐个下发 restart，各自以新 id 重建）。 - - 注意：网关进程内加载的 inline 插件副本经本接口重启，仅重建 worker 协程， - 不会卸载插件；external 插件同理逐个副本重启。 - """ - manager = _get_manager(request) - candidates = [ - w for w in manager.list_workers() - if w["worker_type"] == worker_type - ] - if not candidates: - raise HTTPException(status_code=404, detail=f"无类型为 {worker_type} 的 worker") - results = [] - failed = [] - for w in candidates: - try: - res = await manager.restart_worker(worker_type, w["worker_id"]) - results.append({"worker_id": w["worker_id"], "ok": True, "result": res}) - except Exception as e: # noqa: BLE001 - 批量重启不因单个失败中断 - failed.append({"worker_id": w["worker_id"], "ok": False, "error": str(e)}) - results.append({"worker_id": w["worker_id"], "ok": False, "error": str(e)}) - logger.info( - "重启插件全部副本: %s (成功 %d/%d) 用户=%s", - worker_type, len(results) - len(failed), len(results), - _admin.get("username", _admin.get("sub", "-")), - ) - return {"success": not failed, "total": len(results), "failed": len(failed), "results": results} - - -class ScaleBody(BaseModel): - """副本数量调整请求体：目标副本数。""" - count: int = Field(..., ge=0, description="目标副本数（>=0）") - - -@router.post("/admin/workers/{worker_type}/scale", summary="调整同类 worker 副本数量（扩缩容）") -async def scale_workers( - request: Request, - worker_type: str, - body: ScaleBody, - _admin: dict = Depends(require_admin), -): - """把某类型的副本数调整为 body.count 并持久化到配置。 - - - 扩容新增副本；缩容随机关闭副本，并**优雅排空**（摘除 liveliness/业务 - Queryable 隔绝新请求，等待正在处理的在途请求完成后再关闭）。 - - inline 直接调整本进程 Worker 池；external 经 Zenoh 下发 rescale 到存活副本， - 由其进程内池扩缩并写回自身配置。 - """ - manager = _get_manager(request) - try: - result = await manager.scale_workers(worker_type, body.count) - except Exception as e: # noqa: BLE001 - 无副本/失败映射为 4xx - logger.warning( - "调整副本数量失败: %s -> %d 用户=%s 错误=%s", - worker_type, body.count, - _admin.get("username", _admin.get("sub", "-")), - e, - ) - raise HTTPException(status_code=404, detail=str(e)) - logger.info( - "调整副本数量: %s -> %d 用户=%s 结果=%s", - worker_type, body.count, - _admin.get("username", _admin.get("sub", "-")), - result, - ) - return {"success": True, "result": result} - - -class ScaleInstanceBody(BaseModel): - """逐实例精细扩缩请求体：目标实例 + 该实例的目标副本数。""" - instance_id: str = Field(..., description="外部进程/实例身份（list_instances 返回）") - count: int = Field(..., ge=0, description="该实例的目标副本数（绝对数，>=0）") - - -@router.get( - "/admin/workers/{worker_type}/instances", - summary="按实例分组列出某类型 worker（精细扩缩寻址）", -) -async def list_worker_instances( - request: Request, - worker_type: str, - _admin: dict = Depends(require_admin), -): - """把某类型的在线副本按「外部进程实例」分组返回。 - - 每个实例 = 一个可独立承载副本的执行单元（inline=本网关进程/host， - external=独立进程/容器），含当前副本数与可选中的 worker_id。 - """ - manager = _get_manager(request) - try: - instances = await manager.list_instances(worker_type) - except Exception as e: # noqa: BLE001 - logger.warning("列出实例失败: %s 错误=%s", worker_type, e) - raise HTTPException(status_code=404, detail=str(e)) - return {"success": True, "result": instances} - - -@router.post( - "/admin/workers/{worker_type}/scale_instance", - summary="把某实例精确扩缩到指定副本数（精细扩缩容）", -) -async def scale_worker_instance( - request: Request, - worker_type: str, - body: ScaleInstanceBody, - _admin: dict = Depends(require_admin), -): - """把指定外部进程/**实例**精确调整到 body.count 个副本（绝对数），不影响其它实例。 - - 经该实例任一副本下发 rescale{scale, instance_id, absolute:true}，由其进程内 - WorkerPool 直接扩缩并记入逐实例覆盖表；重启后仍保持该绝对数。 - """ - manager = _get_manager(request) - try: - result = await manager.scale_instance( - worker_type, body.instance_id, body.count - ) - except Exception as e: # noqa: BLE001 - logger.warning( - "逐实例扩缩失败: %s instance=%s -> %d 用户=%s 错误=%s", - worker_type, body.instance_id, body.count, - _admin.get("username", _admin.get("sub", "-")), - e, - ) - raise HTTPException(status_code=404, detail=str(e)) - logger.info( - "逐实例扩缩: %s instance=%s -> %d 用户=%s 结果=%s", - worker_type, body.instance_id, body.count, - _admin.get("username", _admin.get("sub", "-")), - result, - ) - return {"success": True, "result": result} - - -class ConfigUpdateBody(BaseModel): - """配置更新请求体：写目标插件 config.toml 的 [workers.config] 段。""" - updates: dict = Field(..., description="待写入 [workers.config] 的键值（路由/加载控制键除外）") - - -@router.get("/admin/workers/{worker_type}/config", summary="读取插件现有运行时配置") -async def read_worker_config( - request: Request, - worker_type: str, - _admin: dict = Depends(require_admin), -): - """读取某插件 config.toml 的 [workers.config] 现有值（供前端展示）。 - - 经 Zenoh 向该类任一存活副本下发 get_config，由其访问自身缓存（或回退 toml 种子） - 返回 —— 网关不直接接触 external worker 的缓存后端。无存活副本 / 旧版运行时 - 不支持 get_config 时，回退回直读 config/workers//config.toml 的 [workers].config。 - 路由配置（add.toml 等）不在此范围。 - - 响应中的 source/mode 供前端如实标注展示值的来路（避免把回退读到的 toml 值当成缓存值）： - - source="cache"：值来自运行时缓存；source="toml"：缓存缺失或副本未响应，值取自文件； - - mode：该类插件的配置优先来源模式（toml 模式以文件为准、cache 模式以缓存为准）。 - - - worker_type: 插件类型（如 example） - """ - manager = _get_manager(request) - try: - info = await manager.get_worker_config(worker_type) - cfg = info.get("config") - if isinstance(cfg, dict) and info.get("status") in ("ok", None): - return { - "worker_type": worker_type, - "worker_id": info.get("worker_id"), - "source": info.get("source", "cache"), - "mode": info.get("mode") or manager.config_mode, - "config": cfg, - "success": True, - } - logger.debug("worker get_config 未返回有效配置（%s），回退读 toml", info) - except Exception as e: # noqa: BLE001 - 副本未响应等一律回退读 toml - logger.debug("worker get_config 查询失败，回退读 toml: %s", e) - - path = pathlib.Path("config/workers") / worker_type / "config.toml" - if not path.exists(): - raise HTTPException(status_code=404, detail=f"插件 {worker_type} 的配置文件不存在: {path}") - with open(path, "rb") as f: - data = tomllib.load(f) - workers = data.get("workers") or {} - cfg = workers.get("config") - return { - "worker_type": worker_type, - "config": cfg if isinstance(cfg, dict) else {}, - "source": "toml", - "mode": manager.config_mode, - "success": True, - } - - -@router.post("/admin/workers/{worker_type}/config", summary="更新插件运行时配置（下发至 worker）") -async def update_worker_config( - request: Request, - worker_type: str, - body: ConfigUpdateBody, - _admin: dict = Depends(require_admin), -): - """更新某插件的运行时配置 `[workers.config]`（路由配置 add.toml 等除外）。 - - 流程：网关经 Zenoh 下发 config_set 给该类型全部副本 → 由 worker 触发写回各自 - 配置文件（仅 index0 落盘避免同文件并发写）→ 写成功则以新配置重启所有对应副本； - 任一失败则返回失败明细（error 信息）。 - - - worker_type: 插件类型（如 example） - - body.updates: 待更新的键值，如 {"data_dir": "./data2"} - """ - manager = _get_manager(request) - if not body.updates: - raise HTTPException(status_code=422, detail="updates 不能为空") - try: - result = await manager.update_worker_config(worker_type, body.updates) - except Exception as e: # noqa: BLE001 - 无副本等运行时错误映射为 4xx - logger.warning( - "更新插件配置失败: %s 用户=%s 错误=%s", - worker_type, - _admin.get("username", _admin.get("sub", "-")), - e, - ) - raise HTTPException(status_code=404, detail=str(e)) - logger.info( - "更新插件配置: %s\n updates=%s\n 结果=成功 %d/%d (失败: %s) 用户=%s", - worker_type, - body.updates, - result["total"] - result["failed"], result["total"], - result["errors"], - _admin.get("username", _admin.get("sub", "-")), - ) - return result \ No newline at end of file diff --git a/test/bench_http.py b/test/bench_http.py index 6735be6..19cfe1f 100644 --- a/test/bench_http.py +++ b/test/bench_http.py @@ -6,7 +6,7 @@ HTTP 端到端 RPC 压测脚本（chongming 网关全链路） 前置启动（均后台运行，需在仓库根目录）: 1. Zenoh router: uv run python utils/worker-rs/scripts/zenoh_router.py 2. 网关(无热重载): uv run uvicorn chongming.app:app --host 0.0.0.0 --port 8000 --no-access-log - 3. Worker: uv run worker_serve --type demo --count 2 + 3. Worker: uv run chongming-worker --type demo --count 2 运行: uv run python test/bench_http.py --concurrency 1,10,50,100,200 --duration 30 diff --git a/test/test_call_stats.py b/test/test_call_stats.py index d723909..42026fc 100644 --- a/test/test_call_stats.py +++ b/test/test_call_stats.py @@ -23,7 +23,7 @@ import orjson from fastapi import FastAPI REPO_ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(REPO_ROOT / "src")) +sys.path.insert(0, str(REPO_ROOT / "gateway" / "src")) sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src")) # ---- 依赖隔离：桩掉 chongming 包链 / auth / chongming_cache / worker_manager ---- @@ -47,7 +47,7 @@ for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub, _wm_stub): sys.modules[mod.__name__] = mod # 加载真实 call_stats（route_generator 依赖其 spawn/record_call） -_cs_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "call_stats.py" +_cs_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "call_stats.py" _cs_spec = importlib.util.spec_from_file_location("chongming.app.core.call_stats", _cs_path) _cs_mod = importlib.util.module_from_spec(_cs_spec) sys.modules["chongming.app.core.call_stats"] = _cs_mod @@ -59,7 +59,7 @@ _cs_spec.loader.exec_module(_cs_mod) # type: ignore[union-attr] _cs_mod._AUDIT_SAMPLE_RATE = 1.0 # 加载真实 route_generator -_rg_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "route_generator.py" +_rg_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "route_generator.py" _rg_spec = importlib.util.spec_from_file_location("chongming.app.core.route_generator", _rg_path) _rg_mod = importlib.util.module_from_spec(_rg_spec) sys.modules["chongming.app.core.route_generator"] = _rg_mod diff --git a/test/test_inline_config_align.py b/test/test_inline_config_align.py index af32881..40b9af5 100644 --- a/test/test_inline_config_align.py +++ b/test/test_inline_config_align.py @@ -13,7 +13,7 @@ import os import sys REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "src")) +sys.path.insert(0, os.path.join(REPO, "gateway", "src")) os.chdir(REPO) # 网关模块导入期读根 config.toml / 日志配置 from chongming.app.core.worker_manager import _CONFIGS_KEY, WorkerManager # noqa: E402 diff --git a/test/test_inline_leader.py b/test/test_inline_leader.py index 875886a..9949b6b 100644 --- a/test/test_inline_leader.py +++ b/test/test_inline_leader.py @@ -18,9 +18,9 @@ import sys import tempfile from pathlib import Path -_HERE = Path(__file__).resolve().parent # chongming-zenoh/ +_HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/ _PROJ = _HERE -for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"): +for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"): sys.path.insert(0, str(_p)) from chongming_cache import Cache diff --git a/test/test_inline_local_dispatch.py b/test/test_inline_local_dispatch.py index de8cf12..31d01ca 100644 --- a/test/test_inline_local_dispatch.py +++ b/test/test_inline_local_dispatch.py @@ -19,7 +19,8 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent # 优先本地框架 checkout（与 test_stream_worker.py 同约定） sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src")) -sys.path.insert(0, str(REPO_ROOT / "src")) +sys.path.insert(0, str(REPO_ROOT / "gateway" / "src")) +sys.path.insert(0, str(REPO_ROOT / "src")) # plugins 插件源码 import orjson # noqa: E402 import zenoh # noqa: E402 diff --git a/test/test_instance_targeting.py b/test/test_instance_targeting.py index 319af7a..7c56c4f 100644 --- a/test/test_instance_targeting.py +++ b/test/test_instance_targeting.py @@ -18,9 +18,9 @@ import asyncio import sys from pathlib import Path -_HERE = Path(__file__).resolve().parent # chongming-zenoh/ +_HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/ _PROJ = _HERE -for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"): +for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"): sys.path.insert(0, str(_p)) from chongming.app.core.worker_manager import WorkerManager # noqa: E402 diff --git a/test/test_instance_targeting_http.py b/test/test_instance_targeting_http.py index d0eb8c9..29b109e 100644 --- a/test/test_instance_targeting_http.py +++ b/test/test_instance_targeting_http.py @@ -23,7 +23,7 @@ from fastapi.testclient import TestClient TEST_DIR = Path(__file__).resolve().parent REPO_ROOT = TEST_DIR.parent -sys.path.insert(0, str(REPO_ROOT / "src")) +sys.path.insert(0, str(REPO_ROOT / "gateway" / "src")) sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src")) # ---- 依赖隔离：桩掉 chongming 包链 / auth / chongming_cache ---- @@ -43,7 +43,7 @@ _cache_stub.DistributedLock = object # worker_manager 顶层导入（本测试 for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub): sys.modules[mod.__name__] = mod -_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py" +_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py" _wm_spec = importlib.util.spec_from_file_location( "chongming.app.core.worker_manager", _wm_path ) @@ -51,7 +51,7 @@ _wm_mod = importlib.util.module_from_spec(_wm_spec) sys.modules["chongming.app.core.worker_manager"] = _wm_mod _wm_spec.loader.exec_module(_wm_mod) # type: ignore[union-attr] -_cs_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "call_stats.py" +_cs_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "call_stats.py" _cs_spec = importlib.util.spec_from_file_location( "chongming.app.core.call_stats", _cs_path ) @@ -59,7 +59,7 @@ _cs_mod = importlib.util.module_from_spec(_cs_spec) sys.modules["chongming.app.core.call_stats"] = _cs_mod _cs_spec.loader.exec_module(_cs_mod) # type: ignore[union-attr] -_rg_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "route_generator.py" +_rg_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "route_generator.py" _rg_spec = importlib.util.spec_from_file_location( "chongming.app.core.route_generator", _rg_path ) diff --git a/test/test_jwt_cache.py b/test/test_jwt_cache.py index 9eba72f..5c227fb 100644 --- a/test/test_jwt_cache.py +++ b/test/test_jwt_cache.py @@ -17,7 +17,7 @@ import tempfile import time from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent / "src")) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "gateway" / "src")) sys.path.insert(0, str(Path(__file__).resolve().parent / "utils" / "worker" / "src")) import orjson diff --git a/test/test_logging_async.py b/test/test_logging_async.py index aa13ecf..f2b71ac 100644 --- a/test/test_logging_async.py +++ b/test/test_logging_async.py @@ -17,7 +17,7 @@ import tempfile import time from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "gateway" / "src")) sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "utils" / "worker" / "src")) _tmp = tempfile.TemporaryDirectory() @@ -25,7 +25,7 @@ _tmp = tempfile.TemporaryDirectory() # 预加载真实 logger 模块（不触发 chongming 包链导入），把日志目录重定向到 # 临时目录，之后 import chongming.app 时 get_logger 即用新路径 _logger_path = ( - Path(__file__).resolve().parent.parent / "src" / "chongming" / "app" / "core" / "logger.py" + Path(__file__).resolve().parent.parent / "gateway" / "src" / "chongming" / "app" / "core" / "logger.py" ) _logger_spec = importlib.util.spec_from_file_location("chongming.app.core.logger", _logger_path) logger_mod = importlib.util.module_from_spec(_logger_spec) diff --git a/test/test_mqtt_auth.py b/test/test_mqtt_auth.py index 8ef4a73..aa67e14 100644 --- a/test/test_mqtt_auth.py +++ b/test/test_mqtt_auth.py @@ -23,8 +23,8 @@ sys.path.insert(0, str(Path(__file__).parent)) from amqtt.client import MQTTClient from amqtt.errors import ConnectError -from src.chongming.app.core.lifespan_setup import _run_mqtt_broker -from src.chongming.app.core.mqtt_bridge import MQTTBridge +from chongming.app.core.lifespan_setup import _run_mqtt_broker +from chongming.app.core.mqtt_bridge import MQTTBridge HOST = "127.0.0.1" USERNAME = "cm_user" diff --git a/test/test_mqtt_bridge_heal.py b/test/test_mqtt_bridge_heal.py index 5eab698..8808edf 100644 --- a/test/test_mqtt_bridge_heal.py +++ b/test/test_mqtt_bridge_heal.py @@ -19,9 +19,9 @@ import tempfile from pathlib import Path from types import SimpleNamespace -_HERE = Path(__file__).resolve().parent # chongming-zenoh/ +_HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/ _PROJ = _HERE -for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"): +for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"): sys.path.insert(0, str(_p)) import chongming.app.core.lifespan_setup as ls diff --git a/test/test_mqtt_broker_lock.py b/test/test_mqtt_broker_lock.py index 9127394..8ef9a6c 100644 --- a/test/test_mqtt_broker_lock.py +++ b/test/test_mqtt_broker_lock.py @@ -22,9 +22,9 @@ import time from pathlib import Path from types import SimpleNamespace -_HERE = Path(__file__).resolve().parent # chongming-zenoh/ +_HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/ _PROJ = _HERE -for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"): +for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"): sys.path.insert(0, str(_p)) import chongming.app.core.lifespan_setup as ls diff --git a/test/test_mqtt_custom_topics.py b/test/test_mqtt_custom_topics.py index 79336f1..d7df337 100644 --- a/test/test_mqtt_custom_topics.py +++ b/test/test_mqtt_custom_topics.py @@ -12,7 +12,7 @@ import sys from pathlib import Path from unittest.mock import MagicMock -sys.path.insert(0, str(Path(__file__).parent / "src")) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "gateway" / "src")) from chongming.app.core.mqtt_bridge import ( MQTTBridge, diff --git a/test/test_passthrough_optimization.py b/test/test_passthrough_optimization.py index d5441dc..01b7048 100644 --- a/test/test_passthrough_optimization.py +++ b/test/test_passthrough_optimization.py @@ -34,7 +34,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient REPO_ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(REPO_ROOT / "src")) +sys.path.insert(0, str(REPO_ROOT / "gateway" / "src")) sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src")) # ---- 依赖隔离：桩掉 chongming 包链 / auth / chongming_cache ---- @@ -55,7 +55,7 @@ for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub): sys.modules[mod.__name__] = mod # 加载真实 worker_manager（route_generator 依赖其 StreamIdleTimeoutError） -_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py" +_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py" _wm_spec = importlib.util.spec_from_file_location( "chongming.app.core.worker_manager", _wm_path ) @@ -64,7 +64,7 @@ sys.modules["chongming.app.core.worker_manager"] = _wm_mod _wm_spec.loader.exec_module(_wm_mod) # type: ignore[union-attr] # 加载真实 call_stats（route_generator 依赖其 spawn/record_call） -_cs_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "call_stats.py" +_cs_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "call_stats.py" _cs_spec = importlib.util.spec_from_file_location( "chongming.app.core.call_stats", _cs_path ) @@ -73,7 +73,7 @@ sys.modules["chongming.app.core.call_stats"] = _cs_mod _cs_spec.loader.exec_module(_cs_mod) # type: ignore[union-attr] # 加载真实 route_generator -_rg_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "route_generator.py" +_rg_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "route_generator.py" _rg_spec = importlib.util.spec_from_file_location( "chongming.app.core.route_generator", _rg_path ) diff --git a/test/test_process_mode_e2e.py b/test/test_process_mode_e2e.py index a5cf0db..edb3d09 100644 --- a/test/test_process_mode_e2e.py +++ b/test/test_process_mode_e2e.py @@ -6,7 +6,7 @@ 2. demo 进程插件（workers/demo/，load_mode = "external"） 本脚本从 demo 插件项目根（= 打包 zip 根）拉起/停止独立 worker 进程 -（worker_serve --type demo），验证: +（chongming-worker --type demo），验证: 1. worker 启动后 → HTTP 路由动态出现且跨进程 RPC 可调 2. worker 优雅退出 → 路由动态摘除（404） 3. worker 重启 → 路由恢复 diff --git a/test/test_route_registry.py b/test/test_route_registry.py index a554ad1..5908aa8 100644 --- a/test/test_route_registry.py +++ b/test/test_route_registry.py @@ -18,7 +18,7 @@ import zenoh from fastapi import FastAPI from chongming_worker.routes_publisher import RoutePublisher -from src.chongming.app.core.route_registry import RouteRegistry +from chongming.app.core.route_registry import RouteRegistry failures = [] diff --git a/test/test_stream_e2e_zenoh.py b/test/test_stream_e2e_zenoh.py index 9a6d226..1078563 100644 --- a/test/test_stream_e2e_zenoh.py +++ b/test/test_stream_e2e_zenoh.py @@ -26,7 +26,7 @@ import zenoh REPO_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src")) -sys.path.insert(0, str(REPO_ROOT / "src")) +sys.path.insert(0, str(REPO_ROOT / "gateway" / "src")) # ---- 网关依赖隔离（同 test_stream_gateway）：桩掉 chongming 包链与 chongming_cache ---- _pkg = types.ModuleType("chongming") @@ -43,7 +43,7 @@ _cache_stub.DistributedLock = object # worker_manager 顶层导入（本测试 for mod in (_pkg, _app, _core, _logger_stub, _cache_stub): sys.modules[mod.__name__] = mod -_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py" +_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py" _spec = importlib.util.spec_from_file_location( "chongming.app.core.worker_manager", _wm_path ) diff --git a/test/test_stream_gateway.py b/test/test_stream_gateway.py index f55da12..04e7c13 100644 --- a/test/test_stream_gateway.py +++ b/test/test_stream_gateway.py @@ -1,7 +1,7 @@ """ 网关 call_stream 单测（依赖隔离：桩掉 chongming 包链与 chongming_cache） -直接加载 src/chongming/app/core/worker_manager.py 模块（避开 chongming.app +直接加载 gateway/src/chongming/app/core/worker_manager.py 模块（避开 chongming.app 整链导入与 config.toml 依赖），验证: 1. 多块顺序产出（ConsolidationMode.NONE 逐块消费） 2. Worker 错误收尾块 {"error": ...}：转发后终止流 @@ -24,7 +24,7 @@ from types import SimpleNamespace import orjson import zenoh -REPO_ROOT = Path(__file__).resolve().parent +REPO_ROOT = Path(__file__).resolve().parent.parent # ---- 依赖隔离：桩掉网关包链（chongming.app 顶层导入 fastapi/uvicorn/config）---- _pkg = types.ModuleType("chongming") @@ -37,10 +37,11 @@ _logger_stub = types.ModuleType("chongming.app.core.logger") _logger_stub.get_logger = lambda name: logging.getLogger(name) _cache_stub = types.ModuleType("chongming_cache") _cache_stub.Cache = object # WorkerManager 仅作类型标注使用 +_cache_stub.DistributedLock = object # 同上（set_if_not_exists 用的分布式锁） for mod in (_pkg, _app, _core, _logger_stub, _cache_stub): sys.modules[mod.__name__] = mod -_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py" +_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py" _spec = importlib.util.spec_from_file_location( "chongming.app.core.worker_manager", _wm_path ) diff --git a/test/test_stream_route.py b/test/test_stream_route.py index d3f37e5..f0ca3de 100644 --- a/test/test_stream_route.py +++ b/test/test_stream_route.py @@ -1,7 +1,7 @@ """ 网关路由层流式端点单测（依赖隔离：桩掉 chongming 包链、auth 与 chongming_cache） -直接加载 src/chongming/app/core/route_generator.py，用 FastAPI TestClient 验证: +直接加载 gateway/src/chongming/app/core/route_generator.py，用 FastAPI TestClient 验证: 1. 流式 POST 端点：SSE 输出（media_type / 逐块 data: 帧 / 响应头） 2. 流式 GET 端点：query 参数正确透传到 payload 3. 初始错误（无可用 worker）-> HTTP 503 @@ -22,7 +22,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient REPO_ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(REPO_ROOT / "src")) +sys.path.insert(0, str(REPO_ROOT / "gateway" / "src")) sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src")) # ---- 依赖隔离：桩掉 chongming 包链 / auth / chongming_cache ---- @@ -43,7 +43,7 @@ for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub): sys.modules[mod.__name__] = mod # 加载真实 worker_manager（route_generator 依赖其 StreamIdleTimeoutError） -_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py" +_wm_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "worker_manager.py" _wm_spec = importlib.util.spec_from_file_location( "chongming.app.core.worker_manager", _wm_path ) @@ -52,7 +52,7 @@ sys.modules["chongming.app.core.worker_manager"] = _wm_mod _wm_spec.loader.exec_module(_wm_mod) # type: ignore[union-attr] # 加载真实 call_stats（route_generator 依赖其 spawn/record_call） -_cs_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "call_stats.py" +_cs_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "call_stats.py" _cs_spec = importlib.util.spec_from_file_location( "chongming.app.core.call_stats", _cs_path ) @@ -61,7 +61,7 @@ sys.modules["chongming.app.core.call_stats"] = _cs_mod _cs_spec.loader.exec_module(_cs_mod) # type: ignore[union-attr] # 加载真实 route_generator -_rg_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "route_generator.py" +_rg_path = REPO_ROOT / "gateway" / "src" / "chongming" / "app" / "core" / "route_generator.py" _rg_spec = importlib.util.spec_from_file_location( "chongming.app.core.route_generator", _rg_path ) diff --git a/test/verify_example_plugin.py b/test/verify_example_plugin.py index f956d6a..cd96331 100644 --- a/test/verify_example_plugin.py +++ b/test/verify_example_plugin.py @@ -2,7 +2,7 @@ import sys from pathlib import Path -SRC = Path(__file__).parent / "src" +SRC = Path(__file__).resolve().parent.parent / "src" # 仓库 src/（plugins 包源码所在） sys.path.insert(0, str(SRC)) failures = [] diff --git a/uv.lock b/uv.lock index 57bb56a..21cb6bd 100644 --- a/uv.lock +++ b/uv.lock @@ -483,7 +483,7 @@ wheels = [ [[package]] name = "chongming-cli" -version = "0.3.2" +version = "0.3.5" source = { registry = "http://124.71.68.6:3000/api/packages/chongming/pypi/simple/" } dependencies = [ { name = "chongming-config" }, @@ -492,9 +492,8 @@ dependencies = [ { name = "sqlite-vfs" }, { name = "tomli" }, ] -sdist = { url = "http://124.71.68.6:3000/api/packages/chongming/pypi/files/chongming-cli/0.3.2/chongming_cli-0.3.2.tar.gz", hash = "sha256:af02210a9fa124eda0533dcb7c833c985e629fc520a70c6fffeb87ee461a1659" } wheels = [ - { url = "http://124.71.68.6:3000/api/packages/chongming/pypi/files/chongming-cli/0.3.2/chongming_cli-0.3.2-py3-none-any.whl", hash = "sha256:57a286c33f364082aa43a71500ec0c918111327053def1d75d36bb3b3acc3eb6" }, + { url = "http://124.71.68.6:3000/api/packages/chongming/pypi/files/chongming-cli/0.3.5/chongming_cli-0.3.5-py3-none-any.whl", hash = "sha256:8c3be6353fff16920bf7691561c04ea353b22e3c26fe10f5e36e351977607e34" }, ] [[package]] @@ -524,6 +523,34 @@ wheels = [ { url = "http://124.71.68.6:3000/api/packages/chongming/pypi/files/chongming-database/0.2.1/chongming_database-0.2.1-py3-none-any.whl", hash = "sha256:64fa23f1d7221806fd3c71d3b6e29e7ffa6c2a35af84233688ca3b35196f0a30" }, ] +[[package]] +name = "chongming-gateway" +version = "0.1.0" +source = { registry = "http://124.71.68.6:3000/api/packages/chongming/pypi/simple/" } +dependencies = [ + { name = "amqtt" }, + { name = "chongming-auth" }, + { name = "chongming-cache" }, + { name = "chongming-config" }, + { name = "chongming-jwt" }, + { name = "chongming-worker" }, + { name = "eclipse-zenoh" }, + { name = "fastapi" }, + { name = "granian", extra = ["reload", "uvloop"] }, + { name = "gunicorn" }, + { name = "httptools" }, + { name = "orjson" }, + { name = "pwdlib" }, + { name = "pydantic" }, + { name = "sqlite-vfs" }, + { name = "starlette" }, + { name = "uvicorn" }, + { name = "uvloop" }, +] +wheels = [ + { url = "http://124.71.68.6:3000/api/packages/chongming/pypi/files/chongming-gateway/0.1.0/chongming_gateway-0.1.0-py3-none-any.whl", hash = "sha256:2d0e32414be400d8bf91d2bcbd3d2353013b3ebfffa08f17f42d1da3909bd8be" }, +] + [[package]] name = "chongming-jwt" version = "0.1.0" @@ -615,31 +642,20 @@ wheels = [ [[package]] name = "chongming-zenoh" version = "0.1.0" -source = { editable = "." } +source = { virtual = "." } dependencies = [ { name = "aiofiles" }, - { name = "amqtt" }, { name = "bytedance" }, - { name = "chongming-auth" }, - { name = "chongming-cache" }, - { name = "chongming-jwt" }, + { name = "chongming-gateway" }, { name = "chongming-mqtt" }, { name = "chongming-password" }, { name = "chongming-request" }, { name = "chongming-stream-manager" }, - { name = "chongming-worker" }, - { name = "eclipse-zenoh" }, - { name = "fastapi" }, - { name = "granian", extra = ["reload", "uvloop"] }, - { name = "gunicorn" }, - { name = "httptools" }, { name = "module-bank" }, { name = "openai" }, { name = "python-multipart" }, - { name = "sqlite-vfs" }, { name = "tomli" }, { name = "typing-extensions" }, - { name = "uvicorn" }, { name = "uvloop" }, { name = "volcengine-python-sdk", extra = ["ark"] }, ] @@ -648,7 +664,8 @@ dependencies = [ dev = [ { name = "chongming-cli" }, { name = "greenlet" }, - { name = "nuitka" }, + { name = "nuitka", extra = ["onefile"] }, + { name = "patchelf", marker = "sys_platform == 'linux'" }, { name = "pyarmor" }, { name = "pyinstaller" }, { name = "ruff" }, @@ -657,28 +674,17 @@ dev = [ [package.metadata] requires-dist = [ { name = "aiofiles" }, - { name = "amqtt" }, { name = "bytedance" }, - { name = "chongming-auth" }, - { name = "chongming-cache" }, - { name = "chongming-jwt" }, + { name = "chongming-gateway" }, { name = "chongming-mqtt" }, { name = "chongming-password" }, { name = "chongming-request" }, { name = "chongming-stream-manager" }, - { name = "chongming-worker" }, - { name = "eclipse-zenoh" }, - { name = "fastapi", extras = ["uvicorn"] }, - { name = "granian", extras = ["reload", "uvloop"], specifier = ">=2.8.4" }, - { name = "gunicorn" }, - { name = "httptools" }, { name = "module-bank" }, { name = "openai" }, { name = "python-multipart" }, - { name = "sqlite-vfs" }, { name = "tomli" }, { name = "typing-extensions" }, - { name = "uvicorn" }, { name = "uvloop" }, { name = "volcengine-python-sdk", extras = ["ark"] }, ] @@ -687,7 +693,8 @@ requires-dist = [ dev = [ { name = "chongming-cli" }, { name = "greenlet", specifier = ">=3.5.6" }, - { name = "nuitka" }, + { name = "nuitka", extras = ["onefile"] }, + { name = "patchelf", marker = "sys_platform == 'linux'" }, { name = "pyarmor" }, { name = "pyinstaller" }, { name = "ruff" }, @@ -1331,6 +1338,11 @@ version = "4.3" source = { registry = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" } sdist = { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/fa/5f/ba7cb858da0c98c24f85c54454a5ea18f2b784f658a91531fa051b2f2ee9/nuitka-4.3.tar.gz", hash = "sha256:8b102c6bf30d9504e82e69674d396972729092b25cefa680e8fd05678fdfa30b", size = 4766934, upload-time = "2026-10-10T11:13:15.577Z" } +[package.optional-dependencies] +onefile = [ + { name = "zstandard" }, +] + [[package]] name = "openai" version = "3.29.0" @@ -1417,6 +1429,21 @@ wheels = [ { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "patchelf" +version = "0.19.1.0" +source = { registry = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" } +sdist = { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/5a/0b/691b49b40201a80cb09dbf2003eef0e36270ae07b00de99baa915df1d274/patchelf-0.19.1.0.tar.gz", hash = "sha256:8976fbdef7d3e461d623e703024b70db6b6e3308f7e389930f39a71a1e347a2c", size = 371787, upload-time = "2026-08-08T05:07:29.253Z" } +wheels = [ + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/83/03/29cd947cc94b2c91f5bd368c1b5d3469bec6b8ea538db67dcaf6e4ac065a/patchelf-0.19.1.0-py3-none-manylinux1_i686.manylinux_2_5_i686.musllinux_1_1_i686.whl", hash = "sha256:f16f44fbcf0da4f2d1dd841d49f849fe9ea1b8fb952d5c4e9b8234f09d3f149a", size = 624780, upload-time = "2026-08-08T05:07:20.401Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/e7/03/afd9bdd2ba5f96997196109f25dadbe5141bc6e70dffd85f3f84cc85d69c/patchelf-0.19.1.0-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:a8f6331ccf40c345507279f755f4a38c2cb00b9efda746fd43c17713cce0aba4", size = 549422, upload-time = "2026-08-08T05:07:21.583Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/cc/e6/1c78ad10adfc4b00a642ab518c246dd88dc826d641eb31faca5eaddc8b97/patchelf-0.19.1.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:b5c3dec03b89db2bb0df301d316360847eeaf72c6fb6ddde0227326655edabb4", size = 542031, upload-time = "2026-08-08T05:07:22.836Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/e3/b8/a8bcf0a627b688ed0798048e585f1c1c8dbef5070db3f6c30c6895d73ce9/patchelf-0.19.1.0-py3-none-manylinux2014_armv7l.manylinux_2_17_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:2ffad3734be73b83ac2f46709264a34dd2aee40a7dd5deccf84856684d606ec6", size = 487087, upload-time = "2026-08-08T05:07:24.03Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/a5/05/816af7bf3a51106e3558d32e10dc021a306d64f257fc0c404e4a9fcb1fba/patchelf-0.19.1.0-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:89760f542fac28c7798e5f56e66601a78ef20e088fc80a493b6a9161438b8dc5", size = 612621, upload-time = "2026-08-08T05:07:25.276Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/99/fc/48c07d0f74251946e62881a89ef105bed242e855928ac4f834a9bde85f73/patchelf-0.19.1.0-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.musllinux_1_1_s390x.whl", hash = "sha256:efed44482d740e86d84e3f40e0ee9876c4249c7ac1128b5b1639e31287fa14a4", size = 674133, upload-time = "2026-08-08T05:07:26.541Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/0c/f3/dffc8a228ff8a2a627c48c7356ae36eb6d0c2497ba549a98a3b5a55f24af/patchelf-0.19.1.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:c2311d99eac1b9c40f5acd673e3b3581cde320bc4647e7f997e5e063731fc438", size = 572865, upload-time = "2026-08-08T05:07:27.867Z" }, +] + [[package]] name = "pefile" version = "2024.8.26" @@ -2122,11 +2149,11 @@ wheels = [ [[package]] name = "truststore" -version = "0.10.4" +version = "0.10.5" source = { registry = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" } -sdist = { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +sdist = { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/ee/9f/c5201d42a484c061e528825fc8e2d565f5abd50a4ced6fb7d29c4ec99b2b/truststore-0.10.5.tar.gz", hash = "sha256:30d36967ccaded5cbb38d602c433f53600036c79d502f4533a49b60a03bbefcd", size = 28091, upload-time = "2026-10-12T22:27:31.808Z" } wheels = [ - { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/51/e9/3a7820be2bb0fe53b6bc9c3be26d3d1158004e4c3ab953aa6840b955b1e9/truststore-0.10.5-py3-none-any.whl", hash = "sha256:9aaaedaefaf06d8b206278cf8b5012bc897f485a874503501e12d776df78951c", size = 19017, upload-time = "2026-10-12T22:27:30.377Z" }, ] [[package]] @@ -2338,3 +2365,60 @@ sdist = { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/b3/8f/70 wheels = [ { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, ] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" } +sdist = { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +]