add first commit

This commit is contained in:
2026-07-21 15:39:47 +08:00
commit 37434cf207
26 changed files with 4378 additions and 0 deletions
+28
View File
@@ -0,0 +1,28 @@
# Python-generated files
__pycache__/
*.py[oc]
build/
build_nuitka/
dist/
wheels/
*.egg-info
# Virtual environments
.venv
data
logs
resources
cache_data
output.md
# plugins
src/plugins/
# chongming-web
src/chongming-web
# chonging-config
config
# chongming-utils
utils
+1
View File
@@ -0,0 +1 @@
3.12
View File
+95
View File
@@ -0,0 +1,95 @@
[default]
app.debug = false
app.description = "基于 FastAPI 的 chongming 打包脚手架"
app.name = "chongming"
app.version = "chongming"
env = "development"
prefix = "/api/v1"
upload_path = "uploads"
[nanio]
nanio_access_key = "minioadmin"
nanio_secret_key = "minioadmin"
storage_path = "./data/storage"
endpoint_url = "http://127.0.0.1:9000"
signature_version = "s3v4"
region_name = "us-east-1"
bucket = "chongming"
[zenoh]
json5_path = "./config/zenoh_config.json5"
[scheduler]
job_store_path = "scheduler_jobs.db"
[cache]
cache_store_path = "./cache_data"
[security]
access_token_expire_minutes = 1440
algorithm = "HS256"
max_sessions_per_user = 5
refresh_token_expire_days = 7
secret_key = "your-secret-key-change-in-production"
[development]
debug = true
env = "development"
[development.server]
access_log = false
host = "0.0.0.0"
port = 8000
reload = true
[development.logging]
file = "logs"
format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
level = "DEBUG"
[development.cors]
allow_credentials = true
allow_headers = ["*"]
allow_methods = ["*"]
allow_origins = ["*"]
[plugins_worker]
config_path = "./config/workers/config.toml"
[pyinstaller_build_import]
base = [
"utils", # public 骨架
"utils.launch",
"utils.config",
"zenoh",
"chongming_worker",
"chongming_request",
"server",
"module_bank", # 模块化打包工具
"sqlite_vfs",
"sqlite_vfs.core",
"fastapi", # web 应用依赖
"fastapi.staticfiles",
"fastapi.middleware.cors",
"uvicorn",
"apscheduler",
"diskcache",
"orjson",
"python_multipart",
"nanio",
"boto3",
"modelvault",
]
linux = [
"gunicorn.glogging",
"gunicorn.app",
"gunicorn.app.wsgiapp",
]
windows = []
[production.module_system]
path = [
["resources/app.mbank", "src/chongming/app", "app"],
["resources/plugins.mbank", "src/plugins", "plugins"],
]
type = "module-bank"
Binary file not shown.

After

Width:  |  Height:  |  Size: 232 KiB

+69
View File
@@ -0,0 +1,69 @@
[default]
app.debug = false
app.description = "基于 FastAPI 的 chongming 打包脚手架"
app.name = "chongming"
app.version = "0.1.0"
env = "production"
prefix = "/api/v1"
upload_path = "uploads"
[nanio]
nanio_access_key = "minioadmin"
nanio_secret_key = "minioadmin"
storage_path = "./data/storage"
endpoint_url = "http://127.0.0.1:9000"
signature_version = "s3v4"
region_name = "us-east-1"
bucket = "chongming"
[zenoh]
json5_path = "./config/zenoh_config.json5"
[scheduler]
job_store_path = "scheduler_jobs.db"
[cache]
cache_store_path = "./"
[security]
access_token_expire_minutes = 30
algorithm = "HS256"
max_sessions_per_user = 5
refresh_token_expire_days = 7
secret_key = "your-secret-key-change-in-production"
[production]
debug = false
env = "production"
[production.server]
access_log = false
host = "0.0.0.0"
port = 8000
reload = false
timeout_keep_alive = 65
workers = 4
[production.logging]
backup_count = 5 # 最多保存 10 个日志文件
file = "logs"
format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
level = "INFO"
max_size = 10485760 # 10MB
[production.cors]
allow_credentials = true
allow_headers = ["*"]
allow_methods = ["*"]
allow_origins = ["*"]
[production.module_system]
path = [
"resources/app.mbank",
"resources/plugins.mbank",
]
type = "module-bank"
[production.file_system]
path = "static.svfs"
type = "svfs"
+100
View File
@@ -0,0 +1,100 @@
# main.py - 修复 reload 问题
#!/usr/bin/env python3
"""
FastAPI chongming 示例
主入口文件,适配 chongming 打包系统
"""
import os
import sys
import platform
from utils.launch import launch
from multiprocessing import freeze_support
import server
use_shell = platform.system() == "Windows"
def app_run(app_config: dict, default_config: dict):
"""
应用运行函数 - 适配模块银行环境
Args:
app_config: 应用配置字典,包含服务器配置、环境设置等
default_config: 默认配置字典,包含应用名称、版本等基础信息
Returns:
None: 此函数不返回值,直接启动服务器并阻塞运行
"""
from app.core.logger import get_logger # type: ignore
logger = get_logger("app")
# 获取服务器配置
server_config = app_config.get("server", {})
host = server_config.get("host", "0.0.0.0")
port = server_config.get("port", 8000)
timeout_keep_alive = server_config.get("timeout_keep_alive", 5)
# 打包环境强制禁用重载
is_frozen = getattr(sys, "frozen", False)
reload = False if is_frozen else server_config.get("reload", False)
# 支持通过环境变量 APP_WORKERS 覆盖 worker 数量(方便嵌入式设备调优)
default_workers = server_config.get("workers", 4)
workers = int(os.environ.get("APP_WORKERS", str(default_workers)))
# 显示启动信息
env = app_config.get("env", "development")
logger.info("=" * 50)
logger.info(f"🚀 启动 {default_config.get('app.name', 'FastAPI 应用')}")
logger.info(f"📦 版本: {default_config.get('app.version', '1.0.0')}")
logger.info(f"🌍 环境: {env}")
logger.info(f"📍 地址: http://{host}:{port}")
logger.info(f"📚 文档: http://{host}:{port}/docs")
logger.info(f"🔧 调试: {'启用' if app_config.get('debug', False) else '禁用'}")
logger.info(f"🔄 重载: {'启用' if reload else '禁用'}")
logger.info(f"👥 Workers: {workers}")
logger.info("=" * 50)
# 根据操作系统选择不同的服务器启动方式
if use_shell:
import uvicorn
uvicorn.run(
"server:app",
host=host,
port=port,
reload=reload,
workers=workers,
log_level="debug" if app_config.get("debug", False) else "info",
access_log=False,
timeout_keep_alive=timeout_keep_alive,
)
else:
from gunicorn.app.wsgiapp import run
sys.argv = [
"gunicorn",
"server:app",
"-k",
"uvicorn.workers.UvicornWorker",
"-w",
str(workers),
"-b",
f"{host}:{port}",
]
run()
def main():
"""
主函数入口 - 初始化多进程支持并启动应用
Returns:
None: 此函数不返回值,调用launch函数启动应用
"""
freeze_support()
launch(app_run, "production")
if __name__ == "__main__":
main()
+21
View File
@@ -0,0 +1,21 @@
# server.py
import sys
from utils.launch import init_module_bank
from utils.config import ConfigLoader
# 加载配置文件并初始化模块银行
config = ConfigLoader("config.toml")
init_module_bank(config)
# 从数据库动态导入 app 模块
# 该模块存储在 SQLite 数据库中,通过自定义导入机制加载
try:
import app # type: ignore
app_instance = app.app
except ImportError as e:
print(f"无法从数据库导入 app 模块: {e}")
sys.exit(1)
# 暴露 ASGI 应用实例,供 uvicorn 服务器使用
app = app_instance
+64
View File
@@ -0,0 +1,64 @@
[build-system]
build-backend = "hatchling.build"
requires = ["hatchling"]
[project]
dependencies = [
"apscheduler",
"boto3",
"chongming-jwt",
"chongming-request",
"chongming-worker",
"diskcache",
"eclipse-zenoh",
"fastapi[uvicorn]",
"gunicorn",
"modelvault",
"module-bank",
"nanio",
"openai",
"orjson",
"python-multipart",
"sqlite-vfs",
"tomli",
"typing-extensions",
"uvicorn",
"volcengine-python-sdk[ark]>=5.0.41",
]
description = "基于 modules-bank 的 fastapi 打包脚手架"
name = "chongming-zenoh"
readme = "README.md"
requires-python = ">=3.12"
version = "0.1.0"
[dependency-groups]
dev = [
"nuitka",
"pyarmor",
"pyinstaller",
"ruff",
]
[tool.hatch.build.targets.wheel]
packages = ["src/chongming"]
[[tool.uv.index]]
url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
[tool.uv.workspace]
members = [
"utils/worker",
"utils/request",
"utils/jwt",
]
[tool.uv.sources]
chongming-request = { workspace = true }
chongming-worker = { workspace = true }
chongming-jwt = { workspace = true }
[project.scripts]
build = "chongming.scripts.build:build"
gen_models = "chongming.scripts.gen_models:gen_models"
gunicorn = "chongming:gunicorn_serve"
serve = "chongming:serve"
+60
View File
@@ -0,0 +1,60 @@
import uvicorn
from .app.core.config import load_config
from .app.core.logger import get_logger
def serve():
"""
使用 uvicorn 启动服务
从配置文件中加载服务器配置和日志配置,然后使用 uvicorn 启动应用服务。
该函数适用于开发环境或单进程部署场景。
Returns:
None: 该函数不返回值,会阻塞运行直到服务停止
"""
config = load_config()
env = config["default"]["env"]
server_config = config[env]["server"]
log_config = config[env]["logging"]
logger = get_logger("app")
logger.info(f"启动服务: {server_config}")
logger.info(f"启动日志: {log_config}")
uvicorn.run("chongming.app:app", **server_config)
def gunicorn_serve():
"""
使用 gunicorn 启动服务
通过 gunicorn 作为进程管理器启动应用,使用 UvicornWorker 作为 worker 类型,
支持多进程部署,适用于生产环境。
从配置文件中读取服务器配置,动态设置 gunicorn 的 worker 数量、绑定地址和端口等参数。
Returns:
None: 该函数不返回值,会阻塞运行直到服务停止
"""
import sys
from gunicorn.app.wsgiapp import run
config = load_config()
env = config["default"]["env"]
server_config = config[env]["server"]
# 构建 gunicorn 命令行参数并启动服务
sys.argv = [
"gunicorn",
"chongming.app:app",
"-k",
"uvicorn.workers.UvicornWorker",
"-w",
f"{server_config.get('workers', 4)}",
"-b",
f"{server_config.get('host', '0.0.0.0')}:{server_config.get('port', 8000)}",
]
run()
if __name__ == "__main__":
serve()
+190
View File
@@ -0,0 +1,190 @@
"""
chongming FastAPI 应用入口
重构为配置驱动的微服务网关:
- WorkerManager: 统一管理 Zenoh worker 存活检测、负载均衡、超时重试、缓存
- RouteGenerator: 根据配置自动生成 FastAPI 端点
- 插件工厂函数动态加载
"""
import time
import asyncio
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from fastapi.middleware.cors import CORSMiddleware
from .core.config import get_config
from .core.logger import get_logger
from .core.static_files import SVFSStaticFiles
from .core import lifespan_setup
from .routes import storage_router
config = get_config()
logger = get_logger("app")
try:
env = config["default"]["env"]
except KeyError: # pragma: no cover
raise ValueError("配置不存在")
@asynccontextmanager
async def lifespan(app: FastAPI):
# ---- 启动阶段 ----
# 1. nanio
lifespan_setup.setup_nanio(app)
await asyncio.sleep(1) # 等待 nanio 启动
# 2. S3
lifespan_setup.setup_s3(app)
# 3. Zenoh
session = lifespan_setup.setup_zenoh(app)
# 4. WorkerManager
manager = lifespan_setup.setup_worker_manager(app, session)
# 5. 加载 workers
worker_tasks = await lifespan_setup.load_workers(app, session, manager)
# 6. 注册路由
await lifespan_setup.register_routes(app, manager)
# 7. 短暂等待,确保 worker 状态同步
await asyncio.sleep(0.5)
# 打印 worker 就绪状态
worker_status = {
wtype: len(workers)
for wtype, workers in manager._workers_cache.items()
}
logger.info(f"Worker 就绪状态: {worker_status}")
# ---- 应用运行 ----
yield
# ---- 关闭阶段 ----
logger.info("正在关闭应用...")
# 1. 清理 workers(调用 cleanup)
await lifespan_setup.cleanup_workers(app)
# 2. 取消刷新任务
refresh_task = getattr(app.state, "refresh_task", None)
if refresh_task:
refresh_task.cancel()
try:
await refresh_task
except asyncio.CancelledError:
pass
# 3. 取消所有 worker 任务
for task in worker_tasks:
task.cancel()
if worker_tasks:
await asyncio.gather(*worker_tasks, return_exceptions=True)
# 4. 关闭 Zenoh 会话
session.close()
logger.info("Zenoh 会话已关闭")
# 5. 终止 nanio
await lifespan_setup.shutdown_nanio(app)
logger.info("应用已关闭")
cors = config[env].get("cors", {})
app = FastAPI(
title=config["default"]["app"]["name"],
version=config["default"]["app"]["version"],
description=config["default"]["app"]["description"],
docs_url="/docs" if config[env]["debug"] else None,
redoc_url="/redoc" if config[env]["debug"] else None,
openapi_url="/openapi.json" if config[env]["debug"] else None,
lifespan=lifespan,
)
app.add_middleware(CORSMiddleware, **cors)
@app.middleware("http")
async def access_log_middleware(request: Request, call_next):
"""
HTTP访问日志中间件,记录每个请求的详细信息。
记录的信息包括:
- 请求方法和路径
- 响应状态码
- 请求处理时长(毫秒)
- 客户端IP地址
参数:
request: FastAPI请求对象
call_next: 下一个中间件或路由处理函数
返回:
Response: 处理后的响应对象
"""
start = time.perf_counter()
response = await call_next(request)
duration = (time.perf_counter() - start) * 1000
client = request.client.host if request.client else "-"
logger.info(
f"{request.method} {request.url.path} status={response.status_code} "
f"duration={duration:.2f}ms client={client}",
extra={"status_code": response.status_code},
)
return response
@app.get("/", summary="根路径")
async def root():
"""
根路径端点,返回应用的基本信息。
返回:
dict: 包含应用名称、版本、文档地址和健康检查地址的字典
"""
return {
"message": f"欢迎使用 {config['default']['app']['name']}",
"version": config["default"]["app"]["version"],
"docs": "/docs" if config[env]["debug"] else None,
"health": f"{config['default']['prefix']}/health",
}
@app.exception_handler(Exception)
async def global_exception_handler(_: Request, exc: Exception):
"""
全局异常处理器,捕获并处理应用中未捕获的异常。
在调试模式下返回详细的异常信息,在生产模式下返回通用错误提示。
参数:
_: FastAPI请求对象(未使用)
exc: 捕获到的异常对象
返回:
JSONResponse: 包含错误信息的JSON响应,状态码为500
"""
logger.error(f"全局异常: {exc}")
return JSONResponse(
status_code=500,
content={
"error": "内部服务器错误",
"detail": str(exc) if config[env]["debug"] else "请查看服务器日志",
},
)
# 静态文件服务(可选)
if vfs_db_path := config[env].get("file_system", {}).get("path", None):
logger.info("静态文件已存在,将使用静态文件服务")
static_files_handler = SVFSStaticFiles(directory="/", vfs_db_path=vfs_db_path)
app.mount("/static", static_files_handler, name="static")
app.include_router(storage_router)
View File
+207
View File
@@ -0,0 +1,207 @@
import sys
from pathlib import Path
from typing import TypedDict, Literal, Optional, cast, List
# 根据Python版本选择合适的TOML解析库和Required类型
if sys.version_info >= (3, 11):
import tomllib
from typing import Required
else:
import tomli as tomllib
from typing_extensions import Required
# 设置 env 为 development 或 production
class DefaultApp(TypedDict):
"""应用默认配置结构"""
name: str
version: str
description: str
debug: bool
class Default(TypedDict):
"""系统默认配置结构,包含应用信息、环境、路径等基础配置"""
app: DefaultApp
env: Literal["production", "development"]
prefix: str
upload_path: str
class SqliteConnectionArgs(TypedDict, total=False):
"""SQLite数据库连接参数配置"""
timeout: int
check_same_thread: bool
class DatabaseSqlite(TypedDict, total=False):
"""SQLite数据库连接池及行为配置"""
echo: bool
pool_size: int
max_overflow: int
pool_timeout: int
pool_recycle: int
pool_pre_ping: bool
connect_args: SqliteConnectionArgs
class Database(TypedDict, total=False):
"""数据库配置结构,支持SQLite、MySQL、PostgreSQL三种数据库类型"""
type: Required[Literal["sqlite", "mysql", "postgresql"]]
url: Required[str]
generate_schemas: bool
sqlite: DatabaseSqlite
mysql: dict
postgresql: dict
database_path: str
class Security(TypedDict):
"""安全相关配置,包括JWT令牌和安全策略设置"""
secret_key: str
algorithm: Literal["HS256"]
access_token_expire_minutes: int
refresh_token_expire_days: int
max_sessions_per_user: int
class Server(TypedDict, total=False):
"""服务器运行配置,包含主机、端口、工作进程等设置"""
host: Required[str]
port: Required[int]
reload: bool
workers: int
access_log: bool
class Logging(TypedDict):
"""日志系统配置,定义日志级别、输出格式和文件管理策略"""
level: str
file: str
format: str
max_size: int
backup_count: int
class Cors(TypedDict, total=False):
"""跨域资源共享(CORS)配置"""
allow_origins: List[str]
allow_credentials: bool
allow_methods: List[str]
allow_headers: List[str]
class ModuleSystem(TypedDict, total=False):
"""模块系统配置,定义模块加载类型和路径"""
type: str
path: List[str]
class FileSystem(TypedDict, total=False):
"""文件系统配置,定义存储类型和路径"""
type: str
path: str
class Env(TypedDict, total=False):
"""环境特定配置结构,用于开发和生产环境的差异化配置"""
debug: Required[bool]
server: Required[Server]
logging: Required[Logging]
cors: Cors
module_system: ModuleSystem
file_system: FileSystem
class Scheduler(TypedDict):
"""任务调度器配置"""
job_store_path: str
class Cache(TypedDict):
"""缓存系统配置"""
cache_store_path: str
class Config(TypedDict):
"""完整配置结构,整合所有配置项,包含默认配置、数据库、调度器、缓存、安全和环境配置"""
default: Default
database: Database
scheduler: Scheduler
cache: Cache
security: Security
development: Env
production: Env
def load_config(config_path: Path = Path("config.toml")) -> Config:
"""
从TOML配置文件加载配置数据
Args:
config_path: 配置文件路径,默认为当前目录下的config.toml
Returns:
Config: 解析后的配置字典对象
Raises:
FileNotFoundError: 当配置文件不存在时抛出异常
"""
if not config_path.exists():
raise FileNotFoundError(f"配置文件不存在: {config_path}")
with open(config_path, "rb") as f:
return cast(Config, tomllib.load(f))
# 全局配置缓存变量,避免重复加载配置文件
_config: Optional[Config] = None
def get_config(config_path: str = "config.toml") -> Config:
"""
获取应用程序配置(带缓存机制)
首次调用时加载配置文件并缓存,后续调用直接返回缓存的配置对象
Args:
config_path: 配置文件路径,默认为config.toml
Returns:
Config: 配置字典对象
"""
global _config
if _config is None:
_config = load_config(Path(config_path))
return _config
def get_workers_config(config_path: str = "config/workers/config.toml") -> dict:
"""
获取Worker插件配置列表
Args:
config_path: Worker配置文件路径,默认为config/workers/config.toml
Returns:
list: Worker配置字典列表
"""
if not Path(config_path).exists():
raise FileNotFoundError(f"Worker配置文件不存在: {config_path}")
with open(config_path, "rb") as f:
return cast(dict, tomllib.load(f))
+173
View File
@@ -0,0 +1,173 @@
import os
import inspect
import subprocess
import asyncio
import boto3
from botocore.exceptions import ClientError
from botocore.config import Config
from diskcache import Cache
from fastapi import FastAPI
import zenoh
from ..core.config import get_config, get_workers_config
from ..core.logger import get_logger
from ..core.route_generator import register_dynamic_routes
from ..core.worker_manager import WorkerManager
config = get_config()
logger = get_logger("app")
try:
env = config["default"]["env"]
except KeyError: # pragma: no cover
raise ValueError("配置不存在")
def setup_nanio(app: FastAPI):
"""启动 nanio 子进程作为 S3 兼容存储服务。"""
nanio_config = config.get("nanio", {})
env = os.environ.copy()
env["NANIO_ACCESS_KEY"] = nanio_config.get("nanio_access_key", "minioadmin")
env["NANIO_SECRET_KEY"] = nanio_config.get("nanio_secret_key", "minioadmin")
proc = subprocess.Popen(
[
"nanio",
"serve",
"--data-dir",
nanio_config.get("storage_path", "./data/storage")
],
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
app.state.nanio_process = proc
logger.info("Nanio 进程已启动")
def setup_s3(app: FastAPI):
"""初始化 S3 客户端,并确保 bucket 存在。"""
nanio_config = config.get("nanio", {})
s3_client = boto3.client(
"s3",
endpoint_url=nanio_config.get("endpoint_url", "http://127.0.0.1:9000"),
aws_access_key_id=nanio_config.get("nanio_access_key", "minioadmin"),
aws_secret_access_key=nanio_config.get("nanio_secret_key", "minioadmin"),
config=Config(signature_version=nanio_config.get("signature_version", "s3v4")),
region_name=nanio_config.get("region_name", "us-east-1")
)
try:
s3_client.head_bucket(Bucket=nanio_config.get("bucket", "chongming"))
except ClientError as e:
if e.response["Error"]["Code"] == "404":
s3_client.create_bucket(Bucket="chongming")
logger.info("Bucket 'chongming' 已创建")
else:
raise
app.state.s3_client = s3_client
app.state.storage_bucket = nanio_config.get("bucket", "chongming")
app.state.storage_presign_expire = 300
app.state.storage_max_size = 100 * 1024 * 1024 # 100MB
def setup_zenoh(app: FastAPI) -> zenoh.Session:
"""建立 Zenoh 会话并存入 app.state。"""
zenoh_config = config.get("zenoh", {})
zconfig = zenoh.Config()
zconfig.from_file(zenoh_config.get("json5_path", "config/zenoh_config.json5"))
try:
session = zenoh.open(zconfig)
logger.info("Zenoh 会话已建立 (TCP 自连接模式)")
except Exception as e:
logger.error(f"Zenoh 会话建立失败: {e}")
raise
app.state.zenoh_session = session
return session
def setup_worker_manager(app: FastAPI, session: zenoh.Session) -> WorkerManager:
"""创建 WorkerManager,启动定期刷新任务,并存入 app.state。"""
cache = Cache(config["cache"]["cache_store_path"])
manager = WorkerManager(session, cache)
app.state.worker_manager = manager
refresh_task = asyncio.create_task(manager.refresh_workers())
app.state.refresh_task = refresh_task
return manager
async def load_workers(app: FastAPI, session: zenoh.Session, manager: WorkerManager):
"""
加载配置中的 worker 插件,启动对应的 worker 任务。
返回任务列表。
"""
workers_config_path = config.get("plugins_worker", {}).get("config_path", "config/workers/config.toml")
workers_conf = get_workers_config(workers_config_path).get("workers", [])
logger.info(f"读取到 {len(workers_conf)} 个 worker 配置")
worker_tasks = []
for wc in workers_conf:
module_name = wc.get("module", "")
factory_name = wc.get("factory", "")
count = wc.get("count", 1)
worker_config = wc.get("config", {})
if not module_name or not factory_name:
logger.warning(f"Worker 配置不完整: {wc}")
continue
try:
module = __import__(module_name, fromlist=[factory_name])
factory = getattr(module, factory_name)
if inspect.iscoroutinefunction(factory):
tasks = await factory(session, worker_config, count, worker_manager=manager)
else:
tasks = factory(session, worker_config, count, worker_manager=manager)
worker_tasks.extend(tasks)
logger.info(f"已加载 Worker 插件: {module_name}.{factory_name} (副本数: {count})")
except Exception as e:
logger.error(f"加载 Worker 插件失败 {module_name}: {e}")
app.state.worker_tasks = worker_tasks
return worker_tasks
async def register_routes(app: FastAPI, manager: WorkerManager):
"""注册动态路由。"""
prefix = config["default"]["prefix"]
register_dynamic_routes(app, manager, prefix=prefix)
# 可选:等待路由注册生效(已在 register_dynamic_routes 内部完成日志)
async def shutdown_nanio(app: FastAPI):
"""终止 nanio 进程。"""
proc = getattr(app.state, "nanio_process", None)
if proc:
proc.terminate()
proc.wait()
logger.info("Nanio 进程已终止")
async def cleanup_workers(app: FastAPI):
"""调用每个 worker 配置中定义的 cleanup 函数。"""
workers_config_path = config.get("plugins_worker", {}).get("config_path", "config/workers/config.toml")
workers_conf = get_workers_config(workers_config_path).get("workers", [])
for wc in workers_conf:
module_name = wc.get("module", "")
cleanup_name = wc.get("cleanup", "")
worker_type = wc.get("type", "")
if not module_name or not cleanup_name:
logger.warning(f"Worker 配置缺少 cleanup: {wc}")
continue
try:
module = __import__(module_name, fromlist=[cleanup_name])
cleanup = getattr(module, cleanup_name)
pid = os.getpid()
worker_id = f"worker/{worker_type}/worker-i-{pid}"
if inspect.iscoroutinefunction(cleanup):
await cleanup(worker_id)
else:
cleanup(worker_id)
except Exception as e:
logger.error(f"cleanup 调用失败 {module_name}: {e}")
+191
View File
@@ -0,0 +1,191 @@
import re
import logging
import datetime
from pathlib import Path
from logging.handlers import RotatingFileHandler
from ..core.config import get_config
config = get_config()
env = config["default"]["env"]
server_config = config[env]["server"]
log_config = config[env]["logging"]
class ColoredFormatter(logging.Formatter):
"""
为 HTTP 状态码添加颜色的日志格式化器
根据 HTTP 状态码的首位数字为日志消息中的状态码添加不同颜色:
- 2xx: 绿色(成功)
- 3xx: 青色(重定向)
- 4xx: 黄色(客户端错误)
- 5xx: 红色(服务器错误)
Attributes:
COLORS: 状态码首位数字到 ANSI 颜色代码的映射字典
RESET: ANSI 重置颜色代码
"""
COLORS = {
"2": "\033[32m", # 绿色 2xx
"3": "\033[36m", # 青色 3xx
"4": "\033[33m", # 黄色 4xx
"5": "\033[31m", # 红色 5xx
}
RESET = "\033[0m"
def format(self, record):
"""
格式化日志记录,为状态码添加颜色
如果日志记录包含 status_code 属性,则查找消息中的 "status=数字" 模式
并为其添加对应的颜色代码。格式化完成后恢复原始消息以避免影响其他 handler。
Args:
record: 日志记录对象,应包含 msg 属性和可选的 status_code 属性
Returns:
str: 格式化后的日志消息字符串,状态码部分可能包含 ANSI 颜色代码
"""
# 保存原始消息
original_msg = record.msg
# 如果日志记录包含 status_code 属性,则为其添加颜色
status_code = getattr(record, "status_code", None)
if status_code and isinstance(status_code, int):
first_digit = str(status_code)[0]
color = self.COLORS.get(first_digit, "")
colored_status = f"{color}{status_code}{self.RESET}"
# 替换消息中的 "status=数字" 为带颜色的版本
record.msg = re.sub(
rf"status={status_code}\b", f"status={colored_status}", original_msg
)
result = super().format(record)
# 恢复原始消息,避免影响其他 handler
record.msg = original_msg
return result
class ExcludeWatchFilesFilter(logging.Filter):
"""
日志过滤器,用于排除第三方库的日志输出
过滤掉来自 watchfiles、aiosqlite、apscheduler、sqlalchemy 等库的日志记录,
避免这些库的详细日志污染应用日志输出。
"""
def __init__(self):
"""
初始化过滤器,设置需要过滤的日志名称前缀列表
"""
super().__init__()
self._filter_starts_with_tulpe = (
"watchfiles",
"aiosqlite",
"apscheduler",
"sqlalchemy",
"sqlitedict",
"sqlalchemy.engine.Engine",
)
def filter(self, record):
"""
判断是否应该保留该日志记录
Args:
record: 日志记录对象,包含 name 属性表示日志来源
Returns:
bool: 如果日志名称不以过滤列表中的任何前缀开头则返回 True(保留),
否则返回 False(过滤掉)
"""
return not record.name.startswith(self._filter_starts_with_tulpe)
_configured_loggers = set()
def setup_logging(logger: logging.Logger, log_name: str = "app"):
"""
配置日志系统,包括文件和控制台输出
为指定的 logger 配置以下功能:
1. 设置日志级别和格式
2. 创建按日期命名的轮转文件处理器(支持文件大小限制和备份数量控制)
3. 创建带颜色格式化的控制台处理器
4. 应用过滤器排除第三方库日志
该函数具有防重复配置机制,同一个 log_name 只会配置一次。
Args:
logger: 需要配置的 logging.Logger 实例
log_name: 日志名称标识,用于区分不同的日志配置和生成日志文件名,默认为 "app"
Returns:
None
Raises:
无显式抛出异常,但可能因文件系统权限问题导致目录创建或文件写入失败
"""
if log_name in _configured_loggers:
return
_configured_loggers.add(log_name)
log_level = getattr(logging, log_config["level"], logging.INFO)
log_format = log_config["format"]
# 创建文件夹
log_file_path = Path(log_config["file"])
if log_file_path != Path("."):
log_file_path.mkdir(parents=True, exist_ok=True)
# 配置根日志记录器
logger.setLevel(log_level)
# 创建格式化器
plain_formatter = logging.Formatter(log_format)
color_formatter = ColoredFormatter(log_format)
# 创建轮转文件处理器 - 限制文件大小为 10MB,保留 5 个备份文件
max_bytes = log_config.get("max_size", 10 * 1024 * 1024) # 默认 10MB
backup_count = log_config.get("backup_count", 5) # 默认保留 5 个备份
file_handler = RotatingFileHandler(
f"{log_config['file']}/{log_name}-{datetime.date.today()}.log",
encoding="utf-8",
maxBytes=max_bytes,
backupCount=backup_count,
)
file_handler.setFormatter(plain_formatter)
file_handler.addFilter(ExcludeWatchFilesFilter())
# 创建控制台处理器并添加过滤器
console_handler = logging.StreamHandler()
console_handler.setFormatter(color_formatter)
console_handler.addFilter(ExcludeWatchFilesFilter())
# 添加处理器到根日志记录器
logger.addHandler(file_handler)
logger.addHandler(console_handler)
print(
f"日志系统已配置 - 级别: {logging.getLevelName(log_level)}, 文件: {log_config['file']} "
f"(最大:{max_bytes / 1024 / 1024:.1f}MB, 备份:{backup_count})"
)
def get_logger(name: str) -> logging.Logger:
"""
获取配置好的日志记录器实例
根据名称获取或创建 logger 实例,并自动调用 setup_logging 进行配置。
如果该名称的 logger 已经配置过,则直接返回已配置的实例。
Args:
name: 日志记录器名称,通常使用模块名如 __name__
Returns:
logging.Logger: 已配置完成的日志记录器实例,可直接用于记录日志
"""
logger = logging.getLogger(name)
setup_logging(logger, name)
return logger
+418
View File
@@ -0,0 +1,418 @@
"""
动态路由生成器
根据配置文件中的 worker 路由定义,动态生成 FastAPI 端点。
为每个端点生成带有正确 Pydantic 类型注解的处理函数,确保 OpenAPI 文档
正确反映参数类型、响应结构,并注册到 components/schemas。
"""
import asyncio
import json
from typing import Any, Dict, List, Optional, get_type_hints
from inspect import Parameter, Signature
from fastapi import FastAPI, HTTPException, APIRouter, Query, Path as FastAPIPath, Depends
from pydantic import create_model, Field, BaseModel
from .worker_manager import WorkerManager
from .logger import get_logger
logger = get_logger("route_generator")
# 简单类型映射
TYPE_MAP: Dict[str, Any] = {
"float": float,
"int": int,
"str": str,
"bool": bool,
"any": Any,
}
def _parse_python_type(type_str: str) -> Any:
"""解析配置中的类型字符串为 Python 类型对象"""
type_str = type_str.strip()
if type_str.startswith("list[") or type_str.startswith("List["):
inner = type_str[5:-1].strip()
inner_type = _parse_python_type(inner)
return List[inner_type] # type: ignore
if type_str.startswith("dict[") or type_str.startswith("Dict["):
inner = type_str[5:-1].strip()
parts = [p.strip() for p in inner.split(",", 1)]
if len(parts) == 2:
key_type = _parse_python_type(parts[0])
val_type = _parse_python_type(parts[1])
return Dict[key_type, val_type] # type: ignore
return Dict[str, Any] # type: ignore
return TYPE_MAP.get(type_str, str)
def generate_request_model(
param_spec: Dict[str, str], model_name: str = "AutoRequest"
) -> type[BaseModel]:
"""生成 Pydantic 请求体模型"""
fields: Dict[str, Any] = {}
for param_name, param_type_str in param_spec.items():
py_type = _parse_python_type(param_type_str)
fields[param_name] = (py_type, Field(...))
return create_model(model_name, **fields)
def generate_response_model(
response_spec: Dict[str, Any], model_name: str = "AutoResponse"
) -> Optional[type[BaseModel]]:
"""生成 Pydantic 响应模型"""
if not response_spec:
return None
fields: Dict[str, Any] = {}
for field_name, field_spec in response_spec.items():
if isinstance(field_spec, dict):
py_type = _parse_python_type(field_spec.get("type", "str"))
required = field_spec.get("required", True)
default_val = field_spec.get("default", ...)
if required:
fields[field_name] = (py_type, Field(...))
else:
fields[field_name] = (py_type, Field(default=default_val if default_val is not ... else None))
else:
py_type = _parse_python_type(str(field_spec))
fields[field_name] = (py_type, Field(...))
return create_model(model_name, **fields)
class RouteConfig:
"""单个路由配置模型"""
def __init__(self, data: Dict[str, Any]):
self.subject: str = data.get("subject", "")
self.path: str = data.get("path", "")
self.method: str = data.get("method", "GET").upper()
self.params: List[str] = data.get("params", [])
self.response_model: Dict[str, Any] = data.get("response_model", {})
self.summary: str = data.get("summary", "")
self.docstring: str = data.get("docstring", "")
self.timeout: float = data.get("timeout", 3.0)
self.cacheable: bool = data.get("cacheable", False)
def parse_params(self) -> Dict[str, str]:
"""解析 params 列表为 {name: type} 字典"""
result: Dict[str, str] = {}
for param in self.params:
if ":" in param:
name, type_str = param.split(":", 1)
result[name.strip()] = type_str.strip()
else:
result[param.strip()] = "str"
return result
def extract_path_params(self) -> List[str]:
"""从路径中提取路径参数,如 /add/{a}/{b} -> ['a', 'b']"""
return [
seg[1:-1] for seg in self.path.split("/")
if seg.startswith("{") and seg.endswith("}")
]
# 默认的 worker 配置目录
WORKER_CONFIG_DIR = "config/workers"
def load_worker_routes_from_dir(worker_type: str) -> List[Dict[str, Any]]:
"""
从 config/workers/{worker_type}/ 目录加载路由配置。
目录结构:
config/workers/{worker_type}/
config.toml # [worker] 分组信息 + registration + defaults
add.toml # 接口名 = 文件名(不含扩展名)
[GET]
summary = ...
params = [...]
response_model = { ... }
[POST]
... # 同一个接口支持多个请求方法
返回: List[Dict], 每个 dict 包含 flatten 后的路由配置,
每个方法节(如 GET) 生成一个独立的路由条目。
"""
from pathlib import Path
import tomllib
base_dir = Path(WORKER_CONFIG_DIR) / worker_type
if not base_dir.is_dir():
logger.warning(f"Worker 配置目录不存在: {base_dir}")
return []
# 1. 读取分组配置
group_cfg_path = base_dir / "config.toml"
group_defaults: Dict[str, Any] = {}
registration: Dict[str, Any] = {}
router_prefix: str = ""
if group_cfg_path.exists():
with open(group_cfg_path, "rb") as f:
cfg = tomllib.load(f)
w = cfg.get("worker", {})
group_defaults = w.get("defaults", {})
registration = w.get("registration", {})
router_prefix = registration.get("router_prefix", "")
# 2. 遍历所有接口 toml 文件
routes: List[Dict[str, Any]] = []
for route_file in sorted(base_dir.glob("*.toml")):
if route_file.name == "config.toml":
continue
# 文件名(不含 .toml)作为路径名
route_name = route_file.stem # 例如 "add"
base_path = router_prefix + "/" + route_name # 例如 "/calc/add"
with open(route_file, "rb") as f:
route_data = tomllib.load(f)
# 每个方法节(GET/POST/PUT/DELETE)生成一个独立路由
for method, spec in route_data.items():
if not isinstance(spec, dict):
logger.warning(f"忽略非 dict 的方法节: {route_file.name}[{method}]")
continue
# 合并: spec 显式字段优先,然后 group_defaults 兜底
route = dict(spec)
for key, default_val in group_defaults.items():
if key not in route:
route[key] = default_val
# 覆盖关键字段 - method 始终从 TOML 节键获取,不被 defaults 覆盖
route["method"] = method.upper()
route["path"] = route.get("path", base_path)
if "subject" not in route:
# 自动生成 subject: worker_type.route_name (如 "calc.add")
route["subject"] = f"{worker_type}.{route_name}"
routes.append(route)
logger.debug(
f" 已加载: {route['method']} {route['path']} -> {route['subject']}"
)
logger.info(f"从 {base_dir} 加载了 {len(routes)} 个路由")
return routes
def register_dynamic_routes(app: FastAPI, worker_manager: WorkerManager, prefix: str = ""):
"""根据 config/workers/ 目录结构注册所有动态路由"""
from pathlib import Path
workers_base = Path(WORKER_CONFIG_DIR)
if not workers_base.is_dir():
logger.info(f"Worker 配置目录不存在: {WORKER_CONFIG_DIR}")
return
router = APIRouter(prefix=prefix)
extra_schemas: Dict[str, type[BaseModel]] = {}
# 扫描所有 worker 子目录(每个子目录名 = worker_type)
for worker_dir in sorted(workers_base.iterdir()):
if not worker_dir.is_dir():
continue
worker_type = worker_dir.name
routes = load_worker_routes_from_dir(worker_type)
if not routes:
continue
for route_conf in routes:
_register_single_route(router, worker_manager, worker_type, route_conf, extra_schemas)
app.include_router(router)
# 将额外 schema 注册到 OpenAPI 的 components/schemas
if extra_schemas:
_inject_schemas_to_openapi(app, extra_schemas)
logger.info(f"动态路由注册完成,worker 配置目录: {WORKER_CONFIG_DIR}")
def _inject_schemas_to_openapi(app: FastAPI, schemas: Dict[str, type[BaseModel]]):
"""
将自定义 Pydantic 模型注册到 FastAPI 的 OpenAPI components/schemas 中。
通过重写 app.openapi() 方法注入,因为 FastAPI 在生命周期内只生成一次
OpenAPI schema,之后会被缓存。
"""
original_openapi = app.openapi
def patched_openapi() -> dict[str, Any]:
if app.openapi_schema is not None:
return app.openapi_schema
schema = original_openapi()
if schema is None:
return {"openapi": "3.1.0", "info": {"title": "Chongming", "version": "1.0.0"}, "paths": {}}
# 注入我们的 schema
schemas_dict = schema.setdefault("components", {}).setdefault("schemas", {})
for name, model in schemas.items():
if name not in schemas_dict:
schemas_dict[name] = model.model_json_schema()
app.openapi_schema = schema
return schema
app.openapi = patched_openapi
def _make_body_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, RequestModel: type[BaseModel]):
"""创建用于 Body 请求(POST/PUT/PATCH)的端点函数"""
path = rc.path
subject = rc.subject
timeout = rc.timeout
http_method = rc.method
cacheable = rc.cacheable
async def endpoint(
body: RequestModel, # type: ignore
) -> Any:
try:
payload = body.model_dump()
payload.setdefault("_method", http_method)
result = await worker_manager.call(
worker_type=worker_type,
subject=subject,
payload=payload,
timeout=timeout,
cacheable=cacheable,
)
return result
except RuntimeError as e:
raise HTTPException(status_code=503, detail=str(e))
except Exception as e:
logger.error(f"路由 {path} 调用异常: {e}")
raise HTTPException(status_code=500, detail="内部服务器错误")
return endpoint
def _make_query_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, path_params: List[str], query_params: Dict[str, str]):
"""
创建用于 Query 请求(GET/DELETE)的端点函数。
通过设置函数 __signature__ 和 __annotations__ 为每个参数赋予正确的 Pydantic 类型,
使 FastAPI 能在 OpenAPI schema 中生成正确的 type / format 信息。
请求模型 schema 通过 extra_schemas 机制单独注入到 components/schemas。
"""
path = rc.path
subject = rc.subject
timeout = rc.timeout
http_method = rc.method
cacheable = rc.cacheable
# 合并路径参数和查询参数
param_spec_combined: Dict[str, str] = {}
param_spec_combined.update({k: query_params.get(k, "str") for k in path_params if k in query_params})
param_spec_combined.update(query_params)
# 无参数情况
if not param_spec_combined:
async def _simple_endpoint() -> Any:
try:
result = await worker_manager.call(
worker_type=worker_type,
subject=subject,
payload={"_method": http_method},
timeout=timeout,
cacheable=cacheable,
)
return result
except RuntimeError as e:
raise HTTPException(status_code=503, detail=str(e))
except Exception as e:
logger.error(f"路由 {path} 调用异常: {e}")
raise HTTPException(status_code=500, detail="内部服务器错误")
return _simple_endpoint
# 构造参数名列表(用于从 kwargs 中提取)
param_names = list(param_spec_combined.keys())
async def endpoint(**kwargs: Any) -> Any:
payload = {k: kwargs[k] for k in param_names}
payload.setdefault("_method", http_method)
try:
result = await worker_manager.call(
worker_type=worker_type,
subject=subject,
payload=payload,
timeout=timeout,
cacheable=cacheable,
)
return result
except RuntimeError as e:
raise HTTPException(status_code=503, detail=str(e))
except Exception as e:
logger.error(f"路由 {path} 调用异常: {e}")
raise HTTPException(status_code=500, detail="内部服务器错误")
# 构建带类型注解的 Parameter 列表
params: List[Parameter] = []
annotations: Dict[str, Any] = {}
for name, type_str in param_spec_combined.items():
py_type = _parse_python_type(type_str)
annotations[name] = py_type
if name in path_params:
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=FastAPIPath(...), annotation=py_type))
else:
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=Query(...), annotation=py_type))
# 设置 __annotations__(FastAPI 通过 get_type_hints 读取)
endpoint.__annotations__ = annotations
# 设置 __signature__(FastAPI 优先于此获取参数列表)
endpoint.__signature__ = Signature(params, return_annotation=Dict[str, Any]) # type: ignore
return endpoint
def _register_single_route(
router: APIRouter,
worker_manager: WorkerManager,
worker_type: str,
route_conf: Dict[str, Any],
extra_schemas: Optional[Dict[str, type[BaseModel]]] = None,
):
"""注册单个路由,生成正确的函数签名以便 OpenAPI 正确生成文档"""
rc = RouteConfig(route_conf)
param_spec = rc.parse_params()
path_params = rc.extract_path_params()
query_params = {k: v for k, v in param_spec.items() if k not in path_params}
# 生成干净的模型名称(替换路径中的 / 和 {} 为 _)
# 模型名称包含 HTTP method 以区分同名路径的不同调用类型
clean_path = rc.path.replace("/", "_").replace("{", "").replace("}", "").strip("_")
if not clean_path:
clean_path = "root"
method_tag = rc.method.upper()
response_model_name = f"{worker_type}_{clean_path}_{method_tag}_Response"
request_model_name = f"{worker_type}_{clean_path}_{method_tag}_Request"
# 生成响应模型
ResponseModel = generate_response_model(
rc.response_model, response_model_name
)
if rc.method in ("POST", "PUT", "PATCH"):
RequestModel = generate_request_model(
param_spec, request_model_name
)
endpoint = _make_body_endpoint(worker_manager, rc, worker_type, RequestModel)
else:
# 对 GET 端点也生成 Request 模型并注册到 extra_schemas
if param_spec:
RequestModel = generate_request_model(
param_spec, request_model_name
)
if extra_schemas is not None and request_model_name not in extra_schemas:
extra_schemas[request_model_name] = RequestModel
endpoint = _make_query_endpoint(worker_manager, rc, worker_type, path_params, query_params)
# 注册路由 - 使用 worker_type 作为 OpenAPI 分组标签
# 每个 worker 类型自动成为一个独立的分组,显示在 Swagger UI 中
router.add_api_route(
path=rc.path,
endpoint=endpoint,
methods=[rc.method],
summary=rc.summary,
description=rc.docstring,
response_model=ResponseModel,
tags=[worker_type],
include_in_schema=True,
)
logger.info(f"已注册动态路由: {rc.method} {rc.path} -> worker {worker_type}.{rc.subject} | tags=[{worker_type}]")
+100
View File
@@ -0,0 +1,100 @@
from fastapi import HTTPException
from fastapi.responses import Response
from fastapi.staticfiles import StaticFiles
from sqlite_vfs.core import SQLiteVFS
class SVFSStaticFiles(StaticFiles):
"""
基于 SQLite VFS 的静态文件服务类
该类继承自 FastAPI 的 StaticFiles,使用 SQLite 虚拟文件系统来存储和提供静态文件。
支持单页应用(SPA)的路由回退机制,当请求的文件不存在时,会自动返回 index.html。
"""
def __init__(self, directory, vfs_db_path="static.svfs"):
"""
初始化 SVFSStaticFiles 实例
参数:
directory (str): 静态文件的根目录路径
vfs_db_path (str): SQLite VFS 数据库文件路径,默认为 "static.svfs"
返回值:
无
"""
vfs_db_path = vfs_db_path
self._vfs = SQLiteVFS(vfs_db_path, compress=False)
super().__init__(directory=directory)
async def get_response(self, path: str, scope):
"""
根据请求路径获取对应的静态文件响应
该方法处理静态文件的请求逻辑,包括:
1. 路径标准化处理
2. 文件存在性检查
3. SPA 路由回退支持
4. 目录索引处理
5. MIME 类型自动识别
参数:
path (str): 请求的文件路径
scope: ASGI 作用域对象,包含请求上下文信息
返回值:
Response: FastAPI 响应对象,包含文件内容和正确的 Content-Type
异常:
HTTPException: 当文件不存在且无法回退到 index.html 时,抛出 404 错误
"""
# 统一路径格式:以 / 开头,并将 Windows 反斜杠转为正斜杠
normalized_path = "/" + path.replace("\\", "/")
# 获取文件信息
file_info = self._vfs.get_file_info(normalized_path)
# 处理文件不存在的情况,尝试 SPA 回退机制
if file_info is None:
# SPA 回退:如果不是静态资源扩展名,尝试返回 index.html
if not any(
normalized_path.endswith(ext)
for ext in [
".js",
".css",
".png",
".jpg",
".ico",
".svg",
".webp",
".json",
]
):
index_info = self._vfs.get_file_info("/index.html")
if index_info:
content = self._vfs.read_file("/index.html")
return Response(content=content, media_type="text/html")
raise HTTPException(status_code=404, detail="File not found")
# 处理目录请求,返回目录下的 index.html
if file_info["is_directory"]:
index_info = self._vfs.get_file_info(f"{normalized_path}/index.html")
if index_info:
content = self._vfs.read_file(f"{normalized_path}/index.html")
return Response(content=content, media_type="text/html")
raise HTTPException(status_code=404, detail="Directory index not found")
# 读取文件内容并确定 MIME 类型
content = self._vfs.read_file(normalized_path)
content_type = file_info.get("content_type") or "application/octet-stream"
# 简单根据扩展名补充 MIME 类型(可选)
if normalized_path.endswith(".js"):
content_type = "application/javascript"
elif normalized_path.endswith(".css"):
content_type = "text/css"
elif normalized_path.endswith(".html"):
content_type = "text/html"
elif normalized_path.endswith(".ico"):
content_type = "image/x-icon"
return Response(content=content, media_type=content_type)
+202
View File
@@ -0,0 +1,202 @@
import asyncio
import random
import json
import orjson
import time
from typing import Any, Dict, List, Optional, Set
import zenoh
from diskcache import Cache
from .logger import get_logger
logger = get_logger("worker_manager")
_SENTINEL = object() # 哨兵值,用于标记"No reply received"
class WorkerManager:
"""
统一 Worker 管理器
封装 Zenoh 会话,维护每个 worker 类型的存活节点列表,
提供负载均衡、超时重试、缓存等能力的统一 RPC 调用入口。
支持两种 worker 注册方式:
1. 分布式:通过 Zenoh liveliness 自动发现
2. 本地:通过 register_worker() 直接注册(用于同进程 worker)
"""
def __init__(self, session: zenoh.Session, cache: Optional[Cache] = None):
self.session = session
self.cache = cache
self._workers_cache: Dict[str, List[str]] = {} # type -> list of worker key_exprs
self._local_workers: Dict[str, List[str]] = {} # type -> list of locally registered workers
self._lock = asyncio.Lock()
self._refresh_interval = 15 # seconds
def register_worker(self, worker_type: str, worker_id: str):
"""
直接注册一个本地 worker(同进程内使用)
这样无需等待 liveliness 发现,立即可用。
同步执行(仅修改内存字典),确保调用后立即可被 get_worker 发现。
"""
key_expr = f"worker/{worker_type}/{worker_id}"
# 同步直接写入 _local_workers,无需异步锁
# 因为 _local_workers 和 _workers_cache 在调用 call() 时通过 await get_worker() 读取
self._local_workers.setdefault(worker_type, []).append(key_expr)
logger.debug(f"本地 worker 已注册: {key_expr}")
def unregister_worker(self, worker_type: str, worker_id: str):
"""取消注册一个本地 worker"""
key_expr = f"worker/{worker_type}/{worker_id}"
if worker_type in self._local_workers:
try:
self._local_workers[worker_type].remove(key_expr)
except ValueError:
pass
async def refresh_workers(self):
"""定期刷新存活 worker 列表(合并 Zenoh liveliness + 本地注册)"""
while True:
try:
# 1. 从 Zenoh liveliness 获取分布式 workers
replies = self.session.liveliness().get("worker/*/*")
new_cache: Dict[str, List[str]] = {}
for reply in replies:
if reply.ok:
key = str(reply.ok.key_expr) # e.g. "worker/example/worker-0"
parts = key.split("/")
if len(parts) >= 3:
worker_type = parts[1]
new_cache.setdefault(worker_type, []).append(key)
# 2. 合并本地注册的 workers
for wtype, workers in self._local_workers.items():
existing = set(new_cache.get(wtype, []))
for w in workers:
if w not in existing:
new_cache.setdefault(wtype, []).append(w)
async with self._lock:
self._workers_cache = new_cache
logger.debug(f"Worker 存活列表更新: {self._workers_cache}")
except Exception as e:
logger.error(f"刷新 worker 列表失败: {e}")
await asyncio.sleep(self._refresh_interval)
async def get_worker(self, worker_type: str) -> Optional[str]:
"""获取一个可用的 worker 地址(简单随机负载均衡)"""
async with self._lock:
workers = self._workers_cache.get(worker_type, [])
# 如果缓存为空,回退到本地注册表(适用于启动初期 refresh_workers 尚未执行时)
if not workers:
local = self._local_workers.get(worker_type, [])
if local:
return random.choice(local)
return None
return random.choice(workers)
def is_success_response(self, result: dict) -> bool:
"""根据业务约定判断响应是否成功。
假设成功响应没有 'error' 字段,或 'code' 为 0。
"""
if "error" in result and result["error"]:
return False
if "code" in result and result["code"] != 0:
return False
return True
async def call(
self,
worker_type: str,
subject: str,
payload: Dict[str, Any],
timeout: float = 3.0,
retries: int = 1,
cacheable: bool = False,
) -> Any:
"""
统一调用入口:缓存 > 负载均衡 > 超时重试
Args:
worker_type: Worker 类型标识(如 "example")
subject: 调用的主题(如 "calc.add")
payload: 请求参数
timeout: 单次调用超时(秒)
retries: 失败重试次数
cacheable: 是否允许缓存结果(若为 False 则跳过缓存读写)
Returns:
Dict[str, Any]: 调用结果
Raises:
RuntimeError: 无可用 worker 或所有重试均失败
"""
# 1. 缓存检查(仅在 cacheable=True 时检查)
cache_key = f"{worker_type}:{subject}:{json.dumps(payload, sort_keys=True)}"
if cacheable and self.cache is not None:
if cache_key in self.cache:
logger.debug(f"命中缓存: {cache_key}")
return self.cache[cache_key]
# 2. 获取可用 worker
worker_expr = await self.get_worker(worker_type)
if not worker_expr:
raise RuntimeError(f"No available worker for type {worker_type}")
full_key = f"{worker_expr}/{subject}"
payload_bytes = orjson.dumps(payload)
for attempt in range(retries + 1):
try:
reply_iter = self.session.get(
full_key,
payload=payload_bytes,
consolidation=zenoh.ConsolidationMode.NONE,
)
# 使用辅助函数获取第一个回复,避免 StopIteration 在 asyncio.to_thread 中传播的问题
def _get_first(iterator):
"""线程安全地获取迭代器的第一个元素"""
for item in iterator:
return item
return _SENTINEL
first_reply = await asyncio.wait_for(
asyncio.to_thread(_get_first, reply_iter),
timeout=timeout,
)
if first_reply is _SENTINEL:
logger.warning(
f"Worker {worker_expr} 无回复,尝试重试 {attempt+1}/{retries+1}"
)
continue
reply = first_reply
if reply.ok: # type: ignore
# result_str = str(reply.ok.payload)
payload_bytes = bytes(reply.ok.payload) if reply.ok.payload else b"" # type: ignore
if not payload_bytes:
result = {} # 或根据业务返回默认值
else:
result = orjson.loads(payload_bytes)
# 缓存结果(仅在 cacheable=True 时写入)
if cacheable and self.cache is not None and self.is_success_response(result):
self.cache[cache_key] = result
return result
else:
error_payload = reply.err.payload # type: ignore
error_msg = str(error_payload) if error_payload else "unknown error"
logger.warning(f"Worker 返回错误: {error_msg} (尝试 {attempt+1}/{retries+1})")
continue # 重试
except asyncio.TimeoutError:
logger.warning(
f"Worker {worker_expr} 超时,尝试重试 {attempt+1}/{retries+1}"
)
continue
except Exception as e:
logger.error(f"调用 worker 异常: {e}")
raise
raise RuntimeError(
f"Failed to call worker {worker_type} after {retries} retries"
)
+3
View File
@@ -0,0 +1,3 @@
from .storage import router as storage_router
__all__ = ["storage_router"]
+63
View File
@@ -0,0 +1,63 @@
# src/chongming/app/routes/storage.py
import uuid
import urllib.parse
from fastapi import APIRouter, UploadFile, File, Form, HTTPException, Request
from fastapi.responses import StreamingResponse
from botocore.exceptions import ClientError
router = APIRouter(prefix="/api/v1/storage", tags=["Storage"])
@router.post("/upload", summary="上传文件")
async def upload_file(
request: Request,
file: UploadFile = File(...),
worker_type: str = Form(...),
):
"""后端代理上传,直接写入 nanio"""
s3_client = request.app.state.s3_client
bucket = request.app.state.storage_bucket
object_key = f"{worker_type}/{uuid.uuid4()}/{file.filename}"
try:
s3_client.upload_fileobj(
file.file,
bucket,
object_key,
ExtraArgs={"ContentType": file.content_type or "application/octet-stream"}
)
except ClientError as e:
raise HTTPException(status_code=500, detail=f"S3 上传失败: {str(e)}")
return {
"file_key": object_key,
"bucket": bucket,
"filename": file.filename,
"content_type": file.content_type,
"size": file.size or 0, # 注意:UploadFile.size 可能为 None,需读取后计算
}
@router.get("/download/{file_key:path}")
async def download_file(request: Request, file_key: str):
s3_client = request.app.state.s3_client
bucket = request.app.state.storage_bucket
try:
response = s3_client.get_object(Bucket=bucket, Key=file_key)
content_type = response.get('ContentType', 'application/octet-stream')
raw_filename = file_key.split('/')[-1]
# 百分号编码,确保仅 ASCII 字符
encoded = urllib.parse.quote(raw_filename, safe='')
# 仅使用 filename*(现代浏览器都支持)
content_disposition = f"attachment; filename*=UTF-8''{encoded}"
return StreamingResponse(
response['Body'].iter_chunks(),
media_type=content_type,
headers={'Content-Disposition': content_disposition}
)
except ClientError as e:
if e.response['Error']['Code'] == 'NoSuchKey':
raise HTTPException(404, "文件不存在")
raise HTTPException(500, f"下载失败: {e}")
+501
View File
@@ -0,0 +1,501 @@
"""
构建脚本 - 将 chongming 应用打包为可执行文件
功能阶段:
1. 模块打包(module-bank): 加密打包 Python 模块
2. 资源准备: 创建 build 目录, 复制必要文件
3. 混淆与打包: pyarmor 混淆 + PyInstaller/Nuitka 打包
4. 数据库初始化: 创建表结构和默认数据
5. 前端构建: 编译 Vue 项目并打包为 VFS
用法:
python -m chongming.scripts.build [pyinstaller|nuitka]
"""
import platform
import shutil
import subprocess
import sys
import asyncio
import time
from dataclasses import dataclass, field
from pathlib import Path
from module_bank import PythonToSQLite
from module_bank.encryption import Encryption
from sqlite_vfs.core import SQLiteVFS
from sqlite_vfs.folder_packer import FolderPacker
from ..app.core.config import get_config, Config
# ── 常量 ────────────────────────────────────────────────────────────────────
PYINSTALLER = "pyinstaller"
NUITKA = "nuitka"
MODES = (PYINSTALLER, NUITKA)
is_windows = platform.system() == "Windows"
# ── 构建结果数据结构 ─────────────────────────────────────────────────────────
@dataclass
class BuildStepResult:
"""单个构建步骤的执行结果"""
name: str # 步骤名称
success: bool # 是否成功
duration: float # 耗时(秒)
message: str = "" # 附加信息(如包大小)
@dataclass
class BuildReport:
"""构建总结报告"""
mode: str
start_time: float
end_time: float = 0.0
steps: list[BuildStepResult] = field(default_factory=list)
@property
def total_duration(self) -> float:
return self.end_time - self.start_time if self.end_time else 0.0
@property
def all_success(self) -> bool:
return all(s.success for s in self.steps)
def add_step(self, name: str, success: bool, duration: float, message: str = ""):
self.steps.append(BuildStepResult(name, success, duration, message))
def print_summary(self):
"""打印结构化的构建总结"""
sep = "=" * 60
print(f"\n{sep}")
print(f" 🏗️ CHONGMING 构建报告")
print(f"{sep}")
print(f" 构建模式 : {self.mode}")
print(f" 总耗时 : {self.total_duration:.2f}s")
print(f" 构建状态 : {'✅ 成功' if self.all_success else '❌ 失败'}")
print(f"{sep}")
print(f" 步骤明细:")
for i, step in enumerate(self.steps, 1):
icon = "✅" if step.success else "❌"
msg = f" ({step.message})" if step.message else ""
print(
f" {icon} [{i}/{len(self.steps)}] {step.name:<20s} {step.duration:>7.2f}s{msg}"
)
print(f"{sep}\n")
# ── 工具函数 ────────────────────────────────────────────────────────────────
def _run_cmd(
cmd: list[str],
cwd: str | None = None,
capture: bool = True,
shell: bool | None = None,
) -> subprocess.CompletedProcess:
"""统一执行外部命令"""
if shell is None:
shell = is_windows
return subprocess.run(
cmd,
cwd=cwd,
capture_output=capture,
text=True,
shell=shell,
encoding="utf-8",
errors="replace",
)
def _format_size(path: str | Path) -> str:
"""获取文件可读大小"""
size = Path(path).stat().st_size
for unit in ("B", "KB", "MB", "GB"):
if size < 1024:
return f"{size:.2f} {unit}"
size /= 1024.0
return f"{size:.2f} TB"
def _ensure_dir(path: str | Path):
"""确保目录存在"""
Path(path).mkdir(parents=True, exist_ok=True)
def _resolve_mode() -> str:
"""解析构建模式"""
mode = PYINSTALLER
if len(sys.argv) >= 2:
mode = sys.argv[1].lower()
if mode not in MODES:
print(f" ❌ 不支持的构建模式: {mode},可选: {', '.join(MODES)}")
sys.exit(1)
return mode
# ── 阶段 1: 模块打包 (module-bank) ──────────────────────────────────────────
def _stage_pack_modules(config: Config) -> str:
"""
使用 module-bank 加密打包 Python 模块为 .mbank 文件。
返回打包过程中的提示信息。
"""
_ensure_dir("resources")
mbank_list = config.get("production", {}).get("module_system", {}).get("path", [])
if not mbank_list:
return "⚠️ 未配置 module_system.path,跳过模块打包"
messages = []
for output_path, source_dir, package_name in mbank_list:
key = Encryption.generate_key()
packer = PythonToSQLite(
output_path,
key,
pyarmor_obfuscate=True,
)
packer.pack_directory(source_dir, package_name)
packer.verify_package_structure()
packer.packer_key(key)
packer.delete_source_code(None)
size = _format_size(output_path)
messages.append(f"{Path(output_path).name} ({size})")
return " - " + "\n - ".join(messages)
def _stage_prepare_build_dir(config: Config, mode: str) -> str:
"""准备 build 目录并复制必要文件"""
build_path = Path("build")
if build_path.exists():
shutil.rmtree(str(build_path))
_ensure_dir("build")
# 复制 utils
utils_src = Path("public/utils")
if utils_src.exists():
shutil.copytree(utils_src, build_path / "utils")
# 复制 resources
resources_src = Path("resources")
if resources_src.exists():
shutil.copytree(resources_src, build_path / "resources")
resources_src = Path("config")
if resources_src.exists():
shutil.copytree(resources_src, build_path / "config")
# 复制核心文件
for f in ("main.py", "server.py", "config.toml"):
shutil.copy(f"public/{f}", f"build/{f}")
# 注入 JWT 密钥
jwt_key = Encryption.generate_key()
cfg_path = build_path / "config.toml"
content = cfg_path.read_text(encoding="utf-8")
content = content.replace("your-secret-key-change-in-production", jwt_key)
cfg_path.write_text(content, encoding="utf-8")
return "资源准备完成"
# ── 阶段 3: PyArmor 混淆 ───────────────────────────────────────────────────
def _stage_pyarmor_obfuscate() -> str:
"""对关键脚本执行 PyArmor 混淆"""
files = [
("main.py", "./"),
("server.py", "./"),
("utils/launch.py", "./utils"),
("utils/config.py", "./utils"),
]
failed = []
for filename, output in files:
result = _run_cmd(
["pyarmor", "gen", f"--output={output}", filename],
cwd="build",
)
if result.returncode != 0:
failed.append(filename)
if failed:
return f"❌ 混淆失败: {', '.join(failed)}"
return "✅ 所有文件混淆成功"
# ── 阶段 4: PyInstaller 打包 ────────────────────────────────────────────────
def _stage_pyinstaller(config: Config) -> tuple[str, str]:
"""执行 PyInstaller 打包,返回 (状态消息, 可执行文件路径)"""
base_imports = config.get("pyinstaller_build_import", {})
build_imports = list(base_imports.get("base", []))
build_imports.extend(
base_imports.get("windows", []) if is_windows else base_imports.get("linux", [])
)
icon = Path("../../public/chongming.ico")
cmd = [
sys.executable,
"-m",
"PyInstaller",
"--onefile",
"--noconfirm",
"--optimize",
"2",
"--name",
"chongming",
"--distpath",
".",
"--workpath",
"./build_temp",
"--specpath",
"./specs",
"--clean",
]
if not is_windows:
cmd.append("--strip")
if icon.exists():
cmd.extend(["--icon", str(icon)])
for imp in build_imports:
cmd.extend(["--hidden-import", imp])
cmd.append("main.py")
result = _run_cmd(cmd, cwd="build", capture=True)
if result.returncode != 0:
return f"❌ PyInstaller 失败", ""
exe_path = Path("build/chongming")
if is_windows:
exe_path = exe_path.with_suffix(".exe")
suffix = ".exe" if is_windows else ""
exe = Path("build") / f"chongming{suffix}"
if exe.exists():
size = _format_size(exe)
return f"✅ PyInstaller 打包成功 ({size})", str(exe)
return "✅ PyInstaller 打包完成 (但未找到可执行文件)", ""
# ── 阶段 5: Nuitka 打包 ────────────────────────────────────────────────────
def _stage_nuitka(config: Config) -> tuple[str, str]:
"""执行 Nuitka 打包,返回 (状态消息, 可执行文件路径)"""
base_imports = config.get("pyinstaller_build_import", {}).get("base", [])
extra = config.get("pyinstaller_build_import", {}).get(
"windows" if is_windows else "linux", []
)
packages = [
"module_bank",
"sqlite_vfs",
"fastapi",
"uvicorn",
"gunicorn",
"diskcache",
"apscheduler",
"tomli",
"typing_extensions",
]
cmd = [
sys.executable,
"-m",
"nuitka",
"--standalone",
"--onefile",
"--follow-imports",
"--static-libpython=no",
"--remove-output",
"--assume-yes-for-downloads",
"--lto=yes",
"--disable-plugin=tk-inter",
"--disable-plugin=numpy",
"--output-dir=.",
"--output-filename=chongming",
"main.py",
"--jobs=2",
]
if shutil.which("upx"):
cmd.append("--enable-plugin=upx")
for pkg in packages:
cmd.append(f"--include-package={pkg}")
for mod in list(base_imports) + extra:
cmd.append(f"--include-module={mod}")
if is_windows:
icon = Path("../../public/chongming.ico")
if icon.exists():
cmd.append(f"--windows-icon-from-ico={icon}")
try:
_run_cmd(cmd, cwd="build", capture=False)
except subprocess.CalledProcessError:
# 降级重试(移除可能不兼容的参数)
fallback = [
arg for arg in cmd if arg not in ("--lto=yes", "--enable-plugin=upx")
]
_run_cmd(fallback, cwd="build", capture=False)
exe = Path("build/chongming")
if exe.exists():
# 后处理瘦身
if not is_windows:
subprocess.run(["strip", str(exe)], check=False)
if shutil.which("upx"):
subprocess.run(["upx", "--best", str(exe)], check=False)
size = _format_size(exe)
return f"✅ Nuitka 编译完成 ({size})", str(exe)
return "❌ Nuitka 编译失败 (未生成可执行文件)", ""
# ── 阶段 6: 清理临时文件 ────────────────────────────────────────────────────
def _stage_cleanup(mode: str) -> str:
"""清理构建过程中的临时文件"""
build = Path("build")
dirs_to_remove = [
"build_temp",
"specs",
"utils",
"pyarmor_runtime_000000",
"__pycache__",
]
files_to_remove = ["main.py", "server.py"]
for d in dirs_to_remove:
target = build / d
if target.exists():
shutil.rmtree(str(target), ignore_errors=True)
for f in files_to_remove:
target = build / f
if target.exists():
target.unlink(missing_ok=True)
return "临时文件已清理"
# ── 阶段 8: Vue 前端构建 ────────────────────────────────────────────────────
def _stage_build_vue() -> str:
"""构建 Vue 前端并打包为 SQLite VFS"""
dist_dir = "./src/chongming-web/dist"
output_db = "./build/static.svfs"
fs_name = "frontend"
# 1. npm build
result = _run_cmd(
["npm", "run", "build"],
cwd="./src/chongming-web",
capture=True,
)
if result.returncode != 0:
err = (result.stderr or "").strip()[:200]
return f"❌ Vue 构建失败: {err}"
# 2. 检查产物
if not Path(dist_dir).exists():
return "❌ Vue 构建成功但未找到 dist 目录"
# 3. 打包为 VFS
vfs = SQLiteVFS(output_db, compress=False)
packer = FolderPacker(sqlite_vfs=vfs, exclude_patterns=[], fs_name=fs_name)
try:
packer.pack_folder(dist_dir)
finally:
vfs.close()
size = _format_size(output_db)
return f"✅ Vue 打包完成 ({size})"
# ── 主流程 ──────────────────────────────────────────────────────────────────
async def main():
"""规范化构建主流程"""
mode = _resolve_mode()
report = BuildReport(mode=mode, start_time=time.time())
config = get_config()
async def run_step(name: str, fn, *args, **kwargs) -> str:
"""执行一个构建步骤并记录结果(支持同步和异步函数)"""
start = time.time()
try:
result = fn(*args, **kwargs)
if asyncio.iscoroutine(result):
msg = await result
else:
msg = result
elapsed = time.time() - start
report.add_step(name, success=True, duration=elapsed, message=msg or "")
print(f" ✅ {name:<25s} {elapsed:>7.2f}s {msg}")
return msg
except Exception as e:
elapsed = time.time() - start
err_msg = str(e).strip()[:200]
report.add_step(name, success=False, duration=elapsed, message=err_msg)
print(f" ❌ {name:<25s} {elapsed:>7.2f}s {err_msg}")
return ""
print("\n" + "=" * 60)
print(" 🏗️ CHONGMING 构建开始")
print(f" 模式: {mode} | 平台: {platform.system()}")
print("=" * 60 + "\n")
# 阶段 1: 模块打包
await run_step("模块打包(module-bank)", _stage_pack_modules, config)
# 阶段 2: 准备构建目录
await run_step("准备构建目录", _stage_prepare_build_dir, config, mode)
# 阶段 3+4: 混淆 + 打包
if mode == PYINSTALLER:
await run_step("PyArmor 混淆", _stage_pyarmor_obfuscate)
start = time.time()
try:
msg, exe_path = _stage_pyinstaller(config)
elapsed = time.time() - start
success = "❌" not in msg
report.add_step(
"PyInstaller 打包", success, elapsed, msg.replace("✅ PyInstaller ", "")
)
print(
f" {'✅' if success else '❌'} PyInstaller 打包{' ' * 12} {elapsed:>7.2f}s {msg}"
)
except Exception as e:
elapsed = time.time() - start
report.add_step("PyInstaller 打包", False, elapsed, str(e)[:200])
print(f" ❌ PyInstaller 打包 {elapsed:>7.2f}s {str(e)[:200]}")
else:
await run_step("Nuitka 编译", lambda: _stage_nuitka(config)[0])
# 阶段 5: 清理
await run_step("清理临时文件", _stage_cleanup, mode)
# 阶段 7: Vue 前端构建
await run_step("Vue 前端构建", _stage_build_vue)
# 总结
report.end_time = time.time()
report.print_summary()
if not report.all_success:
sys.exit(1)
def build():
"""构建入口"""
asyncio.run(main())
if __name__ == "__main__":
build()
+47
View File
@@ -0,0 +1,47 @@
import os
import sys
def generate_markdown_from_py_files(directory, output_file):
with open(output_file, "w", encoding="utf-8") as md_file:
for root, dirs, files in os.walk(directory):
# 排除 venv 目录
dirs[:] = [d for d in dirs if d != ".venv"]
dirs[:] = [d for d in dirs if d != ".vscode"]
dirs[:] = [d for d in dirs if d != "node_modules"]
dirs[:] = [d for d in dirs if d != "build"]
dirs[:] = [d for d in dirs if d != "dist"]
dirs[:] = [d for d in dirs if d != "chongming-web"]
for file in files:
if (
file.endswith(".py")
or file.endswith(".rs")
or file.endswith(".toml")
or file.endswith(".md")
or file.endswith(".css")
or file.endswith(".vue")
or file.endswith(".ts")
or file.endswith(".json")
or file.endswith(".html")
or file.endswith(".sh")
or file == "Dockerfile"
):
file_path = os.path.join(root, file)
if file == "output.md" or file == "package-lock.json":
continue
if "generated" in file_path:
continue
md_file.write(f"`{file_path}`\n")
md_file.write("```\n")
with open(file_path, "r", encoding="utf-8") as py_file:
md_file.write(py_file.read())
md_file.write("\n```\n\n")
if __name__ == "__main__":
# 指定目录和输出文件名
target_directory = sys.argv[1] # 替换为你的目标目录
output_markdown_file = "output.md" # 输出的 Markdown 文件名
generate_markdown_from_py_files(target_directory, output_markdown_file)
print(f"Markdown 文件已生成:{output_markdown_file}")
+333
View File
@@ -0,0 +1,333 @@
"""
模型生成脚本 — 从 config/workers/<plugin_name>/*.toml 生成 Pydantic 模型
用法:
uv run gen_models config/workers/example src/plugins/example
工作流:
1. 编写 config/workers/<plugin_name>/*.toml 配置文件
2. 执行 gen_models 生成 Pydantic 模型
3. 在业务代码中 import 生成的模型
功能:
1. 扫描配置目录下的所有 .toml 文件
2. 解析每个路由的 params 和 response_model
3. 生成 Pydantic 请求/响应模型
4. 写入 src/plugins/<plugin_name>/models/__init__.py 和 models/pydantic.py
"""
import sys
import json
from pathlib import Path
from typing import Any
def parse_type_str(type_str: str) -> str:
"""将 TOML 中的类型字符串映射为 Python 类型注解"""
mapping = {
"float": "float",
"int": "int",
"str": "str",
"bool": "bool",
"string": "str",
"integer": "int",
"number": "float",
"boolean": "bool",
"any": "Any",
}
if type_str.startswith("list[") and type_str.endswith("]"):
inner = type_str[5:-1]
inner_py = mapping.get(inner, inner)
return f"list[{inner_py}]"
if type_str == "dict":
return "dict[str, Any]"
if type_str.startswith("dict[") and type_str.endswith("]"):
inner = type_str[5:-1]
if "," in inner:
parts = [p.strip() for p in inner.split(",", 1)]
inner_py = mapping.get(parts[-1], parts[-1])
else:
inner_py = mapping.get(inner, inner)
return f"dict[str, {inner_py}]"
if type_str.startswith("Optional[") and type_str.endswith("]"):
inner = type_str[9:-1]
inner_py = mapping.get(inner, inner)
return f"Optional[{inner_py}]"
return mapping.get(type_str, type_str)
def parse_params(params: list[str]) -> list[dict[str, str]]:
"""解析 params 列表,如 ['a: float', 'b: float']"""
result = []
for p in params:
if ":" in p:
name, type_str = p.split(":", 1)
result.append({"name": name.strip(), "type": parse_type_str(type_str.strip())})
else:
result.append({"name": p.strip(), "type": "str"})
return result
def parse_response_model(response_model: dict[str, Any]) -> list[dict[str, Any]]:
"""解析 response_model 字典,返回字段列表"""
fields: list[dict[str, Any]] = []
for field_name, field_spec in response_model.items():
entry: dict[str, Any] = {"name": field_name}
if isinstance(field_spec, dict):
entry["type"] = parse_type_str(field_spec.get("type", "str"))
entry["required"] = field_spec.get("required", True)
if "default" in field_spec:
entry["default"] = field_spec["default"]
else:
entry["type"] = parse_type_str(str(field_spec))
entry["required"] = True
fields.append(entry)
return fields
def _model_name(base: str, method: str, suffix: str) -> str:
"""生成模型名: e.g. add + GET + Request -> AddGetRequest"""
return f"{base.title().replace('-', '').replace('_', '')}{method.title()}{suffix}"
def generate_pydantic_code(
plugin_name: str,
routes: list[dict[str, Any]],
) -> str:
"""生成 pydantic.py 文件内容"""
lines = []
lines.append('"""')
lines.append(f"自动生成的 Pydantic 模型 — {plugin_name}")
lines.append(f"来源: config/workers/{plugin_name}/")
lines.append('"""')
lines.append("")
lines.append("from __future__ import annotations")
lines.append("")
lines.append("from typing import Any, Optional")
lines.append("from pydantic import BaseModel, Field")
lines.append("")
seen_models: set[str] = set()
for route in routes:
route_name = route["name"]
method = route["method"]
summary = route.get("summary", "")
# --- Request Model ---
params = route.get("parsed_params", [])
if params:
model_name = _model_name(route_name, method, "Request")
if model_name not in seen_models:
seen_models.add(model_name)
lines.append(f"class {model_name}(BaseModel):")
lines.append(f' """{summary} — 请求体"""')
for p in params:
lines.append(f" {p['name']}: {p['type']} = Field(...)")
lines.append("")
# --- Response Model ---
response_fields = route.get("parsed_response", [])
if response_fields:
model_name = _model_name(route_name, method, "Response")
if model_name not in seen_models:
seen_models.add(model_name)
lines.append(f"class {model_name}(BaseModel):")
lines.append(f' """{summary} — 响应体"""')
for f in response_fields:
field_type = f["type"]
if f.get("required", True):
if "default" in f:
default_repr = (
json.dumps(f["default"])
if not isinstance(f["default"], str)
else f"'{f['default']}'"
)
lines.append(f" {f['name']}: {field_type} = Field(default={default_repr})")
else:
lines.append(f" {f['name']}: {field_type} = Field(...)")
else:
if "default" in f:
default_repr = (
json.dumps(f["default"])
if not isinstance(f["default"], str)
else f"'{f['default']}'"
)
lines.append(f" {f['name']}: Optional[{field_type}] = Field(default={default_repr})")
else:
lines.append(f" {f['name']}: Optional[{field_type}] = Field(default=None)")
lines.append("")
# --- Union type for all responses ---
response_models = [m for m in seen_models if m.endswith("Response")]
if response_models:
plugin_title = plugin_name.title().replace("-", "").replace("_", "")
lines.append("")
lines.append("# 所有响应模型的联合类型")
lines.append(f"{plugin_title}Response = " + " | ".join(response_models))
lines.append("")
return "\n".join(lines)
def generate_init_code(
plugin_name: str,
routes: list[dict[str, Any]],
) -> str:
"""生成 __init__.py 文件内容"""
lines = []
lines.append('"""')
lines.append(f"模型导出 — {plugin_name}")
lines.append("自动生成,请勿手动修改")
lines.append('"""')
lines.append("")
seen_models: set[str] = set()
for route in routes:
rname = route["name"]
method = route["method"]
if route.get("parsed_params"):
seen_models.add(_model_name(rname, method, "Request"))
if route.get("parsed_response"):
seen_models.add(_model_name(rname, method, "Response"))
response_models = [m for m in seen_models if m.endswith("Response")]
if response_models:
plugin_title = plugin_name.title().replace("-", "").replace("_", "")
seen_models.add(f"{plugin_title}Response")
for model_name in sorted(seen_models):
lines.append(f"from .pydantic import {model_name}")
lines.append("")
lines.append("__all__ = [")
for model_name in sorted(seen_models):
lines.append(f' "{model_name}",')
lines.append("]")
lines.append("")
return "\n".join(lines)
def scan_config_dir(config_dir: Path) -> list[dict[str, Any]]:
"""扫描配置目录,解析所有 .toml 文件中的路由定义"""
import tomli
routes = []
toml_files = sorted(config_dir.glob("*.toml"))
group_config = {}
config_file = config_dir / "config.toml"
if config_file.exists():
with open(config_file, "rb") as f:
group_config = tomli.load(f)
defaults = group_config.get("worker", {}).get("defaults", {})
router_prefix = (
group_config.get("worker", {})
.get("registration", {})
.get("router_prefix", "")
)
for toml_file in toml_files:
if toml_file.name == "config.toml":
continue
with open(toml_file, "rb") as f:
data = tomli.load(f)
route_name = toml_file.stem
for method in ("GET", "POST", "PUT", "DELETE", "PATCH"):
if method not in data:
continue
entry = data[method]
params_raw = entry.get("params", [])
response_raw = entry.get("response_model", {})
route = {
"name": route_name,
"method": method,
"summary": entry.get("summary", ""),
"docstring": entry.get("docstring", ""),
"params": params_raw,
"response_model": response_raw,
"parsed_params": parse_params(params_raw),
"parsed_response": parse_response_model(response_raw),
"path": entry.get("path", f"{router_prefix}/{route_name}"),
"timeout": entry.get("timeout", defaults.get("timeout", 3.0)),
"auth_required": entry.get("auth_required", defaults.get("auth_required", False)),
}
routes.append(route)
return routes
def main():
if len(sys.argv) < 3:
print("用法: uv run gen_models <config_dir> <plugin_dir>")
print("示例:")
print(" uv run gen_models config/workers/example src/plugins/example")
sys.exit(1)
config_dir = Path(sys.argv[1])
plugin_dir = Path(sys.argv[2])
plugin_name = plugin_dir.name
if not config_dir.exists():
print(f"❌ 配置目录不存在: {config_dir}")
sys.exit(1)
if not plugin_dir.exists():
print(f"❌ 插件目录不存在: {plugin_dir}")
sys.exit(1)
print(f"🔍 扫描配置目录: {config_dir}")
routes = scan_config_dir(config_dir)
print(f" 发现 {len(routes)} 个路由定义")
if not routes:
print("⚠️ 未找到路由定义,跳过模型生成")
return
models_dir = plugin_dir / "models"
models_dir.mkdir(parents=True, exist_ok=True)
pydantic_code = generate_pydantic_code(plugin_name, routes)
pydantic_file = models_dir / "pydantic.py"
pydantic_file.write_text(pydantic_code, encoding="utf-8")
print(f"✅ 生成: {pydantic_file}")
init_code = generate_init_code(plugin_name, routes)
init_file = models_dir / "__init__.py"
init_file.write_text(init_code, encoding="utf-8")
print(f"✅ 生成: {init_file}")
print(f"\n📋 生成的模型:")
for route in routes:
method = route["method"]
name = route["name"]
params = route.get("parsed_params", [])
response = route.get("parsed_response", [])
if params:
req_model = _model_name(name, method, "Request")
param_strs = [f"{p['name']}: {p['type']}" for p in params]
print(f" 📥 {req_model}: {', '.join(param_strs)}")
if response:
resp_model = _model_name(name, method, "Response")
field_strs = [f"{f['name']}: {f['type']}" for f in response]
print(f" 📤 {resp_model}: {', '.join(field_strs)}")
def gen_models():
"""CLI 入口"""
main()
if __name__ == "__main__":
main()
BIN
View File
Binary file not shown.
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python
import json
from chongming_jwt import JWT
# 使用与 config.toml 相同的密钥(可从配置读取)
SECRET = "your-secret-key-change-in-production"
jwt = JWT(SECRET, "HS256")
# 1. 生成 token
payload = json.dumps({"sub": "test_user", "role": "admin"})
token = jwt.encode(payload, expires_in_seconds=3600)
print(f"✅ Token: {token}")
# 2. 解码验证
decoded = jwt.decode(token)
print(f"✅ Decoded: {decoded}")
# 3. 仅验证(不返回内容)
is_valid = jwt.validate(token)
print(f"✅ Valid: {is_valid}")
# 4. 测试无效 token
try:
jwt.decode(token + "invalid")
except Exception as e:
print(f"❌ Invalid token caught: {e}")
Generated
+1485
View File
File diff suppressed because it is too large Load Diff