add first commit
This commit is contained in:
+28
@@ -0,0 +1,28 @@
|
||||
# Python-generated files
|
||||
__pycache__/
|
||||
*.py[oc]
|
||||
build/
|
||||
build_nuitka/
|
||||
dist/
|
||||
wheels/
|
||||
*.egg-info
|
||||
|
||||
# Virtual environments
|
||||
.venv
|
||||
data
|
||||
logs
|
||||
resources
|
||||
cache_data
|
||||
output.md
|
||||
|
||||
# plugins
|
||||
src/plugins/
|
||||
|
||||
# chongming-web
|
||||
src/chongming-web
|
||||
|
||||
# chonging-config
|
||||
config
|
||||
|
||||
# chongming-utils
|
||||
utils
|
||||
@@ -0,0 +1 @@
|
||||
3.12
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
[default]
|
||||
app.debug = false
|
||||
app.description = "基于 FastAPI 的 chongming 打包脚手架"
|
||||
app.name = "chongming"
|
||||
app.version = "chongming"
|
||||
env = "development"
|
||||
prefix = "/api/v1"
|
||||
upload_path = "uploads"
|
||||
|
||||
[nanio]
|
||||
nanio_access_key = "minioadmin"
|
||||
nanio_secret_key = "minioadmin"
|
||||
storage_path = "./data/storage"
|
||||
endpoint_url = "http://127.0.0.1:9000"
|
||||
signature_version = "s3v4"
|
||||
region_name = "us-east-1"
|
||||
bucket = "chongming"
|
||||
|
||||
[zenoh]
|
||||
json5_path = "./config/zenoh_config.json5"
|
||||
|
||||
[scheduler]
|
||||
job_store_path = "scheduler_jobs.db"
|
||||
|
||||
[cache]
|
||||
cache_store_path = "./cache_data"
|
||||
|
||||
[security]
|
||||
access_token_expire_minutes = 1440
|
||||
algorithm = "HS256"
|
||||
max_sessions_per_user = 5
|
||||
refresh_token_expire_days = 7
|
||||
secret_key = "your-secret-key-change-in-production"
|
||||
|
||||
[development]
|
||||
debug = true
|
||||
env = "development"
|
||||
|
||||
[development.server]
|
||||
access_log = false
|
||||
host = "0.0.0.0"
|
||||
port = 8000
|
||||
reload = true
|
||||
|
||||
[development.logging]
|
||||
file = "logs"
|
||||
format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
level = "DEBUG"
|
||||
|
||||
[development.cors]
|
||||
allow_credentials = true
|
||||
allow_headers = ["*"]
|
||||
allow_methods = ["*"]
|
||||
allow_origins = ["*"]
|
||||
|
||||
[plugins_worker]
|
||||
config_path = "./config/workers/config.toml"
|
||||
|
||||
[pyinstaller_build_import]
|
||||
base = [
|
||||
"utils", # public 骨架
|
||||
"utils.launch",
|
||||
"utils.config",
|
||||
"zenoh",
|
||||
"chongming_worker",
|
||||
"chongming_request",
|
||||
"server",
|
||||
"module_bank", # 模块化打包工具
|
||||
"sqlite_vfs",
|
||||
"sqlite_vfs.core",
|
||||
"fastapi", # web 应用依赖
|
||||
"fastapi.staticfiles",
|
||||
"fastapi.middleware.cors",
|
||||
"uvicorn",
|
||||
"apscheduler",
|
||||
"diskcache",
|
||||
"orjson",
|
||||
"python_multipart",
|
||||
"nanio",
|
||||
"boto3",
|
||||
"modelvault",
|
||||
]
|
||||
linux = [
|
||||
"gunicorn.glogging",
|
||||
"gunicorn.app",
|
||||
"gunicorn.app.wsgiapp",
|
||||
]
|
||||
windows = []
|
||||
|
||||
[production.module_system]
|
||||
path = [
|
||||
["resources/app.mbank", "src/chongming/app", "app"],
|
||||
["resources/plugins.mbank", "src/plugins", "plugins"],
|
||||
]
|
||||
type = "module-bank"
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 232 KiB |
@@ -0,0 +1,69 @@
|
||||
[default]
|
||||
app.debug = false
|
||||
app.description = "基于 FastAPI 的 chongming 打包脚手架"
|
||||
app.name = "chongming"
|
||||
app.version = "0.1.0"
|
||||
env = "production"
|
||||
prefix = "/api/v1"
|
||||
upload_path = "uploads"
|
||||
|
||||
[nanio]
|
||||
nanio_access_key = "minioadmin"
|
||||
nanio_secret_key = "minioadmin"
|
||||
storage_path = "./data/storage"
|
||||
endpoint_url = "http://127.0.0.1:9000"
|
||||
signature_version = "s3v4"
|
||||
region_name = "us-east-1"
|
||||
bucket = "chongming"
|
||||
|
||||
[zenoh]
|
||||
json5_path = "./config/zenoh_config.json5"
|
||||
|
||||
[scheduler]
|
||||
job_store_path = "scheduler_jobs.db"
|
||||
|
||||
[cache]
|
||||
cache_store_path = "./"
|
||||
|
||||
[security]
|
||||
access_token_expire_minutes = 30
|
||||
algorithm = "HS256"
|
||||
max_sessions_per_user = 5
|
||||
refresh_token_expire_days = 7
|
||||
secret_key = "your-secret-key-change-in-production"
|
||||
|
||||
[production]
|
||||
debug = false
|
||||
env = "production"
|
||||
|
||||
[production.server]
|
||||
access_log = false
|
||||
host = "0.0.0.0"
|
||||
port = 8000
|
||||
reload = false
|
||||
timeout_keep_alive = 65
|
||||
workers = 4
|
||||
|
||||
[production.logging]
|
||||
backup_count = 5 # 最多保存 10 个日志文件
|
||||
file = "logs"
|
||||
format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
level = "INFO"
|
||||
max_size = 10485760 # 10MB
|
||||
|
||||
[production.cors]
|
||||
allow_credentials = true
|
||||
allow_headers = ["*"]
|
||||
allow_methods = ["*"]
|
||||
allow_origins = ["*"]
|
||||
|
||||
[production.module_system]
|
||||
path = [
|
||||
"resources/app.mbank",
|
||||
"resources/plugins.mbank",
|
||||
]
|
||||
type = "module-bank"
|
||||
|
||||
[production.file_system]
|
||||
path = "static.svfs"
|
||||
type = "svfs"
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
# main.py - 修复 reload 问题
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
FastAPI chongming 示例
|
||||
主入口文件,适配 chongming 打包系统
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import platform
|
||||
from utils.launch import launch
|
||||
from multiprocessing import freeze_support
|
||||
import server
|
||||
|
||||
use_shell = platform.system() == "Windows"
|
||||
|
||||
|
||||
def app_run(app_config: dict, default_config: dict):
|
||||
"""
|
||||
应用运行函数 - 适配模块银行环境
|
||||
|
||||
Args:
|
||||
app_config: 应用配置字典,包含服务器配置、环境设置等
|
||||
default_config: 默认配置字典,包含应用名称、版本等基础信息
|
||||
|
||||
Returns:
|
||||
None: 此函数不返回值,直接启动服务器并阻塞运行
|
||||
"""
|
||||
from app.core.logger import get_logger # type: ignore
|
||||
|
||||
logger = get_logger("app")
|
||||
|
||||
# 获取服务器配置
|
||||
server_config = app_config.get("server", {})
|
||||
host = server_config.get("host", "0.0.0.0")
|
||||
port = server_config.get("port", 8000)
|
||||
timeout_keep_alive = server_config.get("timeout_keep_alive", 5)
|
||||
|
||||
# 打包环境强制禁用重载
|
||||
is_frozen = getattr(sys, "frozen", False)
|
||||
reload = False if is_frozen else server_config.get("reload", False)
|
||||
# 支持通过环境变量 APP_WORKERS 覆盖 worker 数量(方便嵌入式设备调优)
|
||||
default_workers = server_config.get("workers", 4)
|
||||
workers = int(os.environ.get("APP_WORKERS", str(default_workers)))
|
||||
|
||||
# 显示启动信息
|
||||
env = app_config.get("env", "development")
|
||||
logger.info("=" * 50)
|
||||
logger.info(f"🚀 启动 {default_config.get('app.name', 'FastAPI 应用')}")
|
||||
logger.info(f"📦 版本: {default_config.get('app.version', '1.0.0')}")
|
||||
logger.info(f"🌍 环境: {env}")
|
||||
logger.info(f"📍 地址: http://{host}:{port}")
|
||||
logger.info(f"📚 文档: http://{host}:{port}/docs")
|
||||
logger.info(f"🔧 调试: {'启用' if app_config.get('debug', False) else '禁用'}")
|
||||
logger.info(f"🔄 重载: {'启用' if reload else '禁用'}")
|
||||
logger.info(f"👥 Workers: {workers}")
|
||||
logger.info("=" * 50)
|
||||
|
||||
# 根据操作系统选择不同的服务器启动方式
|
||||
if use_shell:
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(
|
||||
"server:app",
|
||||
host=host,
|
||||
port=port,
|
||||
reload=reload,
|
||||
workers=workers,
|
||||
log_level="debug" if app_config.get("debug", False) else "info",
|
||||
access_log=False,
|
||||
timeout_keep_alive=timeout_keep_alive,
|
||||
)
|
||||
else:
|
||||
from gunicorn.app.wsgiapp import run
|
||||
|
||||
sys.argv = [
|
||||
"gunicorn",
|
||||
"server:app",
|
||||
"-k",
|
||||
"uvicorn.workers.UvicornWorker",
|
||||
"-w",
|
||||
str(workers),
|
||||
"-b",
|
||||
f"{host}:{port}",
|
||||
]
|
||||
run()
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
主函数入口 - 初始化多进程支持并启动应用
|
||||
|
||||
Returns:
|
||||
None: 此函数不返回值,调用launch函数启动应用
|
||||
"""
|
||||
freeze_support()
|
||||
launch(app_run, "production")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,21 @@
|
||||
# server.py
|
||||
import sys
|
||||
from utils.launch import init_module_bank
|
||||
from utils.config import ConfigLoader
|
||||
|
||||
# 加载配置文件并初始化模块银行
|
||||
config = ConfigLoader("config.toml")
|
||||
init_module_bank(config)
|
||||
|
||||
# 从数据库动态导入 app 模块
|
||||
# 该模块存储在 SQLite 数据库中,通过自定义导入机制加载
|
||||
try:
|
||||
import app # type: ignore
|
||||
|
||||
app_instance = app.app
|
||||
except ImportError as e:
|
||||
print(f"无法从数据库导入 app 模块: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# 暴露 ASGI 应用实例,供 uvicorn 服务器使用
|
||||
app = app_instance
|
||||
@@ -0,0 +1,64 @@
|
||||
[build-system]
|
||||
build-backend = "hatchling.build"
|
||||
requires = ["hatchling"]
|
||||
|
||||
[project]
|
||||
dependencies = [
|
||||
"apscheduler",
|
||||
"boto3",
|
||||
"chongming-jwt",
|
||||
"chongming-request",
|
||||
"chongming-worker",
|
||||
"diskcache",
|
||||
"eclipse-zenoh",
|
||||
"fastapi[uvicorn]",
|
||||
"gunicorn",
|
||||
"modelvault",
|
||||
"module-bank",
|
||||
"nanio",
|
||||
"openai",
|
||||
"orjson",
|
||||
"python-multipart",
|
||||
"sqlite-vfs",
|
||||
"tomli",
|
||||
"typing-extensions",
|
||||
"uvicorn",
|
||||
"volcengine-python-sdk[ark]>=5.0.41",
|
||||
]
|
||||
description = "基于 modules-bank 的 fastapi 打包脚手架"
|
||||
name = "chongming-zenoh"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
version = "0.1.0"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"nuitka",
|
||||
"pyarmor",
|
||||
"pyinstaller",
|
||||
"ruff",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/chongming"]
|
||||
|
||||
[[tool.uv.index]]
|
||||
url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
|
||||
[tool.uv.workspace]
|
||||
members = [
|
||||
"utils/worker",
|
||||
"utils/request",
|
||||
"utils/jwt",
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
chongming-request = { workspace = true }
|
||||
chongming-worker = { workspace = true }
|
||||
chongming-jwt = { workspace = true }
|
||||
|
||||
[project.scripts]
|
||||
build = "chongming.scripts.build:build"
|
||||
gen_models = "chongming.scripts.gen_models:gen_models"
|
||||
gunicorn = "chongming:gunicorn_serve"
|
||||
serve = "chongming:serve"
|
||||
@@ -0,0 +1,60 @@
|
||||
import uvicorn
|
||||
from .app.core.config import load_config
|
||||
from .app.core.logger import get_logger
|
||||
|
||||
|
||||
def serve():
|
||||
"""
|
||||
使用 uvicorn 启动服务
|
||||
|
||||
从配置文件中加载服务器配置和日志配置,然后使用 uvicorn 启动应用服务。
|
||||
该函数适用于开发环境或单进程部署场景。
|
||||
|
||||
Returns:
|
||||
None: 该函数不返回值,会阻塞运行直到服务停止
|
||||
"""
|
||||
config = load_config()
|
||||
env = config["default"]["env"]
|
||||
server_config = config[env]["server"]
|
||||
log_config = config[env]["logging"]
|
||||
logger = get_logger("app")
|
||||
logger.info(f"启动服务: {server_config}")
|
||||
logger.info(f"启动日志: {log_config}")
|
||||
uvicorn.run("chongming.app:app", **server_config)
|
||||
|
||||
|
||||
def gunicorn_serve():
|
||||
"""
|
||||
使用 gunicorn 启动服务
|
||||
|
||||
通过 gunicorn 作为进程管理器启动应用,使用 UvicornWorker 作为 worker 类型,
|
||||
支持多进程部署,适用于生产环境。
|
||||
|
||||
从配置文件中读取服务器配置,动态设置 gunicorn 的 worker 数量、绑定地址和端口等参数。
|
||||
|
||||
Returns:
|
||||
None: 该函数不返回值,会阻塞运行直到服务停止
|
||||
"""
|
||||
import sys
|
||||
from gunicorn.app.wsgiapp import run
|
||||
|
||||
config = load_config()
|
||||
env = config["default"]["env"]
|
||||
server_config = config[env]["server"]
|
||||
|
||||
# 构建 gunicorn 命令行参数并启动服务
|
||||
sys.argv = [
|
||||
"gunicorn",
|
||||
"chongming.app:app",
|
||||
"-k",
|
||||
"uvicorn.workers.UvicornWorker",
|
||||
"-w",
|
||||
f"{server_config.get('workers', 4)}",
|
||||
"-b",
|
||||
f"{server_config.get('host', '0.0.0.0')}:{server_config.get('port', 8000)}",
|
||||
]
|
||||
run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
serve()
|
||||
@@ -0,0 +1,190 @@
|
||||
"""
|
||||
chongming FastAPI 应用入口
|
||||
|
||||
重构为配置驱动的微服务网关:
|
||||
- WorkerManager: 统一管理 Zenoh worker 存活检测、负载均衡、超时重试、缓存
|
||||
- RouteGenerator: 根据配置自动生成 FastAPI 端点
|
||||
- 插件工厂函数动态加载
|
||||
"""
|
||||
|
||||
import time
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from .core.config import get_config
|
||||
from .core.logger import get_logger
|
||||
from .core.static_files import SVFSStaticFiles
|
||||
from .core import lifespan_setup
|
||||
from .routes import storage_router
|
||||
|
||||
|
||||
config = get_config()
|
||||
logger = get_logger("app")
|
||||
|
||||
try:
|
||||
env = config["default"]["env"]
|
||||
except KeyError: # pragma: no cover
|
||||
raise ValueError("配置不存在")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# ---- 启动阶段 ----
|
||||
# 1. nanio
|
||||
lifespan_setup.setup_nanio(app)
|
||||
await asyncio.sleep(1) # 等待 nanio 启动
|
||||
|
||||
# 2. S3
|
||||
lifespan_setup.setup_s3(app)
|
||||
|
||||
# 3. Zenoh
|
||||
session = lifespan_setup.setup_zenoh(app)
|
||||
|
||||
# 4. WorkerManager
|
||||
manager = lifespan_setup.setup_worker_manager(app, session)
|
||||
|
||||
# 5. 加载 workers
|
||||
worker_tasks = await lifespan_setup.load_workers(app, session, manager)
|
||||
|
||||
# 6. 注册路由
|
||||
await lifespan_setup.register_routes(app, manager)
|
||||
|
||||
# 7. 短暂等待,确保 worker 状态同步
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# 打印 worker 就绪状态
|
||||
worker_status = {
|
||||
wtype: len(workers)
|
||||
for wtype, workers in manager._workers_cache.items()
|
||||
}
|
||||
logger.info(f"Worker 就绪状态: {worker_status}")
|
||||
|
||||
# ---- 应用运行 ----
|
||||
yield
|
||||
|
||||
# ---- 关闭阶段 ----
|
||||
logger.info("正在关闭应用...")
|
||||
|
||||
# 1. 清理 workers(调用 cleanup)
|
||||
await lifespan_setup.cleanup_workers(app)
|
||||
|
||||
# 2. 取消刷新任务
|
||||
refresh_task = getattr(app.state, "refresh_task", None)
|
||||
if refresh_task:
|
||||
refresh_task.cancel()
|
||||
try:
|
||||
await refresh_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# 3. 取消所有 worker 任务
|
||||
for task in worker_tasks:
|
||||
task.cancel()
|
||||
if worker_tasks:
|
||||
await asyncio.gather(*worker_tasks, return_exceptions=True)
|
||||
|
||||
# 4. 关闭 Zenoh 会话
|
||||
session.close()
|
||||
logger.info("Zenoh 会话已关闭")
|
||||
|
||||
# 5. 终止 nanio
|
||||
await lifespan_setup.shutdown_nanio(app)
|
||||
|
||||
logger.info("应用已关闭")
|
||||
|
||||
|
||||
cors = config[env].get("cors", {})
|
||||
app = FastAPI(
|
||||
title=config["default"]["app"]["name"],
|
||||
version=config["default"]["app"]["version"],
|
||||
description=config["default"]["app"]["description"],
|
||||
docs_url="/docs" if config[env]["debug"] else None,
|
||||
redoc_url="/redoc" if config[env]["debug"] else None,
|
||||
openapi_url="/openapi.json" if config[env]["debug"] else None,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(CORSMiddleware, **cors)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def access_log_middleware(request: Request, call_next):
|
||||
"""
|
||||
HTTP访问日志中间件,记录每个请求的详细信息。
|
||||
|
||||
记录的信息包括:
|
||||
- 请求方法和路径
|
||||
- 响应状态码
|
||||
- 请求处理时长(毫秒)
|
||||
- 客户端IP地址
|
||||
|
||||
参数:
|
||||
request: FastAPI请求对象
|
||||
call_next: 下一个中间件或路由处理函数
|
||||
|
||||
返回:
|
||||
Response: 处理后的响应对象
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
response = await call_next(request)
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
client = request.client.host if request.client else "-"
|
||||
|
||||
logger.info(
|
||||
f"{request.method} {request.url.path} status={response.status_code} "
|
||||
f"duration={duration:.2f}ms client={client}",
|
||||
extra={"status_code": response.status_code},
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@app.get("/", summary="根路径")
|
||||
async def root():
|
||||
"""
|
||||
根路径端点,返回应用的基本信息。
|
||||
|
||||
返回:
|
||||
dict: 包含应用名称、版本、文档地址和健康检查地址的字典
|
||||
"""
|
||||
return {
|
||||
"message": f"欢迎使用 {config['default']['app']['name']}",
|
||||
"version": config["default"]["app"]["version"],
|
||||
"docs": "/docs" if config[env]["debug"] else None,
|
||||
"health": f"{config['default']['prefix']}/health",
|
||||
}
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def global_exception_handler(_: Request, exc: Exception):
|
||||
"""
|
||||
全局异常处理器,捕获并处理应用中未捕获的异常。
|
||||
|
||||
在调试模式下返回详细的异常信息,在生产模式下返回通用错误提示。
|
||||
|
||||
参数:
|
||||
_: FastAPI请求对象(未使用)
|
||||
exc: 捕获到的异常对象
|
||||
|
||||
返回:
|
||||
JSONResponse: 包含错误信息的JSON响应,状态码为500
|
||||
"""
|
||||
logger.error(f"全局异常: {exc}")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={
|
||||
"error": "内部服务器错误",
|
||||
"detail": str(exc) if config[env]["debug"] else "请查看服务器日志",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# 静态文件服务(可选)
|
||||
if vfs_db_path := config[env].get("file_system", {}).get("path", None):
|
||||
logger.info("静态文件已存在,将使用静态文件服务")
|
||||
static_files_handler = SVFSStaticFiles(directory="/", vfs_db_path=vfs_db_path)
|
||||
app.mount("/static", static_files_handler, name="static")
|
||||
|
||||
app.include_router(storage_router)
|
||||
@@ -0,0 +1,207 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TypedDict, Literal, Optional, cast, List
|
||||
|
||||
|
||||
# 根据Python版本选择合适的TOML解析库和Required类型
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
from typing import Required
|
||||
else:
|
||||
import tomli as tomllib
|
||||
from typing_extensions import Required
|
||||
|
||||
# 设置 env 为 development 或 production
|
||||
|
||||
|
||||
class DefaultApp(TypedDict):
|
||||
"""应用默认配置结构"""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
description: str
|
||||
debug: bool
|
||||
|
||||
|
||||
class Default(TypedDict):
|
||||
"""系统默认配置结构,包含应用信息、环境、路径等基础配置"""
|
||||
|
||||
app: DefaultApp
|
||||
env: Literal["production", "development"]
|
||||
prefix: str
|
||||
upload_path: str
|
||||
|
||||
|
||||
class SqliteConnectionArgs(TypedDict, total=False):
|
||||
"""SQLite数据库连接参数配置"""
|
||||
|
||||
timeout: int
|
||||
check_same_thread: bool
|
||||
|
||||
|
||||
class DatabaseSqlite(TypedDict, total=False):
|
||||
"""SQLite数据库连接池及行为配置"""
|
||||
|
||||
echo: bool
|
||||
pool_size: int
|
||||
max_overflow: int
|
||||
pool_timeout: int
|
||||
pool_recycle: int
|
||||
pool_pre_ping: bool
|
||||
connect_args: SqliteConnectionArgs
|
||||
|
||||
|
||||
class Database(TypedDict, total=False):
|
||||
"""数据库配置结构,支持SQLite、MySQL、PostgreSQL三种数据库类型"""
|
||||
|
||||
type: Required[Literal["sqlite", "mysql", "postgresql"]]
|
||||
url: Required[str]
|
||||
generate_schemas: bool
|
||||
sqlite: DatabaseSqlite
|
||||
mysql: dict
|
||||
postgresql: dict
|
||||
database_path: str
|
||||
|
||||
|
||||
class Security(TypedDict):
|
||||
"""安全相关配置,包括JWT令牌和安全策略设置"""
|
||||
|
||||
secret_key: str
|
||||
algorithm: Literal["HS256"]
|
||||
access_token_expire_minutes: int
|
||||
refresh_token_expire_days: int
|
||||
max_sessions_per_user: int
|
||||
|
||||
|
||||
class Server(TypedDict, total=False):
|
||||
"""服务器运行配置,包含主机、端口、工作进程等设置"""
|
||||
|
||||
host: Required[str]
|
||||
port: Required[int]
|
||||
reload: bool
|
||||
workers: int
|
||||
access_log: bool
|
||||
|
||||
|
||||
class Logging(TypedDict):
|
||||
"""日志系统配置,定义日志级别、输出格式和文件管理策略"""
|
||||
|
||||
level: str
|
||||
file: str
|
||||
format: str
|
||||
max_size: int
|
||||
backup_count: int
|
||||
|
||||
|
||||
class Cors(TypedDict, total=False):
|
||||
"""跨域资源共享(CORS)配置"""
|
||||
|
||||
allow_origins: List[str]
|
||||
allow_credentials: bool
|
||||
allow_methods: List[str]
|
||||
allow_headers: List[str]
|
||||
|
||||
|
||||
class ModuleSystem(TypedDict, total=False):
|
||||
"""模块系统配置,定义模块加载类型和路径"""
|
||||
|
||||
type: str
|
||||
path: List[str]
|
||||
|
||||
|
||||
class FileSystem(TypedDict, total=False):
|
||||
"""文件系统配置,定义存储类型和路径"""
|
||||
|
||||
type: str
|
||||
path: str
|
||||
|
||||
|
||||
class Env(TypedDict, total=False):
|
||||
"""环境特定配置结构,用于开发和生产环境的差异化配置"""
|
||||
|
||||
debug: Required[bool]
|
||||
server: Required[Server]
|
||||
logging: Required[Logging]
|
||||
cors: Cors
|
||||
module_system: ModuleSystem
|
||||
file_system: FileSystem
|
||||
|
||||
|
||||
class Scheduler(TypedDict):
|
||||
"""任务调度器配置"""
|
||||
|
||||
job_store_path: str
|
||||
|
||||
|
||||
class Cache(TypedDict):
|
||||
"""缓存系统配置"""
|
||||
|
||||
cache_store_path: str
|
||||
|
||||
|
||||
class Config(TypedDict):
|
||||
"""完整配置结构,整合所有配置项,包含默认配置、数据库、调度器、缓存、安全和环境配置"""
|
||||
|
||||
default: Default
|
||||
database: Database
|
||||
scheduler: Scheduler
|
||||
cache: Cache
|
||||
security: Security
|
||||
development: Env
|
||||
production: Env
|
||||
|
||||
|
||||
def load_config(config_path: Path = Path("config.toml")) -> Config:
|
||||
"""
|
||||
从TOML配置文件加载配置数据
|
||||
|
||||
Args:
|
||||
config_path: 配置文件路径,默认为当前目录下的config.toml
|
||||
|
||||
Returns:
|
||||
Config: 解析后的配置字典对象
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: 当配置文件不存在时抛出异常
|
||||
"""
|
||||
if not config_path.exists():
|
||||
raise FileNotFoundError(f"配置文件不存在: {config_path}")
|
||||
with open(config_path, "rb") as f:
|
||||
return cast(Config, tomllib.load(f))
|
||||
|
||||
|
||||
# 全局配置缓存变量,避免重复加载配置文件
|
||||
_config: Optional[Config] = None
|
||||
|
||||
|
||||
def get_config(config_path: str = "config.toml") -> Config:
|
||||
"""
|
||||
获取应用程序配置(带缓存机制)
|
||||
|
||||
首次调用时加载配置文件并缓存,后续调用直接返回缓存的配置对象
|
||||
|
||||
Args:
|
||||
config_path: 配置文件路径,默认为config.toml
|
||||
|
||||
Returns:
|
||||
Config: 配置字典对象
|
||||
"""
|
||||
global _config
|
||||
if _config is None:
|
||||
_config = load_config(Path(config_path))
|
||||
return _config
|
||||
|
||||
def get_workers_config(config_path: str = "config/workers/config.toml") -> dict:
|
||||
"""
|
||||
获取Worker插件配置列表
|
||||
|
||||
Args:
|
||||
config_path: Worker配置文件路径,默认为config/workers/config.toml
|
||||
|
||||
Returns:
|
||||
list: Worker配置字典列表
|
||||
"""
|
||||
if not Path(config_path).exists():
|
||||
raise FileNotFoundError(f"Worker配置文件不存在: {config_path}")
|
||||
with open(config_path, "rb") as f:
|
||||
return cast(dict, tomllib.load(f))
|
||||
@@ -0,0 +1,173 @@
|
||||
import os
|
||||
|
||||
import inspect
|
||||
import subprocess
|
||||
import asyncio
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
from botocore.config import Config
|
||||
from diskcache import Cache
|
||||
from fastapi import FastAPI
|
||||
import zenoh
|
||||
|
||||
from ..core.config import get_config, get_workers_config
|
||||
from ..core.logger import get_logger
|
||||
from ..core.route_generator import register_dynamic_routes
|
||||
from ..core.worker_manager import WorkerManager
|
||||
|
||||
config = get_config()
|
||||
logger = get_logger("app")
|
||||
|
||||
try:
|
||||
env = config["default"]["env"]
|
||||
except KeyError: # pragma: no cover
|
||||
raise ValueError("配置不存在")
|
||||
|
||||
def setup_nanio(app: FastAPI):
|
||||
"""启动 nanio 子进程作为 S3 兼容存储服务。"""
|
||||
nanio_config = config.get("nanio", {})
|
||||
env = os.environ.copy()
|
||||
env["NANIO_ACCESS_KEY"] = nanio_config.get("nanio_access_key", "minioadmin")
|
||||
env["NANIO_SECRET_KEY"] = nanio_config.get("nanio_secret_key", "minioadmin")
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"nanio",
|
||||
"serve",
|
||||
"--data-dir",
|
||||
nanio_config.get("storage_path", "./data/storage")
|
||||
],
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL
|
||||
)
|
||||
app.state.nanio_process = proc
|
||||
logger.info("Nanio 进程已启动")
|
||||
|
||||
def setup_s3(app: FastAPI):
|
||||
"""初始化 S3 客户端,并确保 bucket 存在。"""
|
||||
nanio_config = config.get("nanio", {})
|
||||
s3_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=nanio_config.get("endpoint_url", "http://127.0.0.1:9000"),
|
||||
aws_access_key_id=nanio_config.get("nanio_access_key", "minioadmin"),
|
||||
aws_secret_access_key=nanio_config.get("nanio_secret_key", "minioadmin"),
|
||||
config=Config(signature_version=nanio_config.get("signature_version", "s3v4")),
|
||||
region_name=nanio_config.get("region_name", "us-east-1")
|
||||
)
|
||||
try:
|
||||
s3_client.head_bucket(Bucket=nanio_config.get("bucket", "chongming"))
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] == "404":
|
||||
s3_client.create_bucket(Bucket="chongming")
|
||||
logger.info("Bucket 'chongming' 已创建")
|
||||
else:
|
||||
raise
|
||||
|
||||
app.state.s3_client = s3_client
|
||||
app.state.storage_bucket = nanio_config.get("bucket", "chongming")
|
||||
app.state.storage_presign_expire = 300
|
||||
app.state.storage_max_size = 100 * 1024 * 1024 # 100MB
|
||||
|
||||
def setup_zenoh(app: FastAPI) -> zenoh.Session:
|
||||
"""建立 Zenoh 会话并存入 app.state。"""
|
||||
zenoh_config = config.get("zenoh", {})
|
||||
|
||||
zconfig = zenoh.Config()
|
||||
zconfig.from_file(zenoh_config.get("json5_path", "config/zenoh_config.json5"))
|
||||
|
||||
try:
|
||||
session = zenoh.open(zconfig)
|
||||
logger.info("Zenoh 会话已建立 (TCP 自连接模式)")
|
||||
except Exception as e:
|
||||
logger.error(f"Zenoh 会话建立失败: {e}")
|
||||
raise
|
||||
|
||||
app.state.zenoh_session = session
|
||||
return session
|
||||
|
||||
def setup_worker_manager(app: FastAPI, session: zenoh.Session) -> WorkerManager:
|
||||
"""创建 WorkerManager,启动定期刷新任务,并存入 app.state。"""
|
||||
cache = Cache(config["cache"]["cache_store_path"])
|
||||
manager = WorkerManager(session, cache)
|
||||
app.state.worker_manager = manager
|
||||
|
||||
refresh_task = asyncio.create_task(manager.refresh_workers())
|
||||
app.state.refresh_task = refresh_task
|
||||
return manager
|
||||
|
||||
async def load_workers(app: FastAPI, session: zenoh.Session, manager: WorkerManager):
|
||||
"""
|
||||
加载配置中的 worker 插件,启动对应的 worker 任务。
|
||||
返回任务列表。
|
||||
"""
|
||||
workers_config_path = config.get("plugins_worker", {}).get("config_path", "config/workers/config.toml")
|
||||
workers_conf = get_workers_config(workers_config_path).get("workers", [])
|
||||
logger.info(f"读取到 {len(workers_conf)} 个 worker 配置")
|
||||
|
||||
worker_tasks = []
|
||||
for wc in workers_conf:
|
||||
module_name = wc.get("module", "")
|
||||
factory_name = wc.get("factory", "")
|
||||
count = wc.get("count", 1)
|
||||
worker_config = wc.get("config", {})
|
||||
|
||||
if not module_name or not factory_name:
|
||||
logger.warning(f"Worker 配置不完整: {wc}")
|
||||
continue
|
||||
|
||||
try:
|
||||
module = __import__(module_name, fromlist=[factory_name])
|
||||
factory = getattr(module, factory_name)
|
||||
if inspect.iscoroutinefunction(factory):
|
||||
tasks = await factory(session, worker_config, count, worker_manager=manager)
|
||||
else:
|
||||
tasks = factory(session, worker_config, count, worker_manager=manager)
|
||||
worker_tasks.extend(tasks)
|
||||
logger.info(f"已加载 Worker 插件: {module_name}.{factory_name} (副本数: {count})")
|
||||
except Exception as e:
|
||||
logger.error(f"加载 Worker 插件失败 {module_name}: {e}")
|
||||
|
||||
app.state.worker_tasks = worker_tasks
|
||||
return worker_tasks
|
||||
|
||||
async def register_routes(app: FastAPI, manager: WorkerManager):
|
||||
"""注册动态路由。"""
|
||||
prefix = config["default"]["prefix"]
|
||||
register_dynamic_routes(app, manager, prefix=prefix)
|
||||
# 可选:等待路由注册生效(已在 register_dynamic_routes 内部完成日志)
|
||||
|
||||
async def shutdown_nanio(app: FastAPI):
|
||||
"""终止 nanio 进程。"""
|
||||
proc = getattr(app.state, "nanio_process", None)
|
||||
if proc:
|
||||
proc.terminate()
|
||||
proc.wait()
|
||||
logger.info("Nanio 进程已终止")
|
||||
|
||||
async def cleanup_workers(app: FastAPI):
|
||||
"""调用每个 worker 配置中定义的 cleanup 函数。"""
|
||||
workers_config_path = config.get("plugins_worker", {}).get("config_path", "config/workers/config.toml")
|
||||
workers_conf = get_workers_config(workers_config_path).get("workers", [])
|
||||
|
||||
for wc in workers_conf:
|
||||
module_name = wc.get("module", "")
|
||||
cleanup_name = wc.get("cleanup", "")
|
||||
worker_type = wc.get("type", "")
|
||||
|
||||
if not module_name or not cleanup_name:
|
||||
logger.warning(f"Worker 配置缺少 cleanup: {wc}")
|
||||
continue
|
||||
|
||||
try:
|
||||
module = __import__(module_name, fromlist=[cleanup_name])
|
||||
cleanup = getattr(module, cleanup_name)
|
||||
pid = os.getpid()
|
||||
worker_id = f"worker/{worker_type}/worker-i-{pid}"
|
||||
if inspect.iscoroutinefunction(cleanup):
|
||||
await cleanup(worker_id)
|
||||
else:
|
||||
cleanup(worker_id)
|
||||
except Exception as e:
|
||||
logger.error(f"cleanup 调用失败 {module_name}: {e}")
|
||||
@@ -0,0 +1,191 @@
|
||||
import re
|
||||
import logging
|
||||
import datetime
|
||||
from pathlib import Path
|
||||
from logging.handlers import RotatingFileHandler
|
||||
|
||||
from ..core.config import get_config
|
||||
|
||||
|
||||
config = get_config()
|
||||
env = config["default"]["env"]
|
||||
server_config = config[env]["server"]
|
||||
log_config = config[env]["logging"]
|
||||
|
||||
|
||||
class ColoredFormatter(logging.Formatter):
|
||||
"""
|
||||
为 HTTP 状态码添加颜色的日志格式化器
|
||||
|
||||
根据 HTTP 状态码的首位数字为日志消息中的状态码添加不同颜色:
|
||||
- 2xx: 绿色(成功)
|
||||
- 3xx: 青色(重定向)
|
||||
- 4xx: 黄色(客户端错误)
|
||||
- 5xx: 红色(服务器错误)
|
||||
|
||||
Attributes:
|
||||
COLORS: 状态码首位数字到 ANSI 颜色代码的映射字典
|
||||
RESET: ANSI 重置颜色代码
|
||||
"""
|
||||
|
||||
COLORS = {
|
||||
"2": "\033[32m", # 绿色 2xx
|
||||
"3": "\033[36m", # 青色 3xx
|
||||
"4": "\033[33m", # 黄色 4xx
|
||||
"5": "\033[31m", # 红色 5xx
|
||||
}
|
||||
RESET = "\033[0m"
|
||||
|
||||
def format(self, record):
|
||||
"""
|
||||
格式化日志记录,为状态码添加颜色
|
||||
|
||||
如果日志记录包含 status_code 属性,则查找消息中的 "status=数字" 模式
|
||||
并为其添加对应的颜色代码。格式化完成后恢复原始消息以避免影响其他 handler。
|
||||
|
||||
Args:
|
||||
record: 日志记录对象,应包含 msg 属性和可选的 status_code 属性
|
||||
|
||||
Returns:
|
||||
str: 格式化后的日志消息字符串,状态码部分可能包含 ANSI 颜色代码
|
||||
"""
|
||||
# 保存原始消息
|
||||
original_msg = record.msg
|
||||
# 如果日志记录包含 status_code 属性,则为其添加颜色
|
||||
status_code = getattr(record, "status_code", None)
|
||||
if status_code and isinstance(status_code, int):
|
||||
first_digit = str(status_code)[0]
|
||||
color = self.COLORS.get(first_digit, "")
|
||||
colored_status = f"{color}{status_code}{self.RESET}"
|
||||
# 替换消息中的 "status=数字" 为带颜色的版本
|
||||
record.msg = re.sub(
|
||||
rf"status={status_code}\b", f"status={colored_status}", original_msg
|
||||
)
|
||||
result = super().format(record)
|
||||
# 恢复原始消息,避免影响其他 handler
|
||||
record.msg = original_msg
|
||||
return result
|
||||
|
||||
|
||||
class ExcludeWatchFilesFilter(logging.Filter):
|
||||
"""
|
||||
日志过滤器,用于排除第三方库的日志输出
|
||||
|
||||
过滤掉来自 watchfiles、aiosqlite、apscheduler、sqlalchemy 等库的日志记录,
|
||||
避免这些库的详细日志污染应用日志输出。
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化过滤器,设置需要过滤的日志名称前缀列表
|
||||
"""
|
||||
super().__init__()
|
||||
self._filter_starts_with_tulpe = (
|
||||
"watchfiles",
|
||||
"aiosqlite",
|
||||
"apscheduler",
|
||||
"sqlalchemy",
|
||||
"sqlitedict",
|
||||
"sqlalchemy.engine.Engine",
|
||||
)
|
||||
|
||||
def filter(self, record):
|
||||
"""
|
||||
判断是否应该保留该日志记录
|
||||
|
||||
Args:
|
||||
record: 日志记录对象,包含 name 属性表示日志来源
|
||||
|
||||
Returns:
|
||||
bool: 如果日志名称不以过滤列表中的任何前缀开头则返回 True(保留),
|
||||
否则返回 False(过滤掉)
|
||||
"""
|
||||
return not record.name.startswith(self._filter_starts_with_tulpe)
|
||||
|
||||
|
||||
_configured_loggers = set()
|
||||
|
||||
|
||||
def setup_logging(logger: logging.Logger, log_name: str = "app"):
|
||||
"""
|
||||
配置日志系统,包括文件和控制台输出
|
||||
|
||||
为指定的 logger 配置以下功能:
|
||||
1. 设置日志级别和格式
|
||||
2. 创建按日期命名的轮转文件处理器(支持文件大小限制和备份数量控制)
|
||||
3. 创建带颜色格式化的控制台处理器
|
||||
4. 应用过滤器排除第三方库日志
|
||||
|
||||
该函数具有防重复配置机制,同一个 log_name 只会配置一次。
|
||||
|
||||
Args:
|
||||
logger: 需要配置的 logging.Logger 实例
|
||||
log_name: 日志名称标识,用于区分不同的日志配置和生成日志文件名,默认为 "app"
|
||||
|
||||
Returns:
|
||||
None
|
||||
|
||||
Raises:
|
||||
无显式抛出异常,但可能因文件系统权限问题导致目录创建或文件写入失败
|
||||
"""
|
||||
if log_name in _configured_loggers:
|
||||
return
|
||||
_configured_loggers.add(log_name)
|
||||
log_level = getattr(logging, log_config["level"], logging.INFO)
|
||||
log_format = log_config["format"]
|
||||
# 创建文件夹
|
||||
log_file_path = Path(log_config["file"])
|
||||
if log_file_path != Path("."):
|
||||
log_file_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 配置根日志记录器
|
||||
logger.setLevel(log_level)
|
||||
|
||||
# 创建格式化器
|
||||
plain_formatter = logging.Formatter(log_format)
|
||||
color_formatter = ColoredFormatter(log_format)
|
||||
|
||||
# 创建轮转文件处理器 - 限制文件大小为 10MB,保留 5 个备份文件
|
||||
max_bytes = log_config.get("max_size", 10 * 1024 * 1024) # 默认 10MB
|
||||
backup_count = log_config.get("backup_count", 5) # 默认保留 5 个备份
|
||||
|
||||
file_handler = RotatingFileHandler(
|
||||
f"{log_config['file']}/{log_name}-{datetime.date.today()}.log",
|
||||
encoding="utf-8",
|
||||
maxBytes=max_bytes,
|
||||
backupCount=backup_count,
|
||||
)
|
||||
file_handler.setFormatter(plain_formatter)
|
||||
file_handler.addFilter(ExcludeWatchFilesFilter())
|
||||
|
||||
# 创建控制台处理器并添加过滤器
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setFormatter(color_formatter)
|
||||
console_handler.addFilter(ExcludeWatchFilesFilter())
|
||||
|
||||
# 添加处理器到根日志记录器
|
||||
logger.addHandler(file_handler)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
print(
|
||||
f"日志系统已配置 - 级别: {logging.getLevelName(log_level)}, 文件: {log_config['file']} "
|
||||
f"(最大:{max_bytes / 1024 / 1024:.1f}MB, 备份:{backup_count})"
|
||||
)
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""
|
||||
获取配置好的日志记录器实例
|
||||
|
||||
根据名称获取或创建 logger 实例,并自动调用 setup_logging 进行配置。
|
||||
如果该名称的 logger 已经配置过,则直接返回已配置的实例。
|
||||
|
||||
Args:
|
||||
name: 日志记录器名称,通常使用模块名如 __name__
|
||||
|
||||
Returns:
|
||||
logging.Logger: 已配置完成的日志记录器实例,可直接用于记录日志
|
||||
"""
|
||||
logger = logging.getLogger(name)
|
||||
setup_logging(logger, name)
|
||||
return logger
|
||||
@@ -0,0 +1,418 @@
|
||||
"""
|
||||
动态路由生成器
|
||||
|
||||
根据配置文件中的 worker 路由定义,动态生成 FastAPI 端点。
|
||||
为每个端点生成带有正确 Pydantic 类型注解的处理函数,确保 OpenAPI 文档
|
||||
正确反映参数类型、响应结构,并注册到 components/schemas。
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, get_type_hints
|
||||
from inspect import Parameter, Signature
|
||||
|
||||
from fastapi import FastAPI, HTTPException, APIRouter, Query, Path as FastAPIPath, Depends
|
||||
from pydantic import create_model, Field, BaseModel
|
||||
|
||||
from .worker_manager import WorkerManager
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger("route_generator")
|
||||
|
||||
# 简单类型映射
|
||||
TYPE_MAP: Dict[str, Any] = {
|
||||
"float": float,
|
||||
"int": int,
|
||||
"str": str,
|
||||
"bool": bool,
|
||||
"any": Any,
|
||||
}
|
||||
|
||||
|
||||
def _parse_python_type(type_str: str) -> Any:
|
||||
"""解析配置中的类型字符串为 Python 类型对象"""
|
||||
type_str = type_str.strip()
|
||||
if type_str.startswith("list[") or type_str.startswith("List["):
|
||||
inner = type_str[5:-1].strip()
|
||||
inner_type = _parse_python_type(inner)
|
||||
return List[inner_type] # type: ignore
|
||||
if type_str.startswith("dict[") or type_str.startswith("Dict["):
|
||||
inner = type_str[5:-1].strip()
|
||||
parts = [p.strip() for p in inner.split(",", 1)]
|
||||
if len(parts) == 2:
|
||||
key_type = _parse_python_type(parts[0])
|
||||
val_type = _parse_python_type(parts[1])
|
||||
return Dict[key_type, val_type] # type: ignore
|
||||
return Dict[str, Any] # type: ignore
|
||||
return TYPE_MAP.get(type_str, str)
|
||||
|
||||
def generate_request_model(
|
||||
param_spec: Dict[str, str], model_name: str = "AutoRequest"
|
||||
) -> type[BaseModel]:
|
||||
"""生成 Pydantic 请求体模型"""
|
||||
fields: Dict[str, Any] = {}
|
||||
for param_name, param_type_str in param_spec.items():
|
||||
py_type = _parse_python_type(param_type_str)
|
||||
fields[param_name] = (py_type, Field(...))
|
||||
return create_model(model_name, **fields)
|
||||
|
||||
def generate_response_model(
|
||||
response_spec: Dict[str, Any], model_name: str = "AutoResponse"
|
||||
) -> Optional[type[BaseModel]]:
|
||||
"""生成 Pydantic 响应模型"""
|
||||
if not response_spec:
|
||||
return None
|
||||
fields: Dict[str, Any] = {}
|
||||
for field_name, field_spec in response_spec.items():
|
||||
if isinstance(field_spec, dict):
|
||||
py_type = _parse_python_type(field_spec.get("type", "str"))
|
||||
required = field_spec.get("required", True)
|
||||
default_val = field_spec.get("default", ...)
|
||||
if required:
|
||||
fields[field_name] = (py_type, Field(...))
|
||||
else:
|
||||
fields[field_name] = (py_type, Field(default=default_val if default_val is not ... else None))
|
||||
else:
|
||||
py_type = _parse_python_type(str(field_spec))
|
||||
fields[field_name] = (py_type, Field(...))
|
||||
return create_model(model_name, **fields)
|
||||
|
||||
class RouteConfig:
|
||||
"""单个路由配置模型"""
|
||||
|
||||
def __init__(self, data: Dict[str, Any]):
|
||||
self.subject: str = data.get("subject", "")
|
||||
self.path: str = data.get("path", "")
|
||||
self.method: str = data.get("method", "GET").upper()
|
||||
self.params: List[str] = data.get("params", [])
|
||||
self.response_model: Dict[str, Any] = data.get("response_model", {})
|
||||
self.summary: str = data.get("summary", "")
|
||||
self.docstring: str = data.get("docstring", "")
|
||||
self.timeout: float = data.get("timeout", 3.0)
|
||||
self.cacheable: bool = data.get("cacheable", False)
|
||||
|
||||
def parse_params(self) -> Dict[str, str]:
|
||||
"""解析 params 列表为 {name: type} 字典"""
|
||||
result: Dict[str, str] = {}
|
||||
for param in self.params:
|
||||
if ":" in param:
|
||||
name, type_str = param.split(":", 1)
|
||||
result[name.strip()] = type_str.strip()
|
||||
else:
|
||||
result[param.strip()] = "str"
|
||||
return result
|
||||
|
||||
def extract_path_params(self) -> List[str]:
|
||||
"""从路径中提取路径参数,如 /add/{a}/{b} -> ['a', 'b']"""
|
||||
return [
|
||||
seg[1:-1] for seg in self.path.split("/")
|
||||
if seg.startswith("{") and seg.endswith("}")
|
||||
]
|
||||
|
||||
|
||||
# 默认的 worker 配置目录
|
||||
WORKER_CONFIG_DIR = "config/workers"
|
||||
|
||||
|
||||
def load_worker_routes_from_dir(worker_type: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
从 config/workers/{worker_type}/ 目录加载路由配置。
|
||||
|
||||
目录结构:
|
||||
config/workers/{worker_type}/
|
||||
config.toml # [worker] 分组信息 + registration + defaults
|
||||
add.toml # 接口名 = 文件名(不含扩展名)
|
||||
[GET]
|
||||
summary = ...
|
||||
params = [...]
|
||||
response_model = { ... }
|
||||
[POST]
|
||||
... # 同一个接口支持多个请求方法
|
||||
|
||||
返回: List[Dict], 每个 dict 包含 flatten 后的路由配置,
|
||||
每个方法节(如 GET) 生成一个独立的路由条目。
|
||||
"""
|
||||
from pathlib import Path
|
||||
import tomllib
|
||||
|
||||
base_dir = Path(WORKER_CONFIG_DIR) / worker_type
|
||||
if not base_dir.is_dir():
|
||||
logger.warning(f"Worker 配置目录不存在: {base_dir}")
|
||||
return []
|
||||
|
||||
# 1. 读取分组配置
|
||||
group_cfg_path = base_dir / "config.toml"
|
||||
group_defaults: Dict[str, Any] = {}
|
||||
registration: Dict[str, Any] = {}
|
||||
router_prefix: str = ""
|
||||
if group_cfg_path.exists():
|
||||
with open(group_cfg_path, "rb") as f:
|
||||
cfg = tomllib.load(f)
|
||||
w = cfg.get("worker", {})
|
||||
group_defaults = w.get("defaults", {})
|
||||
registration = w.get("registration", {})
|
||||
router_prefix = registration.get("router_prefix", "")
|
||||
|
||||
# 2. 遍历所有接口 toml 文件
|
||||
routes: List[Dict[str, Any]] = []
|
||||
for route_file in sorted(base_dir.glob("*.toml")):
|
||||
if route_file.name == "config.toml":
|
||||
continue
|
||||
|
||||
# 文件名(不含 .toml)作为路径名
|
||||
route_name = route_file.stem # 例如 "add"
|
||||
base_path = router_prefix + "/" + route_name # 例如 "/calc/add"
|
||||
|
||||
with open(route_file, "rb") as f:
|
||||
route_data = tomllib.load(f)
|
||||
|
||||
# 每个方法节(GET/POST/PUT/DELETE)生成一个独立路由
|
||||
for method, spec in route_data.items():
|
||||
if not isinstance(spec, dict):
|
||||
logger.warning(f"忽略非 dict 的方法节: {route_file.name}[{method}]")
|
||||
continue
|
||||
|
||||
# 合并: spec 显式字段优先,然后 group_defaults 兜底
|
||||
route = dict(spec)
|
||||
for key, default_val in group_defaults.items():
|
||||
if key not in route:
|
||||
route[key] = default_val
|
||||
|
||||
# 覆盖关键字段 - method 始终从 TOML 节键获取,不被 defaults 覆盖
|
||||
route["method"] = method.upper()
|
||||
route["path"] = route.get("path", base_path)
|
||||
if "subject" not in route:
|
||||
# 自动生成 subject: worker_type.route_name (如 "calc.add")
|
||||
route["subject"] = f"{worker_type}.{route_name}"
|
||||
|
||||
routes.append(route)
|
||||
logger.debug(
|
||||
f" 已加载: {route['method']} {route['path']} -> {route['subject']}"
|
||||
)
|
||||
|
||||
logger.info(f"从 {base_dir} 加载了 {len(routes)} 个路由")
|
||||
return routes
|
||||
|
||||
|
||||
def register_dynamic_routes(app: FastAPI, worker_manager: WorkerManager, prefix: str = ""):
|
||||
"""根据 config/workers/ 目录结构注册所有动态路由"""
|
||||
from pathlib import Path
|
||||
|
||||
workers_base = Path(WORKER_CONFIG_DIR)
|
||||
if not workers_base.is_dir():
|
||||
logger.info(f"Worker 配置目录不存在: {WORKER_CONFIG_DIR}")
|
||||
return
|
||||
|
||||
router = APIRouter(prefix=prefix)
|
||||
extra_schemas: Dict[str, type[BaseModel]] = {}
|
||||
|
||||
# 扫描所有 worker 子目录(每个子目录名 = worker_type)
|
||||
for worker_dir in sorted(workers_base.iterdir()):
|
||||
if not worker_dir.is_dir():
|
||||
continue
|
||||
worker_type = worker_dir.name
|
||||
routes = load_worker_routes_from_dir(worker_type)
|
||||
if not routes:
|
||||
continue
|
||||
|
||||
for route_conf in routes:
|
||||
_register_single_route(router, worker_manager, worker_type, route_conf, extra_schemas)
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
# 将额外 schema 注册到 OpenAPI 的 components/schemas
|
||||
if extra_schemas:
|
||||
_inject_schemas_to_openapi(app, extra_schemas)
|
||||
|
||||
logger.info(f"动态路由注册完成,worker 配置目录: {WORKER_CONFIG_DIR}")
|
||||
|
||||
|
||||
def _inject_schemas_to_openapi(app: FastAPI, schemas: Dict[str, type[BaseModel]]):
|
||||
"""
|
||||
将自定义 Pydantic 模型注册到 FastAPI 的 OpenAPI components/schemas 中。
|
||||
|
||||
通过重写 app.openapi() 方法注入,因为 FastAPI 在生命周期内只生成一次
|
||||
OpenAPI schema,之后会被缓存。
|
||||
"""
|
||||
original_openapi = app.openapi
|
||||
|
||||
def patched_openapi() -> dict[str, Any]:
|
||||
if app.openapi_schema is not None:
|
||||
return app.openapi_schema
|
||||
schema = original_openapi()
|
||||
if schema is None:
|
||||
return {"openapi": "3.1.0", "info": {"title": "Chongming", "version": "1.0.0"}, "paths": {}}
|
||||
# 注入我们的 schema
|
||||
schemas_dict = schema.setdefault("components", {}).setdefault("schemas", {})
|
||||
for name, model in schemas.items():
|
||||
if name not in schemas_dict:
|
||||
schemas_dict[name] = model.model_json_schema()
|
||||
app.openapi_schema = schema
|
||||
return schema
|
||||
|
||||
app.openapi = patched_openapi
|
||||
|
||||
|
||||
def _make_body_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, RequestModel: type[BaseModel]):
|
||||
"""创建用于 Body 请求(POST/PUT/PATCH)的端点函数"""
|
||||
path = rc.path
|
||||
subject = rc.subject
|
||||
timeout = rc.timeout
|
||||
http_method = rc.method
|
||||
cacheable = rc.cacheable
|
||||
|
||||
async def endpoint(
|
||||
body: RequestModel, # type: ignore
|
||||
) -> Any:
|
||||
try:
|
||||
payload = body.model_dump()
|
||||
payload.setdefault("_method", http_method)
|
||||
result = await worker_manager.call(
|
||||
worker_type=worker_type,
|
||||
subject=subject,
|
||||
payload=payload,
|
||||
timeout=timeout,
|
||||
cacheable=cacheable,
|
||||
)
|
||||
return result
|
||||
except RuntimeError as e:
|
||||
raise HTTPException(status_code=503, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"路由 {path} 调用异常: {e}")
|
||||
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||
|
||||
return endpoint
|
||||
|
||||
|
||||
def _make_query_endpoint(worker_manager: WorkerManager, rc: RouteConfig, worker_type: str, path_params: List[str], query_params: Dict[str, str]):
|
||||
"""
|
||||
创建用于 Query 请求(GET/DELETE)的端点函数。
|
||||
|
||||
通过设置函数 __signature__ 和 __annotations__ 为每个参数赋予正确的 Pydantic 类型,
|
||||
使 FastAPI 能在 OpenAPI schema 中生成正确的 type / format 信息。
|
||||
请求模型 schema 通过 extra_schemas 机制单独注入到 components/schemas。
|
||||
"""
|
||||
path = rc.path
|
||||
subject = rc.subject
|
||||
timeout = rc.timeout
|
||||
http_method = rc.method
|
||||
cacheable = rc.cacheable
|
||||
|
||||
# 合并路径参数和查询参数
|
||||
param_spec_combined: Dict[str, str] = {}
|
||||
param_spec_combined.update({k: query_params.get(k, "str") for k in path_params if k in query_params})
|
||||
param_spec_combined.update(query_params)
|
||||
|
||||
# 无参数情况
|
||||
if not param_spec_combined:
|
||||
async def _simple_endpoint() -> Any:
|
||||
try:
|
||||
result = await worker_manager.call(
|
||||
worker_type=worker_type,
|
||||
subject=subject,
|
||||
payload={"_method": http_method},
|
||||
timeout=timeout,
|
||||
cacheable=cacheable,
|
||||
)
|
||||
return result
|
||||
except RuntimeError as e:
|
||||
raise HTTPException(status_code=503, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"路由 {path} 调用异常: {e}")
|
||||
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||
return _simple_endpoint
|
||||
|
||||
# 构造参数名列表(用于从 kwargs 中提取)
|
||||
param_names = list(param_spec_combined.keys())
|
||||
|
||||
async def endpoint(**kwargs: Any) -> Any:
|
||||
payload = {k: kwargs[k] for k in param_names}
|
||||
payload.setdefault("_method", http_method)
|
||||
try:
|
||||
result = await worker_manager.call(
|
||||
worker_type=worker_type,
|
||||
subject=subject,
|
||||
payload=payload,
|
||||
timeout=timeout,
|
||||
cacheable=cacheable,
|
||||
)
|
||||
return result
|
||||
except RuntimeError as e:
|
||||
raise HTTPException(status_code=503, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"路由 {path} 调用异常: {e}")
|
||||
raise HTTPException(status_code=500, detail="内部服务器错误")
|
||||
|
||||
# 构建带类型注解的 Parameter 列表
|
||||
params: List[Parameter] = []
|
||||
annotations: Dict[str, Any] = {}
|
||||
for name, type_str in param_spec_combined.items():
|
||||
py_type = _parse_python_type(type_str)
|
||||
annotations[name] = py_type
|
||||
if name in path_params:
|
||||
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=FastAPIPath(...), annotation=py_type))
|
||||
else:
|
||||
params.append(Parameter(name, Parameter.KEYWORD_ONLY, default=Query(...), annotation=py_type))
|
||||
|
||||
# 设置 __annotations__(FastAPI 通过 get_type_hints 读取)
|
||||
endpoint.__annotations__ = annotations
|
||||
# 设置 __signature__(FastAPI 优先于此获取参数列表)
|
||||
endpoint.__signature__ = Signature(params, return_annotation=Dict[str, Any]) # type: ignore
|
||||
|
||||
return endpoint
|
||||
|
||||
|
||||
def _register_single_route(
|
||||
router: APIRouter,
|
||||
worker_manager: WorkerManager,
|
||||
worker_type: str,
|
||||
route_conf: Dict[str, Any],
|
||||
extra_schemas: Optional[Dict[str, type[BaseModel]]] = None,
|
||||
):
|
||||
"""注册单个路由,生成正确的函数签名以便 OpenAPI 正确生成文档"""
|
||||
rc = RouteConfig(route_conf)
|
||||
param_spec = rc.parse_params()
|
||||
path_params = rc.extract_path_params()
|
||||
query_params = {k: v for k, v in param_spec.items() if k not in path_params}
|
||||
|
||||
# 生成干净的模型名称(替换路径中的 / 和 {} 为 _)
|
||||
# 模型名称包含 HTTP method 以区分同名路径的不同调用类型
|
||||
clean_path = rc.path.replace("/", "_").replace("{", "").replace("}", "").strip("_")
|
||||
if not clean_path:
|
||||
clean_path = "root"
|
||||
method_tag = rc.method.upper()
|
||||
response_model_name = f"{worker_type}_{clean_path}_{method_tag}_Response"
|
||||
request_model_name = f"{worker_type}_{clean_path}_{method_tag}_Request"
|
||||
|
||||
# 生成响应模型
|
||||
ResponseModel = generate_response_model(
|
||||
rc.response_model, response_model_name
|
||||
)
|
||||
|
||||
if rc.method in ("POST", "PUT", "PATCH"):
|
||||
RequestModel = generate_request_model(
|
||||
param_spec, request_model_name
|
||||
)
|
||||
endpoint = _make_body_endpoint(worker_manager, rc, worker_type, RequestModel)
|
||||
else:
|
||||
# 对 GET 端点也生成 Request 模型并注册到 extra_schemas
|
||||
if param_spec:
|
||||
RequestModel = generate_request_model(
|
||||
param_spec, request_model_name
|
||||
)
|
||||
if extra_schemas is not None and request_model_name not in extra_schemas:
|
||||
extra_schemas[request_model_name] = RequestModel
|
||||
endpoint = _make_query_endpoint(worker_manager, rc, worker_type, path_params, query_params)
|
||||
|
||||
# 注册路由 - 使用 worker_type 作为 OpenAPI 分组标签
|
||||
# 每个 worker 类型自动成为一个独立的分组,显示在 Swagger UI 中
|
||||
router.add_api_route(
|
||||
path=rc.path,
|
||||
endpoint=endpoint,
|
||||
methods=[rc.method],
|
||||
summary=rc.summary,
|
||||
description=rc.docstring,
|
||||
response_model=ResponseModel,
|
||||
tags=[worker_type],
|
||||
include_in_schema=True,
|
||||
)
|
||||
logger.info(f"已注册动态路由: {rc.method} {rc.path} -> worker {worker_type}.{rc.subject} | tags=[{worker_type}]")
|
||||
@@ -0,0 +1,100 @@
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlite_vfs.core import SQLiteVFS
|
||||
|
||||
|
||||
class SVFSStaticFiles(StaticFiles):
|
||||
"""
|
||||
基于 SQLite VFS 的静态文件服务类
|
||||
|
||||
该类继承自 FastAPI 的 StaticFiles,使用 SQLite 虚拟文件系统来存储和提供静态文件。
|
||||
支持单页应用(SPA)的路由回退机制,当请求的文件不存在时,会自动返回 index.html。
|
||||
"""
|
||||
|
||||
def __init__(self, directory, vfs_db_path="static.svfs"):
|
||||
"""
|
||||
初始化 SVFSStaticFiles 实例
|
||||
|
||||
参数:
|
||||
directory (str): 静态文件的根目录路径
|
||||
vfs_db_path (str): SQLite VFS 数据库文件路径,默认为 "static.svfs"
|
||||
|
||||
返回值:
|
||||
无
|
||||
"""
|
||||
vfs_db_path = vfs_db_path
|
||||
self._vfs = SQLiteVFS(vfs_db_path, compress=False)
|
||||
super().__init__(directory=directory)
|
||||
|
||||
async def get_response(self, path: str, scope):
|
||||
"""
|
||||
根据请求路径获取对应的静态文件响应
|
||||
|
||||
该方法处理静态文件的请求逻辑,包括:
|
||||
1. 路径标准化处理
|
||||
2. 文件存在性检查
|
||||
3. SPA 路由回退支持
|
||||
4. 目录索引处理
|
||||
5. MIME 类型自动识别
|
||||
|
||||
参数:
|
||||
path (str): 请求的文件路径
|
||||
scope: ASGI 作用域对象,包含请求上下文信息
|
||||
|
||||
返回值:
|
||||
Response: FastAPI 响应对象,包含文件内容和正确的 Content-Type
|
||||
|
||||
异常:
|
||||
HTTPException: 当文件不存在且无法回退到 index.html 时,抛出 404 错误
|
||||
"""
|
||||
# 统一路径格式:以 / 开头,并将 Windows 反斜杠转为正斜杠
|
||||
normalized_path = "/" + path.replace("\\", "/")
|
||||
|
||||
# 获取文件信息
|
||||
file_info = self._vfs.get_file_info(normalized_path)
|
||||
|
||||
# 处理文件不存在的情况,尝试 SPA 回退机制
|
||||
if file_info is None:
|
||||
# SPA 回退:如果不是静态资源扩展名,尝试返回 index.html
|
||||
if not any(
|
||||
normalized_path.endswith(ext)
|
||||
for ext in [
|
||||
".js",
|
||||
".css",
|
||||
".png",
|
||||
".jpg",
|
||||
".ico",
|
||||
".svg",
|
||||
".webp",
|
||||
".json",
|
||||
]
|
||||
):
|
||||
index_info = self._vfs.get_file_info("/index.html")
|
||||
if index_info:
|
||||
content = self._vfs.read_file("/index.html")
|
||||
return Response(content=content, media_type="text/html")
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
|
||||
# 处理目录请求,返回目录下的 index.html
|
||||
if file_info["is_directory"]:
|
||||
index_info = self._vfs.get_file_info(f"{normalized_path}/index.html")
|
||||
if index_info:
|
||||
content = self._vfs.read_file(f"{normalized_path}/index.html")
|
||||
return Response(content=content, media_type="text/html")
|
||||
raise HTTPException(status_code=404, detail="Directory index not found")
|
||||
|
||||
# 读取文件内容并确定 MIME 类型
|
||||
content = self._vfs.read_file(normalized_path)
|
||||
content_type = file_info.get("content_type") or "application/octet-stream"
|
||||
# 简单根据扩展名补充 MIME 类型(可选)
|
||||
if normalized_path.endswith(".js"):
|
||||
content_type = "application/javascript"
|
||||
elif normalized_path.endswith(".css"):
|
||||
content_type = "text/css"
|
||||
elif normalized_path.endswith(".html"):
|
||||
content_type = "text/html"
|
||||
elif normalized_path.endswith(".ico"):
|
||||
content_type = "image/x-icon"
|
||||
|
||||
return Response(content=content, media_type=content_type)
|
||||
@@ -0,0 +1,202 @@
|
||||
import asyncio
|
||||
import random
|
||||
import json
|
||||
import orjson
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
import zenoh
|
||||
from diskcache import Cache
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger("worker_manager")
|
||||
|
||||
_SENTINEL = object() # 哨兵值,用于标记"No reply received"
|
||||
|
||||
|
||||
class WorkerManager:
|
||||
"""
|
||||
统一 Worker 管理器
|
||||
|
||||
封装 Zenoh 会话,维护每个 worker 类型的存活节点列表,
|
||||
提供负载均衡、超时重试、缓存等能力的统一 RPC 调用入口。
|
||||
|
||||
支持两种 worker 注册方式:
|
||||
1. 分布式:通过 Zenoh liveliness 自动发现
|
||||
2. 本地:通过 register_worker() 直接注册(用于同进程 worker)
|
||||
"""
|
||||
|
||||
def __init__(self, session: zenoh.Session, cache: Optional[Cache] = None):
|
||||
self.session = session
|
||||
self.cache = cache
|
||||
self._workers_cache: Dict[str, List[str]] = {} # type -> list of worker key_exprs
|
||||
self._local_workers: Dict[str, List[str]] = {} # type -> list of locally registered workers
|
||||
self._lock = asyncio.Lock()
|
||||
self._refresh_interval = 15 # seconds
|
||||
|
||||
def register_worker(self, worker_type: str, worker_id: str):
|
||||
"""
|
||||
直接注册一个本地 worker(同进程内使用)
|
||||
这样无需等待 liveliness 发现,立即可用。
|
||||
|
||||
同步执行(仅修改内存字典),确保调用后立即可被 get_worker 发现。
|
||||
"""
|
||||
key_expr = f"worker/{worker_type}/{worker_id}"
|
||||
# 同步直接写入 _local_workers,无需异步锁
|
||||
# 因为 _local_workers 和 _workers_cache 在调用 call() 时通过 await get_worker() 读取
|
||||
self._local_workers.setdefault(worker_type, []).append(key_expr)
|
||||
logger.debug(f"本地 worker 已注册: {key_expr}")
|
||||
|
||||
def unregister_worker(self, worker_type: str, worker_id: str):
|
||||
"""取消注册一个本地 worker"""
|
||||
key_expr = f"worker/{worker_type}/{worker_id}"
|
||||
if worker_type in self._local_workers:
|
||||
try:
|
||||
self._local_workers[worker_type].remove(key_expr)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
async def refresh_workers(self):
|
||||
"""定期刷新存活 worker 列表(合并 Zenoh liveliness + 本地注册)"""
|
||||
while True:
|
||||
try:
|
||||
# 1. 从 Zenoh liveliness 获取分布式 workers
|
||||
replies = self.session.liveliness().get("worker/*/*")
|
||||
new_cache: Dict[str, List[str]] = {}
|
||||
for reply in replies:
|
||||
if reply.ok:
|
||||
key = str(reply.ok.key_expr) # e.g. "worker/example/worker-0"
|
||||
parts = key.split("/")
|
||||
if len(parts) >= 3:
|
||||
worker_type = parts[1]
|
||||
new_cache.setdefault(worker_type, []).append(key)
|
||||
|
||||
# 2. 合并本地注册的 workers
|
||||
for wtype, workers in self._local_workers.items():
|
||||
existing = set(new_cache.get(wtype, []))
|
||||
for w in workers:
|
||||
if w not in existing:
|
||||
new_cache.setdefault(wtype, []).append(w)
|
||||
|
||||
async with self._lock:
|
||||
self._workers_cache = new_cache
|
||||
logger.debug(f"Worker 存活列表更新: {self._workers_cache}")
|
||||
except Exception as e:
|
||||
logger.error(f"刷新 worker 列表失败: {e}")
|
||||
await asyncio.sleep(self._refresh_interval)
|
||||
|
||||
async def get_worker(self, worker_type: str) -> Optional[str]:
|
||||
"""获取一个可用的 worker 地址(简单随机负载均衡)"""
|
||||
async with self._lock:
|
||||
workers = self._workers_cache.get(worker_type, [])
|
||||
# 如果缓存为空,回退到本地注册表(适用于启动初期 refresh_workers 尚未执行时)
|
||||
if not workers:
|
||||
local = self._local_workers.get(worker_type, [])
|
||||
if local:
|
||||
return random.choice(local)
|
||||
return None
|
||||
return random.choice(workers)
|
||||
|
||||
def is_success_response(self, result: dict) -> bool:
|
||||
"""根据业务约定判断响应是否成功。
|
||||
假设成功响应没有 'error' 字段,或 'code' 为 0。
|
||||
"""
|
||||
if "error" in result and result["error"]:
|
||||
return False
|
||||
if "code" in result and result["code"] != 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def call(
|
||||
self,
|
||||
worker_type: str,
|
||||
subject: str,
|
||||
payload: Dict[str, Any],
|
||||
timeout: float = 3.0,
|
||||
retries: int = 1,
|
||||
cacheable: bool = False,
|
||||
) -> Any:
|
||||
"""
|
||||
统一调用入口:缓存 > 负载均衡 > 超时重试
|
||||
|
||||
Args:
|
||||
worker_type: Worker 类型标识(如 "example")
|
||||
subject: 调用的主题(如 "calc.add")
|
||||
payload: 请求参数
|
||||
timeout: 单次调用超时(秒)
|
||||
retries: 失败重试次数
|
||||
cacheable: 是否允许缓存结果(若为 False 则跳过缓存读写)
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 调用结果
|
||||
|
||||
Raises:
|
||||
RuntimeError: 无可用 worker 或所有重试均失败
|
||||
"""
|
||||
# 1. 缓存检查(仅在 cacheable=True 时检查)
|
||||
cache_key = f"{worker_type}:{subject}:{json.dumps(payload, sort_keys=True)}"
|
||||
if cacheable and self.cache is not None:
|
||||
if cache_key in self.cache:
|
||||
logger.debug(f"命中缓存: {cache_key}")
|
||||
return self.cache[cache_key]
|
||||
|
||||
# 2. 获取可用 worker
|
||||
worker_expr = await self.get_worker(worker_type)
|
||||
if not worker_expr:
|
||||
raise RuntimeError(f"No available worker for type {worker_type}")
|
||||
|
||||
full_key = f"{worker_expr}/{subject}"
|
||||
payload_bytes = orjson.dumps(payload)
|
||||
|
||||
for attempt in range(retries + 1):
|
||||
try:
|
||||
reply_iter = self.session.get(
|
||||
full_key,
|
||||
payload=payload_bytes,
|
||||
consolidation=zenoh.ConsolidationMode.NONE,
|
||||
)
|
||||
# 使用辅助函数获取第一个回复,避免 StopIteration 在 asyncio.to_thread 中传播的问题
|
||||
def _get_first(iterator):
|
||||
"""线程安全地获取迭代器的第一个元素"""
|
||||
for item in iterator:
|
||||
return item
|
||||
return _SENTINEL
|
||||
|
||||
first_reply = await asyncio.wait_for(
|
||||
asyncio.to_thread(_get_first, reply_iter),
|
||||
timeout=timeout,
|
||||
)
|
||||
if first_reply is _SENTINEL:
|
||||
logger.warning(
|
||||
f"Worker {worker_expr} 无回复,尝试重试 {attempt+1}/{retries+1}"
|
||||
)
|
||||
continue
|
||||
|
||||
reply = first_reply
|
||||
if reply.ok: # type: ignore
|
||||
# result_str = str(reply.ok.payload)
|
||||
payload_bytes = bytes(reply.ok.payload) if reply.ok.payload else b"" # type: ignore
|
||||
if not payload_bytes:
|
||||
result = {} # 或根据业务返回默认值
|
||||
else:
|
||||
result = orjson.loads(payload_bytes)
|
||||
# 缓存结果(仅在 cacheable=True 时写入)
|
||||
if cacheable and self.cache is not None and self.is_success_response(result):
|
||||
self.cache[cache_key] = result
|
||||
return result
|
||||
else:
|
||||
error_payload = reply.err.payload # type: ignore
|
||||
error_msg = str(error_payload) if error_payload else "unknown error"
|
||||
logger.warning(f"Worker 返回错误: {error_msg} (尝试 {attempt+1}/{retries+1})")
|
||||
continue # 重试
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
f"Worker {worker_expr} 超时,尝试重试 {attempt+1}/{retries+1}"
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"调用 worker 异常: {e}")
|
||||
raise
|
||||
|
||||
raise RuntimeError(
|
||||
f"Failed to call worker {worker_type} after {retries} retries"
|
||||
)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .storage import router as storage_router
|
||||
|
||||
__all__ = ["storage_router"]
|
||||
@@ -0,0 +1,63 @@
|
||||
# src/chongming/app/routes/storage.py
|
||||
import uuid
|
||||
import urllib.parse
|
||||
from fastapi import APIRouter, UploadFile, File, Form, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
router = APIRouter(prefix="/api/v1/storage", tags=["Storage"])
|
||||
|
||||
|
||||
@router.post("/upload", summary="上传文件")
|
||||
async def upload_file(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
worker_type: str = Form(...),
|
||||
):
|
||||
"""后端代理上传,直接写入 nanio"""
|
||||
s3_client = request.app.state.s3_client
|
||||
bucket = request.app.state.storage_bucket
|
||||
object_key = f"{worker_type}/{uuid.uuid4()}/{file.filename}"
|
||||
|
||||
try:
|
||||
s3_client.upload_fileobj(
|
||||
file.file,
|
||||
bucket,
|
||||
object_key,
|
||||
ExtraArgs={"ContentType": file.content_type or "application/octet-stream"}
|
||||
)
|
||||
except ClientError as e:
|
||||
raise HTTPException(status_code=500, detail=f"S3 上传失败: {str(e)}")
|
||||
|
||||
return {
|
||||
"file_key": object_key,
|
||||
"bucket": bucket,
|
||||
"filename": file.filename,
|
||||
"content_type": file.content_type,
|
||||
"size": file.size or 0, # 注意:UploadFile.size 可能为 None,需读取后计算
|
||||
}
|
||||
|
||||
|
||||
@router.get("/download/{file_key:path}")
|
||||
async def download_file(request: Request, file_key: str):
|
||||
s3_client = request.app.state.s3_client
|
||||
bucket = request.app.state.storage_bucket
|
||||
|
||||
try:
|
||||
response = s3_client.get_object(Bucket=bucket, Key=file_key)
|
||||
content_type = response.get('ContentType', 'application/octet-stream')
|
||||
raw_filename = file_key.split('/')[-1]
|
||||
# 百分号编码,确保仅 ASCII 字符
|
||||
encoded = urllib.parse.quote(raw_filename, safe='')
|
||||
# 仅使用 filename*(现代浏览器都支持)
|
||||
content_disposition = f"attachment; filename*=UTF-8''{encoded}"
|
||||
|
||||
return StreamingResponse(
|
||||
response['Body'].iter_chunks(),
|
||||
media_type=content_type,
|
||||
headers={'Content-Disposition': content_disposition}
|
||||
)
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == 'NoSuchKey':
|
||||
raise HTTPException(404, "文件不存在")
|
||||
raise HTTPException(500, f"下载失败: {e}")
|
||||
@@ -0,0 +1,501 @@
|
||||
"""
|
||||
构建脚本 - 将 chongming 应用打包为可执行文件
|
||||
|
||||
功能阶段:
|
||||
1. 模块打包(module-bank): 加密打包 Python 模块
|
||||
2. 资源准备: 创建 build 目录, 复制必要文件
|
||||
3. 混淆与打包: pyarmor 混淆 + PyInstaller/Nuitka 打包
|
||||
4. 数据库初始化: 创建表结构和默认数据
|
||||
5. 前端构建: 编译 Vue 项目并打包为 VFS
|
||||
|
||||
用法:
|
||||
python -m chongming.scripts.build [pyinstaller|nuitka]
|
||||
"""
|
||||
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from module_bank import PythonToSQLite
|
||||
from module_bank.encryption import Encryption
|
||||
from sqlite_vfs.core import SQLiteVFS
|
||||
from sqlite_vfs.folder_packer import FolderPacker
|
||||
from ..app.core.config import get_config, Config
|
||||
|
||||
# ── 常量 ────────────────────────────────────────────────────────────────────
|
||||
|
||||
PYINSTALLER = "pyinstaller"
|
||||
NUITKA = "nuitka"
|
||||
MODES = (PYINSTALLER, NUITKA)
|
||||
|
||||
is_windows = platform.system() == "Windows"
|
||||
|
||||
# ── 构建结果数据结构 ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildStepResult:
|
||||
"""单个构建步骤的执行结果"""
|
||||
|
||||
name: str # 步骤名称
|
||||
success: bool # 是否成功
|
||||
duration: float # 耗时(秒)
|
||||
message: str = "" # 附加信息(如包大小)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildReport:
|
||||
"""构建总结报告"""
|
||||
|
||||
mode: str
|
||||
start_time: float
|
||||
end_time: float = 0.0
|
||||
steps: list[BuildStepResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total_duration(self) -> float:
|
||||
return self.end_time - self.start_time if self.end_time else 0.0
|
||||
|
||||
@property
|
||||
def all_success(self) -> bool:
|
||||
return all(s.success for s in self.steps)
|
||||
|
||||
def add_step(self, name: str, success: bool, duration: float, message: str = ""):
|
||||
self.steps.append(BuildStepResult(name, success, duration, message))
|
||||
|
||||
def print_summary(self):
|
||||
"""打印结构化的构建总结"""
|
||||
sep = "=" * 60
|
||||
print(f"\n{sep}")
|
||||
print(f" 🏗️ CHONGMING 构建报告")
|
||||
print(f"{sep}")
|
||||
print(f" 构建模式 : {self.mode}")
|
||||
print(f" 总耗时 : {self.total_duration:.2f}s")
|
||||
print(f" 构建状态 : {'✅ 成功' if self.all_success else '❌ 失败'}")
|
||||
print(f"{sep}")
|
||||
print(f" 步骤明细:")
|
||||
for i, step in enumerate(self.steps, 1):
|
||||
icon = "✅" if step.success else "❌"
|
||||
msg = f" ({step.message})" if step.message else ""
|
||||
print(
|
||||
f" {icon} [{i}/{len(self.steps)}] {step.name:<20s} {step.duration:>7.2f}s{msg}"
|
||||
)
|
||||
print(f"{sep}\n")
|
||||
|
||||
|
||||
# ── 工具函数 ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _run_cmd(
|
||||
cmd: list[str],
|
||||
cwd: str | None = None,
|
||||
capture: bool = True,
|
||||
shell: bool | None = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
"""统一执行外部命令"""
|
||||
if shell is None:
|
||||
shell = is_windows
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
cwd=cwd,
|
||||
capture_output=capture,
|
||||
text=True,
|
||||
shell=shell,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
|
||||
|
||||
def _format_size(path: str | Path) -> str:
|
||||
"""获取文件可读大小"""
|
||||
size = Path(path).stat().st_size
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if size < 1024:
|
||||
return f"{size:.2f} {unit}"
|
||||
size /= 1024.0
|
||||
return f"{size:.2f} TB"
|
||||
|
||||
|
||||
def _ensure_dir(path: str | Path):
|
||||
"""确保目录存在"""
|
||||
Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _resolve_mode() -> str:
|
||||
"""解析构建模式"""
|
||||
mode = PYINSTALLER
|
||||
if len(sys.argv) >= 2:
|
||||
mode = sys.argv[1].lower()
|
||||
if mode not in MODES:
|
||||
print(f" ❌ 不支持的构建模式: {mode},可选: {', '.join(MODES)}")
|
||||
sys.exit(1)
|
||||
return mode
|
||||
|
||||
|
||||
# ── 阶段 1: 模块打包 (module-bank) ──────────────────────────────────────────
|
||||
|
||||
|
||||
def _stage_pack_modules(config: Config) -> str:
|
||||
"""
|
||||
使用 module-bank 加密打包 Python 模块为 .mbank 文件。
|
||||
返回打包过程中的提示信息。
|
||||
"""
|
||||
_ensure_dir("resources")
|
||||
mbank_list = config.get("production", {}).get("module_system", {}).get("path", [])
|
||||
if not mbank_list:
|
||||
return "⚠️ 未配置 module_system.path,跳过模块打包"
|
||||
|
||||
messages = []
|
||||
for output_path, source_dir, package_name in mbank_list:
|
||||
key = Encryption.generate_key()
|
||||
packer = PythonToSQLite(
|
||||
output_path,
|
||||
key,
|
||||
pyarmor_obfuscate=True,
|
||||
)
|
||||
packer.pack_directory(source_dir, package_name)
|
||||
packer.verify_package_structure()
|
||||
packer.packer_key(key)
|
||||
packer.delete_source_code(None)
|
||||
size = _format_size(output_path)
|
||||
messages.append(f"{Path(output_path).name} ({size})")
|
||||
return " - " + "\n - ".join(messages)
|
||||
|
||||
|
||||
def _stage_prepare_build_dir(config: Config, mode: str) -> str:
|
||||
"""准备 build 目录并复制必要文件"""
|
||||
build_path = Path("build")
|
||||
if build_path.exists():
|
||||
shutil.rmtree(str(build_path))
|
||||
_ensure_dir("build")
|
||||
|
||||
# 复制 utils
|
||||
utils_src = Path("public/utils")
|
||||
if utils_src.exists():
|
||||
shutil.copytree(utils_src, build_path / "utils")
|
||||
|
||||
# 复制 resources
|
||||
resources_src = Path("resources")
|
||||
if resources_src.exists():
|
||||
shutil.copytree(resources_src, build_path / "resources")
|
||||
|
||||
resources_src = Path("config")
|
||||
if resources_src.exists():
|
||||
shutil.copytree(resources_src, build_path / "config")
|
||||
|
||||
# 复制核心文件
|
||||
for f in ("main.py", "server.py", "config.toml"):
|
||||
shutil.copy(f"public/{f}", f"build/{f}")
|
||||
|
||||
# 注入 JWT 密钥
|
||||
jwt_key = Encryption.generate_key()
|
||||
cfg_path = build_path / "config.toml"
|
||||
content = cfg_path.read_text(encoding="utf-8")
|
||||
content = content.replace("your-secret-key-change-in-production", jwt_key)
|
||||
cfg_path.write_text(content, encoding="utf-8")
|
||||
|
||||
return "资源准备完成"
|
||||
|
||||
|
||||
# ── 阶段 3: PyArmor 混淆 ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _stage_pyarmor_obfuscate() -> str:
|
||||
"""对关键脚本执行 PyArmor 混淆"""
|
||||
files = [
|
||||
("main.py", "./"),
|
||||
("server.py", "./"),
|
||||
("utils/launch.py", "./utils"),
|
||||
("utils/config.py", "./utils"),
|
||||
]
|
||||
failed = []
|
||||
for filename, output in files:
|
||||
result = _run_cmd(
|
||||
["pyarmor", "gen", f"--output={output}", filename],
|
||||
cwd="build",
|
||||
)
|
||||
if result.returncode != 0:
|
||||
failed.append(filename)
|
||||
if failed:
|
||||
return f"❌ 混淆失败: {', '.join(failed)}"
|
||||
return "✅ 所有文件混淆成功"
|
||||
|
||||
|
||||
# ── 阶段 4: PyInstaller 打包 ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _stage_pyinstaller(config: Config) -> tuple[str, str]:
|
||||
"""执行 PyInstaller 打包,返回 (状态消息, 可执行文件路径)"""
|
||||
base_imports = config.get("pyinstaller_build_import", {})
|
||||
build_imports = list(base_imports.get("base", []))
|
||||
build_imports.extend(
|
||||
base_imports.get("windows", []) if is_windows else base_imports.get("linux", [])
|
||||
)
|
||||
|
||||
icon = Path("../../public/chongming.ico")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"PyInstaller",
|
||||
"--onefile",
|
||||
"--noconfirm",
|
||||
"--optimize",
|
||||
"2",
|
||||
"--name",
|
||||
"chongming",
|
||||
"--distpath",
|
||||
".",
|
||||
"--workpath",
|
||||
"./build_temp",
|
||||
"--specpath",
|
||||
"./specs",
|
||||
"--clean",
|
||||
]
|
||||
if not is_windows:
|
||||
cmd.append("--strip")
|
||||
if icon.exists():
|
||||
cmd.extend(["--icon", str(icon)])
|
||||
for imp in build_imports:
|
||||
cmd.extend(["--hidden-import", imp])
|
||||
|
||||
cmd.append("main.py")
|
||||
|
||||
result = _run_cmd(cmd, cwd="build", capture=True)
|
||||
if result.returncode != 0:
|
||||
return f"❌ PyInstaller 失败", ""
|
||||
|
||||
exe_path = Path("build/chongming")
|
||||
if is_windows:
|
||||
exe_path = exe_path.with_suffix(".exe")
|
||||
|
||||
suffix = ".exe" if is_windows else ""
|
||||
exe = Path("build") / f"chongming{suffix}"
|
||||
if exe.exists():
|
||||
size = _format_size(exe)
|
||||
return f"✅ PyInstaller 打包成功 ({size})", str(exe)
|
||||
return "✅ PyInstaller 打包完成 (但未找到可执行文件)", ""
|
||||
|
||||
|
||||
# ── 阶段 5: Nuitka 打包 ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _stage_nuitka(config: Config) -> tuple[str, str]:
|
||||
"""执行 Nuitka 打包,返回 (状态消息, 可执行文件路径)"""
|
||||
base_imports = config.get("pyinstaller_build_import", {}).get("base", [])
|
||||
extra = config.get("pyinstaller_build_import", {}).get(
|
||||
"windows" if is_windows else "linux", []
|
||||
)
|
||||
|
||||
packages = [
|
||||
"module_bank",
|
||||
"sqlite_vfs",
|
||||
"fastapi",
|
||||
"uvicorn",
|
||||
"gunicorn",
|
||||
"diskcache",
|
||||
"apscheduler",
|
||||
"tomli",
|
||||
"typing_extensions",
|
||||
]
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"nuitka",
|
||||
"--standalone",
|
||||
"--onefile",
|
||||
"--follow-imports",
|
||||
"--static-libpython=no",
|
||||
"--remove-output",
|
||||
"--assume-yes-for-downloads",
|
||||
"--lto=yes",
|
||||
"--disable-plugin=tk-inter",
|
||||
"--disable-plugin=numpy",
|
||||
"--output-dir=.",
|
||||
"--output-filename=chongming",
|
||||
"main.py",
|
||||
"--jobs=2",
|
||||
]
|
||||
if shutil.which("upx"):
|
||||
cmd.append("--enable-plugin=upx")
|
||||
for pkg in packages:
|
||||
cmd.append(f"--include-package={pkg}")
|
||||
for mod in list(base_imports) + extra:
|
||||
cmd.append(f"--include-module={mod}")
|
||||
|
||||
if is_windows:
|
||||
icon = Path("../../public/chongming.ico")
|
||||
if icon.exists():
|
||||
cmd.append(f"--windows-icon-from-ico={icon}")
|
||||
|
||||
try:
|
||||
_run_cmd(cmd, cwd="build", capture=False)
|
||||
except subprocess.CalledProcessError:
|
||||
# 降级重试(移除可能不兼容的参数)
|
||||
fallback = [
|
||||
arg for arg in cmd if arg not in ("--lto=yes", "--enable-plugin=upx")
|
||||
]
|
||||
_run_cmd(fallback, cwd="build", capture=False)
|
||||
|
||||
exe = Path("build/chongming")
|
||||
if exe.exists():
|
||||
# 后处理瘦身
|
||||
if not is_windows:
|
||||
subprocess.run(["strip", str(exe)], check=False)
|
||||
if shutil.which("upx"):
|
||||
subprocess.run(["upx", "--best", str(exe)], check=False)
|
||||
size = _format_size(exe)
|
||||
return f"✅ Nuitka 编译完成 ({size})", str(exe)
|
||||
|
||||
return "❌ Nuitka 编译失败 (未生成可执行文件)", ""
|
||||
|
||||
|
||||
# ── 阶段 6: 清理临时文件 ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _stage_cleanup(mode: str) -> str:
|
||||
"""清理构建过程中的临时文件"""
|
||||
build = Path("build")
|
||||
dirs_to_remove = [
|
||||
"build_temp",
|
||||
"specs",
|
||||
"utils",
|
||||
"pyarmor_runtime_000000",
|
||||
"__pycache__",
|
||||
]
|
||||
files_to_remove = ["main.py", "server.py"]
|
||||
|
||||
for d in dirs_to_remove:
|
||||
target = build / d
|
||||
if target.exists():
|
||||
shutil.rmtree(str(target), ignore_errors=True)
|
||||
|
||||
for f in files_to_remove:
|
||||
target = build / f
|
||||
if target.exists():
|
||||
target.unlink(missing_ok=True)
|
||||
|
||||
return "临时文件已清理"
|
||||
|
||||
# ── 阶段 8: Vue 前端构建 ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _stage_build_vue() -> str:
|
||||
"""构建 Vue 前端并打包为 SQLite VFS"""
|
||||
dist_dir = "./src/chongming-web/dist"
|
||||
output_db = "./build/static.svfs"
|
||||
fs_name = "frontend"
|
||||
|
||||
# 1. npm build
|
||||
result = _run_cmd(
|
||||
["npm", "run", "build"],
|
||||
cwd="./src/chongming-web",
|
||||
capture=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
err = (result.stderr or "").strip()[:200]
|
||||
return f"❌ Vue 构建失败: {err}"
|
||||
|
||||
# 2. 检查产物
|
||||
if not Path(dist_dir).exists():
|
||||
return "❌ Vue 构建成功但未找到 dist 目录"
|
||||
|
||||
# 3. 打包为 VFS
|
||||
vfs = SQLiteVFS(output_db, compress=False)
|
||||
packer = FolderPacker(sqlite_vfs=vfs, exclude_patterns=[], fs_name=fs_name)
|
||||
try:
|
||||
packer.pack_folder(dist_dir)
|
||||
finally:
|
||||
vfs.close()
|
||||
|
||||
size = _format_size(output_db)
|
||||
return f"✅ Vue 打包完成 ({size})"
|
||||
|
||||
|
||||
# ── 主流程 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def main():
|
||||
"""规范化构建主流程"""
|
||||
mode = _resolve_mode()
|
||||
report = BuildReport(mode=mode, start_time=time.time())
|
||||
config = get_config()
|
||||
|
||||
async def run_step(name: str, fn, *args, **kwargs) -> str:
|
||||
"""执行一个构建步骤并记录结果(支持同步和异步函数)"""
|
||||
start = time.time()
|
||||
try:
|
||||
result = fn(*args, **kwargs)
|
||||
if asyncio.iscoroutine(result):
|
||||
msg = await result
|
||||
else:
|
||||
msg = result
|
||||
elapsed = time.time() - start
|
||||
report.add_step(name, success=True, duration=elapsed, message=msg or "")
|
||||
print(f" ✅ {name:<25s} {elapsed:>7.2f}s {msg}")
|
||||
return msg
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start
|
||||
err_msg = str(e).strip()[:200]
|
||||
report.add_step(name, success=False, duration=elapsed, message=err_msg)
|
||||
print(f" ❌ {name:<25s} {elapsed:>7.2f}s {err_msg}")
|
||||
return ""
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print(" 🏗️ CHONGMING 构建开始")
|
||||
print(f" 模式: {mode} | 平台: {platform.system()}")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
# 阶段 1: 模块打包
|
||||
await run_step("模块打包(module-bank)", _stage_pack_modules, config)
|
||||
|
||||
# 阶段 2: 准备构建目录
|
||||
await run_step("准备构建目录", _stage_prepare_build_dir, config, mode)
|
||||
|
||||
# 阶段 3+4: 混淆 + 打包
|
||||
if mode == PYINSTALLER:
|
||||
await run_step("PyArmor 混淆", _stage_pyarmor_obfuscate)
|
||||
start = time.time()
|
||||
try:
|
||||
msg, exe_path = _stage_pyinstaller(config)
|
||||
elapsed = time.time() - start
|
||||
success = "❌" not in msg
|
||||
report.add_step(
|
||||
"PyInstaller 打包", success, elapsed, msg.replace("✅ PyInstaller ", "")
|
||||
)
|
||||
print(
|
||||
f" {'✅' if success else '❌'} PyInstaller 打包{' ' * 12} {elapsed:>7.2f}s {msg}"
|
||||
)
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start
|
||||
report.add_step("PyInstaller 打包", False, elapsed, str(e)[:200])
|
||||
print(f" ❌ PyInstaller 打包 {elapsed:>7.2f}s {str(e)[:200]}")
|
||||
else:
|
||||
await run_step("Nuitka 编译", lambda: _stage_nuitka(config)[0])
|
||||
|
||||
# 阶段 5: 清理
|
||||
await run_step("清理临时文件", _stage_cleanup, mode)
|
||||
|
||||
# 阶段 7: Vue 前端构建
|
||||
await run_step("Vue 前端构建", _stage_build_vue)
|
||||
|
||||
# 总结
|
||||
report.end_time = time.time()
|
||||
report.print_summary()
|
||||
|
||||
if not report.all_success:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def build():
|
||||
"""构建入口"""
|
||||
asyncio.run(main())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
build()
|
||||
@@ -0,0 +1,47 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def generate_markdown_from_py_files(directory, output_file):
|
||||
with open(output_file, "w", encoding="utf-8") as md_file:
|
||||
for root, dirs, files in os.walk(directory):
|
||||
# 排除 venv 目录
|
||||
dirs[:] = [d for d in dirs if d != ".venv"]
|
||||
dirs[:] = [d for d in dirs if d != ".vscode"]
|
||||
dirs[:] = [d for d in dirs if d != "node_modules"]
|
||||
dirs[:] = [d for d in dirs if d != "build"]
|
||||
dirs[:] = [d for d in dirs if d != "dist"]
|
||||
dirs[:] = [d for d in dirs if d != "chongming-web"]
|
||||
for file in files:
|
||||
if (
|
||||
file.endswith(".py")
|
||||
or file.endswith(".rs")
|
||||
or file.endswith(".toml")
|
||||
or file.endswith(".md")
|
||||
or file.endswith(".css")
|
||||
or file.endswith(".vue")
|
||||
or file.endswith(".ts")
|
||||
or file.endswith(".json")
|
||||
or file.endswith(".html")
|
||||
or file.endswith(".sh")
|
||||
or file == "Dockerfile"
|
||||
):
|
||||
file_path = os.path.join(root, file)
|
||||
if file == "output.md" or file == "package-lock.json":
|
||||
continue
|
||||
if "generated" in file_path:
|
||||
continue
|
||||
md_file.write(f"`{file_path}`\n")
|
||||
md_file.write("```\n")
|
||||
with open(file_path, "r", encoding="utf-8") as py_file:
|
||||
md_file.write(py_file.read())
|
||||
md_file.write("\n```\n\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 指定目录和输出文件名
|
||||
target_directory = sys.argv[1] # 替换为你的目标目录
|
||||
output_markdown_file = "output.md" # 输出的 Markdown 文件名
|
||||
|
||||
generate_markdown_from_py_files(target_directory, output_markdown_file)
|
||||
print(f"Markdown 文件已生成:{output_markdown_file}")
|
||||
@@ -0,0 +1,333 @@
|
||||
"""
|
||||
模型生成脚本 — 从 config/workers/<plugin_name>/*.toml 生成 Pydantic 模型
|
||||
|
||||
用法:
|
||||
uv run gen_models config/workers/example src/plugins/example
|
||||
|
||||
工作流:
|
||||
1. 编写 config/workers/<plugin_name>/*.toml 配置文件
|
||||
2. 执行 gen_models 生成 Pydantic 模型
|
||||
3. 在业务代码中 import 生成的模型
|
||||
|
||||
功能:
|
||||
1. 扫描配置目录下的所有 .toml 文件
|
||||
2. 解析每个路由的 params 和 response_model
|
||||
3. 生成 Pydantic 请求/响应模型
|
||||
4. 写入 src/plugins/<plugin_name>/models/__init__.py 和 models/pydantic.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def parse_type_str(type_str: str) -> str:
|
||||
"""将 TOML 中的类型字符串映射为 Python 类型注解"""
|
||||
mapping = {
|
||||
"float": "float",
|
||||
"int": "int",
|
||||
"str": "str",
|
||||
"bool": "bool",
|
||||
"string": "str",
|
||||
"integer": "int",
|
||||
"number": "float",
|
||||
"boolean": "bool",
|
||||
"any": "Any",
|
||||
}
|
||||
|
||||
if type_str.startswith("list[") and type_str.endswith("]"):
|
||||
inner = type_str[5:-1]
|
||||
inner_py = mapping.get(inner, inner)
|
||||
return f"list[{inner_py}]"
|
||||
|
||||
if type_str == "dict":
|
||||
return "dict[str, Any]"
|
||||
|
||||
if type_str.startswith("dict[") and type_str.endswith("]"):
|
||||
inner = type_str[5:-1]
|
||||
if "," in inner:
|
||||
parts = [p.strip() for p in inner.split(",", 1)]
|
||||
inner_py = mapping.get(parts[-1], parts[-1])
|
||||
else:
|
||||
inner_py = mapping.get(inner, inner)
|
||||
return f"dict[str, {inner_py}]"
|
||||
|
||||
if type_str.startswith("Optional[") and type_str.endswith("]"):
|
||||
inner = type_str[9:-1]
|
||||
inner_py = mapping.get(inner, inner)
|
||||
return f"Optional[{inner_py}]"
|
||||
|
||||
return mapping.get(type_str, type_str)
|
||||
|
||||
|
||||
def parse_params(params: list[str]) -> list[dict[str, str]]:
|
||||
"""解析 params 列表,如 ['a: float', 'b: float']"""
|
||||
result = []
|
||||
for p in params:
|
||||
if ":" in p:
|
||||
name, type_str = p.split(":", 1)
|
||||
result.append({"name": name.strip(), "type": parse_type_str(type_str.strip())})
|
||||
else:
|
||||
result.append({"name": p.strip(), "type": "str"})
|
||||
return result
|
||||
|
||||
|
||||
def parse_response_model(response_model: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""解析 response_model 字典,返回字段列表"""
|
||||
fields: list[dict[str, Any]] = []
|
||||
for field_name, field_spec in response_model.items():
|
||||
entry: dict[str, Any] = {"name": field_name}
|
||||
if isinstance(field_spec, dict):
|
||||
entry["type"] = parse_type_str(field_spec.get("type", "str"))
|
||||
entry["required"] = field_spec.get("required", True)
|
||||
if "default" in field_spec:
|
||||
entry["default"] = field_spec["default"]
|
||||
else:
|
||||
entry["type"] = parse_type_str(str(field_spec))
|
||||
entry["required"] = True
|
||||
fields.append(entry)
|
||||
return fields
|
||||
|
||||
|
||||
def _model_name(base: str, method: str, suffix: str) -> str:
|
||||
"""生成模型名: e.g. add + GET + Request -> AddGetRequest"""
|
||||
return f"{base.title().replace('-', '').replace('_', '')}{method.title()}{suffix}"
|
||||
|
||||
|
||||
def generate_pydantic_code(
|
||||
plugin_name: str,
|
||||
routes: list[dict[str, Any]],
|
||||
) -> str:
|
||||
"""生成 pydantic.py 文件内容"""
|
||||
lines = []
|
||||
lines.append('"""')
|
||||
lines.append(f"自动生成的 Pydantic 模型 — {plugin_name}")
|
||||
lines.append(f"来源: config/workers/{plugin_name}/")
|
||||
lines.append('"""')
|
||||
lines.append("")
|
||||
lines.append("from __future__ import annotations")
|
||||
lines.append("")
|
||||
lines.append("from typing import Any, Optional")
|
||||
lines.append("from pydantic import BaseModel, Field")
|
||||
lines.append("")
|
||||
|
||||
seen_models: set[str] = set()
|
||||
|
||||
for route in routes:
|
||||
route_name = route["name"]
|
||||
method = route["method"]
|
||||
summary = route.get("summary", "")
|
||||
|
||||
# --- Request Model ---
|
||||
params = route.get("parsed_params", [])
|
||||
if params:
|
||||
model_name = _model_name(route_name, method, "Request")
|
||||
if model_name not in seen_models:
|
||||
seen_models.add(model_name)
|
||||
lines.append(f"class {model_name}(BaseModel):")
|
||||
lines.append(f' """{summary} — 请求体"""')
|
||||
for p in params:
|
||||
lines.append(f" {p['name']}: {p['type']} = Field(...)")
|
||||
lines.append("")
|
||||
|
||||
# --- Response Model ---
|
||||
response_fields = route.get("parsed_response", [])
|
||||
if response_fields:
|
||||
model_name = _model_name(route_name, method, "Response")
|
||||
if model_name not in seen_models:
|
||||
seen_models.add(model_name)
|
||||
lines.append(f"class {model_name}(BaseModel):")
|
||||
lines.append(f' """{summary} — 响应体"""')
|
||||
for f in response_fields:
|
||||
field_type = f["type"]
|
||||
if f.get("required", True):
|
||||
if "default" in f:
|
||||
default_repr = (
|
||||
json.dumps(f["default"])
|
||||
if not isinstance(f["default"], str)
|
||||
else f"'{f['default']}'"
|
||||
)
|
||||
lines.append(f" {f['name']}: {field_type} = Field(default={default_repr})")
|
||||
else:
|
||||
lines.append(f" {f['name']}: {field_type} = Field(...)")
|
||||
else:
|
||||
if "default" in f:
|
||||
default_repr = (
|
||||
json.dumps(f["default"])
|
||||
if not isinstance(f["default"], str)
|
||||
else f"'{f['default']}'"
|
||||
)
|
||||
lines.append(f" {f['name']}: Optional[{field_type}] = Field(default={default_repr})")
|
||||
else:
|
||||
lines.append(f" {f['name']}: Optional[{field_type}] = Field(default=None)")
|
||||
lines.append("")
|
||||
|
||||
# --- Union type for all responses ---
|
||||
response_models = [m for m in seen_models if m.endswith("Response")]
|
||||
if response_models:
|
||||
plugin_title = plugin_name.title().replace("-", "").replace("_", "")
|
||||
lines.append("")
|
||||
lines.append("# 所有响应模型的联合类型")
|
||||
lines.append(f"{plugin_title}Response = " + " | ".join(response_models))
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_init_code(
|
||||
plugin_name: str,
|
||||
routes: list[dict[str, Any]],
|
||||
) -> str:
|
||||
"""生成 __init__.py 文件内容"""
|
||||
lines = []
|
||||
lines.append('"""')
|
||||
lines.append(f"模型导出 — {plugin_name}")
|
||||
lines.append("自动生成,请勿手动修改")
|
||||
lines.append('"""')
|
||||
lines.append("")
|
||||
|
||||
seen_models: set[str] = set()
|
||||
for route in routes:
|
||||
rname = route["name"]
|
||||
method = route["method"]
|
||||
if route.get("parsed_params"):
|
||||
seen_models.add(_model_name(rname, method, "Request"))
|
||||
if route.get("parsed_response"):
|
||||
seen_models.add(_model_name(rname, method, "Response"))
|
||||
|
||||
response_models = [m for m in seen_models if m.endswith("Response")]
|
||||
if response_models:
|
||||
plugin_title = plugin_name.title().replace("-", "").replace("_", "")
|
||||
seen_models.add(f"{plugin_title}Response")
|
||||
|
||||
for model_name in sorted(seen_models):
|
||||
lines.append(f"from .pydantic import {model_name}")
|
||||
|
||||
lines.append("")
|
||||
lines.append("__all__ = [")
|
||||
for model_name in sorted(seen_models):
|
||||
lines.append(f' "{model_name}",')
|
||||
lines.append("]")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def scan_config_dir(config_dir: Path) -> list[dict[str, Any]]:
|
||||
"""扫描配置目录,解析所有 .toml 文件中的路由定义"""
|
||||
import tomli
|
||||
|
||||
routes = []
|
||||
toml_files = sorted(config_dir.glob("*.toml"))
|
||||
|
||||
group_config = {}
|
||||
config_file = config_dir / "config.toml"
|
||||
if config_file.exists():
|
||||
with open(config_file, "rb") as f:
|
||||
group_config = tomli.load(f)
|
||||
|
||||
defaults = group_config.get("worker", {}).get("defaults", {})
|
||||
router_prefix = (
|
||||
group_config.get("worker", {})
|
||||
.get("registration", {})
|
||||
.get("router_prefix", "")
|
||||
)
|
||||
|
||||
for toml_file in toml_files:
|
||||
if toml_file.name == "config.toml":
|
||||
continue
|
||||
|
||||
with open(toml_file, "rb") as f:
|
||||
data = tomli.load(f)
|
||||
|
||||
route_name = toml_file.stem
|
||||
|
||||
for method in ("GET", "POST", "PUT", "DELETE", "PATCH"):
|
||||
if method not in data:
|
||||
continue
|
||||
|
||||
entry = data[method]
|
||||
params_raw = entry.get("params", [])
|
||||
response_raw = entry.get("response_model", {})
|
||||
|
||||
route = {
|
||||
"name": route_name,
|
||||
"method": method,
|
||||
"summary": entry.get("summary", ""),
|
||||
"docstring": entry.get("docstring", ""),
|
||||
"params": params_raw,
|
||||
"response_model": response_raw,
|
||||
"parsed_params": parse_params(params_raw),
|
||||
"parsed_response": parse_response_model(response_raw),
|
||||
"path": entry.get("path", f"{router_prefix}/{route_name}"),
|
||||
"timeout": entry.get("timeout", defaults.get("timeout", 3.0)),
|
||||
"auth_required": entry.get("auth_required", defaults.get("auth_required", False)),
|
||||
}
|
||||
routes.append(route)
|
||||
|
||||
return routes
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
print("用法: uv run gen_models <config_dir> <plugin_dir>")
|
||||
print("示例:")
|
||||
print(" uv run gen_models config/workers/example src/plugins/example")
|
||||
sys.exit(1)
|
||||
|
||||
config_dir = Path(sys.argv[1])
|
||||
plugin_dir = Path(sys.argv[2])
|
||||
plugin_name = plugin_dir.name
|
||||
|
||||
if not config_dir.exists():
|
||||
print(f"❌ 配置目录不存在: {config_dir}")
|
||||
sys.exit(1)
|
||||
if not plugin_dir.exists():
|
||||
print(f"❌ 插件目录不存在: {plugin_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"🔍 扫描配置目录: {config_dir}")
|
||||
routes = scan_config_dir(config_dir)
|
||||
print(f" 发现 {len(routes)} 个路由定义")
|
||||
|
||||
if not routes:
|
||||
print("⚠️ 未找到路由定义,跳过模型生成")
|
||||
return
|
||||
|
||||
models_dir = plugin_dir / "models"
|
||||
models_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pydantic_code = generate_pydantic_code(plugin_name, routes)
|
||||
pydantic_file = models_dir / "pydantic.py"
|
||||
pydantic_file.write_text(pydantic_code, encoding="utf-8")
|
||||
print(f"✅ 生成: {pydantic_file}")
|
||||
|
||||
init_code = generate_init_code(plugin_name, routes)
|
||||
init_file = models_dir / "__init__.py"
|
||||
init_file.write_text(init_code, encoding="utf-8")
|
||||
print(f"✅ 生成: {init_file}")
|
||||
|
||||
print(f"\n📋 生成的模型:")
|
||||
for route in routes:
|
||||
method = route["method"]
|
||||
name = route["name"]
|
||||
params = route.get("parsed_params", [])
|
||||
response = route.get("parsed_response", [])
|
||||
if params:
|
||||
req_model = _model_name(name, method, "Request")
|
||||
param_strs = [f"{p['name']}: {p['type']}" for p in params]
|
||||
print(f" 📥 {req_model}: {', '.join(param_strs)}")
|
||||
if response:
|
||||
resp_model = _model_name(name, method, "Response")
|
||||
field_strs = [f"{f['name']}: {f['type']}" for f in response]
|
||||
print(f" 📤 {resp_model}: {', '.join(field_strs)}")
|
||||
|
||||
|
||||
def gen_models():
|
||||
"""CLI 入口"""
|
||||
main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python
|
||||
import json
|
||||
from chongming_jwt import JWT
|
||||
|
||||
# 使用与 config.toml 相同的密钥(可从配置读取)
|
||||
SECRET = "your-secret-key-change-in-production"
|
||||
|
||||
jwt = JWT(SECRET, "HS256")
|
||||
|
||||
# 1. 生成 token
|
||||
payload = json.dumps({"sub": "test_user", "role": "admin"})
|
||||
token = jwt.encode(payload, expires_in_seconds=3600)
|
||||
print(f"✅ Token: {token}")
|
||||
|
||||
# 2. 解码验证
|
||||
decoded = jwt.decode(token)
|
||||
print(f"✅ Decoded: {decoded}")
|
||||
|
||||
# 3. 仅验证(不返回内容)
|
||||
is_valid = jwt.validate(token)
|
||||
print(f"✅ Valid: {is_valid}")
|
||||
|
||||
# 4. 测试无效 token
|
||||
try:
|
||||
jwt.decode(token + "invalid")
|
||||
except Exception as e:
|
||||
print(f"❌ Invalid token caught: {e}")
|
||||
Reference in New Issue
Block a user