modify: rust 插件

This commit is contained in:
2026-08-31 17:18:03 +08:00
commit e59ecfc0e3
22 changed files with 3689 additions and 0 deletions
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
# 清理本地验证进程(脚本化避免 pkill -f 自匹配问题)
set -u
for pat in "target/debug/examples/demo" ".venv/bin/worker_serve" "scripts/zenoh_router.py" ".venv/bin/serve" "target/debug/gen_models"; do
pids=$(pgrep -f "$pat" || true)
if [ -n "$pids" ]; then
kill $pids 2>/dev/null
echo "killed [$pat]: $(echo $pids | tr '\n' ' ')"
fi
done
sleep 1
remaining=$(pgrep -f "target/debug/examples/demo|worker_serve|zenoh_router|\.venv/bin/serve" || true)
if [ -n "$remaining" ]; then
echo "WARN 仍存活: $remaining"
kill -9 $remaining 2>/dev/null
else
echo "清理完成"
fi
+294
View File
@@ -0,0 +1,294 @@
"""Rust 进程插件端到端协议验证(多语言演进规划阶段 1 验收)
前置条件:
1. 本地 Zenoh Router 已运行(scripts/zenoh_router.py)
2. Rust 参考插件已编译(cargo build --examples 或 workers/rsdemo)
验证内容(docs/plugin-wire-protocol.md 全协议面):
1. 路由描述符发布(keyexpr 编码、字段、schema_version/type)
2. liveliness 服务发现(worker/rsdemo/*)
3. 普通 RPC(GET 信封 / POST user_info 透传)
4. 非信封载荷兼容(method 默认 GET)
5. 流式多回复(逐块)+ 流中异常 {"error": ...} 收尾块
6. 非流式异常回复格式(result/operation/error/worker)
7. SIGTERM 优雅退出 → 描述符 DELETE 摘除
8. 重启 → 10s 内重发布自愈
用法(项目根目录):
.venv/bin/python utils/worker-rs/scripts/verify_e2e.py [--type rsdemo] [--count 1] [--bin <路径>]
"""
import argparse
import orjson
import os
import signal
import subprocess
import sys
import time
import zenoh
PASS = 0
FAIL = 0
DEFAULT_BIN = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "target", "debug", "examples", "demo")
)
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
ROUTES_PREFIX = "chongming/routes"
def report(name, cond, detail=""):
global PASS, FAIL
if cond:
PASS += 1
print(f"✅ {name}" + (f" — {detail}" if detail else ""))
else:
FAIL += 1
print(f"❌ {name}" + (f" — {detail}" if detail else ""))
def start_worker(worker_type, count=1, bin_path=None):
"""启动 Rust 插件进程(CWD=项目根,读取全局 config.toml)"""
binary = bin_path or DEFAULT_BIN
proc = subprocess.Popen(
[binary, "--type", worker_type, "--count", str(count)],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
# 等待 Queryable 就绪(日志出现"进程就绪")
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
if proc.poll() is not None:
out = proc.stdout.read().decode() if proc.stdout else ""
raise RuntimeError(f"Worker 启动失败,退出码 {proc.returncode}:\n{out}")
time.sleep(0.2)
line = proc.stdout.readline() if proc.stdout else b""
if "等待退出信号" in line.decode("utf-8", "ignore"):
return proc
raise RuntimeError("Worker 20s 内未就绪")
def wait_worker_exit(proc, timeout=10):
try:
proc.wait(timeout=timeout)
return True
except subprocess.TimeoutExpired:
return False
def make_subscriber(session, worker_type):
"""建立常驻订阅,持续记录 PUT 描述符与 DELETE 事件(不重放历史值,
调用方须在事件发生前建立订阅)"""
samples = {}
def cb(sample):
key = str(sample.key_expr)
if sample.kind == zenoh.SampleKind.DELETE:
samples[key] = None # 摘除标记
else:
samples[key] = orjson.loads(bytes(sample.payload))
sub = session.declare_subscriber(f"{ROUTES_PREFIX}/{worker_type}/**", cb)
return sub, samples
def rpc(session, worker_expr, subject, payload, timeout=10.0):
"""按信封协议发起 Zenoh Query,返回所有回复块 JSON 列表"""
full_key = f"{worker_expr}/{subject}"
replies = session.get(
full_key,
payload=orjson.dumps(payload),
consolidation=zenoh.ConsolidationMode.NONE,
timeout=timeout,
)
result = []
for reply in replies:
if reply.ok:
result.append(orjson.loads(bytes(reply.ok.payload)))
else:
result.append({"__zenoh_err__": str(reply.err.payload)})
return result
def get_worker_exprs(session, worker_type):
"""liveliness 发现 worker/worker_type/* 副本列表"""
exprs = []
for reply in session.liveliness().get(f"worker/{worker_type}/*"):
if reply.ok:
exprs.append(str(reply.ok.key_expr))
return exprs
def main():
global PASS, FAIL
parser = argparse.ArgumentParser()
parser.add_argument("--type", default="rsdemo")
parser.add_argument("--count", type=int, default=1)
parser.add_argument("--bin", default=None, help="插件二进制路径(默认 SDK 示例 demo)")
args = parser.parse_args()
wt = args.type
session = zenoh.open(zenoh.Config.from_file("config/zenoh_config.json5"))
print(f"=== Rust 进程插件协议验证 (type={wt}, bin={args.bin or DEFAULT_BIN}) ===")
# ========== 1. 路由描述符发布(先订阅再启动,zenoh 不重放历史值) ==========
sub1, samples1 = make_subscriber(session, wt)
proc = start_worker(wt, args.count, args.bin)
time.sleep(1.5)
expect_keys = {
"rsdemo": {
"chongming/routes/rsdemo/rsdemo_hello__GET",
"chongming/routes/rsdemo/rsdemo_echo__POST",
"chongming/routes/rsdemo/rsdemo_stream__POST",
"chongming/routes/rsdemo/rsdemo_error__GET",
},
"demo": {
"chongming/routes/demo/demo_hello__GET",
"chongming/routes/demo/demo_storage_upload_bytes__POST",
"chongming/routes/demo/demo_storage_download_bytes__GET",
"chongming/routes/demo/demo_storage_presign_upload__POST",
"chongming/routes/demo/demo_storage_presign_download__GET",
},
}[wt]
missing = expect_keys - set(samples1.keys())
report(
"路由描述符 keyexpr 编码正确",
not missing,
f"缺失: {missing or '无'}",
)
if "chongming/routes/rsdemo/rsdemo_hello__GET" in samples1:
d = samples1["chongming/routes/rsdemo/rsdemo_hello__GET"]
report("描述符 schema_version=1", d.get("schema_version") == 1, f"实际: {d.get('schema_version')}")
report("描述符 type=插件名", d.get("type") == wt, f"实际: {d.get('type')}")
report(
"描述符字段完整(subject/method/path/params/response_model)",
d.get("subject") == "rsdemo.hello"
and d.get("method") == "GET"
and d.get("path") == "/rsdemo/hello"
and d.get("params") == ["name: str"]
and isinstance(d.get("response_model"), dict),
)
report(
"defaults 合并(timeout=30.0/auth_required=false)",
d.get("timeout") == 30.0 and d.get("auth_required") is False,
)
sub1.undeclare()
# ========== 2. liveliness 服务发现 ==========
exprs = get_worker_exprs(session, wt)
report(
"liveliness 服务发现副本",
len(exprs) == args.count,
f"实际: {exprs}",
)
worker_expr = exprs[0] if exprs else f"worker/{wt}/missing"
# ========== 3. 普通 RPC:GET 信封 ==========
replies = rpc(session, worker_expr, "rsdemo.hello", {"meta": {"method": "GET"}, "data": {"name": "rust"}})
ok = (
len(replies) == 1
and replies[0].get("greeting") == "Hello, rust!"
and replies[0].get("worker_id") == worker_expr.split("/")[-1]
and isinstance(replies[0].get("timestamp"), (int, float))
)
report("RPC GET(信封分派 + worker_id 注入)", ok, f"实际: {replies}")
# ========== 4. 非信封载荷兼容(method 默认 GET) ==========
replies = rpc(session, worker_expr, "rsdemo.hello", {"name": "legacy"})
report(
"非信封载荷兼容(按 GET 处理)",
len(replies) == 1 and replies[0].get("greeting") == "Hello, legacy!",
f"实际: {replies}",
)
# ========== 5. POST + user_info 透传 ==========
replies = rpc(
session, worker_expr, "rsdemo.echo",
{"meta": {"method": "POST", "user_info": {"username": "alice"}}, "data": {"msg": "hi"}},
)
ok = (
len(replies) == 1
and replies[0].get("echo") == "hi"
and replies[0].get("user_info") == {"username": "alice"}
)
report("RPC POST(user_info 信封透传)", ok, f"实际: {replies}")
# ========== 6. 流式多回复 ==========
replies = rpc(
session, worker_expr, "rsdemo.stream",
{"meta": {"method": "POST"}, "data": {"prompt": "x", "tokens": 5}},
timeout=15.0,
)
report(
"流式多回复(5 块逐块推送)",
len(replies) == 5
and all(r.get("index") == i for i, r in enumerate(replies))
and all("error" not in r for r in replies),
f"块数: {len(replies)}",
)
# 流中异常 → {"error": ...} 收尾块(第 26 块抛错,共 26 块)
replies = rpc(
session, worker_expr, "rsdemo.stream",
{"meta": {"method": "POST"}, "data": {"prompt": "x", "tokens": 60}},
timeout=20.0,
)
ok = (
len(replies) == 26
and all("error" not in r for r in replies[:25])
and isinstance(replies[25].get("error"), str)
and "模拟流中异常" in replies[25]["error"]
)
report("流中异常 → error 收尾块终止", ok, f"块数: {len(replies)}, 末块: {replies[-1] if replies else None}")
# ========== 7. 非流式异常回复格式 ==========
replies = rpc(session, worker_expr, "rsdemo.error", {"meta": {"method": "GET"}, "data": {}})
ok = (
len(replies) == 1
and replies[0].get("result") == 0.0
and replies[0].get("operation") == "error"
and "模拟业务异常" in replies[0].get("error", "")
and replies[0].get("worker") == worker_expr.split("/")[-1]
)
report("非流式异常回复格式(result/operation/error/worker)", ok, f"实际: {replies}")
# ========== 8. SIGTERM 优雅退出 → 描述符摘除(先订阅再 kill) ==========
sub2, samples2 = make_subscriber(session, wt)
os.kill(proc.pid, signal.SIGTERM)
exited = wait_worker_exit(proc, timeout=10)
report("SIGTERM 优雅退出", exited, f"退出码: {proc.returncode if exited else '超时'}")
time.sleep(1.5)
removed = [k for k, v in samples2.items() if v is None]
report(
"退出时 DELETE 摘除全部描述符",
len(removed) == len(expect_keys),
f"摘除: {sorted(removed)}",
)
sub2.undeclare()
# ========== 9. 重启 → 10s 内重发布自愈 ==========
sub3, samples3 = make_subscriber(session, wt)
proc = start_worker(wt, args.count, args.bin)
time.sleep(11.5) # 覆盖一个完整重发布周期
ok = all(k in samples3 and samples3[k] is not None for k in expect_keys)
report(
"重启后周期重发布自愈(≤10s)",
ok,
f"恢复: {sorted(k for k in samples3 if samples3[k] is not None)}",
)
sub3.undeclare()
# 清理
os.kill(proc.pid, signal.SIGTERM)
wait_worker_exit(proc, timeout=10)
session.close()
print(f"\n=== 结果: {PASS} 通过, {FAIL} 失败 ===")
sys.exit(1 if FAIL else 0)
if __name__ == "__main__":
main()
+188
View File
@@ -0,0 +1,188 @@
"""Python 与 Rust 进程插件同 type 混布对拍(多语言演进规划 §3.4 场景 5)
前置条件:
1. 本地 Zenoh Router 已运行(scripts/zenoh_router.py)
2. Rust 参考插件已编译;venv 已安装(worker_serve console script)
验证内容:
1. Python 与 Rust 对同一 TOML 配置发布的描述符内容 hash 一致
(网关据此幂等去重,内容不一致会告警冲突)
2. 同一 type 下两个实现副本均被 liveliness 发现(负载均衡透明)
3. 两个实现的 RPC 均返回等价结果(worker_id 区分副本)
4. SIGTERM 后各自摘除(不互相干扰)
用法(项目根目录):
.venv/bin/python utils/worker-rs/scripts/verify_mixed.py
"""
import hashlib
import orjson
import os
import signal
import subprocess
import sys
import time
import zenoh
PASS = 0
FAIL = 0
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
RUST_BIN = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "target", "debug", "examples", "demo")
)
PY_WORKER = os.path.join(ROOT, ".venv", "bin", "worker_serve")
def report(name, cond, detail=""):
global PASS, FAIL
if cond:
PASS += 1
print(f"✅ {name}" + (f" — {detail}" if detail else ""))
else:
FAIL += 1
print(f"❌ {name}" + (f" — {detail}" if detail else ""))
def start(cmd, worker_type):
proc = subprocess.Popen(
cmd + ["--type", worker_type, "--count", "1"],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
deadline = time.monotonic() + 25
while time.monotonic() < deadline:
if proc.poll() is not None:
out = proc.stdout.read().decode() if proc.stdout else ""
raise RuntimeError(f"Worker 启动失败 (退出码 {proc.returncode}):\n{out}")
line = proc.stdout.readline() if proc.stdout else b""
if "等待退出信号" in line.decode("utf-8", "ignore"):
return proc
raise RuntimeError("Worker 25s 未就绪")
def rpc(session, worker_expr, subject, payload, timeout=10.0):
full_key = f"{worker_expr}/{subject}"
replies = session.get(
full_key,
payload=orjson.dumps(payload),
consolidation=zenoh.ConsolidationMode.NONE,
timeout=timeout,
)
return [orjson.loads(bytes(r.ok.payload)) for r in replies if r.ok]
def main():
session = zenoh.open(zenoh.Config.from_file("config/zenoh_config.json5"))
print("=== Python + Rust 同 type(demo) 混布对拍 ===")
# 先建订阅(zenoh 不重放历史值)
samples = {}
def cb(sample):
if not sample.payload:
return # 空载荷样本(如 DELETE 预发布期),忽略
key = str(sample.key_expr)
samples[key] = orjson.loads(bytes(sample.payload))
sub = session.declare_subscriber("chongming/routes/demo/**", cb)
# 同时启动 Python 与 Rust 两个实现
py_proc = start([PY_WORKER], "demo")
rs_proc = start([RUST_BIN], "demo")
time.sleep(2.0)
# 1. 描述符内容一致性:同一 route_id 多次发布(Python/Rust 各一次)内容 hash 必须一致
expect = {
"demo_hello__GET",
"demo_storage_upload_bytes__POST",
"demo_storage_download_bytes__GET",
"demo_storage_presign_upload__POST",
"demo_storage_presign_download__GET",
}
seen = {k: [] for k in expect}
for key, desc in samples.items():
rid = key.split("/")[-1]
if rid in seen:
seen[rid].append(orjson.dumps(desc, option=orjson.OPT_SORT_KEYS).decode())
all_same = True
for rid, versions in seen.items():
if len(versions) < 1:
continue
hashes = {hashlib.sha1(v.encode("utf-8")).hexdigest()[:8] for v in versions}
if len(hashes) > 1:
all_same = False
print(f" ⚠ {rid} 描述符内容不一致: {hashes}")
report(
"Python/Rust 描述符逐字段一致(内容 hash 唯一,无冲突告警)",
all_same and all(seen.values()),
f"各 route_id 版本数: { {k: len(v) for k, v in seen.items()} }",
)
# 2. liveliness:同 type 两个实现副本均被发现
exprs = [
str(r.ok.key_expr)
for r in session.liveliness().get("worker/demo/*")
if r.ok
]
report(
"liveliness 发现 Python + Rust 副本(共 2 个)",
len(exprs) == 2,
f"实际: {exprs}",
)
if len(exprs) < 2:
print("混布副本不足,跳过 RPC 对拍")
else:
# 区分 Python 副本与 Rust 副本:Rust worker_id 格式同为 worker-0-<pid>,
# 无法从 key 区分实现,但可分别对两个副本调用并验证都响应
for expr in exprs:
replies = rpc(
session, expr, "demo.hello",
{"meta": {"method": "GET"}, "data": {"name": "mixed"}},
)
ok = (
len(replies) == 1
and replies[0].get("greeting") == "Hello, mixed!"
and replies[0].get("worker_id") == expr.split("/")[-1]
)
report(f"RPC 对拍 {expr}", ok, f"实际: {replies}")
# storage 端点:两实现均应返回错误(Python 未配置 storage 降级;Rust 未实现降级)
replies = rpc(
session, exprs[0], "demo.storage.upload_bytes",
{"meta": {"method": "POST"}, "data": {"object_key": "a.txt", "data_b64": "eA=="}},
)
report(
"storage 端点两实现均降级为错误",
len(replies) == 1
and replies[0].get("operation") == "error",
f"实际: {replies}",
)
# 3. SIGTERM 顺序退出:互不干扰
os.kill(rs_proc.pid, signal.SIGTERM)
rs_proc.wait(timeout=10)
time.sleep(1.5)
remain = [
str(r.ok.key_expr)
for r in session.liveliness().get("worker/demo/*")
if r.ok
]
report(
"Rust 退出后 Python 副本不受影响",
len(remain) == 1,
f"剩余: {remain}",
)
os.kill(py_proc.pid, signal.SIGTERM)
py_proc.wait(timeout=10)
sub.undeclare()
session.close()
print(f"\n=== 结果: {PASS} 通过, {FAIL} 失败 ===")
sys.exit(1 if FAIL else 0)
if __name__ == "__main__":
main()
+20
View File
@@ -0,0 +1,20 @@
"""本地测试用 Zenoh Router(mode=router,监听 tcp/127.0.0.1:7447)。
替代 zenohd:环境无独立 router 二进制时,用 Python zenoh 开一个 router 节点,
供网关(peer + connect 7447)与 worker 进程共同接入。仅用于本地联调。
"""
import time
import zenoh
config = zenoh.Config()
config.insert_json5("mode", '"router"')
session = zenoh.open(config)
print(">>> zenoh router listening on tcp/127.0.0.1:7447", flush=True)
try:
while True:
time.sleep(3600)
except KeyboardInterrupt:
pass
finally:
session.close()