modify: 微服务演进

test/test_call_stats.py

@@ -22,7 +22,7 @@ import httpx
 import orjson
 from fastapi import FastAPI
 
-REPO_ROOT = Path(__file__).resolve().parent
+REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "src"))
 sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
 
@@ -53,6 +53,11 @@ _cs_mod = importlib.util.module_from_spec(_cs_spec)
 sys.modules["chongming.app.core.call_stats"] = _cs_mod
 _cs_spec.loader.exec_module(_cs_mod)  # type: ignore[union-attr]
 
+# 审计采样率固定为全量：生产 config 的 access_sample_rate=0.1 下，成功请求的审计行
+# 本就是按概率写的，断言"几条请求 = 几条审计行"会随机失败。本测试验证的是审计链路
+# 是否工作，与采样策略无关，故固定为 1.0。
+_cs_mod._AUDIT_SAMPLE_RATE = 1.0
+
 # 加载真实 route_generator
 _rg_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "route_generator.py"
 _rg_spec = importlib.util.spec_from_file_location("chongming.app.core.route_generator", _rg_path)
@@ -95,12 +100,14 @@ class FakeWorkerManager:
         self.reply = reply
         self.error = error
 
-    async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False):
+    async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False,
+                       instance_id=None):
         if self.error == "no_worker":
             raise RuntimeError(f"类型 '{worker_type}' 没有可用 Worker。")
         return self.reply
 
-    async def call_stream(self, worker_type, subject, payload, timeout=300.0, idle_timeout=10.0):
+    async def call_stream(self, worker_type, subject, payload, timeout=300.0, idle_timeout=10.0,
+                          instance_id=None):
         if self.error == "no_worker":
             raise RuntimeError(f"类型 '{worker_type}' 没有可用 Worker。")
 

test/test_instance_targeting.py

@@ -0,0 +1,220 @@
+"""
+worker 实例定向（把请求固定到某个进程实例）测试
+
+验证 WorkerManager 的实例定向语义（HTTP 的 X-Chongming-Instance 头 / MQTT 的
+target_instance 字段 / 程序调用 instance_id 参数最终都走这里）：
+  1. 不指定实例：与以往一致，在全部候选副本里随机挑
+  2. 指定实例：只在其实例的在线副本里挑，不跨实例
+  3. 身份来源：副本经 admin status 上报 instance_id，网关按需查询并缓存
+     （本地注册的 inline 副本直接用网关自身实例，无需查询）
+  4. 目标实例不在线：报错而不回退到别的实例（定向即严格）
+  5. 调用失败重试：限定在同一实例内换副本，绝不跨实例
+  6. 实例内已无其它副本可换：立即失败，不重复打同一个副本
+  7. 实例维度缓存键：定向调用的缓存与其它实例隔离
+
+运行: .venv/bin/python test/test_instance_targeting.py
+"""
+import asyncio
+import sys
+from pathlib import Path
+
+_HERE = Path(__file__).resolve().parent          # chongming-zenoh/
+_PROJ = _HERE
+for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"):
+    sys.path.insert(0, str(_p))
+
+from chongming.app.core.worker_manager import WorkerManager  # noqa: E402
+
+failures = []
+
+
+def check(name, cond, detail=""):
+    status = "PASS" if cond else "FAIL"
+    print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
+    if not cond:
+        failures.append(name)
+
+
+class _OkPayload:
+    def __init__(self, data: bytes):
+        self.payload = data
+
+
+class _Ok:
+    def __init__(self, data: bytes):
+        self.ok = _OkPayload(data)
+
+
+class _Reply:
+    """一次 get 的回复：ok 与 err 互斥（与 Zenoh reply 的形态一致）。"""
+
+    def __init__(self, data: bytes = b"{}", err: bytes | None = None):
+        if err is None:
+            self.ok = _OkPayload(data)
+            self.err = None
+        else:
+            self.ok = None
+            self.err = _OkPayload(err)
+
+
+class FakeSession:
+    """记录每次 get 的 key，并按 key 决定回复。
+
+    admin 查询（worker/<type>/<id>/__admin__）由 admin_instances 决定 instance_id；
+    业务调用（worker/<type>/<id>/<subject>）可按 call_errors 注入错误回复。
+    """
+
+    def __init__(self, admin_instances: dict | None = None,
+                 call_errors: set | None = None):
+        self.admin_instances = admin_instances or {}
+        self.call_errors = set(call_errors or ())
+        self.calls: list = []
+
+    def get(self, key, payload=None, **kwargs):
+        self.calls.append(key)
+        if key.endswith("/__admin__"):
+            wid = key.split("/")[2]
+            iid = self.admin_instances.get(wid)
+            body = (
+                f'{{"worker_id":"{wid}","instance_id":"{iid}","status":"running"}}'.encode()
+                if iid else f'{{"worker_id":"{wid}","status":"running"}}'.encode()
+            )
+            return iter([_Reply(body)])
+        if key in self.call_errors:
+            return iter([_Reply(err=b'{"error":"boom"}')])
+        return iter([_Reply(b'{"ok":true}')])
+
+    def liveliness(self):
+        raise RuntimeError("本测试不触及 liveliness")
+
+    def close(self):
+        pass
+
+
+def new_manager(session, gateway_id="gw-local"):
+    m = WorkerManager(session=session)
+    m._gateway_id = gateway_id
+    return m
+
+
+async def main():
+    # ── 1. 不定向：与以往一致，全候选集随机 ─────────────────
+    sess = FakeSession(admin_instances={"w-a": "edge-01", "w-b": "edge-02"})
+    m = new_manager(sess)
+    m._workers_cache["demo"] = ["worker/demo/w-a", "worker/demo/w-b"]
+    picked = {await m.get_worker("demo") for _ in range(60)}
+    check("1.1 不定向时可命中全部副本", picked == set(m._workers_cache["demo"]),
+          f"picked={sorted(picked)}")
+    check("1.2 不定向不触发实例身份查询（零额外 RPC）",
+          not [c for c in sess.calls if c.endswith("__admin__")], f"calls={sess.calls}")
+    check("1.3 不定向调用走随机副本",
+          await m.call_raw("demo", "ping", b"{}") == b'{"ok":true}')
+
+    # ── 2. 定向：只在目标实例内挑副本 ────────────────────────
+    sess = FakeSession(admin_instances={"w-a": "edge-01", "w-b": "edge-02",
+                                        "w-c": "edge-01"})
+    m = new_manager(sess)
+    m._workers_cache["demo"] = [
+        "worker/demo/w-a", "worker/demo/w-b", "worker/demo/w-c",
+    ]
+    picked = {await m.get_worker("demo", instance_id="edge-01") for _ in range(60)}
+    check("2.1 定向只命中目标实例的副本",
+          picked == {"worker/demo/w-a", "worker/demo/w-c"}, f"picked={sorted(picked)}")
+
+    # ── 3. 身份查询：按需查询并缓存，实例归属正确 ─────────────
+    sess = FakeSession(admin_instances={"w-a": "edge-01", "w-b": "edge-02",
+                                        "w-c": "edge-01"})
+    m = new_manager(sess)
+    m._workers_cache["demo"] = [
+        "worker/demo/w-a", "worker/demo/w-b", "worker/demo/w-c",
+    ]
+    index = await m._instance_workers("demo", "edge-01")
+    check("3.1 实例→副本反向索引正确",
+          sorted(index) == ["worker/demo/w-a", "worker/demo/w-c"], f"index={sorted(index)}")
+    admin_calls = [c for c in sess.calls if c.endswith("__admin__")]
+    check("3.2 每个副本只查询一次身份", len(admin_calls) == 3, f"n={len(admin_calls)}")
+    await m._refresh_instance_map("demo")
+    check("3.3 重复刷新复用缓存不重复查询",
+          len([c for c in sess.calls if c.endswith("__admin__")]) == len(admin_calls))
+
+    # ── 3b. inline 本地注册副本：直接用网关实例，零查询 ──────
+    sess = FakeSession()
+    m = new_manager(sess, gateway_id="gw-1")
+    m.register_worker("demo", "worker-0-1-999")
+    index = await m._instance_workers("demo", "gw-1")
+    check("3.4 inline 副本归入网关实例", index == ["worker/demo/worker-0-1-999"],
+          f"index={index}")
+    check("3.5 inline 副本身份查询零 RPC",
+          not [c for c in sess.calls if c.endswith("__admin__")], f"calls={sess.calls}")
+
+    # ── 4. 定向实例不在线：严格报错，不回退 ──────────────────
+    sess = FakeSession(admin_instances={"w-a": "edge-01"})
+    m = new_manager(sess)
+    m._workers_cache["demo"] = ["worker/demo/w-a"]
+    check("4.1 目标实例无副本时 get_worker 返回 None",
+          await m.get_worker("demo", instance_id="edge-99") is None)
+    try:
+        await m.call_raw("demo", "ping", b"{}", instance_id="edge-99")
+        check("4.2 定向到离线实例应报错", False, "未抛错")
+    except RuntimeError as e:
+        msg = str(e)
+        check("4.2 定向到离线实例报错且说明不跨实例回退",
+              "edge-99" in msg and "不" in msg, f"msg={msg[:90]}")
+
+    # ── 5. 重试锚定同实例：绝不跨实例 ────────────────────────
+    sess = FakeSession(
+        admin_instances={"w-a": "edge-01", "w-b": "edge-02", "w-c": "edge-01"},
+        call_errors={"worker/demo/w-b/ping"},  # 只让别实例的副本出错，验证不会被选中
+    )
+    m = new_manager(sess)
+    m._workers_cache["demo"] = [
+        "worker/demo/w-a", "worker/demo/w-b", "worker/demo/w-c",
+    ]
+    # 目标实例 edge-01 内的 w-a 失败 -> 应换到同实例的 w-c，不碰 edge-02 的 w-b
+    sess.call_errors = {"worker/demo/w-a/ping"}
+    out = await m.call_raw("demo", "ping", b"{}", instance_id="edge-01", retries=2)
+    business = [c for c in sess.calls if not c.endswith("__admin__")]
+    check("5.1 重试不跨实例（未打其它实例的副本）",
+          all("/w-b/" not in c for c in business), f"calls={business}")
+    check("5.2 同实例内换副本后成功", out == b'{"ok":true}', f"out={out!r}")
+
+    # ── 6. 实例内无其它副本可换：立即失败 ────────────────────
+    sess = FakeSession(admin_instances={"w-a": "edge-01"},
+                       call_errors={"worker/demo/w-a/ping"})
+    m = new_manager(sess)
+    m._workers_cache["demo"] = ["worker/demo/w-a", "worker/demo/w-b"]
+    sess.admin_instances["w-b"] = "edge-02"
+    raised = None
+    try:
+        await m.call_raw("demo", "ping", b"{}", instance_id="edge-01", retries=2)
+    except RuntimeError as e:
+        raised = e
+    check("6.1 实例内无副本可换应报错", raised is not None, f"raised={raised!r}")
+    n = len([c for c in sess.calls if not c.endswith("__admin__")])
+    check("6.2 不重复打同一个副本（只调用一次）", n == 1, f"业务调用次数={n}")
+    check("6.3 错误信息点名实例与原因",
+          raised is not None and "edge-01" in str(raised), f"msg={str(raised)[:90]}")
+
+    # ── 7. 定向调用的缓存按实例隔离 ──────────────────────────
+    sess = FakeSession(admin_instances={"w-a": "edge-01", "w-b": "edge-02"})
+    m = new_manager(sess)
+    m._workers_cache["demo"] = ["worker/demo/w-a", "worker/demo/w-b"]
+    r1 = await m.call_raw("demo", "state", b"{}", cacheable=True, instance_id="edge-01")
+    r2 = await m.call_raw("demo", "state", b"{}", cacheable=True, instance_id="edge-02")
+    keys = [f"{iid}@demo:state:" for iid in ("edge-01", "edge-02")]
+    check("7.1 不同实例的定向缓存键互不影响",
+          all(any(k.startswith(p) for k in m._rpc_mem_cache._d) for p in keys),
+          f"cache_keys={list(m._rpc_mem_cache._d)}")
+    check("7.2 定向调用结果正常返回", r1 == b'{"ok":true}' and r2 == b'{"ok":true}',
+          f"r1={r1!r}, r2={r2!r}")
+
+    print()
+    if failures:
+        print(f"FAILED: {len(failures)} 项 -> {failures}")
+        return 1
+    print("全部通过")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

test/test_instance_targeting_http.py

@@ -0,0 +1,207 @@
+"""
+HTTP 实例定向头（X-Chongming-Instance）透传测试
+
+依赖隔离：桩掉 chongming 包链、auth 与 chongming_cache，直接加载真实
+route_generator.py 与 worker_manager.py，用 FastAPI TestClient 验证:
+  1. 携带 X-Chongming-Instance 的请求，该实例 id 透传到 call_raw / call_stream
+  2. 未携带该头时 instance_id 为 None（行为与改动前完全一致，兼容旧调用方）
+  3. 覆盖全部端点形态：GET query / POST body / validate=false 原样透传 / SSE 流式
+  4. 头值为空白时视为未指定
+  5. 定向到离线实例（RuntimeError）-> HTTP 503，错误信息透出
+
+运行: .venv/bin/python test/test_instance_targeting_http.py
+"""
+import importlib.util
+import logging
+import sys
+import types
+from pathlib import Path
+
+import orjson
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+TEST_DIR = Path(__file__).resolve().parent
+REPO_ROOT = TEST_DIR.parent
+sys.path.insert(0, str(REPO_ROOT / "src"))
+sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
+
+# ---- 依赖隔离：桩掉 chongming 包链 / auth / chongming_cache ----
+_pkg = types.ModuleType("chongming")
+_pkg.__path__ = []
+_app = types.ModuleType("chongming.app")
+_app.__path__ = []
+_core = types.ModuleType("chongming.app.core")
+_core.__path__ = []
+_logger_stub = types.ModuleType("chongming.app.core.logger")
+_logger_stub.get_logger = lambda name: logging.getLogger(name)
+_auth_stub = types.ModuleType("chongming.app.core.auth")
+_auth_stub.store_user_to_request = lambda x: x
+_cache_stub = types.ModuleType("chongming_cache")
+_cache_stub.Cache = object
+_cache_stub.DistributedLock = object  # worker_manager 顶层导入（本测试不触达）
+for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub):
+    sys.modules[mod.__name__] = mod
+
+_wm_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "worker_manager.py"
+_wm_spec = importlib.util.spec_from_file_location(
+    "chongming.app.core.worker_manager", _wm_path
+)
+_wm_mod = importlib.util.module_from_spec(_wm_spec)
+sys.modules["chongming.app.core.worker_manager"] = _wm_mod
+_wm_spec.loader.exec_module(_wm_mod)  # type: ignore[union-attr]
+
+_cs_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "call_stats.py"
+_cs_spec = importlib.util.spec_from_file_location(
+    "chongming.app.core.call_stats", _cs_path
+)
+_cs_mod = importlib.util.module_from_spec(_cs_spec)
+sys.modules["chongming.app.core.call_stats"] = _cs_mod
+_cs_spec.loader.exec_module(_cs_mod)  # type: ignore[union-attr]
+
+_rg_path = REPO_ROOT / "src" / "chongming" / "app" / "core" / "route_generator.py"
+_rg_spec = importlib.util.spec_from_file_location(
+    "chongming.app.core.route_generator", _rg_path
+)
+_rg_mod = importlib.util.module_from_spec(_rg_spec)
+sys.modules["chongming.app.core.route_generator"] = _rg_mod
+_rg_spec.loader.exec_module(_rg_mod)  # type: ignore[union-attr]
+
+register_route_group = _rg_mod.register_route_group
+INSTANCE_HEADER = _rg_mod.INSTANCE_HEADER
+
+failures = []
+
+
+def check(name, cond, detail=""):
+    status = "PASS" if cond else "FAIL"
+    print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
+    if not cond:
+        failures.append(name)
+
+
+class FakeWorkerManager:
+    """记录每次调用收到的 instance_id；可指定定点失败以验证 503 映射。"""
+
+    def __init__(self, fail_instance=None):
+        self.fail_instance = fail_instance            # 指定实例报错（模拟实例离线）
+        self.raw_instances = []                       # call_raw 收到的 instance_id
+        self.stream_instances = []                    # call_stream 收到的 instance_id
+
+    async def call_raw(self, worker_type, subject, payload, timeout=3.0,
+                       retries=1, cacheable=False, instance_id=None):
+        self.raw_instances.append(instance_id)
+        if instance_id and instance_id == self.fail_instance:
+            raise RuntimeError(
+                f"类型 '{worker_type}' 的实例 '{instance_id}' 没有在线副本"
+                "（定向调用不跨实例回退）。"
+            )
+        return orjson.dumps({"op": subject, "instance": instance_id})
+
+    async def call_stream(self, worker_type, subject, payload, timeout=300.0,
+                          idle_timeout=10.0, instance_id=None):
+        self.stream_instances.append(instance_id)
+        if instance_id and instance_id == self.fail_instance:
+            raise RuntimeError(f"实例 '{instance_id}' 没有在线副本")
+
+        async def _gen():
+            yield orjson.dumps({"instance": instance_id})
+
+        return _gen()
+
+
+ROUTES = [
+    {"subject": "t.get", "path": "/calc/add", "method": "GET",
+     "params": ["a: float", "b: float"], "summary": "add",
+     "timeout": 3.0, "auth_required": False},
+    {"subject": "t.post", "path": "/calc/echo", "method": "POST",
+     "params": ["msg: str"], "summary": "echo",
+     "timeout": 3.0, "auth_required": False},
+    {"subject": "t.raw", "path": "/calc/raw", "method": "POST",
+     "params": ["payload: any"], "summary": "原样透传",
+     "validate": False, "timeout": 3.0, "auth_required": False},
+    {"subject": "t.stream", "path": "/stream/chat", "method": "POST",
+     "params": ["prompt: str"], "summary": "sse",
+     "stream": True, "timeout": 60.0, "stream_idle_timeout": 1.0,
+     "auth_required": False},
+]
+
+
+def build_client(fwm):
+    app = FastAPI()
+    register_route_group(app, fwm, "demotype", ROUTES, prefix="/api/v1")
+    return TestClient(app)
+
+
+def main():
+    # ── 1. GET query 端点 ────────────────────────────────────
+    fwm = FakeWorkerManager()
+    c = build_client(fwm)
+    r = c.get("/api/v1/calc/add?a=1&b=2", headers={INSTANCE_HEADER: "edge-01"})
+    check("1.1 GET 携带头 -> 200", r.status_code == 200, f"status={r.status_code}")
+    check("1.2 GET 头透传到 call_raw", fwm.raw_instances == ["edge-01"],
+          f"instances={fwm.raw_instances}")
+    check("1.3 响应体回带实例（证明打到目标实例）",
+          r.json().get("instance") == "edge-01", f"body={r.json()}")
+
+    # ── 2. 未携带头：兼容旧调用方 ────────────────────────────
+    fwm = FakeWorkerManager()
+    c = build_client(fwm)
+    r = c.get("/api/v1/calc/add?a=1&b=2")
+    check("2.1 未携带头 -> 200（默认随机均衡路径不变）", r.status_code == 200)
+    check("2.2 未携带头时 instance_id=None", fwm.raw_instances == [None],
+          f"instances={fwm.raw_instances}")
+    check("2.3 响应体不含实例", r.json().get("instance") is None, f"body={r.json()}")
+
+    # ── 3. POST body 端点 ────────────────────────────────────
+    fwm = FakeWorkerManager()
+    c = build_client(fwm)
+    r = c.post("/api/v1/calc/echo", json={"msg": "hi"},
+               headers={INSTANCE_HEADER: "edge-02"})
+    check("3.1 POST body 头透传", fwm.raw_instances == ["edge-02"],
+          f"instances={fwm.raw_instances}")
+
+    # ── 4. validate=false 原样透传端点 ───────────────────────
+    fwm = FakeWorkerManager()
+    c = build_client(fwm)
+    r = c.post("/api/v1/calc/raw", content=b'{"anything": [1, 2]}',
+               headers={INSTANCE_HEADER: "edge-03",
+                        "content-type": "application/json"})
+    check("4.1 原样透传端点头透传", fwm.raw_instances == ["edge-03"],
+          f"instances={fwm.raw_instances}")
+
+    # ── 5. SSE 流式端点 ──────────────────────────────────────
+    fwm = FakeWorkerManager()
+    c = build_client(fwm)
+    with c.stream("POST", "/api/v1/stream/chat", json={"prompt": "x"},
+                  headers={INSTANCE_HEADER: "edge-04"}) as r:
+        body = b"".join(r.iter_bytes())
+    check("5.1 流式端点头透传", fwm.stream_instances == ["edge-04"],
+          f"instances={fwm.stream_instances}")
+    check("5.2 流式响应含目标实例", b"edge-04" in body, f"body={body[:80]!r}")
+
+    # ── 6. 空白头值视为未指定 ────────────────────────────────
+    fwm = FakeWorkerManager()
+    c = build_client(fwm)
+    r = c.get("/api/v1/calc/add?a=1&b=2", headers={INSTANCE_HEADER: "   "})
+    check("6.1 空白头值 -> instance_id=None", fwm.raw_instances == [None],
+          f"instances={fwm.raw_instances}")
+
+    # ── 7. 定向实例离线 -> 503 ───────────────────────────────
+    fwm = FakeWorkerManager(fail_instance="edge-gone")
+    c = build_client(fwm)
+    r = c.get("/api/v1/calc/add?a=1&b=2", headers={INSTANCE_HEADER: "edge-gone"})
+    check("7.1 定向实例离线 -> 503", r.status_code == 503, f"status={r.status_code}")
+    check("7.2 503 详情点名实例", "edge-gone" in str(r.json().get("detail", "")),
+          f"detail={r.json().get('detail')}")
+
+    print()
+    if failures:
+        print(f"FAILED: {len(failures)} 项 -> {failures}")
+        return 1
+    print("全部通过")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

test/test_mqtt_custom_topics.py

@@ -33,7 +33,8 @@ class FakeWM:
         self.calls = []
         self.result = {"result": "ok"}
 
-    async def call(self, worker_type, subject, payload, timeout=30, cacheable=False):
+    async def call(self, worker_type, subject, payload, timeout=30, cacheable=False,
+                   instance_id=None):
         self.calls.append(
             {
                 "worker_type": worker_type,
@@ -41,6 +42,7 @@ class FakeWM:
                 "payload": payload,
                 "timeout": timeout,
                 "cacheable": cacheable,
+                "instance_id": instance_id,
             }
         )
         return self.result
@@ -153,6 +155,41 @@ async def test_flat_payload_routed():
     print("✅ 非信封（扁平）载荷按原始报文路由通过")
 
 
+async def test_target_instance():
+    """实例定向：载荷顶层 target_instance 作为控制字段摘除并透传给 worker_manager.call"""
+    wm = FakeWM()
+    bridge, published = make_bridge(
+        wm, [{"topic": "devices/+/telemetry", "worker": "device"}]
+    )
+    # 1. 带 target_instance：透传实例名，且控制字段不进业务 payload
+    await bridge._handle(make_msg(
+        "devices/d1/telemetry",
+        {"meta": {}, "data": {"v": 1}, "target_instance": "edge-01"},
+    ))
+    assert wm.calls[-1]["instance_id"] == "edge-01", wm.calls[-1]
+    assert "target_instance" not in wm.calls[-1]["payload"], wm.calls[-1]["payload"]
+    assert wm.calls[-1]["payload"]["data"] == {"v": 1}
+
+    # 2. 不带该字段：instance_id 为 None（跨实例随机均衡，与改动前一致）
+    await bridge._handle(make_msg("devices/d1/telemetry", {"meta": {}, "data": {"v": 1}}))
+    assert wm.calls[-1]["instance_id"] is None, wm.calls[-1]
+
+    # 3. 空白值视为未指定
+    await bridge._handle(make_msg(
+        "devices/d1/telemetry",
+        {"meta": {}, "data": {"v": 1}, "target_instance": "   "},
+    ))
+    assert wm.calls[-1]["instance_id"] is None, wm.calls[-1]
+
+    # 4. 内置 chongming/... 主题同样支持
+    await bridge._handle(make_msg(
+        f"{MQTT_TOPIC_PREFIX}/example/add",
+        {"meta": {}, "data": {"a": 1}, "target_instance": "edge-02"},
+    ))
+    assert wm.calls[-1]["instance_id"] == "edge-02", wm.calls[-1]
+    print("✅ 实例定向（target_instance 控制字段）透传通过")
+
+
 async def test_non_json_dropped():
     wm = FakeWM()
     bridge, published = make_bridge(
@@ -219,6 +256,7 @@ async def main():
     await test_custom_routing()
     await test_envelope_meta_fields()
     await test_flat_payload_routed()
+    await test_target_instance()
     await test_non_json_dropped()
     await test_board_ping_topic()
     test_plugin_config_driven_collection()

test/test_passthrough_optimization.py

@@ -16,12 +16,13 @@
   D. WorkerManager 单测（FakeSession）:
      9. call_raw 零解析返回原始字节；call() 仍解析为 dict（MQTT 语义保留）
      10. cacheable: 缓存原始 bytes、二次命中不再发查询、业务错误不缓存
-     11. reply.err -> 重试 -> RuntimeError；空回复兜底 b"{}"
+     11. reply.err -> 换副本重试/无副本可换时立即 RuntimeError；空回复兜底 b"{}"
 
 运行: uv run python test_passthrough_optimization.py
 """
 import asyncio
 import importlib.util
+import random
 import logging
 import sys
 import types
@@ -32,7 +33,7 @@ import orjson
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
-REPO_ROOT = Path(__file__).resolve().parent
+REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "src"))
 sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
 
@@ -49,6 +50,7 @@ _auth_stub = types.ModuleType("chongming.app.core.auth")
 _auth_stub.store_user_to_request = lambda x: x  # 仅用于 Depends，桩掉鉴权
 _cache_stub = types.ModuleType("chongming_cache")
 _cache_stub.Cache = object
+_cache_stub.DistributedLock = object  # worker_manager 顶层导入（本测试不触达）
 for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub):
     sys.modules[mod.__name__] = mod
 
@@ -101,7 +103,8 @@ class FakeWorkerManager:
         self.error = error
         self.call_raw_payloads = []  # 捕获的预序列化请求字节
 
-    async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False):
+    async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False,
+                       instance_id=None):
         self.call_raw_payloads.append(payload)
         if self.error == "no_worker":
             raise RuntimeError(f"类型 '{worker_type}' 没有可用 Worker。")
@@ -282,16 +285,27 @@ def test_openapi_request_body_injected():
 # ---- D. WorkerManager 单测（FakeSession）----
 
 class FakeSession:
-    """模拟 zenoh Session：get() 返回单回复迭代器"""
+    """模拟 zenoh Session：get() 返回单回复迭代器
 
-    def __init__(self, payload=b'{"result": 5}', err_payload=None):
+    err_payload 非空时整体报错；err_target 给定时只对该 key 前缀报错（其余正常回复），
+    用于验证「失败换副本重试」。
+    """
+
+    def __init__(self, payload=b'{"result": 5}', err_payload=None, err_target=None):
         self.payload = payload
         self.err_payload = err_payload
+        self.err_target = err_target
         self.get_calls = 0
+        self.call_keys = []
 
     def get(self, *args, **kwargs):
         self.get_calls += 1
-        if self.err_payload is not None:
+        key = args[0] if args else kwargs.get("key_expr", "")
+        self.call_keys.append(str(key))
+        failing = self.err_payload is not None and (
+            self.err_target is None or str(key).startswith(self.err_target)
+        )
+        if failing:
             reply = SimpleNamespace(ok=None, err=SimpleNamespace(payload=self.err_payload))
         else:
             reply = SimpleNamespace(ok=SimpleNamespace(payload=self.payload), err=None)
@@ -341,14 +355,49 @@ async def test_wm_cache_bytes():
 
 
 async def test_wm_reply_err_retry():
+    # 单副本：无其它副本可换 —— 立即失败（不空转重试同一个坏副本），保留具体错误
     session = FakeSession(err_payload=b'{"error": "boom"}')
     wm = make_wm(session)
     try:
         await wm.call_raw("demo", "calc.add", b'{}')
         check("reply.err 应抛 RuntimeError", False)
     except RuntimeError as e:
-        check("reply.err 重试后抛 RuntimeError", "boom" in str(e), str(e))
-    check("reply.err 触发重试（2 次查询）", session.get_calls == 2, str(session.get_calls))
+        check("reply.err 抛 RuntimeError 且保留具体错误", "boom" in str(e), str(e))
+    check("单副本无其它可换时不空转重试（1 次查询）",
+          session.get_calls == 1, str(session.get_calls))
+
+
+async def test_wm_reply_err_switches_worker():
+    """多副本：先打的副本报错则换另一个副本重试，且不重复打已失败的副本。
+
+    首个副本由 random.choice 决定，故对多个 seed 试跑：必有一个覆盖「先失败后成功」
+    路径（另一个覆盖「首个即成功」），两条路径都不允许重复打同一个失败的副本。
+    """
+    ok, detail = False, "无 seed 覆盖到失败换副本路径"
+    for seed in (0, 1):
+        random.seed(seed)
+        session = FakeSession(
+            payload=b'{"result": 5}', err_payload=b'{"error": "boom"}',
+            err_target="worker/demo/w-dead",
+        )
+        wm = make_wm(session)
+        wm._workers_cache = {"demo": ["worker/demo/w-dead", "worker/demo/w-alive"]}
+        try:
+            result = await wm.call_raw("demo", "calc.add", b'{}', retries=2)
+        except RuntimeError as e:  # 未换到正常副本（不该发生）
+            detail = f"seed={seed} 意外失败: {e}"
+            continue
+        assert result == b'{"result": 5}', result
+        dead_calls = session.call_keys.count("worker/demo/w-dead/calc.add")
+        if dead_calls > 1:
+            detail = f"seed={seed} 重复打失败副本: {session.call_keys}"
+            continue
+        if session.call_keys == [
+            "worker/demo/w-dead/calc.add",
+            "worker/demo/w-alive/calc.add",
+        ]:
+            ok, detail = True, f"seed={seed} keys={session.call_keys}"
+    check("失败副本 -> 换副本重试成功（且不重复打失败副本）", ok, detail)
 
 
 async def test_wm_empty_reply_fallback():
@@ -375,6 +424,7 @@ def run():
     asyncio.run(test_wm_call_dict_semantics())
     asyncio.run(test_wm_cache_bytes())
     asyncio.run(test_wm_reply_err_retry())
+    asyncio.run(test_wm_reply_err_switches_worker())
     asyncio.run(test_wm_empty_reply_fallback())
 
     print(f"\n结果: {len(failures)} 个失败")

test/test_stream_e2e_zenoh.py

@@ -24,7 +24,7 @@ from pathlib import Path
 import orjson
 import zenoh
 
-REPO_ROOT = Path(__file__).resolve().parent
+REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
 sys.path.insert(0, str(REPO_ROOT / "src"))
 
@@ -39,6 +39,7 @@ _logger_stub = types.ModuleType("chongming.app.core.logger")
 _logger_stub.get_logger = lambda name: logging.getLogger(name)
 _cache_stub = types.ModuleType("chongming_cache")
 _cache_stub.Cache = object
+_cache_stub.DistributedLock = object  # worker_manager 顶层导入（本测试不触达）
 for mod in (_pkg, _app, _core, _logger_stub, _cache_stub):
     sys.modules[mod.__name__] = mod
 
@@ -67,7 +68,8 @@ def check(name, cond, detail=""):
         failures.append(name)
 
 
-LISTEN_PORT = 7447
+import os
+LISTEN_PORT = int(os.environ.get("CM_STREAM_TEST_PORT", "7447"))
 LISTEN = f"tcp/127.0.0.1:{LISTEN_PORT}"
 
 

test/test_stream_route.py

@@ -21,7 +21,7 @@ import orjson
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
-REPO_ROOT = Path(__file__).resolve().parent
+REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "src"))
 sys.path.insert(0, str(REPO_ROOT / "utils" / "worker" / "src"))
 
@@ -38,6 +38,7 @@ _auth_stub = types.ModuleType("chongming.app.core.auth")
 _auth_stub.store_user_to_request = lambda x: x  # 仅用于 Depends，桩掉鉴权
 _cache_stub = types.ModuleType("chongming_cache")
 _cache_stub.Cache = object
+_cache_stub.DistributedLock = object  # worker_manager 顶层导入（本测试不触达）
 for mod in (_pkg, _app, _core, _logger_stub, _auth_stub, _cache_stub):
     sys.modules[mod.__name__] = mod
 
@@ -90,18 +91,21 @@ class FakeWorkerManager:
         self.call_payloads = []
         self.stream_payloads = []
 
-    async def call(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False):
+    async def call(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False,
+                   instance_id=None):
         self.call_payloads.append((worker_type, subject, payload))
         data = payload["data"]
         return {"result": data.get("a", 0) + data.get("b", 0)}
 
-    async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False):
+    async def call_raw(self, worker_type, subject, payload, timeout=3.0, retries=1, cacheable=False,
+                       instance_id=None):
         # 网关透传路径：payload 为预序列化字节，回复原始字节
         self.call_payloads.append((worker_type, subject, orjson.loads(payload)))
         data = orjson.loads(payload)["data"]
         return orjson.dumps({"result": data.get("a", 0) + data.get("b", 0)})
 
-    async def call_stream(self, worker_type, subject, payload, timeout=300.0, idle_timeout=10.0):
+    async def call_stream(self, worker_type, subject, payload, timeout=300.0, idle_timeout=10.0,
+                          instance_id=None):
         if self.stream_error == "no_worker":
             raise RuntimeError(f"类型 '{worker_type}' 没有可用 Worker。")
         self.stream_payloads.append((worker_type, subject, payload))