modify: 分布式方案

test/test_mqtt_bridge_heal.py

@@ -0,0 +1,146 @@
+"""
+MQTT Bridge 持有者崩溃自愈巡检测试
+
+验证 mqtt_bridge_supervisor 的接管语义（基于共享 cache 的分布式锁 "mqtt_bridge"）：
+  1. mqtt 启用、本进程非持有者、锁空闲（原持有者租约过期=已死）→ 抢锁接管，
+     重新拉起 Broker（setup_mqtt_broker）并重建 Bridge（_start_mqtt）
+  2. 已持有一存活 Bridge（本进程即持有者）→ 正常托管，不重复拉起
+  3. 本进程已持锁但 Bridge 停止运行 → 不重新抢锁，直接重建 Bridge（自愈自身池）
+  4. mqtt 禁用 → 巡检直接退出，不做任何动作
+
+注：通过 monkeypatch setup_mqtt_broker / _start_mqtt 隔离真实 Broker/Bridge；
+分布式锁本体用真实 diskcache，验证 set_nx 唯一性与租约语义。
+
+运行: uv run python test/test_mqtt_bridge_heal.py
+"""
+import asyncio
+import sys
+import tempfile
+from pathlib import Path
+from types import SimpleNamespace
+
+_HERE = Path(__file__).resolve().parent          # chongming-zenoh/
+_PROJ = _HERE
+for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"):
+    sys.path.insert(0, str(_p))
+
+import chongming.app.core.lifespan_setup as ls
+from chongming_cache import Cache
+from chongming.app.core.worker_manager import WorkerManager
+
+failures = []
+
+
+def check(name, cond, detail=""):
+    status = "PASS" if cond else "FAIL"
+    print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
+    if not cond:
+        failures.append(name)
+
+
+class DummySession:
+    pass
+
+
+def make_app():
+    return SimpleNamespace(state=SimpleNamespace(
+        mqtt_bridge=None, mqtt_bridge_lock=None, plugin_mqtt_topics=[],
+        route_registry=None, mqtt_broker_lock_acquired=False,
+        mqtt_broker_process=None,
+    ))
+
+
+async def drive_supervisor(app, manager, steps, enabled=True, interval=0.02):
+    """运行 supervisor，让其控制权让出 steps 次（每让出可经过 interval 睡眠）后取消。"""
+    ls.env = "testenv"
+    ls.config = {"testenv": {"mqtt": {
+        "enabled": enabled, "bridge_supervise_interval": interval,
+    }}}
+    counts = {"broker": 0, "start": 0}
+
+    async def fake_start(aw, wm, lock):
+        counts["start"] += 1
+        aw.state.mqtt_bridge = "RUNNING"       # 模拟 Bridge 已运行
+        aw.state.mqtt_bridge_lock = lock
+        return "RUNNING"
+
+    async def _broker_stub(aw, wm):
+        counts["broker"] += 1
+
+    ls.setup_mqtt_broker = _broker_stub          # async：supervisor 会 await 拉起 Broker
+    ls._start_mqtt = fake_start
+
+    task = asyncio.create_task(ls.mqtt_bridge_supervisor(app, manager))
+    try:
+        for _ in range(steps):
+            await asyncio.sleep(interval * 2)
+    finally:
+        task.cancel()
+        try:
+            await task
+        except (asyncio.CancelledError, Exception):
+            pass
+    return counts
+
+
+async def main():
+    c = Cache(backend="diskcache", cache_store_path=tempfile.mkdtemp())
+
+    # ── 1. mqtt 禁用：巡检直接退出，无任何动作 ──────────────
+    mgr_dis = WorkerManager(session=DummySession(), cache=c)
+    app_dis = make_app()
+    cnt = await drive_supervisor(app_dis, mgr_dis, 3, enabled=False)
+    check("1.1 mqtt 禁用：不拉起 Broker/Bridge", cnt["broker"] == 0 and cnt["start"] == 0)
+
+    # ── 2. 非持有者 + 锁空闲 → 接管 ─────────────────────────
+    mgr = WorkerManager(session=DummySession(), cache=c)
+    app = make_app()
+    cnt = await drive_supervisor(app, mgr, 8)
+    check("2.1 锁空闲时接管：拉起 Broker", cnt["broker"] >= 1)
+    check("2.2 锁空闲时接管：重建 Bridge", cnt["start"] >= 1)
+    check("2.3 接管后本进程变为持有者（持锁）", app.state.mqtt_bridge_lock is not None)
+    # 释放 scenario 2 抢得的那把分布式锁（含其续约任务），避免污染后续场景
+    _lk = getattr(app.state, "mqtt_bridge_lock", None)
+    if _lk is not None:
+        await _lk.stop()
+        _lk.release()
+
+    # ── 3. 本进程已是存活持有者：不再重复拉起 ────────────────
+    app2 = make_app()
+    app2.state.mqtt_bridge = "RUNNING"         # 模拟已持有存活 Bridge
+    app2.state.mqtt_bridge_lock = object()      # 持锁标记
+    cnt2 = await drive_supervisor(app2, mgr, 8)
+    check("3.1 存活持有者不重复接管", cnt2["broker"] == 0 and cnt2["start"] == 0)
+
+    # ── 4. 本进程已持锁但 Bridge 停止 → 不带锁重建（池自愈）─
+    app3 = make_app()
+    app3.state.mqtt_bridge = SimpleNamespace(is_running=False)
+    app3.state.mqtt_bridge_lock = object()
+    cnt3 = await drive_supervisor(app3, mgr, 8)
+    check("4.1 持锁但 Bridge 停：重建 Bridge", cnt3["start"] >= 1,
+          detail=f"{cnt3}")
+    check("4.2 持锁但 Bridge 停：拉起 Broker", cnt3["broker"] >= 1,
+          detail=f"{cnt3}")
+
+    # ── 5. 持有者在场时（锁被把持），非持有者不接管 ──────────
+    app5 = make_app()
+    from chongming_cache import DistributedLock as _DL
+    held = _DL(c, "mqtt_bridge")
+    held.acquire()          # 模拟一个存活的持有者把持着桥锁
+    cnt5 = await drive_supervisor(app5, mgr, 5)
+    check("5.1 持有者存活：非持有者不拉起", cnt5["broker"] == 0 and cnt5["start"] == 0,
+          detail=f"{cnt5}")
+    await held.stop()
+    held.release()
+
+    c.close()
+    print()
+    if failures:
+        print(f"FAILED {len(failures)}: {failures}")
+        return 1
+    print("ALL PASS")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(main()))