""" MQTT Bridge 持有者崩溃自愈巡检测试 验证 mqtt_bridge_supervisor 的接管语义(基于共享 cache 的分布式锁 "mqtt_bridge"): 1. mqtt 启用、本进程非持有者、锁空闲(原持有者租约过期=已死)→ 抢锁接管, 重新拉起 Broker(setup_mqtt_broker)并重建 Bridge(_start_mqtt) 2. 已持有一存活 Bridge(本进程即持有者)→ 正常托管,不重复拉起 3. 本进程已持锁但 Bridge 停止运行 → 不重新抢锁,直接重建 Bridge(自愈自身池) 4. mqtt 禁用 → 巡检直接退出,不做任何动作 注:通过 monkeypatch setup_mqtt_broker / _start_mqtt 隔离真实 Broker/Bridge; 分布式锁本体用真实 diskcache,验证 set_nx 唯一性与租约语义。 运行: uv run python test/test_mqtt_bridge_heal.py """ import asyncio import sys import tempfile from pathlib import Path from types import SimpleNamespace _HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/ _PROJ = _HERE for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"): sys.path.insert(0, str(_p)) import chongming.app.core.lifespan_setup as ls from chongming_cache import Cache from chongming.app.core.worker_manager import WorkerManager failures = [] def check(name, cond, detail=""): status = "PASS" if cond else "FAIL" print(f"[{status}] {name}" + (f" — {detail}" if detail else "")) if not cond: failures.append(name) class DummySession: pass def make_app(): return SimpleNamespace(state=SimpleNamespace( mqtt_bridge=None, mqtt_bridge_lock=None, plugin_mqtt_topics=[], route_registry=None, mqtt_broker_lock_acquired=False, mqtt_broker_process=None, )) async def drive_supervisor(app, manager, steps, enabled=True, interval=0.02): """运行 supervisor,让其控制权让出 steps 次(每让出可经过 interval 睡眠)后取消。""" ls.env = "testenv" ls.config = {"testenv": {"mqtt": { "enabled": enabled, "bridge_supervise_interval": interval, }}} counts = {"broker": 0, "start": 0} async def fake_start(aw, wm, lock): counts["start"] += 1 aw.state.mqtt_bridge = "RUNNING" # 模拟 Bridge 已运行 aw.state.mqtt_bridge_lock = lock return "RUNNING" async def _broker_stub(aw, wm): counts["broker"] += 1 ls.setup_mqtt_broker = _broker_stub # async:supervisor 会 await 拉起 Broker ls._start_mqtt = fake_start task = asyncio.create_task(ls.mqtt_bridge_supervisor(app, manager)) try: for _ in range(steps): await asyncio.sleep(interval * 2) finally: task.cancel() try: await task except (asyncio.CancelledError, Exception): pass return counts async def main(): c = Cache(backend="diskcache", cache_store_path=tempfile.mkdtemp()) # ── 1. mqtt 禁用:巡检直接退出,无任何动作 ────────────── mgr_dis = WorkerManager(session=DummySession(), cache=c) app_dis = make_app() cnt = await drive_supervisor(app_dis, mgr_dis, 3, enabled=False) check("1.1 mqtt 禁用:不拉起 Broker/Bridge", cnt["broker"] == 0 and cnt["start"] == 0) # ── 2. 非持有者 + 锁空闲 → 接管 ───────────────────────── mgr = WorkerManager(session=DummySession(), cache=c) app = make_app() cnt = await drive_supervisor(app, mgr, 8) check("2.1 锁空闲时接管:拉起 Broker", cnt["broker"] >= 1) check("2.2 锁空闲时接管:重建 Bridge", cnt["start"] >= 1) check("2.3 接管后本进程变为持有者(持锁)", app.state.mqtt_bridge_lock is not None) # 释放 scenario 2 抢得的那把分布式锁(含其续约任务),避免污染后续场景 _lk = getattr(app.state, "mqtt_bridge_lock", None) if _lk is not None: await _lk.stop() _lk.release() # ── 3. 本进程已是存活持有者:不再重复拉起 ──────────────── app2 = make_app() app2.state.mqtt_bridge = "RUNNING" # 模拟已持有存活 Bridge app2.state.mqtt_bridge_lock = object() # 持锁标记 cnt2 = await drive_supervisor(app2, mgr, 8) check("3.1 存活持有者不重复接管", cnt2["broker"] == 0 and cnt2["start"] == 0) # ── 4. 本进程已持锁但 Bridge 停止 → 不带锁重建(池自愈)─ app3 = make_app() app3.state.mqtt_bridge = SimpleNamespace(is_running=False) app3.state.mqtt_bridge_lock = object() cnt3 = await drive_supervisor(app3, mgr, 8) check("4.1 持锁但 Bridge 停:重建 Bridge", cnt3["start"] >= 1, detail=f"{cnt3}") check("4.2 持锁但 Bridge 停:拉起 Broker", cnt3["broker"] >= 1, detail=f"{cnt3}") # ── 5. 持有者在场时(锁被把持),非持有者不接管 ────────── app5 = make_app() from chongming_cache import DistributedLock as _DL held = _DL(c, "mqtt_bridge") held.acquire() # 模拟一个存活的持有者把持着桥锁 cnt5 = await drive_supervisor(app5, mgr, 5) check("5.1 持有者存活:非持有者不拉起", cnt5["broker"] == 0 and cnt5["start"] == 0, detail=f"{cnt5}") await held.stop() held.release() c.close() print() if failures: print(f"FAILED {len(failures)}: {failures}") return 1 print("ALL PASS") return 0 if __name__ == "__main__": sys.exit(asyncio.run(main()))