146 lines
5.5 KiB
Python
146 lines
5.5 KiB
Python
"""
|
||
MQTT Bridge 持有者崩溃自愈巡检测试
|
||
|
||
验证 mqtt_bridge_supervisor 的接管语义（基于共享 cache 的分布式锁 "mqtt_bridge"）：
|
||
1. mqtt 启用、本进程非持有者、锁空闲（原持有者租约过期=已死）→ 抢锁接管，
|
||
重新拉起 Broker（setup_mqtt_broker）并重建 Bridge（_start_mqtt）
|
||
2. 已持有一存活 Bridge（本进程即持有者）→ 正常托管，不重复拉起
|
||
3. 本进程已持锁但 Bridge 停止运行 → 不重新抢锁，直接重建 Bridge（自愈自身池）
|
||
4. mqtt 禁用 → 巡检直接退出，不做任何动作
|
||
|
||
注：通过 monkeypatch setup_mqtt_broker / _start_mqtt 隔离真实 Broker/Bridge；
|
||
分布式锁本体用真实 diskcache，验证 set_nx 唯一性与租约语义。
|
||
|
||
运行: uv run python test/test_mqtt_bridge_heal.py
|
||
"""
|
||
import asyncio
|
||
import sys
|
||
import tempfile
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
_HERE = Path(__file__).resolve().parent.parent # chongming-zenoh/
|
||
_PROJ = _HERE
|
||
for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"):
|
||
sys.path.insert(0, str(_p))
|
||
|
||
import chongming.app.core.lifespan_setup as ls
|
||
from chongming_cache import Cache
|
||
from chongming.app.core.worker_manager import WorkerManager
|
||
|
||
failures = []
|
||
|
||
|
||
def check(name, cond, detail=""):
|
||
status = "PASS" if cond else "FAIL"
|
||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||
if not cond:
|
||
failures.append(name)
|
||
|
||
|
||
class DummySession:
|
||
pass
|
||
|
||
|
||
def make_app():
|
||
return SimpleNamespace(state=SimpleNamespace(
|
||
mqtt_bridge=None, mqtt_bridge_lock=None, plugin_mqtt_topics=[],
|
||
route_registry=None, mqtt_broker_lock_acquired=False,
|
||
mqtt_broker_process=None,
|
||
))
|
||
|
||
|
||
async def drive_supervisor(app, manager, steps, enabled=True, interval=0.02):
|
||
"""运行 supervisor，让其控制权让出 steps 次（每让出可经过 interval 睡眠）后取消。"""
|
||
ls.env = "testenv"
|
||
ls.config = {"testenv": {"mqtt": {
|
||
"enabled": enabled, "bridge_supervise_interval": interval,
|
||
}}}
|
||
counts = {"broker": 0, "start": 0}
|
||
|
||
async def fake_start(aw, wm, lock):
|
||
counts["start"] += 1
|
||
aw.state.mqtt_bridge = "RUNNING" # 模拟 Bridge 已运行
|
||
aw.state.mqtt_bridge_lock = lock
|
||
return "RUNNING"
|
||
|
||
async def _broker_stub(aw, wm):
|
||
counts["broker"] += 1
|
||
|
||
ls.setup_mqtt_broker = _broker_stub # async：supervisor 会 await 拉起 Broker
|
||
ls._start_mqtt = fake_start
|
||
|
||
task = asyncio.create_task(ls.mqtt_bridge_supervisor(app, manager))
|
||
try:
|
||
for _ in range(steps):
|
||
await asyncio.sleep(interval * 2)
|
||
finally:
|
||
task.cancel()
|
||
try:
|
||
await task
|
||
except (asyncio.CancelledError, Exception):
|
||
pass
|
||
return counts
|
||
|
||
|
||
async def main():
|
||
c = Cache(backend="diskcache", cache_store_path=tempfile.mkdtemp())
|
||
|
||
# ── 1. mqtt 禁用：巡检直接退出，无任何动作 ──────────────
|
||
mgr_dis = WorkerManager(session=DummySession(), cache=c)
|
||
app_dis = make_app()
|
||
cnt = await drive_supervisor(app_dis, mgr_dis, 3, enabled=False)
|
||
check("1.1 mqtt 禁用：不拉起 Broker/Bridge", cnt["broker"] == 0 and cnt["start"] == 0)
|
||
|
||
# ── 2. 非持有者 + 锁空闲 → 接管 ─────────────────────────
|
||
mgr = WorkerManager(session=DummySession(), cache=c)
|
||
app = make_app()
|
||
cnt = await drive_supervisor(app, mgr, 8)
|
||
check("2.1 锁空闲时接管：拉起 Broker", cnt["broker"] >= 1)
|
||
check("2.2 锁空闲时接管：重建 Bridge", cnt["start"] >= 1)
|
||
check("2.3 接管后本进程变为持有者（持锁）", app.state.mqtt_bridge_lock is not None)
|
||
# 释放 scenario 2 抢得的那把分布式锁（含其续约任务），避免污染后续场景
|
||
_lk = getattr(app.state, "mqtt_bridge_lock", None)
|
||
if _lk is not None:
|
||
await _lk.stop()
|
||
_lk.release()
|
||
|
||
# ── 3. 本进程已是存活持有者：不再重复拉起 ────────────────
|
||
app2 = make_app()
|
||
app2.state.mqtt_bridge = "RUNNING" # 模拟已持有存活 Bridge
|
||
app2.state.mqtt_bridge_lock = object() # 持锁标记
|
||
cnt2 = await drive_supervisor(app2, mgr, 8)
|
||
check("3.1 存活持有者不重复接管", cnt2["broker"] == 0 and cnt2["start"] == 0)
|
||
|
||
# ── 4. 本进程已持锁但 Bridge 停止 → 不带锁重建（池自愈）─
|
||
app3 = make_app()
|
||
app3.state.mqtt_bridge = SimpleNamespace(is_running=False)
|
||
app3.state.mqtt_bridge_lock = object()
|
||
cnt3 = await drive_supervisor(app3, mgr, 8)
|
||
check("4.1 持锁但 Bridge 停：重建 Bridge", cnt3["start"] >= 1,
|
||
detail=f"{cnt3}")
|
||
check("4.2 持锁但 Bridge 停：拉起 Broker", cnt3["broker"] >= 1,
|
||
detail=f"{cnt3}")
|
||
|
||
# ── 5. 持有者在场时（锁被把持），非持有者不接管 ──────────
|
||
app5 = make_app()
|
||
from chongming_cache import DistributedLock as _DL
|
||
held = _DL(c, "mqtt_bridge")
|
||
held.acquire() # 模拟一个存活的持有者把持着桥锁
|
||
cnt5 = await drive_supervisor(app5, mgr, 5)
|
||
check("5.1 持有者存活：非持有者不拉起", cnt5["broker"] == 0 and cnt5["start"] == 0,
|
||
detail=f"{cnt5}")
|
||
await held.stop()
|
||
held.release()
|
||
|
||
c.close()
|
||
print()
|
||
if failures:
|
||
print(f"FAILED {len(failures)}: {failures}")
|
||
return 1
|
||
print("ALL PASS")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(asyncio.run(main())) |