chongming-core/test/test_mqtt_bridge_heal.py

"""
MQTT Bridge 持有者崩溃自愈巡检测试

验证 mqtt_bridge_supervisor 的接管语义（基于共享 cache 的分布式锁 "mqtt_bridge"）：
  1. mqtt 启用、本进程非持有者、锁空闲（原持有者租约过期=已死）→ 抢锁接管，
     重新拉起 Broker（setup_mqtt_broker）并重建 Bridge（_start_mqtt）
  2. 已持有一存活 Bridge（本进程即持有者）→ 正常托管，不重复拉起
  3. 本进程已持锁但 Bridge 停止运行 → 不重新抢锁，直接重建 Bridge（自愈自身池）
  4. mqtt 禁用 → 巡检直接退出，不做任何动作

注：通过 monkeypatch setup_mqtt_broker / _start_mqtt 隔离真实 Broker/Bridge；
分布式锁本体用真实 diskcache，验证 set_nx 唯一性与租约语义。

运行: uv run python test/test_mqtt_bridge_heal.py
"""
import asyncio
import sys
import tempfile
from pathlib import Path
from types import SimpleNamespace

_HERE = Path(__file__).resolve().parent.parent   # chongming-zenoh/
_PROJ = _HERE
for _p in (_PROJ / "gateway" / "src", _PROJ / "utils" / "worker" / "src"):
    sys.path.insert(0, str(_p))

import chongming.app.core.lifespan_setup as ls
from chongming_cache import Cache
from chongming.app.core.worker_manager import WorkerManager

failures = []


def check(name, cond, detail=""):
    status = "PASS" if cond else "FAIL"
    print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
    if not cond:
        failures.append(name)


class DummySession:
    pass


def make_app():
    return SimpleNamespace(state=SimpleNamespace(
        mqtt_bridge=None, mqtt_bridge_lock=None, plugin_mqtt_topics=[],
        route_registry=None, mqtt_broker_lock_acquired=False,
        mqtt_broker_process=None,
    ))


async def drive_supervisor(app, manager, steps, enabled=True, interval=0.02):
    """运行 supervisor，让其控制权让出 steps 次（每让出可经过 interval 睡眠）后取消。"""
    ls.env = "testenv"
    ls.config = {"testenv": {"mqtt": {
        "enabled": enabled, "bridge_supervise_interval": interval,
    }}}
    counts = {"broker": 0, "start": 0}

    async def fake_start(aw, wm, lock):
        counts["start"] += 1
        aw.state.mqtt_bridge = "RUNNING"       # 模拟 Bridge 已运行
        aw.state.mqtt_bridge_lock = lock
        return "RUNNING"

    async def _broker_stub(aw, wm):
        counts["broker"] += 1

    ls.setup_mqtt_broker = _broker_stub          # async：supervisor 会 await 拉起 Broker
    ls._start_mqtt = fake_start

    task = asyncio.create_task(ls.mqtt_bridge_supervisor(app, manager))
    try:
        for _ in range(steps):
            await asyncio.sleep(interval * 2)
    finally:
        task.cancel()
        try:
            await task
        except (asyncio.CancelledError, Exception):
            pass
    return counts


async def main():
    c = Cache(backend="diskcache", cache_store_path=tempfile.mkdtemp())

    # ── 1. mqtt 禁用：巡检直接退出，无任何动作 ──────────────
    mgr_dis = WorkerManager(session=DummySession(), cache=c)
    app_dis = make_app()
    cnt = await drive_supervisor(app_dis, mgr_dis, 3, enabled=False)
    check("1.1 mqtt 禁用：不拉起 Broker/Bridge", cnt["broker"] == 0 and cnt["start"] == 0)

    # ── 2. 非持有者 + 锁空闲 → 接管 ─────────────────────────
    mgr = WorkerManager(session=DummySession(), cache=c)
    app = make_app()
    cnt = await drive_supervisor(app, mgr, 8)
    check("2.1 锁空闲时接管：拉起 Broker", cnt["broker"] >= 1)
    check("2.2 锁空闲时接管：重建 Bridge", cnt["start"] >= 1)
    check("2.3 接管后本进程变为持有者（持锁）", app.state.mqtt_bridge_lock is not None)
    # 释放 scenario 2 抢得的那把分布式锁（含其续约任务），避免污染后续场景
    _lk = getattr(app.state, "mqtt_bridge_lock", None)
    if _lk is not None:
        await _lk.stop()
        _lk.release()

    # ── 3. 本进程已是存活持有者：不再重复拉起 ────────────────
    app2 = make_app()
    app2.state.mqtt_bridge = "RUNNING"         # 模拟已持有存活 Bridge
    app2.state.mqtt_bridge_lock = object()      # 持锁标记
    cnt2 = await drive_supervisor(app2, mgr, 8)
    check("3.1 存活持有者不重复接管", cnt2["broker"] == 0 and cnt2["start"] == 0)

    # ── 4. 本进程已持锁但 Bridge 停止 → 不带锁重建（池自愈）─
    app3 = make_app()
    app3.state.mqtt_bridge = SimpleNamespace(is_running=False)
    app3.state.mqtt_bridge_lock = object()
    cnt3 = await drive_supervisor(app3, mgr, 8)
    check("4.1 持锁但 Bridge 停：重建 Bridge", cnt3["start"] >= 1,
          detail=f"{cnt3}")
    check("4.2 持锁但 Bridge 停：拉起 Broker", cnt3["broker"] >= 1,
          detail=f"{cnt3}")

    # ── 5. 持有者在场时（锁被把持），非持有者不接管 ──────────
    app5 = make_app()
    from chongming_cache import DistributedLock as _DL
    held = _DL(c, "mqtt_bridge")
    held.acquire()          # 模拟一个存活的持有者把持着桥锁
    cnt5 = await drive_supervisor(app5, mgr, 5)
    check("5.1 持有者存活：非持有者不拉起", cnt5["broker"] == 0 and cnt5["start"] == 0,
          detail=f"{cnt5}")
    await held.stop()
    held.release()

    c.close()
    print()
    if failures:
        print(f"FAILED {len(failures)}: {failures}")
        return 1
    print("ALL PASS")
    return 0


if __name__ == "__main__":
    sys.exit(asyncio.run(main()))