modify: 分布式方案
This commit is contained in:
@@ -0,0 +1,146 @@
|
||||
"""
|
||||
MQTT Bridge 持有者崩溃自愈巡检测试
|
||||
|
||||
验证 mqtt_bridge_supervisor 的接管语义（基于共享 cache 的分布式锁 "mqtt_bridge"）：
|
||||
1. mqtt 启用、本进程非持有者、锁空闲（原持有者租约过期=已死）→ 抢锁接管，
|
||||
重新拉起 Broker（setup_mqtt_broker）并重建 Bridge（_start_mqtt）
|
||||
2. 已持有一存活 Bridge（本进程即持有者）→ 正常托管，不重复拉起
|
||||
3. 本进程已持锁但 Bridge 停止运行 → 不重新抢锁，直接重建 Bridge（自愈自身池）
|
||||
4. mqtt 禁用 → 巡检直接退出，不做任何动作
|
||||
|
||||
注：通过 monkeypatch setup_mqtt_broker / _start_mqtt 隔离真实 Broker/Bridge；
|
||||
分布式锁本体用真实 diskcache，验证 set_nx 唯一性与租约语义。
|
||||
|
||||
运行: uv run python test/test_mqtt_bridge_heal.py
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
_HERE = Path(__file__).resolve().parent # chongming-zenoh/
|
||||
_PROJ = _HERE
|
||||
for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"):
|
||||
sys.path.insert(0, str(_p))
|
||||
|
||||
import chongming.app.core.lifespan_setup as ls
|
||||
from chongming_cache import Cache
|
||||
from chongming.app.core.worker_manager import WorkerManager
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
class DummySession:
|
||||
pass
|
||||
|
||||
|
||||
def make_app():
|
||||
return SimpleNamespace(state=SimpleNamespace(
|
||||
mqtt_bridge=None, mqtt_bridge_lock=None, plugin_mqtt_topics=[],
|
||||
route_registry=None, mqtt_broker_lock_acquired=False,
|
||||
mqtt_broker_process=None,
|
||||
))
|
||||
|
||||
|
||||
async def drive_supervisor(app, manager, steps, enabled=True, interval=0.02):
|
||||
"""运行 supervisor，让其控制权让出 steps 次（每让出可经过 interval 睡眠）后取消。"""
|
||||
ls.env = "testenv"
|
||||
ls.config = {"testenv": {"mqtt": {
|
||||
"enabled": enabled, "bridge_supervise_interval": interval,
|
||||
}}}
|
||||
counts = {"broker": 0, "start": 0}
|
||||
|
||||
async def fake_start(aw, wm, lock):
|
||||
counts["start"] += 1
|
||||
aw.state.mqtt_bridge = "RUNNING" # 模拟 Bridge 已运行
|
||||
aw.state.mqtt_bridge_lock = lock
|
||||
return "RUNNING"
|
||||
|
||||
async def _broker_stub(aw, wm):
|
||||
counts["broker"] += 1
|
||||
|
||||
ls.setup_mqtt_broker = _broker_stub # async：supervisor 会 await 拉起 Broker
|
||||
ls._start_mqtt = fake_start
|
||||
|
||||
task = asyncio.create_task(ls.mqtt_bridge_supervisor(app, manager))
|
||||
try:
|
||||
for _ in range(steps):
|
||||
await asyncio.sleep(interval * 2)
|
||||
finally:
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
return counts
|
||||
|
||||
|
||||
async def main():
|
||||
c = Cache(backend="diskcache", cache_store_path=tempfile.mkdtemp())
|
||||
|
||||
# ── 1. mqtt 禁用：巡检直接退出，无任何动作 ──────────────
|
||||
mgr_dis = WorkerManager(session=DummySession(), cache=c)
|
||||
app_dis = make_app()
|
||||
cnt = await drive_supervisor(app_dis, mgr_dis, 3, enabled=False)
|
||||
check("1.1 mqtt 禁用：不拉起 Broker/Bridge", cnt["broker"] == 0 and cnt["start"] == 0)
|
||||
|
||||
# ── 2. 非持有者 + 锁空闲 → 接管 ─────────────────────────
|
||||
mgr = WorkerManager(session=DummySession(), cache=c)
|
||||
app = make_app()
|
||||
cnt = await drive_supervisor(app, mgr, 8)
|
||||
check("2.1 锁空闲时接管：拉起 Broker", cnt["broker"] >= 1)
|
||||
check("2.2 锁空闲时接管：重建 Bridge", cnt["start"] >= 1)
|
||||
check("2.3 接管后本进程变为持有者（持锁）", app.state.mqtt_bridge_lock is not None)
|
||||
# 释放 scenario 2 抢得的那把分布式锁（含其续约任务），避免污染后续场景
|
||||
_lk = getattr(app.state, "mqtt_bridge_lock", None)
|
||||
if _lk is not None:
|
||||
await _lk.stop()
|
||||
_lk.release()
|
||||
|
||||
# ── 3. 本进程已是存活持有者：不再重复拉起 ────────────────
|
||||
app2 = make_app()
|
||||
app2.state.mqtt_bridge = "RUNNING" # 模拟已持有存活 Bridge
|
||||
app2.state.mqtt_bridge_lock = object() # 持锁标记
|
||||
cnt2 = await drive_supervisor(app2, mgr, 8)
|
||||
check("3.1 存活持有者不重复接管", cnt2["broker"] == 0 and cnt2["start"] == 0)
|
||||
|
||||
# ── 4. 本进程已持锁但 Bridge 停止 → 不带锁重建（池自愈）─
|
||||
app3 = make_app()
|
||||
app3.state.mqtt_bridge = SimpleNamespace(is_running=False)
|
||||
app3.state.mqtt_bridge_lock = object()
|
||||
cnt3 = await drive_supervisor(app3, mgr, 8)
|
||||
check("4.1 持锁但 Bridge 停：重建 Bridge", cnt3["start"] >= 1,
|
||||
detail=f"{cnt3}")
|
||||
check("4.2 持锁但 Bridge 停：拉起 Broker", cnt3["broker"] >= 1,
|
||||
detail=f"{cnt3}")
|
||||
|
||||
# ── 5. 持有者在场时（锁被把持），非持有者不接管 ──────────
|
||||
app5 = make_app()
|
||||
from chongming_cache import DistributedLock as _DL
|
||||
held = _DL(c, "mqtt_bridge")
|
||||
held.acquire() # 模拟一个存活的持有者把持着桥锁
|
||||
cnt5 = await drive_supervisor(app5, mgr, 5)
|
||||
check("5.1 持有者存活：非持有者不拉起", cnt5["broker"] == 0 and cnt5["start"] == 0,
|
||||
detail=f"{cnt5}")
|
||||
await held.stop()
|
||||
held.release()
|
||||
|
||||
c.close()
|
||||
print()
|
||||
if failures:
|
||||
print(f"FAILED {len(failures)}: {failures}")
|
||||
return 1
|
||||
print("ALL PASS")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,158 @@
|
||||
"""
|
||||
内嵌 MQTT Broker 宿主级分布式锁 + 孤儿接管测试
|
||||
|
||||
验证 setup_mqtt_broker 用 chongming-cache DistruibutedLock 取代文件锁后的语义：
|
||||
1. 外部 Broker（use_embedded_broker=false）→ 跳过锁，直接返回
|
||||
2. 未配置共享 cache → 无法取锁，跳过（并给出警告）
|
||||
3. 非持有者：锁被他人持有 → 等待期间端口已就绪即复用，不启动子进程
|
||||
4. 持有者 & 端口空闲 → 启动子进程并登记 PID
|
||||
5. 持有者 & 端口被占 & 无孤儿登记 → 复用（不托管、不杀进程）
|
||||
6. 持有者 & 端口被占 & 孤儿 PID 存活 → 终止孤儿、接管重启、登记新 PID
|
||||
7. 关停 → 终止子进程、释放锁、清理登记
|
||||
|
||||
通过 monkeypatch is_port_in_use / _pid_alive / _terminate_pid / _wait_port_free 隔离
|
||||
真实端口与进程；子进程目标替换为休眠桩，验证编排逻辑而非真实 amqtt 收发。
|
||||
|
||||
运行: uv run python test/test_mqtt_broker_lock.py
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
_HERE = Path(__file__).resolve().parent # chongming-zenoh/
|
||||
_PROJ = _HERE
|
||||
for _p in (_PROJ / "src", _PROJ / "utils" / "worker" / "src"):
|
||||
sys.path.insert(0, str(_p))
|
||||
|
||||
import chongming.app.core.lifespan_setup as ls
|
||||
from chongming_cache import Cache, DistributedLock
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
status = "PASS" if cond else "FAIL"
|
||||
print(f"[{status}] {name}" + (f" — {detail}" if detail else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
def _dummy_broker(*_a, **_k):
|
||||
"""休眠桩子进程，替代真实 amqtt Broker（验证编排，不做 MQTT 收发）。"""
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
def make_manager(cache):
|
||||
return SimpleNamespace(cache=cache)
|
||||
|
||||
|
||||
def make_app():
|
||||
return SimpleNamespace(state=SimpleNamespace(
|
||||
mqtt_broker_process=None, mqtt_broker_lock=None,
|
||||
mqtt_broker_lock_acquired=False,
|
||||
))
|
||||
|
||||
|
||||
def _port_script(*seq):
|
||||
"""一次性的 is_port_in_use 脚本（耗尽后一律 True，防止等待循环无限跑）。"""
|
||||
it = iter(seq)
|
||||
|
||||
def fake(port, host="127.0.0.1"):
|
||||
try:
|
||||
return next(it)
|
||||
except StopIteration:
|
||||
return True
|
||||
return fake
|
||||
|
||||
|
||||
async def run_setup(cache, conf, port_script, patch=()):
|
||||
"""用给定配置跑一遍 setup_mqtt_broker，返回 app。"""
|
||||
ls.env = "testenv"
|
||||
ls.config = {"testenv": {"mqtt": conf}}
|
||||
ls._run_mqtt_broker = _dummy_broker
|
||||
ls.is_port_in_use = port_script
|
||||
ls._pid_alive = lambda pid: True
|
||||
ls._terminate_pid = lambda pid: None
|
||||
ls._wait_port_free = lambda *a, **k: None
|
||||
for name, val in patch:
|
||||
setattr(ls, name, val)
|
||||
app = make_app()
|
||||
mgr = make_manager(cache)
|
||||
await ls.setup_mqtt_broker(app, mgr)
|
||||
return app
|
||||
|
||||
|
||||
async def main():
|
||||
c = Cache(backend="diskcache", cache_store_path=tempfile.mkdtemp())
|
||||
base = {"enabled": True, "host": "127.0.0.1", "port": 18883,
|
||||
"use_embedded_broker": True}
|
||||
|
||||
# ── 1. 外部 Broker：跳过 ────────────────────────────────
|
||||
ext = dict(base, use_embedded_broker=False)
|
||||
app = await run_setup(c, ext, _port_script(False))
|
||||
check("1.1 外部 Broker 跳过（不启动）", app.state.mqtt_broker_process is None
|
||||
and app.state.mqtt_broker_lock is None)
|
||||
|
||||
# ── 2. 未配 cache：跳过 ─────────────────────────────────
|
||||
app = await run_setup(None, base, _port_script(False))
|
||||
check("2.1 未配 cache 跳过（不取锁）", app.state.mqtt_broker_lock is None
|
||||
and app.state.mqtt_broker_process is None)
|
||||
|
||||
# ── 3. 非持有者：锁被他人持有 → 已监听则复用 ────────────
|
||||
actor_lock = DistributedLock(c, ls._broker_lock_key()) # 先被别人抢到
|
||||
actor_lock.acquire()
|
||||
app = await run_setup(c, base, _port_script(True)) # 端口已被"占用"
|
||||
check("3.1 非持有者复用（不取锁、不启动）",
|
||||
app.state.mqtt_broker_lock is None
|
||||
and app.state.mqtt_broker_process is None)
|
||||
await actor_lock.stop()
|
||||
actor_lock.release()
|
||||
|
||||
# ── 4. 持有者 & 端口空闲 → 启动并登记 PID ──────────────
|
||||
app = await run_setup(c, base, _port_script(False, True))
|
||||
check("4.1 持锁启动子进程", app.state.mqtt_broker_process is not None)
|
||||
check("4.2 已登记子进程 PID",
|
||||
c.get(ls._broker_owner_key()) == str(app.state.mqtt_broker_process.pid))
|
||||
await ls.shutdown_mqtt_broker(app)
|
||||
check("4.3 关停后释放锁", app.state.mqtt_broker_lock is None)
|
||||
check("4.4 关停后清理登记", c.get(ls._broker_owner_key()) is None)
|
||||
|
||||
# ── 5. 持有者 & 端口被占 & 无孤儿登记 → 复用 ───────────
|
||||
app = await run_setup(c, base, _port_script(True))
|
||||
check("5.1 端口被占无孤儿：复用（不托管不杀）",
|
||||
app.state.mqtt_broker_process is None
|
||||
and app.state.mqtt_broker_lock is not None)
|
||||
await ls.shutdown_mqtt_broker(app)
|
||||
check("5.2 复用路径关停释放锁", app.state.mqtt_broker_lock is None)
|
||||
|
||||
# ── 6. 持有者 & 端口被占 & 孤儿 PID 存活 → 终止并接管 ──
|
||||
# 端口脚本：入口=占用(True) → 终止后=空闲(False) → 新进程监听(True)
|
||||
c.set(ls._broker_owner_key(), "99999") # 模拟前任登记的孤儿 PID
|
||||
terminated = []
|
||||
app = await run_setup(
|
||||
c, base, _port_script(True, False, True),
|
||||
patch=[("_pid_alive", lambda pid: pid == 99999),
|
||||
("_terminate_pid", lambda pid: terminated.append(pid))],
|
||||
)
|
||||
check("6.1 检出并终止孤儿 PID 99999", 99999 in terminated, detail=f"{terminated}")
|
||||
check("6.2 接管后启动并登记新 PID",
|
||||
app.state.mqtt_broker_process is not None
|
||||
and c.get(ls._broker_owner_key()) == str(app.state.mqtt_broker_process.pid))
|
||||
await ls.shutdown_mqtt_broker(app)
|
||||
check("6.3 接管后关停释放并清理", app.state.mqtt_broker_lock is None
|
||||
and c.get(ls._broker_owner_key()) is None)
|
||||
|
||||
c.close()
|
||||
print()
|
||||
if failures:
|
||||
print(f"FAILED {len(failures)}: {failures}")
|
||||
return 1
|
||||
print("ALL PASS")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
Reference in New Issue
Block a user