chakcy
f4de43c743
单任务受控执行（15 态状态机 + 审计轨迹）、DAG 管道、Controller 门面。
执行器三个：线程（默认，含 async def 任务）、内联（同步 / 测试）、进程
（spawn 子进程：硬性停止、绕开 GIL、崩溃隔离）。
调度：并发额度 + 每桶限流 + 具名资源信号量（按标签）+ 优先级排队。
持久层：TaskStore 契约 + 内存 / SQLite / Postgres 实现，契约测试随包发布。
跨进程与多节点：写穿 / 读穿、指令通道与租约、owned_buckets 划界、reconcile。
CLI、12 个可直接运行的示例、docs/ 下 5 份文档。
门禁：ruff / ruff format / mypy(strict) 干净；
pytest 563 passed（44 skipped 是需要真实 Postgres 的那批）；
带一台可连的 Postgres 时 607 passed。
391 lines
14 KiB
Python
391 lines
14 KiB
Python
"""进程执行器测试。
|
|
|
|
这些用例全部用**真实子进程**（``spawn``）跑：执行器的全部价值就在于子进程能不能被
|
|
停住、能不能被强杀、父进程能不能看见它。用假的进程模型测不出这些。
|
|
|
|
三条约束写在前面，它们是这套设计的一部分而不是测试的毛病：
|
|
|
|
* 目标函数必须是**模块级**的——spawn 靠"模块 + 限定名"在子进程里定位它；
|
|
* 所以子进程会重新 import 本模块（模块顶层不能有副作用）；
|
|
* 每个用例给足超时（5 秒），子进程数量控制在小规模，测试整体仍然是秒级。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import threading
|
|
import time
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from chongming_task.errors import ConfigurationError
|
|
from chongming_task.models import FailureKind
|
|
from chongming_task.runtime.executors.base import ExecutionRequest
|
|
from chongming_task.runtime.executors.process import ProcessExecutor
|
|
from chongming_task.testing import FakeClock
|
|
|
|
_WINDOW = 10.0
|
|
|
|
#: 子进程启动本身就要几百毫秒（新解释器 + import），协作路径的窗口要比线程宽。
|
|
_START = 15.0
|
|
|
|
|
|
def _request(target: Any, *args: Any, **kwargs: Any) -> ExecutionRequest:
|
|
return ExecutionRequest(
|
|
task_id="t-proc",
|
|
name="t-proc",
|
|
target=target,
|
|
args=args,
|
|
kwargs=kwargs,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 子进程里跑的模块级函数（spawn 要求）
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _add(ctx: Any, left: int, right: int, *, scale: int = 1) -> int:
|
|
"""两数相加。"""
|
|
del ctx
|
|
return (left + right) * scale
|
|
|
|
|
|
def _boom(ctx: Any) -> None:
|
|
"""抛异常。"""
|
|
del ctx
|
|
raise ValueError("子进程里炸了")
|
|
|
|
|
|
def _describe(ctx: Any) -> dict[str, Any]:
|
|
"""把注入进来的上下文信息带回来，验证它是真的被注入了。"""
|
|
return {
|
|
"task_id": ctx.task_id,
|
|
"name": ctx.name,
|
|
"attempt": ctx.attempt,
|
|
"pid": os.getpid(),
|
|
"has_token": ctx.token is not None,
|
|
}
|
|
|
|
|
|
def _beater(ctx: Any) -> str:
|
|
"""持续 checkpoint，直到进程退出。"""
|
|
for _ in range(10_000):
|
|
ctx.checkpoint()
|
|
time.sleep(0.01)
|
|
return "never"
|
|
|
|
|
|
def _bounded_beater(ctx: Any) -> str:
|
|
"""有限次 checkpoint 后正常返回——用于验证"暂停没生效时不能把它卡住"。"""
|
|
for _ in range(5):
|
|
ctx.checkpoint()
|
|
time.sleep(0.1)
|
|
return "done"
|
|
|
|
|
|
def _late_checkpointer(ctx: Any) -> str:
|
|
"""先干一段**不 checkpoint** 的活，之后才正常推进。
|
|
|
|
用来测暂停回滚：暂停请求会在它到达第一个检查点之前就超时。
|
|
"""
|
|
time.sleep(0.5)
|
|
for _ in range(5):
|
|
ctx.checkpoint()
|
|
time.sleep(0.05)
|
|
return "done"
|
|
|
|
|
|
def _progress_reporter(ctx: Any) -> str:
|
|
"""报进度，父进程应当收到。"""
|
|
for index in range(1, 4):
|
|
ctx.progress(index, total=3, unit="批")
|
|
time.sleep(0.02)
|
|
return "ok"
|
|
|
|
|
|
def _ignore_everything(ctx: Any) -> str:
|
|
"""**不**调用 checkpoint 的死循环：只能靠强杀停下。"""
|
|
del ctx
|
|
end = time.monotonic() + 60
|
|
while time.monotonic() < end:
|
|
pass
|
|
return "never"
|
|
|
|
|
|
def _abrupt_exit(ctx: Any) -> str:
|
|
"""直接消失，连一句结论都不留。"""
|
|
del ctx
|
|
os._exit(3)
|
|
return "never" # pragma: no cover
|
|
|
|
|
|
def _unpicklable_result(ctx: Any) -> Any:
|
|
"""返回值不可 pickle（线程执行器能接住它，进程执行器不行）。"""
|
|
del ctx
|
|
return lambda: None
|
|
|
|
|
|
def _explode_on_unpickle() -> None: # pragma: no cover - 只在子进程里被调用
|
|
raise RuntimeError("反序列化时炸了")
|
|
|
|
|
|
class _CannotCrossTheBorder:
|
|
"""能 pickle 出去、却在子进程里反序列化失败的对象。
|
|
|
|
``__reduce__`` 只在**加载**时调用那个工厂函数，所以父进程的预检（只 dump）会放行
|
|
——正好用来测"子进程根本没跑起来"那条分支。
|
|
"""
|
|
|
|
def __reduce__(self) -> Any:
|
|
return (_explode_on_unpickle, ())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 载荷校验：把"子进程里神秘失败"变成一句能照做的话
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_a_lambda_target_is_rejected_with_an_actionable_error() -> None:
|
|
with pytest.raises(ConfigurationError, match="可 pickle"):
|
|
ProcessExecutor().launch(_request(lambda ctx: "nope"))
|
|
|
|
|
|
def test_a_closure_target_is_rejected() -> None:
|
|
captured = "closure"
|
|
|
|
def inner(ctx: Any) -> str:
|
|
del ctx
|
|
return captured
|
|
|
|
with pytest.raises(ConfigurationError, match="ThreadExecutor"):
|
|
ProcessExecutor().launch(_request(inner))
|
|
|
|
|
|
def test_an_unpicklable_argument_is_rejected() -> None:
|
|
with pytest.raises(ConfigurationError, match="参数"):
|
|
ProcessExecutor().launch(_request(_add, threading.Lock(), 1))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 正常路径
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_target_runs_in_a_child_process_and_returns_its_value() -> None:
|
|
handle = ProcessExecutor().launch(
|
|
ExecutionRequest(
|
|
task_id="t-args",
|
|
name="t-args",
|
|
target=_add,
|
|
args=(2, 3),
|
|
kwargs={"scale": 10},
|
|
)
|
|
)
|
|
outcome = handle.wait(timeout=_START)
|
|
|
|
assert outcome is not None
|
|
assert outcome.ok
|
|
assert outcome.value == 50
|
|
assert outcome.abandoned is False, "进程没有强杀这个词，结论是已知的"
|
|
|
|
|
|
def test_the_child_has_its_own_pid_and_gets_the_injected_context() -> None:
|
|
handle = ProcessExecutor().launch(_request(_describe))
|
|
outcome = handle.wait(timeout=_START)
|
|
|
|
assert outcome is not None and outcome.ok
|
|
payload = outcome.value
|
|
assert payload["task_id"] == "t-proc"
|
|
assert payload["name"] == "t-proc"
|
|
assert payload["has_token"] is True
|
|
assert payload["pid"] == handle.pid, "子进程里看到的 pid 就是句柄报出来的那个"
|
|
assert handle.pid != os.getpid(), "任务确实不在监督进程里跑"
|
|
|
|
|
|
def test_the_handle_knows_the_child_pid() -> None:
|
|
handle = ProcessExecutor().launch(_request(_add, 1, 1))
|
|
assert handle.pid is not None
|
|
assert handle.running is True
|
|
assert handle.wait(timeout=_START) is not None
|
|
assert handle.running is False, "有结论之后就不该再报'在运行'"
|
|
|
|
|
|
def test_a_task_failure_crosses_the_boundary_as_a_structured_failure() -> None:
|
|
outcome = ProcessExecutor().launch(_request(_boom)).wait(timeout=_START)
|
|
|
|
assert outcome is not None
|
|
assert not outcome.ok
|
|
assert outcome.failure is not None
|
|
assert outcome.failure.kind is FailureKind.EXCEPTION
|
|
assert "子进程里炸了" in outcome.failure.message
|
|
assert outcome.failure.traceback, "子进程里的 traceback 必须带回来"
|
|
|
|
|
|
def test_an_unpicklable_result_is_reported_as_a_result_problem() -> None:
|
|
"""返回值过不去，报的是"产出接不住"，不是"任务出错了"——两者的修法完全不同。"""
|
|
outcome = ProcessExecutor().launch(_request(_unpicklable_result)).wait(timeout=_START)
|
|
|
|
assert outcome is not None
|
|
assert not outcome.ok
|
|
assert outcome.failure is not None
|
|
assert outcome.failure.kind is FailureKind.RESULT_INVALID
|
|
assert outcome.failure.retryable is False, "同一个返回值重跑多少次都还是过不去"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 心跳：idle 超时靠它，而它必须真的跨过进程边界
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_heartbeats_cross_the_process_boundary() -> None:
|
|
"""把父进程的时钟推远，再等子进程的一次 heartbeat 把它拉回来。
|
|
|
|
用假时钟是为了**确定性地**区分"子进程真的报了心跳"与"父进程自己刚记了一次"。
|
|
"""
|
|
clock = FakeClock()
|
|
handle = ProcessExecutor(clock=clock).launch(_request(_beater))
|
|
try:
|
|
assert handle.wait(timeout=0.2) is None, "子进程不该这么快结束"
|
|
clock.advance(300)
|
|
assert handle.heartbeat.idle_seconds() >= 300
|
|
|
|
deadline = time.monotonic() + _START
|
|
while handle.heartbeat.idle_seconds() >= 300 and time.monotonic() < deadline:
|
|
time.sleep(0.02)
|
|
assert handle.heartbeat.idle_seconds() == 0, "子进程的 checkpoint 应当刷新心跳"
|
|
finally:
|
|
handle.stop(graceful_timeout=_WINDOW)
|
|
|
|
|
|
def test_progress_reports_are_forwarded_to_the_parent() -> None:
|
|
seen: list[Any] = []
|
|
request = ExecutionRequest(
|
|
task_id="t-progress",
|
|
name="t-progress",
|
|
target=_progress_reporter,
|
|
on_progress=seen.append,
|
|
)
|
|
outcome = ProcessExecutor().launch(request).wait(timeout=_START)
|
|
|
|
assert outcome is not None and outcome.ok
|
|
assert [item.done for item in seen] == [1, 2, 3]
|
|
assert seen[-1].unit == "批"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 暂停：确认了才算，没确认就必须回滚
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_pause_suspends_the_child_and_resume_lets_it_finish() -> None:
|
|
handle = ProcessExecutor().launch(_request(_bounded_beater))
|
|
assert handle.pause(timeout=_START) is True, "子进程应当在下一次 checkpoint 停下"
|
|
|
|
# 若没真的暂停，这个任务 0.5 秒内就该跑完了。
|
|
time.sleep(0.8)
|
|
assert handle.wait(timeout=0) is None, "被暂停的任务不该继续推进"
|
|
assert handle.running is True
|
|
|
|
handle.resume()
|
|
outcome = handle.wait(timeout=_START)
|
|
assert outcome is not None and outcome.ok
|
|
assert outcome.value == "done"
|
|
|
|
|
|
def test_pause_rolls_back_when_the_child_never_reaches_a_checkpoint() -> None:
|
|
"""暂停请求超时后必须**撤销**。
|
|
|
|
不撤销的话，调用方已经按"暂停没生效"把状态回滚成 RUNNING，而子进程会在稍后的
|
|
某个检查点自己停住：那是一个没人知道、也没人会恢复的挂起——比暂停失败更糟。
|
|
"""
|
|
handle = ProcessExecutor().launch(_request(_late_checkpointer))
|
|
assert handle.wait(timeout=0.1) is None, "它还在那段不 checkpoint 的活里"
|
|
assert handle.pause(timeout=0.2) is False, "窗口内它压根没到检查点"
|
|
|
|
outcome = handle.wait(timeout=_START)
|
|
assert outcome is not None, "回滚之后它必须继续跑完，而不是停在下一个检查点上"
|
|
assert outcome.ok, f"不该被自己撤销掉的暂停影响：{outcome.failure}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 停止：进程执行器存在的理由
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_cooperative_stop_ends_the_task_without_killing_it() -> None:
|
|
handle = ProcessExecutor().launch(_request(_beater))
|
|
assert handle.wait(timeout=0.3) is None # 先让它跑起来
|
|
|
|
outcome = handle.stop(graceful_timeout=_WINDOW)
|
|
|
|
assert not outcome.ok
|
|
assert outcome.abandoned is False, "它是自己收尾的，不是被放弃"
|
|
assert outcome.failure is not None
|
|
assert outcome.failure.kind is FailureKind.CANCELLED
|
|
assert handle.running is False
|
|
|
|
|
|
def test_a_task_that_never_checks_the_token_is_actually_killed() -> None:
|
|
"""**这是进程执行器唯一不可替代的能力。** 线程只能放弃等待并记 LOST；
|
|
子进程会真的被停掉，结论因此是已知的。"""
|
|
handle = ProcessExecutor().launch(_request(_ignore_everything))
|
|
pid = handle.pid
|
|
assert pid is not None
|
|
assert handle.wait(timeout=0.3) is None # 它连 checkpoint 都不调，只能强杀
|
|
|
|
outcome = handle.stop(graceful_timeout=0.3)
|
|
|
|
assert not outcome.ok
|
|
assert outcome.abandoned is False, "任务确实停了，不是'不知道停没停'"
|
|
assert outcome.failure is not None
|
|
assert outcome.failure.kind is FailureKind.CANCELLED
|
|
assert handle.running is False, "载体已经死了，不能谎报还在跑"
|
|
assert handle.process is not None and handle.process.exitcode is not None
|
|
|
|
|
|
def test_stop_is_also_delivered_through_the_handle_token() -> None:
|
|
"""控制面有几条路径直接调 ``handle.token.request_stop()``，绕过了 ``stop()``。
|
|
|
|
令牌不转发的话子进程永远收不到那次停止——表现出来就是"停止按钮没反应"。
|
|
"""
|
|
handle = ProcessExecutor().launch(_request(_beater))
|
|
assert handle.wait(timeout=0.3) is None
|
|
|
|
handle.token.request_stop("走令牌这条路")
|
|
|
|
outcome = handle.wait(timeout=_WINDOW)
|
|
assert outcome is not None
|
|
assert not outcome.ok
|
|
assert outcome.failure is not None
|
|
assert outcome.failure.kind is FailureKind.CANCELLED
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 崩溃隔离：父进程必须活着，并且如实说清楚发生了什么
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_an_abrupt_child_exit_is_reported_with_its_exit_code() -> None:
|
|
outcome = ProcessExecutor().launch(_request(_abrupt_exit)).wait(timeout=_START)
|
|
|
|
assert outcome is not None
|
|
assert not outcome.ok
|
|
assert outcome.failure is not None
|
|
assert outcome.failure.kind is FailureKind.NONZERO_EXIT
|
|
assert outcome.failure.exit_code == 3, "退出码是唯一能解释'它为什么没了'的事实"
|
|
assert "副作用" in outcome.failure.message, "结果未知必须说清楚，别让人放心重跑"
|
|
|
|
|
|
def test_a_child_that_never_boots_is_reported_distinctly() -> None:
|
|
"""载荷在子进程里反序列化失败：任务**从未开始**，与"跑一半没了"是两回事。"""
|
|
handle = ProcessExecutor().launch(_request(_add, _CannotCrossTheBorder(), 1))
|
|
outcome = handle.wait(timeout=_START)
|
|
|
|
assert outcome is not None
|
|
assert not outcome.ok
|
|
assert outcome.failure is not None
|
|
assert outcome.failure.kind is FailureKind.NONZERO_EXIT
|
|
assert "从未开始" in outcome.failure.message
|