初始提交：chongming-task 任务控制库
单任务受控执行（15 态状态机 + 审计轨迹）、DAG 管道、Controller 门面。
执行器三个：线程（默认，含 async def 任务）、内联（同步 / 测试）、进程
（spawn 子进程：硬性停止、绕开 GIL、崩溃隔离）。
调度：并发额度 + 每桶限流 + 具名资源信号量（按标签）+ 优先级排队。
持久层：TaskStore 契约 + 内存 / SQLite / Postgres 实现，契约测试随包发布。
跨进程与多节点：写穿 / 读穿、指令通道与租约、owned_buckets 划界、reconcile。
CLI、12 个可直接运行的示例、docs/ 下 5 份文档。
门禁：ruff / ruff format / mypy(strict) 干净；
pytest 563 passed（44 skipped 是需要真实 Postgres 的那批）；
带一台可连的 Postgres 时 607 passed。
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
"""分桶策略测试。
|
||||
|
||||
最重要的一条是**跨进程稳定性**：内置 ``hash()`` 带进程级随机盐，若图省事用它，
|
||||
同一个键在不同进程里会落到不同的桶，分布式下"同桶任务在同一台机器"这条约定会
|
||||
**静默**失效。这里用真实子进程与不同的 ``PYTHONHASHSEED`` 把它钉死。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.policies import BucketPolicy
|
||||
|
||||
|
||||
def _run_with_hash_seed(script: str, seed: str) -> str:
|
||||
"""在指定 ``PYTHONHASHSEED`` 的子进程里跑一段脚本，返回其标准输出。"""
|
||||
environment = {**os.environ, "PYTHONHASHSEED": seed}
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
check=True,
|
||||
timeout=60,
|
||||
)
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 稳定性（分桶能用于分布式的全部前提）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_same_key_always_maps_to_the_same_bucket() -> None:
|
||||
policy = BucketPolicy(64)
|
||||
for _ in range(50):
|
||||
assert policy.bucket_for("tenant-42") == policy.bucket_for("tenant-42")
|
||||
|
||||
|
||||
def test_assignment_is_stable_across_processes_with_different_hash_seeds() -> None:
|
||||
"""内置 ``hash()`` 带随机盐，本实现必须不受它影响。"""
|
||||
script = (
|
||||
"from chongming_task.policies import BucketPolicy;"
|
||||
"print(' '.join(str(BucketPolicy(64).bucket_for(f'tenant-{i}')) for i in range(20)))"
|
||||
)
|
||||
first = _run_with_hash_seed(script, "0")
|
||||
second = _run_with_hash_seed(script, "1")
|
||||
assert first == second, "同一个键在不同进程里落到了不同的桶"
|
||||
|
||||
local = " ".join(str(BucketPolicy(64).bucket_for(f"tenant-{i}")) for i in range(20))
|
||||
assert first == local, "子进程与本进程的结果不一致"
|
||||
|
||||
|
||||
def test_builtin_hash_would_have_been_unusable() -> None:
|
||||
"""留一条反例，说明为什么不能图省事用内置 ``hash()``。
|
||||
|
||||
不是"可能不一样"，而是**确实不一样**——这就是那个坑本身。
|
||||
"""
|
||||
script = "print(' '.join(str(hash(f'k{i}') % 64) for i in range(20)))"
|
||||
assert _run_with_hash_seed(script, "0") != _run_with_hash_seed(script, "1")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 取值范围与分布
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_bucket_is_always_within_range() -> None:
|
||||
for num_buckets in (1, 2, 7, 64, 1000):
|
||||
policy = BucketPolicy(num_buckets)
|
||||
for index in range(200):
|
||||
bucket = policy.bucket_for(f"key-{index}")
|
||||
assert 0 <= bucket < num_buckets
|
||||
|
||||
|
||||
def test_single_bucket_policy_puts_everything_in_bucket_zero() -> None:
|
||||
policy = BucketPolicy(1)
|
||||
assert all(policy.bucket_for(key) == 0 for key in ("a", "b", "c"))
|
||||
|
||||
|
||||
def test_distribution_is_reasonably_uniform() -> None:
|
||||
"""分桶明显倾斜会让某些机器空转、另一些排队。
|
||||
|
||||
这里只做一个粗检：64 个桶、4096 个键，每个桶的落点不应当偏离均值太多。
|
||||
"""
|
||||
policy = BucketPolicy(64)
|
||||
counts = [0] * 64
|
||||
for index in range(4096):
|
||||
counts[policy.bucket_for(f"tenant-{index}")] += 1
|
||||
|
||||
expected = 4096 / 64
|
||||
assert all(count > 0 for count in counts), "有桶完全没分到键，分布明显异常"
|
||||
assert max(counts) < expected * 3, f"分布过于集中：max={max(counts)}"
|
||||
|
||||
|
||||
def test_similar_keys_do_not_cluster() -> None:
|
||||
"""``tenant-1`` 与 ``tenant-2`` 这类只差一个字符的键必须被打散。"""
|
||||
policy = BucketPolicy(64)
|
||||
buckets = {policy.bucket_for(f"tenant-{index}") for index in range(50)}
|
||||
assert len(buckets) > 20, "相邻键挤在少数几个桶里，哈希的雪崩性不足"
|
||||
|
||||
|
||||
def test_policy_validates_bucket_range() -> None:
|
||||
policy = BucketPolicy(8)
|
||||
assert policy.is_valid(0)
|
||||
assert policy.is_valid(7)
|
||||
assert not policy.is_valid(8)
|
||||
assert not policy.is_valid(-1)
|
||||
|
||||
|
||||
def test_policy_exposes_all_buckets() -> None:
|
||||
assert list(BucketPolicy(4).buckets) == [0, 1, 2, 3]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 参数校验
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_num_buckets_must_be_positive() -> None:
|
||||
with pytest.raises(ValueError, match="num_buckets"):
|
||||
BucketPolicy(0)
|
||||
|
||||
|
||||
def test_empty_key_is_rejected() -> None:
|
||||
"""空键会让所有"没填租户"的任务挤进同一个桶。"""
|
||||
with pytest.raises(ValueError, match="不能为空"):
|
||||
BucketPolicy().bucket_for("")
|
||||
|
||||
|
||||
def test_policy_is_hashable_and_comparable() -> None:
|
||||
"""策略会作为配置传递，值语义让它能被比较与当作字典键。"""
|
||||
assert BucketPolicy(64) == BucketPolicy(64)
|
||||
assert BucketPolicy(64) != BucketPolicy(128)
|
||||
assert len({BucketPolicy(64), BucketPolicy(64)}) == 1
|
||||
@@ -0,0 +1,281 @@
|
||||
"""协作式取消令牌的行为测试。
|
||||
|
||||
这里必须用**真实线程**：令牌的价值恰恰在于跨线程正确握手（任务线程阻塞在
|
||||
暂停点、调度线程请求停止并唤醒它）。用假的线程模型测不出竞态。
|
||||
|
||||
因此这些用例带小超时并断言线程确实退出——**同时也在验证没有死锁**。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.errors import TaskCancelled
|
||||
from chongming_task.runtime.cancellation import CancellationToken
|
||||
|
||||
_JOIN_TIMEOUT = 5.0
|
||||
|
||||
|
||||
def _run_in_thread(target: object) -> threading.Thread:
|
||||
thread = threading.Thread(target=target) # type: ignore[arg-type]
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 停止
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_checkpoint_is_noop_without_any_request() -> None:
|
||||
"""没有请求时 checkpoint 必须立即返回，否则每轮循环都在付代价。"""
|
||||
CancellationToken().checkpoint()
|
||||
|
||||
|
||||
def test_checkpoint_raises_when_stop_requested() -> None:
|
||||
token = CancellationToken()
|
||||
token.request_stop("发布窗口")
|
||||
with pytest.raises(TaskCancelled) as info:
|
||||
token.checkpoint()
|
||||
assert info.value.reason == "发布窗口"
|
||||
|
||||
|
||||
def test_stop_is_idempotent_and_keeps_the_first_reason() -> None:
|
||||
"""首个原因通常最接近真实起因，后续请求不应覆盖它。"""
|
||||
token = CancellationToken()
|
||||
token.request_stop("first")
|
||||
token.request_stop("second")
|
||||
with pytest.raises(TaskCancelled) as info:
|
||||
token.checkpoint()
|
||||
assert info.value.reason == "first"
|
||||
|
||||
|
||||
def test_raise_if_stopped_ignores_pause() -> None:
|
||||
"""用于清理路径：只响应停止，绝不阻塞。"""
|
||||
token = CancellationToken()
|
||||
token.request_pause()
|
||||
token.raise_if_stopped() # 不应阻塞
|
||||
token.request_stop()
|
||||
with pytest.raises(TaskCancelled):
|
||||
token.raise_if_stopped()
|
||||
|
||||
|
||||
def test_is_stop_requested_reflects_state() -> None:
|
||||
token = CancellationToken()
|
||||
assert not token.is_stop_requested
|
||||
token.request_stop()
|
||||
assert token.is_stop_requested
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 暂停
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_pause_blocks_until_resume() -> None:
|
||||
token = CancellationToken()
|
||||
token.request_pause() # 先请求，消除"任务先跑到检查点"的竞态
|
||||
observed: list[str] = []
|
||||
|
||||
def worker() -> None:
|
||||
token.checkpoint()
|
||||
observed.append("resumed")
|
||||
|
||||
thread = _run_in_thread(worker)
|
||||
assert token.wait_paused(timeout=_JOIN_TIMEOUT) is True
|
||||
assert observed == [], "暂停请求生效前任务不应继续执行"
|
||||
|
||||
token.resume()
|
||||
thread.join(_JOIN_TIMEOUT)
|
||||
assert not thread.is_alive(), "resume() 必须唤醒阻塞中的任务"
|
||||
assert observed == ["resumed"]
|
||||
|
||||
|
||||
def test_stop_wakes_a_paused_task() -> None:
|
||||
"""暂停中的任务必须能被停止，否则"暂停"就成了不可撤销的挂起。"""
|
||||
token = CancellationToken()
|
||||
token.request_pause()
|
||||
reasons: list[str | None] = []
|
||||
|
||||
def worker() -> None:
|
||||
try:
|
||||
token.checkpoint()
|
||||
except TaskCancelled as exc:
|
||||
reasons.append(exc.reason)
|
||||
|
||||
thread = _run_in_thread(worker)
|
||||
assert token.wait_paused(timeout=_JOIN_TIMEOUT) is True
|
||||
|
||||
token.request_stop("运维介入")
|
||||
thread.join(_JOIN_TIMEOUT)
|
||||
assert not thread.is_alive(), "停止请求必须能唤醒暂停点"
|
||||
assert reasons == ["运维介入"]
|
||||
|
||||
|
||||
def test_repeated_pause_does_not_clear_acknowledgement() -> None:
|
||||
"""重复请求暂停不应把"已确认"重置掉，否则 wait_paused 会误判为未生效。"""
|
||||
token = CancellationToken()
|
||||
token.request_pause()
|
||||
|
||||
def worker() -> None:
|
||||
token.checkpoint()
|
||||
|
||||
thread = _run_in_thread(worker)
|
||||
assert token.wait_paused(timeout=_JOIN_TIMEOUT) is True
|
||||
token.request_pause()
|
||||
assert token.is_pause_acknowledged is True
|
||||
|
||||
token.resume()
|
||||
thread.join(_JOIN_TIMEOUT)
|
||||
|
||||
|
||||
def test_wait_paused_times_out_when_task_never_checkpoints() -> None:
|
||||
"""调度器必须能区分"已暂停"与"暂停没生效"——后者要回滚为 RUNNING 并告警。"""
|
||||
token = CancellationToken()
|
||||
token.request_pause()
|
||||
assert token.wait_paused(timeout=0.05) is False
|
||||
assert token.is_pause_acknowledged is False
|
||||
|
||||
|
||||
def test_wait_paused_reports_false_when_stopped_instead() -> None:
|
||||
"""请求暂停后又被停止：这不是"暂停成功"，必须返回 False。"""
|
||||
token = CancellationToken()
|
||||
token.request_pause()
|
||||
token.request_stop()
|
||||
assert token.wait_paused(timeout=_JOIN_TIMEOUT) is False
|
||||
|
||||
|
||||
def test_resume_without_pause_is_noop() -> None:
|
||||
token = CancellationToken()
|
||||
token.resume()
|
||||
assert not token.is_pause_requested
|
||||
token.checkpoint() # 仍应立即返回
|
||||
|
||||
|
||||
def _start_pausing_worker(token: CancellationToken) -> threading.Event:
|
||||
"""起一个持续调用 ``checkpoint()`` 的工作线程，返回"完成过一轮"的事件。
|
||||
|
||||
线程必须**持续**调用 checkpoint——只调一次就退出的话，之后的暂停请求自然没人
|
||||
确认，那是测试写错而不是缺陷。
|
||||
"""
|
||||
|
||||
def worker() -> None:
|
||||
# 收尾时的 stop 会让 checkpoint 抛 TaskCancelled：那正是预期行为，
|
||||
# 但线程里未捕获的异常会被 pytest 转成错误。
|
||||
with contextlib.suppress(TaskCancelled):
|
||||
while True:
|
||||
token.checkpoint()
|
||||
ran.set()
|
||||
|
||||
ran = threading.Event()
|
||||
threading.Thread(target=worker, daemon=True).start()
|
||||
return ran
|
||||
|
||||
|
||||
def test_pause_is_acknowledged_again_after_a_resume() -> None:
|
||||
"""**回归用例**：暂停 → 恢复 → 再暂停里，第二次暂停曾经永久失效。
|
||||
|
||||
``checkpoint()`` 原先只在**进入**暂停分支时设置 ``_ack``。``resume()`` 把
|
||||
``_pause`` 置回 False 并通知后，任务线程还没来得及重新取到锁，新的
|
||||
``request_pause()`` 就把 ``_pause`` 又置成了 True——线程重新检查循环条件时发现
|
||||
"还要暂停"，于是继续等待，而它已经越过了设置 ``_ack`` 的那一行。
|
||||
|
||||
结果 ``wait_paused()`` 永远等不到确认，暂停超时回滚。表现出来就是"暂停一次成功，
|
||||
之后再也暂停不了"。
|
||||
"""
|
||||
token = CancellationToken()
|
||||
ran = _start_pausing_worker(token)
|
||||
assert ran.wait(_JOIN_TIMEOUT), "工作线程没有跑起来"
|
||||
|
||||
for round_index in range(5):
|
||||
token.request_pause()
|
||||
assert token.wait_paused(timeout=_JOIN_TIMEOUT) is True, (
|
||||
f"第 {round_index + 1} 轮暂停没有得到确认"
|
||||
)
|
||||
|
||||
ran.clear()
|
||||
token.resume()
|
||||
assert ran.wait(_JOIN_TIMEOUT), f"第 {round_index + 1} 轮恢复后任务没有继续推进"
|
||||
|
||||
token.request_stop()
|
||||
|
||||
|
||||
def test_a_pause_right_after_a_resume_is_still_acknowledged() -> None:
|
||||
"""恢复与再次暂停之间不留任何间隔——最容易踩中的时序。"""
|
||||
token = CancellationToken()
|
||||
ran = _start_pausing_worker(token)
|
||||
assert ran.wait(_JOIN_TIMEOUT)
|
||||
|
||||
token.request_pause()
|
||||
assert token.wait_paused(timeout=_JOIN_TIMEOUT) is True
|
||||
|
||||
# 立刻恢复再立刻暂停：不给任务线程任何重新取锁的机会。
|
||||
token.resume()
|
||||
token.request_pause()
|
||||
assert token.wait_paused(timeout=_JOIN_TIMEOUT) is True, "紧接着的第二次暂停丢了"
|
||||
|
||||
token.resume()
|
||||
token.request_stop()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 异步路径
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_acheckpoint_fast_path_does_not_offload_to_thread() -> None:
|
||||
"""紧密循环里的每次检查都丢线程会拖垮事件循环，快速路径必须留在本线程。"""
|
||||
|
||||
async def main() -> str:
|
||||
token = CancellationToken()
|
||||
for _ in range(2000):
|
||||
await token.acheckpoint()
|
||||
return "done"
|
||||
|
||||
assert asyncio.run(main()) == "done"
|
||||
|
||||
|
||||
def test_acheckpoint_raises_when_stopped() -> None:
|
||||
token = CancellationToken()
|
||||
token.request_stop("已取消")
|
||||
|
||||
async def main() -> None:
|
||||
await token.acheckpoint()
|
||||
|
||||
with pytest.raises(TaskCancelled) as info:
|
||||
asyncio.run(main())
|
||||
assert info.value.reason == "已取消"
|
||||
|
||||
|
||||
def test_acheckpoint_suspends_and_resumes_without_blocking_loop() -> None:
|
||||
"""暂停期间事件循环必须仍能推进——这正是异步版本存在的理由。"""
|
||||
token = CancellationToken()
|
||||
token.request_pause()
|
||||
|
||||
async def main() -> list[str]:
|
||||
events: list[str] = []
|
||||
|
||||
async def heartbeat() -> None:
|
||||
while True:
|
||||
events.append("tick")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
ticker = asyncio.create_task(heartbeat())
|
||||
waiter = asyncio.create_task(token.acheckpoint())
|
||||
|
||||
while not token.is_pause_acknowledged:
|
||||
await asyncio.sleep(0.01)
|
||||
ticks_while_paused = len(events)
|
||||
assert ticks_while_paused > 0, "暂停期间事件循环不应被阻塞"
|
||||
|
||||
token.resume()
|
||||
await asyncio.wait_for(waiter, timeout=_JOIN_TIMEOUT)
|
||||
ticker.cancel()
|
||||
return events
|
||||
|
||||
assert asyncio.run(main())
|
||||
@@ -0,0 +1,678 @@
|
||||
"""任务管理门面（Controller / Registry）的测试。
|
||||
|
||||
重点覆盖三件事：**白名单**（只能启动已注册的任务）、**并发额度**（超限排队而不是
|
||||
无限起线程）、以及**收尾的诚实性**（排队中的任务被取消时不应真的执行任务体）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.controller import Controller, StatusCounts, TaskRegistry, render_table
|
||||
from chongming_task.errors import ConfigurationError
|
||||
from chongming_task.policies import BucketPolicy, ConcurrencyPolicy
|
||||
from chongming_task.run import TaskRun
|
||||
from chongming_task.states import TaskStatus
|
||||
|
||||
_WINDOW = 10.0
|
||||
|
||||
|
||||
def _wait_until(predicate: Any, timeout: float = _WINDOW) -> bool:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return True
|
||||
time.sleep(0.01)
|
||||
return False
|
||||
|
||||
|
||||
def _wait_for_status(run: TaskRun, status: TaskStatus, timeout: float = _WINDOW) -> bool:
|
||||
return _wait_until(lambda: run.status is status, timeout)
|
||||
|
||||
|
||||
# --- 注册用任务（模块级，便于复用） ---
|
||||
|
||||
|
||||
def add(ctx: Any, left: int, right: int) -> int:
|
||||
"""两数相加。"""
|
||||
del ctx
|
||||
return left + right
|
||||
|
||||
|
||||
def slow_loop(ctx: Any) -> str:
|
||||
"""持续小步推进直到被停止。"""
|
||||
for _ in range(1_000_000):
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.005)
|
||||
return "done"
|
||||
|
||||
|
||||
def never_runs(ctx: Any) -> str:
|
||||
"""排队期间被取消时，这个函数体不应该被执行。"""
|
||||
marker.append("ran")
|
||||
return "ran"
|
||||
|
||||
|
||||
marker: list[str] = []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 注册表：白名单
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_registry_rejects_duplicate_names() -> None:
|
||||
"""静默覆盖会让"我改了实现，跑的还是旧的"这类问题极难发现。"""
|
||||
registry = TaskRegistry()
|
||||
registry.register("add", add)
|
||||
with pytest.raises(ConfigurationError, match="已注册"):
|
||||
registry.register("add", add)
|
||||
|
||||
|
||||
def test_registry_requires_a_callable() -> None:
|
||||
with pytest.raises(ConfigurationError, match="可调用"):
|
||||
TaskRegistry().register("bad", "not-callable") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_registry_rejects_blank_name() -> None:
|
||||
with pytest.raises(ConfigurationError, match="不能为空"):
|
||||
TaskRegistry().register(" ", add)
|
||||
|
||||
|
||||
def test_registry_picks_up_the_docstring_as_description() -> None:
|
||||
registration = TaskRegistry().register("add", add)
|
||||
assert registration.description == "两数相加。"
|
||||
|
||||
|
||||
def test_submitting_an_unregistered_task_is_rejected() -> None:
|
||||
"""安全底线：入口只接受**名字**，绝不接受代码。
|
||||
|
||||
一个能启动任意可调用对象的入口等于任意代码执行——这是将来接 CLI 还是
|
||||
HTTP 都不能破的规则。
|
||||
"""
|
||||
controller = Controller()
|
||||
with pytest.raises(ConfigurationError, match="未注册"):
|
||||
controller.submit("nope")
|
||||
assert controller.registry.names() == ()
|
||||
|
||||
|
||||
def test_registry_lists_registered_names() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
controller.register("slow", slow_loop)
|
||||
assert controller.registry.names() == ("add", "slow")
|
||||
assert len(controller.registry) == 2
|
||||
assert "add" in controller.registry
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 提交、查询、统计
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_submit_runs_and_reports_snapshot() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
run = controller.submit("add", 2, 3, task_id="t1", label="加法")
|
||||
assert isinstance(run, TaskRun)
|
||||
|
||||
outcome = controller.wait("t1", timeout=_WINDOW)
|
||||
assert outcome.ok
|
||||
assert outcome.value == 5
|
||||
|
||||
snapshot = controller.get("t1")
|
||||
assert snapshot is not None
|
||||
assert snapshot.task_id == "t1"
|
||||
assert snapshot.name == "加法"
|
||||
assert snapshot.status is TaskStatus.SUCCEEDED
|
||||
|
||||
|
||||
def test_get_returns_none_for_unknown_task() -> None:
|
||||
assert Controller().get("nope") is None
|
||||
|
||||
|
||||
def test_stats_counts_by_status() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
for index in range(3):
|
||||
controller.submit("add", index, 1, task_id=f"t{index}")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
stats = controller.stats()
|
||||
assert isinstance(stats, StatusCounts)
|
||||
assert stats.total == 3
|
||||
assert stats.of(TaskStatus.SUCCEEDED) == 3
|
||||
assert stats.finished == 3
|
||||
assert stats.unfinished == 0
|
||||
|
||||
|
||||
def test_list_filters_by_status_and_name_prefix() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
controller.submit("add", 1, 1, task_id="a1", label="抽取-甲")
|
||||
controller.submit("add", 1, 1, task_id="a2", label="抽取-乙")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
assert len(controller.list_tasks()) == 2
|
||||
assert len(controller.list_tasks(name_prefix="抽取")) == 2
|
||||
assert controller.list_tasks(name_prefix="加载") == []
|
||||
assert len(controller.list_tasks(status=TaskStatus.SUCCEEDED)) == 2
|
||||
assert controller.list_tasks(status=TaskStatus.FAILED) == []
|
||||
assert len(controller.list_tasks(limit=1)) == 1
|
||||
|
||||
|
||||
def test_list_filters_by_tag() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add, tags={"io"})
|
||||
controller.submit("add", 1, 1, task_id="a1")
|
||||
controller.submit("add", 1, 1, task_id="a2", tags={"cpu"})
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
assert len(controller.list_tasks(tag="io")) == 2, "注册时的标签应当被继承"
|
||||
assert [s.task_id for s in controller.list_tasks(tag="cpu")] == ["a2"]
|
||||
|
||||
|
||||
def test_duplicate_task_id_is_rejected() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
controller.submit("add", 1, 1, task_id="dup")
|
||||
with pytest.raises(ConfigurationError, match="已存在"):
|
||||
controller.submit("add", 1, 1, task_id="dup")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 并发额度
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_excess_submissions_queue_instead_of_starting() -> None:
|
||||
"""超限的任务状态是 ``queued``（已被受理、等待执行槽），不是 ``pending``。"""
|
||||
controller = Controller(max_parallel=1)
|
||||
controller.register("slow", slow_loop)
|
||||
|
||||
first = controller.submit("slow", task_id="t1")
|
||||
assert _wait_for_status(first, TaskStatus.RUNNING)
|
||||
|
||||
second = controller.submit("slow", task_id="t2")
|
||||
third = controller.submit("slow", task_id="t3")
|
||||
|
||||
assert second.status is TaskStatus.QUEUED
|
||||
assert third.status is TaskStatus.QUEUED
|
||||
assert not second.started, "排队中的任务不应被启动"
|
||||
assert controller.active_count == 1
|
||||
assert controller.queued_count == 2
|
||||
|
||||
controller.stop_all(reason="测试结束")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
assert controller.queued_count == 0
|
||||
|
||||
|
||||
def test_queued_tasks_start_as_slots_free_up() -> None:
|
||||
controller = Controller(max_parallel=2)
|
||||
controller.register("add", add)
|
||||
runs = [controller.submit("add", index, 1, task_id=f"t{index}") for index in range(4)]
|
||||
|
||||
outcomes = controller.wait_all(timeout=_WINDOW)
|
||||
assert len(outcomes) == 4
|
||||
assert all(outcome.ok for outcome in outcomes)
|
||||
assert sorted(outcome.value for outcome in outcomes) == [1, 2, 3, 4]
|
||||
assert controller.active_count == 0
|
||||
assert all(run.status is TaskStatus.SUCCEEDED for run in runs)
|
||||
|
||||
|
||||
def test_max_parallel_must_be_positive() -> None:
|
||||
with pytest.raises(ConfigurationError, match="max_parallel"):
|
||||
Controller(max_parallel=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 具名资源信号量与优先级（ConcurrencyPolicy）
|
||||
#
|
||||
# 这三个字段曾经只是"声明了、校验了、文档里写着支持"——调度器从不读它，于是
|
||||
# ``limits`` 与 ``priority`` 设了等于没设，而且**不报任何错**。下面几条把它们钉住。
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_named_resource_limit_caps_only_the_tagged_tasks() -> None:
|
||||
"""``limits={"gpu": 1}`` 只约束带 ``gpu`` 标签的任务，不是给全局加一道闸。"""
|
||||
controller = Controller(concurrency=ConcurrencyPolicy(limits={"gpu": 1}))
|
||||
controller.register("slow", slow_loop)
|
||||
|
||||
first = controller.submit("slow", task_id="gpu-1", tags=["gpu"])
|
||||
assert _wait_for_status(first, TaskStatus.RUNNING)
|
||||
second = controller.submit("slow", task_id="gpu-2", tags=["gpu"])
|
||||
assert second.status is TaskStatus.QUEUED, "gpu 名额已满，第二个必须排队"
|
||||
|
||||
# 不带该标签的任务不受它约束——具名信号量不是全局额度。
|
||||
plain = controller.submit("slow", task_id="plain")
|
||||
assert _wait_for_status(plain, TaskStatus.RUNNING)
|
||||
assert controller.active_count == 2
|
||||
|
||||
controller.stop_all(reason="测试结束")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_a_task_needs_room_in_every_limited_tag_it_carries() -> None:
|
||||
"""多标签任务要**每个**受限期资源都还有余量才放行。"""
|
||||
controller = Controller(concurrency=ConcurrencyPolicy(limits={"gpu": 1, "db": 1}))
|
||||
controller.register("slow", slow_loop)
|
||||
|
||||
gpu = controller.submit("slow", task_id="gpu-holder", tags=["gpu"])
|
||||
assert _wait_for_status(gpu, TaskStatus.RUNNING)
|
||||
db = controller.submit("slow", task_id="db-holder", tags=["db"])
|
||||
assert _wait_for_status(db, TaskStatus.RUNNING)
|
||||
|
||||
both = controller.submit("slow", task_id="needs-both", tags=["gpu", "db"])
|
||||
assert both.status is TaskStatus.QUEUED, "两个资源都满了，带两个标签的任务更不能放行"
|
||||
|
||||
# 只空出一个：db 还占着，所以它依旧不能启动。等额度记账真的落下来再断言，
|
||||
# 否则这句话可能在调度器还没醒的时候"通过"，什么都没证明。
|
||||
assert controller.stop("gpu-holder") is True
|
||||
controller.wait("gpu-holder", timeout=_WINDOW)
|
||||
assert _wait_until(lambda: controller.active_count == 1), "额度应当已归还"
|
||||
assert both.status is TaskStatus.QUEUED, "gpu 空了但 db 还占着，不能放行"
|
||||
|
||||
# 两个都空出来才轮到它。
|
||||
assert controller.stop("db-holder") is True
|
||||
controller.wait("db-holder", timeout=_WINDOW)
|
||||
assert _wait_for_status(both, TaskStatus.RUNNING)
|
||||
|
||||
controller.stop_all(reason="测试结束")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_tags_without_a_limit_do_not_throttle_anything() -> None:
|
||||
"""没写进 ``limits`` 的标签不受约束（常见误解：以为打个标签就会被限流）。"""
|
||||
controller = Controller(concurrency=ConcurrencyPolicy(limits={"gpu": 1}))
|
||||
controller.register("slow", slow_loop)
|
||||
|
||||
first = controller.submit("slow", task_id="io-1", tags=["io"])
|
||||
second = controller.submit("slow", task_id="io-2", tags=["io"])
|
||||
assert _wait_for_status(first, TaskStatus.RUNNING)
|
||||
assert _wait_for_status(second, TaskStatus.RUNNING)
|
||||
|
||||
controller.stop_all(reason="测试结束")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_priority_picks_the_highest_first() -> None:
|
||||
"""都能启动时先取优先级最高的；数值越大越优先。"""
|
||||
controller = Controller(max_parallel=1)
|
||||
controller.register("slow", slow_loop)
|
||||
|
||||
blocker = controller.submit("slow", task_id="blocker")
|
||||
assert _wait_for_status(blocker, TaskStatus.RUNNING)
|
||||
low = controller.submit("slow", task_id="low", priority=1)
|
||||
high = controller.submit("slow", task_id="high", priority=5)
|
||||
|
||||
assert controller.stop("blocker") is True
|
||||
controller.wait("blocker", timeout=_WINDOW)
|
||||
|
||||
assert _wait_for_status(high, TaskStatus.RUNNING), "高优先级的应当先被取走"
|
||||
assert low.status is TaskStatus.QUEUED, "低优先级的不该插到前面"
|
||||
|
||||
controller.stop_all(reason="测试结束")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_equal_priority_keeps_submission_order() -> None:
|
||||
"""同优先级仍然先进先出——优先级机制不该打乱时间序。"""
|
||||
controller = Controller(max_parallel=1)
|
||||
controller.register("slow", slow_loop)
|
||||
|
||||
blocker = controller.submit("slow", task_id="blocker")
|
||||
assert _wait_for_status(blocker, TaskStatus.RUNNING)
|
||||
first_queued = controller.submit("slow", task_id="first")
|
||||
second_queued = controller.submit("slow", task_id="second")
|
||||
|
||||
assert controller.stop("blocker") is True
|
||||
controller.wait("blocker", timeout=_WINDOW)
|
||||
|
||||
assert _wait_for_status(first_queued, TaskStatus.RUNNING), "先提交的应当先跑"
|
||||
assert second_queued.status is TaskStatus.QUEUED
|
||||
|
||||
controller.stop_all(reason="测试结束")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_policy_default_priority_applies_to_submissions() -> None:
|
||||
"""``ConcurrencyPolicy.priority`` 是 ``submit`` 不给 ``priority`` 时的默认值。"""
|
||||
controller = Controller(concurrency=ConcurrencyPolicy(max_parallel=1, priority=7))
|
||||
controller.register("slow", slow_loop)
|
||||
|
||||
blocker = controller.submit("slow", task_id="blocker")
|
||||
assert _wait_for_status(blocker, TaskStatus.RUNNING)
|
||||
assert blocker.priority == 7, "没给 priority 时应当取策略里的默认值"
|
||||
|
||||
explicit_low = controller.submit("slow", task_id="low", priority=1)
|
||||
assert explicit_low.priority == 1, "显式给的优先级必须能覆盖默认值"
|
||||
default_high = controller.submit("slow", task_id="high")
|
||||
assert default_high.priority == 7
|
||||
|
||||
assert controller.stop("blocker") is True
|
||||
controller.wait("blocker", timeout=_WINDOW)
|
||||
assert _wait_for_status(default_high, TaskStatus.RUNNING)
|
||||
assert explicit_low.status is TaskStatus.QUEUED
|
||||
|
||||
controller.stop_all(reason="测试结束")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_concurrency_policy_max_parallel_is_honoured() -> None:
|
||||
"""策略里的 ``max_parallel`` 与 ``Controller(max_parallel=...)`` 等价。"""
|
||||
controller = Controller(concurrency=ConcurrencyPolicy(max_parallel=1))
|
||||
controller.register("slow", slow_loop)
|
||||
|
||||
first = controller.submit("slow", task_id="t1")
|
||||
assert _wait_for_status(first, TaskStatus.RUNNING)
|
||||
second = controller.submit("slow", task_id="t2")
|
||||
assert second.status is TaskStatus.QUEUED
|
||||
assert controller.active_count == 1
|
||||
|
||||
controller.stop_all(reason="测试结束")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_policy_and_max_parallel_are_mutually_exclusive() -> None:
|
||||
"""两份配置只能给一份：谁说了算必须明确，不能靠"后者覆盖前者"这种猜法。"""
|
||||
with pytest.raises(ConfigurationError, match="只能给一个"):
|
||||
Controller(concurrency=ConcurrencyPolicy(max_parallel=2), max_parallel=1)
|
||||
|
||||
|
||||
def test_policy_rejects_a_non_positive_resource_limit() -> None:
|
||||
with pytest.raises(ValueError, match="并发上限"):
|
||||
ConcurrencyPolicy(limits={"gpu": 0})
|
||||
|
||||
|
||||
def test_a_non_integer_priority_is_rejected_at_submit_time() -> None:
|
||||
"""非整数优先级必须在提交时拦下。
|
||||
|
||||
放过去的话，它会在**派发线程**里抛 ``TypeError``——线程直接死掉，此后所有排队
|
||||
任务都不再启动，而没有任何报错指向真正的原因。
|
||||
"""
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
with pytest.raises(ConfigurationError, match="priority"):
|
||||
controller.submit("add", 1, 2, task_id="t1", priority="high") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 控制
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_stop_routes_to_the_right_task() -> None:
|
||||
controller = Controller()
|
||||
controller.register("slow", slow_loop)
|
||||
victim = controller.submit("slow", task_id="victim")
|
||||
bystander = controller.submit("slow", task_id="bystander")
|
||||
assert _wait_for_status(victim, TaskStatus.RUNNING)
|
||||
assert _wait_for_status(bystander, TaskStatus.RUNNING)
|
||||
|
||||
assert controller.stop("victim", reason="发布窗口") is True
|
||||
outcome = controller.wait("victim", timeout=_WINDOW)
|
||||
assert outcome.status is TaskStatus.STOPPED
|
||||
assert bystander.status is TaskStatus.RUNNING, "不该误伤别的任务"
|
||||
|
||||
controller.stop_all()
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_stop_returns_false_for_unknown_task() -> None:
|
||||
assert Controller().stop("nope") is False
|
||||
|
||||
|
||||
def test_pause_and_resume_route_through_the_controller() -> None:
|
||||
controller = Controller()
|
||||
controller.register("slow", slow_loop)
|
||||
run = controller.submit("slow", task_id="p1")
|
||||
assert _wait_for_status(run, TaskStatus.RUNNING)
|
||||
|
||||
assert controller.pause("p1", timeout=_WINDOW) is True
|
||||
assert run.status is TaskStatus.PAUSED
|
||||
|
||||
assert controller.resume("p1") is True
|
||||
assert run.status is TaskStatus.RUNNING
|
||||
|
||||
controller.stop_all()
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_stop_all_reports_how_many_were_affected() -> None:
|
||||
controller = Controller()
|
||||
controller.register("slow", slow_loop)
|
||||
for index in range(3):
|
||||
controller.submit("slow", task_id=f"t{index}")
|
||||
assert _wait_until(lambda: controller.active_count == 3)
|
||||
|
||||
assert controller.stop_all(reason="批量收工") == 3
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
assert controller.stop_all() == 0, "已无未落终态的任务"
|
||||
|
||||
|
||||
def test_submit_after_close_is_rejected() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
controller.close(timeout=_WINDOW)
|
||||
with pytest.raises(ConfigurationError, match="已关闭"):
|
||||
controller.submit("add", 1, 1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 收尾
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_close_does_not_execute_queued_task_bodies() -> None:
|
||||
"""排队中的任务被取消时不该真的跑起来——否则"停止"只是把副作用推迟了。"""
|
||||
marker.clear()
|
||||
controller = Controller(max_parallel=1)
|
||||
controller.register("slow", slow_loop)
|
||||
controller.register("never", never_runs)
|
||||
|
||||
controller.submit("slow", task_id="t-block")
|
||||
assert _wait_until(lambda: controller.active_count == 1)
|
||||
controller.submit("never", task_id="t-queued")
|
||||
assert controller.queued_count == 1
|
||||
|
||||
outcomes = controller.close(reason="收工", timeout=_WINDOW)
|
||||
|
||||
assert marker == [], "排队中的任务被启动了，任务体被执行"
|
||||
statuses = {outcome.snapshot.task_id: outcome.status for outcome in outcomes}
|
||||
assert statuses["t-queued"] is TaskStatus.STOPPED
|
||||
assert statuses["t-block"] is TaskStatus.STOPPED
|
||||
|
||||
|
||||
def test_prune_drops_oldest_terminal_records() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
for index in range(5):
|
||||
controller.submit("add", index, 1, task_id=f"t{index}")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
assert len(controller.list_tasks()) == 5
|
||||
|
||||
assert controller.prune(keep_terminal=2) == 3
|
||||
assert len(controller.list_tasks()) == 2
|
||||
assert controller.prune(keep_terminal=2) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 渲染
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_render_table_aligns_chinese_names_by_display_width() -> None:
|
||||
"""中日韩全角字符占两列；不按显示宽度对齐，中文任务名会把整张表挤歪。"""
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
controller.submit("add", 1, 1, task_id="a1", label="抽取")
|
||||
controller.submit("add", 1, 1, task_id="a2", label="a-much-longer-name")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
table = render_table(controller.list_tasks())
|
||||
lines = table.splitlines()
|
||||
|
||||
assert lines[0].startswith("TASK_ID")
|
||||
assert set(lines[1]) <= {"-", " "}, "表头下方应当是分隔线"
|
||||
widths = {len(line.rstrip()) for line in lines}
|
||||
assert len(widths) > 1, "各行列宽应当随内容变化，而不是靠不齐"
|
||||
|
||||
|
||||
def test_render_table_reports_failure_kind() -> None:
|
||||
def boom(ctx: Any) -> None:
|
||||
raise RuntimeError("炸了")
|
||||
|
||||
controller = Controller()
|
||||
controller.register("boom", boom)
|
||||
controller.submit("boom", task_id="b1")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
table = render_table(controller.list_tasks())
|
||||
assert "exception" in table
|
||||
assert "failed" in table
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 分桶
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _two_keys_in_different_buckets(policy: BucketPolicy) -> tuple[str, str]:
|
||||
"""找两个确实落在不同桶里的键。
|
||||
|
||||
硬编码两个键是不行的：它们有约 1/64 的概率撞进同一个桶，那会让"分桶隔离"
|
||||
的用例变成一句空话，而且只在特定版本上偶发失败。
|
||||
"""
|
||||
first = "tenant-0"
|
||||
first_bucket = policy.bucket_for(first)
|
||||
for index in range(1, 500):
|
||||
candidate = f"tenant-{index}"
|
||||
if policy.bucket_for(candidate) != first_bucket:
|
||||
return first, candidate
|
||||
raise AssertionError("在 500 个键里找不到第二个桶，分桶策略有问题")
|
||||
|
||||
|
||||
def test_bucket_is_derived_from_the_key_and_recorded() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
run = controller.submit("add", 1, 1, task_id="a1", bucket_key="tenant-42")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
snapshot = run.snapshot
|
||||
assert snapshot.bucket == controller.bucket_policy.bucket_for("tenant-42")
|
||||
assert snapshot.bucket_key == "tenant-42", "原始键必须留存，否则改桶数时无法迁移"
|
||||
|
||||
|
||||
def test_same_key_lands_in_the_same_bucket() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
for index in range(3):
|
||||
controller.submit("add", index, 1, task_id=f"t{index}", bucket_key="tenant-9")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
buckets = {snapshot.bucket for snapshot in controller.list_tasks()}
|
||||
assert len(buckets) == 1
|
||||
|
||||
|
||||
def test_task_without_bucket_key_is_unbucketed() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
controller.submit("add", 1, 1, task_id="a1")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
assert controller.get("a1").bucket is None # type: ignore[union-attr]
|
||||
assert controller.bucket_counts() == {None: 1}
|
||||
|
||||
|
||||
def test_per_bucket_limit_isolates_buckets_from_each_other() -> None:
|
||||
"""一个桶把额度占满不应饿死其他桶——这是分桶在**本地**就有的价值。
|
||||
|
||||
若调度器直接取队首而不是扫描可运行者，一个积压的桶会阻塞所有桶，把分桶要解决
|
||||
的问题又原样引入一遍。
|
||||
"""
|
||||
controller = Controller(max_parallel=10, max_parallel_per_bucket=1)
|
||||
controller.register("slow", slow_loop)
|
||||
hot, cold = _two_keys_in_different_buckets(controller.bucket_policy)
|
||||
|
||||
first = controller.submit("slow", task_id="a1", bucket_key=hot)
|
||||
second = controller.submit("slow", task_id="a2", bucket_key=hot)
|
||||
other = controller.submit("slow", task_id="b1", bucket_key=cold)
|
||||
|
||||
assert _wait_for_status(first, TaskStatus.RUNNING)
|
||||
assert _wait_for_status(other, TaskStatus.RUNNING), "别的桶被积压的桶挡住了"
|
||||
assert second.status is TaskStatus.QUEUED, "同桶超出额度应当排队"
|
||||
assert controller.active_in_bucket(controller.bucket_policy.bucket_for(hot)) == 1
|
||||
|
||||
controller.stop_all()
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_per_bucket_limit_still_respects_the_global_limit() -> None:
|
||||
controller = Controller(max_parallel=1, max_parallel_per_bucket=5)
|
||||
controller.register("slow", slow_loop)
|
||||
hot, cold = _two_keys_in_different_buckets(controller.bucket_policy)
|
||||
|
||||
first = controller.submit("slow", task_id="a1", bucket_key=hot)
|
||||
second = controller.submit("slow", task_id="b1", bucket_key=cold)
|
||||
assert _wait_for_status(first, TaskStatus.RUNNING)
|
||||
assert second.status is TaskStatus.QUEUED, "总额度才是硬上限"
|
||||
|
||||
controller.stop_all()
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_per_bucket_limit_must_be_positive() -> None:
|
||||
with pytest.raises(ConfigurationError, match="max_parallel_per_bucket"):
|
||||
Controller(max_parallel_per_bucket=0)
|
||||
|
||||
|
||||
def test_list_tasks_filters_by_bucket_and_bucket_set() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
hot, cold = _two_keys_in_different_buckets(controller.bucket_policy)
|
||||
controller.submit("add", 1, 1, task_id="a1", bucket_key=hot)
|
||||
controller.submit("add", 1, 1, task_id="b1", bucket_key=cold)
|
||||
controller.submit("add", 1, 1, task_id="c1")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
hot_bucket = controller.bucket_policy.bucket_for(hot)
|
||||
cold_bucket = controller.bucket_policy.bucket_for(cold)
|
||||
|
||||
assert [s.task_id for s in controller.list_tasks(bucket=hot_bucket)] == ["a1"]
|
||||
assert [s.task_id for s in controller.list_tasks(buckets=[hot_bucket, cold_bucket])] == [
|
||||
"a1",
|
||||
"b1",
|
||||
]
|
||||
assert controller.list_tasks(bucket=9999) == []
|
||||
|
||||
|
||||
def test_bucket_counts_summarises_each_bucket() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
hot, cold = _two_keys_in_different_buckets(controller.bucket_policy)
|
||||
controller.submit("add", 1, 1, task_id="a1", bucket_key=hot)
|
||||
controller.submit("add", 1, 1, task_id="a2", bucket_key=hot)
|
||||
controller.submit("add", 1, 1, task_id="b1", bucket_key=cold)
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
counts = controller.bucket_counts()
|
||||
assert counts[controller.bucket_policy.bucket_for(hot)] == 2
|
||||
assert counts[controller.bucket_policy.bucket_for(cold)] == 1
|
||||
|
||||
|
||||
def test_render_table_shows_the_bucket_column() -> None:
|
||||
controller = Controller()
|
||||
controller.register("add", add)
|
||||
controller.submit("add", 1, 1, task_id="a1", bucket_key="tenant-42")
|
||||
controller.submit("add", 1, 1, task_id="a2")
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
|
||||
table = render_table(controller.list_tasks())
|
||||
lines = table.splitlines()
|
||||
assert "BUCKET" in lines[0]
|
||||
expected = str(controller.bucket_policy.bucket_for("tenant-42"))
|
||||
assert expected in lines[2]
|
||||
assert lines[3].split()[2] == "-", "未分桶的任务应当显示 -"
|
||||
@@ -0,0 +1,119 @@
|
||||
"""内联执行器测试。
|
||||
|
||||
重点是两件事：它**确实**同步跑完（违反 launch 不阻塞的约定），以及它对无法
|
||||
支持的暂停能力**如实返回 False** 而不是假装成功。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from chongming_task.models import FailureKind
|
||||
from chongming_task.runtime.executors.base import ExecutionRequest
|
||||
from chongming_task.runtime.executors.inline import InlineExecutor
|
||||
|
||||
|
||||
def _request(target: Any, *args: Any) -> ExecutionRequest:
|
||||
return ExecutionRequest(task_id="t-inline", name="t-inline", target=target, args=args)
|
||||
|
||||
|
||||
def test_launch_runs_to_completion_before_returning() -> None:
|
||||
"""这是它与 ``Executor`` 协议唯一的差异，也是它存在的意义。"""
|
||||
order: list[str] = []
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
del ctx
|
||||
order.append("task")
|
||||
return "done"
|
||||
|
||||
handle = InlineExecutor().launch(_request(work))
|
||||
order.append("after-launch")
|
||||
|
||||
assert order == ["task", "after-launch"], "内联执行必须在 launch 返回前跑完"
|
||||
assert handle.running is False
|
||||
outcome = handle.wait()
|
||||
assert outcome is not None
|
||||
assert outcome.value == "done"
|
||||
|
||||
|
||||
def test_arguments_are_forwarded_after_the_context() -> None:
|
||||
def work(ctx: Any, left: int, right: int, *, scale: int = 1) -> int:
|
||||
del ctx
|
||||
return (left + right) * scale
|
||||
|
||||
handle = InlineExecutor().launch(
|
||||
ExecutionRequest(
|
||||
task_id="t-inline-args",
|
||||
name="t-inline-args",
|
||||
target=work,
|
||||
args=(2, 3),
|
||||
kwargs={"scale": 10},
|
||||
)
|
||||
)
|
||||
outcome = handle.wait()
|
||||
assert outcome is not None
|
||||
assert outcome.value == 50
|
||||
|
||||
|
||||
def test_failure_is_reported_like_any_other_executor() -> None:
|
||||
def boom(ctx: Any) -> None:
|
||||
del ctx
|
||||
raise KeyError("missing")
|
||||
|
||||
outcome = InlineExecutor().launch(_request(boom)).wait()
|
||||
assert outcome is not None
|
||||
assert not outcome.ok
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.EXCEPTION
|
||||
assert "KeyError" in outcome.failure.message
|
||||
assert outcome.failure.traceback is not None
|
||||
|
||||
|
||||
def test_pause_reports_false_because_it_cannot_be_honoured() -> None:
|
||||
"""如实返回 False，调用方据此回滚为 RUNNING，而不是以为暂停成功了。"""
|
||||
handle = InlineExecutor().launch(_request(lambda ctx: "ok"))
|
||||
assert handle.pause(timeout=0.01) is False
|
||||
|
||||
|
||||
def test_resume_is_harmless_after_the_fact() -> None:
|
||||
handle = InlineExecutor().launch(_request(lambda ctx: "ok"))
|
||||
handle.resume()
|
||||
assert handle.running is False
|
||||
|
||||
|
||||
def test_stop_on_a_finished_inline_task_returns_its_real_outcome() -> None:
|
||||
"""已结束的任务应当返回真实结果，而不是落进"兜底放弃"分支。"""
|
||||
handle = InlineExecutor().launch(_request(lambda ctx: 7))
|
||||
outcome = handle.stop(graceful_timeout=1.0)
|
||||
assert outcome.ok
|
||||
assert outcome.value == 7
|
||||
assert outcome.abandoned is False
|
||||
|
||||
|
||||
def test_context_is_injected() -> None:
|
||||
seen: dict[str, Any] = {}
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
seen["task_id"] = ctx.task_id
|
||||
seen["token"] = ctx.token
|
||||
return "ok"
|
||||
|
||||
handle = InlineExecutor().launch(_request(work))
|
||||
assert seen["task_id"] == "t-inline"
|
||||
assert seen["token"] is handle.token
|
||||
|
||||
|
||||
def test_async_target_runs_to_completion_before_launch_returns() -> None:
|
||||
"""``async def`` 任务在内联执行器下同样是同步跑完的（本线程起一个事件循环）。"""
|
||||
|
||||
async def work(ctx: Any) -> int:
|
||||
await asyncio.sleep(0)
|
||||
return 42
|
||||
|
||||
handle = InlineExecutor().launch(_request(work))
|
||||
assert handle.running is False
|
||||
outcome = handle.wait()
|
||||
assert outcome is not None
|
||||
assert outcome.ok
|
||||
assert outcome.value == 42
|
||||
@@ -0,0 +1,143 @@
|
||||
"""快照与正交维度模型的不变量。
|
||||
|
||||
重点覆盖那些"容易被合并成单一状态枚举"的维度：进度、失败原因、请求动作。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.models import (
|
||||
AttemptRecord,
|
||||
Failure,
|
||||
FailureKind,
|
||||
Progress,
|
||||
RequestedAction,
|
||||
TaskSnapshot,
|
||||
TimeoutScope,
|
||||
)
|
||||
from chongming_task.states import TaskStatus
|
||||
|
||||
|
||||
def test_progress_fraction_is_none_when_total_unknown() -> None:
|
||||
"""total 未知必须返回 None 而非 0.0——后者会被渲染成"尚未开始"。"""
|
||||
assert Progress(done=5).fraction is None
|
||||
assert Progress(done=0, total=0).fraction is None
|
||||
assert Progress(done=25, total=100).fraction == 0.25
|
||||
|
||||
|
||||
def test_progress_rejects_done_beyond_total() -> None:
|
||||
with pytest.raises(ValueError, match="不能超过"):
|
||||
Progress(done=11, total=10)
|
||||
|
||||
|
||||
def test_progress_rejects_negative_values() -> None:
|
||||
with pytest.raises(ValueError, match="不能为负"):
|
||||
Progress(done=-1)
|
||||
with pytest.raises(ValueError, match="不能为负"):
|
||||
Progress(total=-1)
|
||||
|
||||
|
||||
def test_timeout_scope_maps_to_failure_kind() -> None:
|
||||
"""超时是失败原因而非生命周期状态，两者的对应关系必须稳定。"""
|
||||
assert FailureKind.for_timeout(TimeoutScope.START) is FailureKind.TIMEOUT_START
|
||||
assert FailureKind.for_timeout(TimeoutScope.ATTEMPT) is FailureKind.TIMEOUT_ATTEMPT
|
||||
assert FailureKind.for_timeout(TimeoutScope.IDLE) is FailureKind.TIMEOUT_IDLE
|
||||
|
||||
|
||||
def test_attempt_record_numbering_starts_at_one() -> None:
|
||||
now = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
with pytest.raises(ValueError, match="从 1 开始"):
|
||||
AttemptRecord(task_id="t1", attempt=0, started_at=now)
|
||||
|
||||
|
||||
def test_snapshot_requires_identity() -> None:
|
||||
with pytest.raises(ValueError, match="task_id"):
|
||||
TaskSnapshot(task_id="", name="x")
|
||||
with pytest.raises(ValueError, match="name"):
|
||||
TaskSnapshot(task_id="t1", name="")
|
||||
|
||||
|
||||
def test_snapshot_rejects_attempt_beyond_max_attempts() -> None:
|
||||
with pytest.raises(ValueError, match="max_attempts"):
|
||||
TaskSnapshot(task_id="t1", name="x", attempt=3, max_attempts=2)
|
||||
|
||||
|
||||
def test_snapshot_is_frozen() -> None:
|
||||
"""快照不可变，才能安全跨线程/协程传递并直接作为持久化记录。"""
|
||||
snap = TaskSnapshot(task_id="t1", name="x")
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
snap.status = TaskStatus.RUNNING # type: ignore[misc]
|
||||
|
||||
|
||||
def test_validate_flags_terminal_missing_ended_at() -> None:
|
||||
snap = TaskSnapshot(task_id="t1", name="x", status=TaskStatus.SUCCEEDED)
|
||||
assert any("ended_at" in problem for problem in snap.validate())
|
||||
|
||||
|
||||
def test_validate_flags_leftover_requested_action_on_terminal() -> None:
|
||||
"""终态残留请求会让 UI 永远显示"停止中"。"""
|
||||
now = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
snap = TaskSnapshot(
|
||||
task_id="t1",
|
||||
name="x",
|
||||
status=TaskStatus.STOPPED,
|
||||
ended_at=now,
|
||||
requested_action=RequestedAction.STOP,
|
||||
)
|
||||
assert any("requested_action" in problem for problem in snap.validate())
|
||||
|
||||
|
||||
def test_retry_wait_may_carry_failure() -> None:
|
||||
"""RETRY_WAIT 必须能携带上次失败原因，否则界面无法回答"它为什么在等"。"""
|
||||
snap = TaskSnapshot(
|
||||
task_id="t1",
|
||||
name="x",
|
||||
status=TaskStatus.RETRY_WAIT,
|
||||
failure=Failure(kind=FailureKind.EXCEPTION, message="boom"),
|
||||
)
|
||||
assert snap.validate() == ()
|
||||
|
||||
|
||||
def test_retry_wait_without_failure_is_inconsistent() -> None:
|
||||
snap = TaskSnapshot(task_id="t1", name="x", status=TaskStatus.RETRY_WAIT)
|
||||
assert any("RETRY_WAIT" in problem for problem in snap.validate())
|
||||
|
||||
|
||||
def test_success_must_not_carry_failure() -> None:
|
||||
now = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
snap = TaskSnapshot(
|
||||
task_id="t1",
|
||||
name="x",
|
||||
status=TaskStatus.SUCCEEDED,
|
||||
ended_at=now,
|
||||
failure=Failure(kind=FailureKind.EXCEPTION, message="boom"),
|
||||
)
|
||||
assert any("SUCCEEDED" in problem for problem in snap.validate())
|
||||
|
||||
|
||||
def test_consistent_terminal_snapshot_validates_clean() -> None:
|
||||
now = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
snap = TaskSnapshot(
|
||||
task_id="t1",
|
||||
name="x",
|
||||
status=TaskStatus.FAILED,
|
||||
ended_at=now,
|
||||
attempt=1,
|
||||
max_attempts=1,
|
||||
failure=Failure(kind=FailureKind.EXCEPTION, message="boom"),
|
||||
)
|
||||
assert snap.validate() == ()
|
||||
assert snap.is_terminal
|
||||
assert snap.is_revivable
|
||||
|
||||
|
||||
def test_running_snapshot_requires_started_at_only_on_validate() -> None:
|
||||
"""时间线完整性刻意不放在 __post_init__，以免逐步构造快照时处处受限。"""
|
||||
snap = TaskSnapshot(task_id="t1", name="x", status=TaskStatus.RUNNING)
|
||||
assert any("started_at" in problem for problem in snap.validate())
|
||||
# 但构造本身不报错
|
||||
assert snap.status is TaskStatus.RUNNING
|
||||
@@ -0,0 +1,590 @@
|
||||
"""DAG 管道测试。
|
||||
|
||||
分三层：
|
||||
|
||||
1. **定义期校验**——环、悬空依赖、未注册任务、quorum 都要在创建任何任务之前拒绝，
|
||||
否则存储里会留下一堆看起来正常的半成品任务；
|
||||
2. **推进**——屏障、扇出、扇入、三种汇合语义；
|
||||
3. **收场**——fail_fast 连坐、下游跳过、整体状态推导。
|
||||
|
||||
约定：节点 id 一律用 ``node_id=`` 显式给出。第一个位置参数是**已注册的任务名**，
|
||||
其后才是传给任务的位置参数——这两者很容易看混，显式写出来既清楚又自成文档。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.controller import Controller
|
||||
from chongming_task.errors import ConfigurationError
|
||||
from chongming_task.pipeline import (
|
||||
Pipeline,
|
||||
PipelineStatus,
|
||||
render_pipeline,
|
||||
)
|
||||
from chongming_task.policies import AggregationPolicy, JoinMode
|
||||
from chongming_task.states import TaskStatus
|
||||
from chongming_task.stores.memory import InMemoryTaskStore
|
||||
|
||||
_WINDOW = 15.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. 定义期校验
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_empty_pipeline_is_rejected() -> None:
|
||||
with pytest.raises(ConfigurationError, match="没有任何节点"):
|
||||
Pipeline("empty").validate()
|
||||
|
||||
|
||||
def test_dangling_dependency_is_rejected() -> None:
|
||||
pipeline = Pipeline("dangling")
|
||||
pipeline.node("add", node_id="a", depends_on=["no-such-node"])
|
||||
with pytest.raises(ConfigurationError, match="不存在的节点"):
|
||||
pipeline.validate()
|
||||
|
||||
|
||||
def test_self_dependency_is_rejected() -> None:
|
||||
pipeline = Pipeline("self")
|
||||
pipeline.node("add", node_id="a", depends_on=["a"])
|
||||
with pytest.raises(ConfigurationError, match="依赖了自己"):
|
||||
pipeline.validate()
|
||||
|
||||
|
||||
def test_cycle_is_rejected() -> None:
|
||||
"""有环时必须显式报错。
|
||||
|
||||
不检测的话调度器只会安静地什么都不做（所有节点都在等对方），表现为"管道卡住了"
|
||||
——那种故障最难查。
|
||||
"""
|
||||
pipeline = Pipeline("cyclic")
|
||||
first = pipeline.node("add", node_id="a")
|
||||
second = pipeline.node("add", node_id="b", depends_on=[first])
|
||||
# 人为制造环：a 反过来依赖 b。
|
||||
object.__setattr__(pipeline.node_by_id("a"), "depends_on", ("b",))
|
||||
assert second.depends_on == ("a",)
|
||||
|
||||
with pytest.raises(ConfigurationError, match="环形依赖"):
|
||||
pipeline.validate()
|
||||
|
||||
|
||||
def test_unregistered_task_is_rejected_before_creating_anything() -> None:
|
||||
"""控制面只接受任务名。写错的 DAG 不能留下半成品任务。"""
|
||||
store = InMemoryTaskStore()
|
||||
controller = Controller(store=store)
|
||||
controller.register("add", _add)
|
||||
|
||||
pipeline = Pipeline("bad")
|
||||
pipeline.node("add", node_id="ok")
|
||||
pipeline.node("nope", node_id="typo")
|
||||
|
||||
with pytest.raises(ConfigurationError):
|
||||
controller.submit_pipeline(pipeline)
|
||||
assert store.query() == [], "校验失败前不该创建任何任务"
|
||||
|
||||
|
||||
def test_quorum_settlement_is_rejected_with_an_explanation() -> None:
|
||||
"""诚实拒绝，而不是把 quorum 硬套到 DAG 上发明一个没人能预测的语义。"""
|
||||
pipeline = Pipeline("q", aggregation=AggregationPolicy.of_quorum(2))
|
||||
pipeline.node("add")
|
||||
with pytest.raises(ConfigurationError, match="不支持 quorum"):
|
||||
pipeline.validate()
|
||||
|
||||
|
||||
def test_duplicate_node_id_is_rejected() -> None:
|
||||
pipeline = Pipeline("dup")
|
||||
pipeline.node("add", node_id="same")
|
||||
with pytest.raises(ConfigurationError, match="节点 id 重复"):
|
||||
pipeline.node("add", node_id="same")
|
||||
|
||||
|
||||
def test_same_task_can_appear_many_times_with_auto_ids() -> None:
|
||||
"""扇出靠"同名多次"表达，不该强迫调用方给每个节点起名。"""
|
||||
pipeline = Pipeline("fanout")
|
||||
pipeline.node("add")
|
||||
pipeline.node("add")
|
||||
pipeline.node("add")
|
||||
assert [node.node_id for node in pipeline.nodes] == ["add", "add#2", "add#3"]
|
||||
|
||||
|
||||
def test_positional_args_belong_to_the_task_not_the_node_id() -> None:
|
||||
"""位置参数是传给任务的，不是节点名——这正是把参数叫 ``node_id`` 的原因。"""
|
||||
pipeline = Pipeline("args")
|
||||
node = pipeline.node("add", 2, 3, node_id="sum")
|
||||
assert node.node_id == "sum"
|
||||
assert node.args == (2, 3)
|
||||
|
||||
|
||||
def test_topological_order_puts_upstream_first() -> None:
|
||||
pipeline = Pipeline("order")
|
||||
last = pipeline.node("add", node_id="last")
|
||||
first = pipeline.node("add", node_id="first")
|
||||
pipeline.node("add", node_id="middle", depends_on=[first, last])
|
||||
|
||||
order = [node.node_id for node in pipeline.topological_order()]
|
||||
assert order.index("first") < order.index("middle")
|
||||
assert order.index("last") < order.index("middle")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. 推进
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_linear_pipeline_runs_in_dependency_order() -> None:
|
||||
controller = Controller(max_parallel=4)
|
||||
controller.register("mark", _mark)
|
||||
|
||||
pipeline = Pipeline("linear")
|
||||
extract = pipeline.node("mark", node_id="extract")
|
||||
transform = pipeline.node("mark", node_id="transform", depends_on=[extract])
|
||||
pipeline.node("mark", node_id="load", depends_on=[transform])
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p1")
|
||||
assert run.wait(timeout=_WINDOW).status is PipelineStatus.SUCCEEDED
|
||||
assert _finished_order(controller, "p1") == ["extract", "transform", "load"]
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_a_downstream_node_waits_in_blocked_until_its_upstream_finishes() -> None:
|
||||
gate = threading.Event()
|
||||
controller = Controller(max_parallel=4)
|
||||
controller.register("wait_gate", _wait_gate)
|
||||
controller.register("mark", _mark)
|
||||
|
||||
pipeline = Pipeline("blocking")
|
||||
upstream = pipeline.node("wait_gate", gate, node_id="upstream")
|
||||
pipeline.node("mark", node_id="downstream", depends_on=[upstream])
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p2")
|
||||
try:
|
||||
assert _wait_until(lambda: run.node_status("upstream") is TaskStatus.RUNNING)
|
||||
time.sleep(0.1)
|
||||
|
||||
assert run.node_status("downstream") is TaskStatus.BLOCKED, "上游没完成前必须停在 blocked"
|
||||
assert run.status is PipelineStatus.RUNNING
|
||||
|
||||
gate.set()
|
||||
assert run.wait(timeout=_WINDOW).status is PipelineStatus.SUCCEEDED
|
||||
finally:
|
||||
gate.set()
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_fan_out_runs_siblings() -> None:
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("mark", _mark)
|
||||
|
||||
pipeline = Pipeline("fanout")
|
||||
root = pipeline.node("mark", node_id="root")
|
||||
for index in range(4):
|
||||
pipeline.node("mark", node_id=f"shard{index}", depends_on=[root])
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p3")
|
||||
assert run.wait(timeout=_WINDOW).status is PipelineStatus.SUCCEEDED
|
||||
assert set(run.task_ids) == {"root", "shard0", "shard1", "shard2", "shard3"}
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_fan_in_with_all_succeeded_waits_for_every_upstream() -> None:
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("mark", _mark)
|
||||
|
||||
pipeline = Pipeline("fanin")
|
||||
root = pipeline.node("mark", node_id="root")
|
||||
left = pipeline.node("mark", node_id="left", depends_on=[root])
|
||||
right = pipeline.node("mark", node_id="right", depends_on=[root])
|
||||
pipeline.node("mark", node_id="join", depends_on=[left, right])
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p4")
|
||||
assert run.wait(timeout=_WINDOW).status is PipelineStatus.SUCCEEDED
|
||||
|
||||
order = _finished_order(controller, "p4")
|
||||
assert order.index("join") > order.index("left")
|
||||
assert order.index("join") > order.index("right")
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_any_succeeded_join_runs_after_the_first_success() -> None:
|
||||
"""先到先得：任一上游成功即可放行，不必等另一个失败的上游。"""
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("mark", _mark)
|
||||
controller.register("boom", _boom)
|
||||
|
||||
pipeline = Pipeline("any")
|
||||
good = pipeline.node("mark", node_id="good")
|
||||
bad = pipeline.node("boom", node_id="bad")
|
||||
winner = pipeline.node(
|
||||
"mark", node_id="winner", depends_on=[good, bad], join=JoinMode.ANY_SUCCEEDED
|
||||
)
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p5")
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
|
||||
assert winner.join is JoinMode.ANY_SUCCEEDED
|
||||
assert outcome.nodes["winner"] is TaskStatus.SUCCEEDED, "有一个上游成功就该放行"
|
||||
assert outcome.status is PipelineStatus.FAILED, "另一个上游失败，整条管道仍失败"
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_any_succeeded_join_skips_when_nothing_succeeds() -> None:
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("boom", _boom)
|
||||
controller.register("mark", _mark)
|
||||
|
||||
pipeline = Pipeline("any-none")
|
||||
bad = pipeline.node("boom", node_id="bad")
|
||||
pipeline.node("mark", node_id="winner", depends_on=[bad], join=JoinMode.ANY_SUCCEEDED)
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p6")
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
assert outcome.nodes["winner"] is TaskStatus.SKIPPED
|
||||
assert outcome.status is PipelineStatus.FAILED
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_all_settled_join_runs_even_after_an_upstream_failure() -> None:
|
||||
"""清理/通知型节点：不管上游成败都要跑。"""
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("boom", _boom)
|
||||
controller.register("mark", _mark)
|
||||
|
||||
pipeline = Pipeline("cleanup", aggregation=AggregationPolicy.all_settled())
|
||||
failed = pipeline.node("boom", node_id="work")
|
||||
cleanup = pipeline.node(
|
||||
"mark", node_id="cleanup", depends_on=[failed], join=JoinMode.ALL_SETTLED
|
||||
)
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p7")
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
assert cleanup.join is JoinMode.ALL_SETTLED
|
||||
assert outcome.nodes["cleanup"] is TaskStatus.SUCCEEDED, "ALL_SETTLED 下游必须照跑"
|
||||
assert outcome.status is PipelineStatus.FAILED, "上游失败仍让整条管道失败"
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. 收场
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_failed_upstream_skips_downstream_with_a_reason() -> None:
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("boom", _boom)
|
||||
controller.register("mark", _mark)
|
||||
|
||||
pipeline = Pipeline("skip")
|
||||
bad = pipeline.node("boom", node_id="bad")
|
||||
pipeline.node("mark", node_id="after", depends_on=[bad])
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p8")
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
|
||||
assert outcome.nodes["after"] is TaskStatus.SKIPPED
|
||||
snapshot = controller.get("p8:after")
|
||||
assert snapshot is not None
|
||||
assert snapshot.status is TaskStatus.SKIPPED
|
||||
assert snapshot.failure is None, "跳过不是失败，不该带 failure"
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_fail_fast_cancels_independent_running_nodes() -> None:
|
||||
"""真正需要连坐的场景：另一个分支已经在跑，必须被取消。
|
||||
|
||||
被连坐的必须落 ``CANCELLED`` 而不是 ``STOPPED``——"上游挂了"与"值班同学手动停的"
|
||||
是完全不同的事，混成一个状态就再也分不出来了。
|
||||
"""
|
||||
gate = threading.Event()
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("boom", _boom)
|
||||
controller.register("wait_gate", _wait_gate)
|
||||
|
||||
pipeline = Pipeline("cascade", aggregation=AggregationPolicy.fail_fast())
|
||||
pipeline.node("wait_gate", gate, node_id="independent")
|
||||
pipeline.node("boom", node_id="bad")
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p9")
|
||||
try:
|
||||
assert _wait_until(lambda: run.node_status("independent") is TaskStatus.RUNNING)
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
|
||||
assert outcome.status is PipelineStatus.FAILED
|
||||
assert outcome.nodes["independent"] is TaskStatus.CANCELLED, (
|
||||
"被连坐的应当是 cancelled，而不是 stopped"
|
||||
)
|
||||
snapshot = controller.get("p9:independent")
|
||||
assert snapshot is not None
|
||||
assert snapshot.failure is not None
|
||||
assert snapshot.failure.kind.value == "cancelled"
|
||||
finally:
|
||||
gate.set()
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_fail_fast_never_settles_while_a_cancelled_node_is_still_running() -> None:
|
||||
"""``status`` 必须先看终态、再看失败标志。
|
||||
|
||||
顺序反了，等待者会被提前唤醒，看到一个"已经失败、但还有任务在跑"的矛盾状态。
|
||||
"""
|
||||
gate = threading.Event()
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("boom", _boom)
|
||||
controller.register("wait_gate", _wait_gate)
|
||||
|
||||
pipeline = Pipeline("ordering", aggregation=AggregationPolicy.fail_fast())
|
||||
pipeline.node("wait_gate", gate, node_id="independent")
|
||||
pipeline.node("boom", node_id="bad")
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p10")
|
||||
try:
|
||||
assert _wait_until(lambda: run.node_status("independent") is TaskStatus.RUNNING)
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
for node_id in outcome.nodes:
|
||||
assert outcome.nodes[node_id] in {
|
||||
TaskStatus.SUCCEEDED,
|
||||
TaskStatus.FAILED,
|
||||
TaskStatus.SKIPPED,
|
||||
TaskStatus.CANCELLED,
|
||||
TaskStatus.STOPPED,
|
||||
TaskStatus.LOST,
|
||||
}, f"{node_id} 在管道给出结论时还没落终态"
|
||||
finally:
|
||||
gate.set()
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_all_settled_does_not_cancel_siblings() -> None:
|
||||
gate = threading.Event()
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("boom", _boom)
|
||||
controller.register("wait_gate", _wait_gate)
|
||||
|
||||
pipeline = Pipeline("settled", aggregation=AggregationPolicy.all_settled())
|
||||
pipeline.node("wait_gate", gate, node_id="independent")
|
||||
pipeline.node("boom", node_id="bad")
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p11")
|
||||
try:
|
||||
assert _wait_until(lambda: run.node_status("independent") is TaskStatus.RUNNING)
|
||||
time.sleep(0.2)
|
||||
assert run.node_status("independent") is TaskStatus.RUNNING, "all_settled 不该连坐"
|
||||
|
||||
gate.set()
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
assert outcome.nodes["independent"] is TaskStatus.SUCCEEDED
|
||||
assert outcome.status is PipelineStatus.FAILED, "有人失败过，整体就是失败"
|
||||
finally:
|
||||
gate.set()
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_pipeline_tasks_carry_the_run_id() -> None:
|
||||
controller = Controller(max_parallel=4)
|
||||
controller.register("mark", _mark)
|
||||
pipeline = Pipeline("tagged")
|
||||
pipeline.node("mark", node_id="one")
|
||||
pipeline.node("mark", node_id="two")
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="tagged-1")
|
||||
run.wait(timeout=_WINDOW)
|
||||
|
||||
for node_id, task_id in run.task_ids.items():
|
||||
snapshot = controller.get(task_id)
|
||||
assert snapshot is not None
|
||||
assert snapshot.pipeline_run_id == "tagged-1"
|
||||
assert task_id == f"tagged-1:{node_id}"
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_cancelling_the_whole_pipeline() -> None:
|
||||
gate = threading.Event()
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("wait_gate", _wait_gate)
|
||||
|
||||
pipeline = Pipeline("cancelme")
|
||||
root = pipeline.node("wait_gate", gate, node_id="root")
|
||||
pipeline.node("wait_gate", gate, node_id="child", depends_on=[root])
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p12")
|
||||
try:
|
||||
assert _wait_until(lambda: run.node_status("root") is TaskStatus.RUNNING)
|
||||
assert run.cancel(reason="运维取消") >= 1
|
||||
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
assert outcome.status is PipelineStatus.CANCELLED
|
||||
assert outcome.nodes["root"] is TaskStatus.CANCELLED
|
||||
assert outcome.nodes["child"] is TaskStatus.CANCELLED
|
||||
finally:
|
||||
gate.set()
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_wait_timeout_does_not_change_anything() -> None:
|
||||
gate = threading.Event()
|
||||
controller = Controller(max_parallel=4)
|
||||
controller.register("wait_gate", _wait_gate)
|
||||
|
||||
pipeline = Pipeline("slow")
|
||||
pipeline.node("wait_gate", gate, node_id="root")
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p13")
|
||||
try:
|
||||
outcome = run.wait(timeout=0.2)
|
||||
assert outcome.status is PipelineStatus.RUNNING
|
||||
assert run.node_status("root") in {TaskStatus.STARTING, TaskStatus.RUNNING}
|
||||
assert not run.settled
|
||||
finally:
|
||||
gate.set()
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_pipeline_run_is_registered_on_the_controller() -> None:
|
||||
controller = Controller(max_parallel=4)
|
||||
controller.register("mark", _mark)
|
||||
pipeline = Pipeline("reg")
|
||||
pipeline.node("mark", node_id="only")
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="reg-1")
|
||||
run.wait(timeout=_WINDOW)
|
||||
assert controller.get_pipeline("reg-1") is run
|
||||
assert controller.get_pipeline("nope") is None
|
||||
assert controller.pipelines() == (run,)
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_a_diamond_pipeline_works_end_to_end() -> None:
|
||||
"""完整形状：一个源 → 两个分支 → 汇合。"""
|
||||
controller = Controller(max_parallel=8)
|
||||
controller.register("mark", _mark)
|
||||
|
||||
pipeline = Pipeline("diamond")
|
||||
source = pipeline.node("mark", node_id="source")
|
||||
left = pipeline.node("mark", node_id="left", depends_on=[source])
|
||||
right = pipeline.node("mark", node_id="right", depends_on=[source])
|
||||
pipeline.node("mark", node_id="sink", depends_on=[left, right])
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p14")
|
||||
assert run.wait(timeout=_WINDOW).status is PipelineStatus.SUCCEEDED
|
||||
|
||||
order = _finished_order(controller, "p14")
|
||||
assert order[0] == "source"
|
||||
assert order[-1] == "sink"
|
||||
assert "sink" in render_pipeline(run)
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_two_runs_of_the_same_pipeline_are_independent() -> None:
|
||||
controller = Controller(max_parallel=4)
|
||||
controller.register("mark", _mark)
|
||||
pipeline = Pipeline("reusable")
|
||||
pipeline.node("mark", node_id="only")
|
||||
|
||||
first = controller.submit_pipeline(pipeline, pipeline_run_id="r1")
|
||||
second = controller.submit_pipeline(pipeline, pipeline_run_id="r2")
|
||||
assert first.wait(timeout=_WINDOW).status is PipelineStatus.SUCCEEDED
|
||||
assert second.wait(timeout=_WINDOW).status is PipelineStatus.SUCCEEDED
|
||||
assert set(first.task_ids.values()).isdisjoint(second.task_ids.values())
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_pipeline_node_args_reach_the_task() -> None:
|
||||
controller = Controller(max_parallel=4)
|
||||
controller.register("add", _add)
|
||||
pipeline = Pipeline("args")
|
||||
pipeline.node("add", 20, 22, node_id="sum")
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p15")
|
||||
assert run.wait(timeout=_WINDOW).status is PipelineStatus.SUCCEEDED
|
||||
snapshot = controller.get("p15:sum")
|
||||
assert snapshot is not None
|
||||
assert snapshot.outcome == {"value": 42}
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 与并发额度协同
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_deferred_nodes_do_not_consume_concurrency_slots() -> None:
|
||||
"""等待上游的节点不该占额度——否则一条长管道会把额度全锁死。"""
|
||||
gate = threading.Event()
|
||||
controller = Controller(max_parallel=2)
|
||||
controller.register("wait_gate", _wait_gate)
|
||||
controller.register("mark", _mark)
|
||||
|
||||
pipeline = Pipeline("slots")
|
||||
root = pipeline.node("wait_gate", gate, node_id="root")
|
||||
for index in range(5):
|
||||
pipeline.node("mark", node_id=f"child{index}", depends_on=[root])
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="p16")
|
||||
try:
|
||||
assert _wait_until(lambda: run.node_status("root") is TaskStatus.RUNNING)
|
||||
time.sleep(0.1)
|
||||
assert controller.active_count == 1, "只有 root 该占额度"
|
||||
assert controller.queued_count == 0, "下游还在 blocked，不是 queued"
|
||||
|
||||
gate.set()
|
||||
assert run.wait(timeout=_WINDOW).status is PipelineStatus.SUCCEEDED
|
||||
finally:
|
||||
gate.set()
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 工具
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _finished_order(controller: Controller, run_id: str) -> list[str]:
|
||||
"""按结束时间排出某个管道 run 里各节点的完成顺序。"""
|
||||
snapshots = [
|
||||
item
|
||||
for item in controller.list_tasks()
|
||||
if item.pipeline_run_id == run_id and item.ended_at is not None
|
||||
]
|
||||
snapshots.sort(key=lambda item: item.ended_at or item.created_at) # type: ignore[arg-type,return-value]
|
||||
return [item.name.split("/", 1)[-1] for item in snapshots]
|
||||
|
||||
|
||||
def _wait_until(predicate: Any, timeout: float = _WINDOW) -> bool:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return True
|
||||
time.sleep(0.01)
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 任务体
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mark(ctx: Any) -> str:
|
||||
del ctx
|
||||
return "ok"
|
||||
|
||||
|
||||
def _add(ctx: Any, left: int, right: int) -> int:
|
||||
del ctx
|
||||
return left + right
|
||||
|
||||
|
||||
def _boom(ctx: Any) -> str:
|
||||
del ctx
|
||||
raise RuntimeError("故意失败")
|
||||
|
||||
|
||||
def _wait_gate(ctx: Any, gate: threading.Event) -> str:
|
||||
while not gate.wait(0.01):
|
||||
ctx.checkpoint()
|
||||
ctx.checkpoint()
|
||||
return "released"
|
||||
@@ -0,0 +1,98 @@
|
||||
"""``PostgresTaskStore`` 的惰性导入契约——**不需要数据库**。
|
||||
|
||||
钉住两条承诺：
|
||||
|
||||
1. 取到 ``chongming_task.PostgresTaskStore`` 之前，iss 不导入 psycopg——核心包保持
|
||||
零第三方运行时依赖（``import chongming_task`` 不会因为没装驱动而变慢或失败）；
|
||||
2. 没装可选依赖时，报错必须**可操作**：给出安装命令，而不是一个光秃秃的
|
||||
``ModuleNotFoundError``。
|
||||
|
||||
真正的行为测试在 ``tests/integration/test_postgres_store.py``（需要 DSN）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import chongming_task
|
||||
|
||||
|
||||
def _run(script: str) -> subprocess.CompletedProcess[str]:
|
||||
environment = {**os.environ}
|
||||
return subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def test_star_import_does_not_pull_the_driver() -> None:
|
||||
"""``from chongming_task import *`` 不该要求安装数据库驱动。"""
|
||||
assert "PostgresTaskStore" not in chongming_task.__all__
|
||||
|
||||
script = (
|
||||
"import sys\n"
|
||||
"from chongming_task import *\n"
|
||||
"assert 'psycopg' not in sys.modules, '星号导入拉起了 psycopg'\n"
|
||||
"assert 'PostgresTaskStore' not in dir(), '它不该出现在星号导入里'\n"
|
||||
"print('ok')\n"
|
||||
)
|
||||
completed = _run(script)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert completed.stdout.strip() == "ok"
|
||||
|
||||
|
||||
def test_lazy_attribute_does_not_import_the_driver_until_used() -> None:
|
||||
script = (
|
||||
"import sys\n"
|
||||
"import chongming_task\n"
|
||||
"assert 'psycopg' not in sys.modules, '导入核心包就拉起了 psycopg'\n"
|
||||
"print('ok')\n"
|
||||
)
|
||||
completed = _run(script)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert completed.stdout.strip() == "ok"
|
||||
|
||||
|
||||
def test_missing_driver_gives_an_actionable_error() -> None:
|
||||
"""把 psycopg 屏蔽成"未安装"，取 ``PostgresTaskStore`` 必须报出安装指引。"""
|
||||
script = (
|
||||
"import sys\n"
|
||||
"sys.modules['psycopg'] = None\n"
|
||||
"sys.modules['psycopg_pool'] = None\n"
|
||||
"import chongming_task\n"
|
||||
"try:\n"
|
||||
" chongming_task.PostgresTaskStore\n"
|
||||
"except ImportError as exc:\n"
|
||||
" text = str(exc)\n"
|
||||
" assert 'chongming-task[postgres]' in text, text\n"
|
||||
" print('ok')\n"
|
||||
"else:\n"
|
||||
" raise SystemExit('没有抛出 ImportError')\n"
|
||||
)
|
||||
completed = _run(script)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert completed.stdout.strip() == "ok"
|
||||
|
||||
|
||||
def test_unknown_attribute_still_raises_attribute_error() -> None:
|
||||
"""``__getattr__`` 只放行已知的可选导出，其余照常抛 AttributeError。"""
|
||||
with pytest.raises(AttributeError, match="no attribute"):
|
||||
_ = chongming_task.NoSuchThing # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_postgres_store_resolves_when_the_extra_is_installed() -> None:
|
||||
"""装了可选依赖时必须能取到——并在 stores 包与顶层给出同一个类。"""
|
||||
pytest.importorskip("psycopg")
|
||||
|
||||
from chongming_task.stores import PostgresTaskStore as FromStores
|
||||
from chongming_task.stores.postgres import PostgresTaskStore as FromModule
|
||||
|
||||
assert chongming_task.PostgresTaskStore is FromModule
|
||||
assert FromStores is FromModule
|
||||
@@ -0,0 +1,390 @@
|
||||
"""进程执行器测试。
|
||||
|
||||
这些用例全部用**真实子进程**（``spawn``）跑：执行器的全部价值就在于子进程能不能被
|
||||
停住、能不能被强杀、父进程能不能看见它。用假的进程模型测不出这些。
|
||||
|
||||
三条约束写在前面，它们是这套设计的一部分而不是测试的毛病：
|
||||
|
||||
* 目标函数必须是**模块级**的——spawn 靠"模块 + 限定名"在子进程里定位它；
|
||||
* 所以子进程会重新 import 本模块（模块顶层不能有副作用）；
|
||||
* 每个用例给足超时（5 秒），子进程数量控制在小规模，测试整体仍然是秒级。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.errors import ConfigurationError
|
||||
from chongming_task.models import FailureKind
|
||||
from chongming_task.runtime.executors.base import ExecutionRequest
|
||||
from chongming_task.runtime.executors.process import ProcessExecutor
|
||||
from chongming_task.testing import FakeClock
|
||||
|
||||
_WINDOW = 10.0
|
||||
|
||||
#: 子进程启动本身就要几百毫秒（新解释器 + import），协作路径的窗口要比线程宽。
|
||||
_START = 15.0
|
||||
|
||||
|
||||
def _request(target: Any, *args: Any, **kwargs: Any) -> ExecutionRequest:
|
||||
return ExecutionRequest(
|
||||
task_id="t-proc",
|
||||
name="t-proc",
|
||||
target=target,
|
||||
args=args,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 子进程里跑的模块级函数（spawn 要求）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _add(ctx: Any, left: int, right: int, *, scale: int = 1) -> int:
|
||||
"""两数相加。"""
|
||||
del ctx
|
||||
return (left + right) * scale
|
||||
|
||||
|
||||
def _boom(ctx: Any) -> None:
|
||||
"""抛异常。"""
|
||||
del ctx
|
||||
raise ValueError("子进程里炸了")
|
||||
|
||||
|
||||
def _describe(ctx: Any) -> dict[str, Any]:
|
||||
"""把注入进来的上下文信息带回来，验证它是真的被注入了。"""
|
||||
return {
|
||||
"task_id": ctx.task_id,
|
||||
"name": ctx.name,
|
||||
"attempt": ctx.attempt,
|
||||
"pid": os.getpid(),
|
||||
"has_token": ctx.token is not None,
|
||||
}
|
||||
|
||||
|
||||
def _beater(ctx: Any) -> str:
|
||||
"""持续 checkpoint，直到进程退出。"""
|
||||
for _ in range(10_000):
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.01)
|
||||
return "never"
|
||||
|
||||
|
||||
def _bounded_beater(ctx: Any) -> str:
|
||||
"""有限次 checkpoint 后正常返回——用于验证"暂停没生效时不能把它卡住"。"""
|
||||
for _ in range(5):
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.1)
|
||||
return "done"
|
||||
|
||||
|
||||
def _late_checkpointer(ctx: Any) -> str:
|
||||
"""先干一段**不 checkpoint** 的活，之后才正常推进。
|
||||
|
||||
用来测暂停回滚：暂停请求会在它到达第一个检查点之前就超时。
|
||||
"""
|
||||
time.sleep(0.5)
|
||||
for _ in range(5):
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.05)
|
||||
return "done"
|
||||
|
||||
|
||||
def _progress_reporter(ctx: Any) -> str:
|
||||
"""报进度，父进程应当收到。"""
|
||||
for index in range(1, 4):
|
||||
ctx.progress(index, total=3, unit="批")
|
||||
time.sleep(0.02)
|
||||
return "ok"
|
||||
|
||||
|
||||
def _ignore_everything(ctx: Any) -> str:
|
||||
"""**不**调用 checkpoint 的死循环：只能靠强杀停下。"""
|
||||
del ctx
|
||||
end = time.monotonic() + 60
|
||||
while time.monotonic() < end:
|
||||
pass
|
||||
return "never"
|
||||
|
||||
|
||||
def _abrupt_exit(ctx: Any) -> str:
|
||||
"""直接消失，连一句结论都不留。"""
|
||||
del ctx
|
||||
os._exit(3)
|
||||
return "never" # pragma: no cover
|
||||
|
||||
|
||||
def _unpicklable_result(ctx: Any) -> Any:
|
||||
"""返回值不可 pickle（线程执行器能接住它，进程执行器不行）。"""
|
||||
del ctx
|
||||
return lambda: None
|
||||
|
||||
|
||||
def _explode_on_unpickle() -> None: # pragma: no cover - 只在子进程里被调用
|
||||
raise RuntimeError("反序列化时炸了")
|
||||
|
||||
|
||||
class _CannotCrossTheBorder:
|
||||
"""能 pickle 出去、却在子进程里反序列化失败的对象。
|
||||
|
||||
``__reduce__`` 只在**加载**时调用那个工厂函数，所以父进程的预检（只 dump）会放行
|
||||
——正好用来测"子进程根本没跑起来"那条分支。
|
||||
"""
|
||||
|
||||
def __reduce__(self) -> Any:
|
||||
return (_explode_on_unpickle, ())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 载荷校验：把"子进程里神秘失败"变成一句能照做的话
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_lambda_target_is_rejected_with_an_actionable_error() -> None:
|
||||
with pytest.raises(ConfigurationError, match="可 pickle"):
|
||||
ProcessExecutor().launch(_request(lambda ctx: "nope"))
|
||||
|
||||
|
||||
def test_a_closure_target_is_rejected() -> None:
|
||||
captured = "closure"
|
||||
|
||||
def inner(ctx: Any) -> str:
|
||||
del ctx
|
||||
return captured
|
||||
|
||||
with pytest.raises(ConfigurationError, match="ThreadExecutor"):
|
||||
ProcessExecutor().launch(_request(inner))
|
||||
|
||||
|
||||
def test_an_unpicklable_argument_is_rejected() -> None:
|
||||
with pytest.raises(ConfigurationError, match="参数"):
|
||||
ProcessExecutor().launch(_request(_add, threading.Lock(), 1))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 正常路径
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_target_runs_in_a_child_process_and_returns_its_value() -> None:
|
||||
handle = ProcessExecutor().launch(
|
||||
ExecutionRequest(
|
||||
task_id="t-args",
|
||||
name="t-args",
|
||||
target=_add,
|
||||
args=(2, 3),
|
||||
kwargs={"scale": 10},
|
||||
)
|
||||
)
|
||||
outcome = handle.wait(timeout=_START)
|
||||
|
||||
assert outcome is not None
|
||||
assert outcome.ok
|
||||
assert outcome.value == 50
|
||||
assert outcome.abandoned is False, "进程没有强杀这个词，结论是已知的"
|
||||
|
||||
|
||||
def test_the_child_has_its_own_pid_and_gets_the_injected_context() -> None:
|
||||
handle = ProcessExecutor().launch(_request(_describe))
|
||||
outcome = handle.wait(timeout=_START)
|
||||
|
||||
assert outcome is not None and outcome.ok
|
||||
payload = outcome.value
|
||||
assert payload["task_id"] == "t-proc"
|
||||
assert payload["name"] == "t-proc"
|
||||
assert payload["has_token"] is True
|
||||
assert payload["pid"] == handle.pid, "子进程里看到的 pid 就是句柄报出来的那个"
|
||||
assert handle.pid != os.getpid(), "任务确实不在监督进程里跑"
|
||||
|
||||
|
||||
def test_the_handle_knows_the_child_pid() -> None:
|
||||
handle = ProcessExecutor().launch(_request(_add, 1, 1))
|
||||
assert handle.pid is not None
|
||||
assert handle.running is True
|
||||
assert handle.wait(timeout=_START) is not None
|
||||
assert handle.running is False, "有结论之后就不该再报'在运行'"
|
||||
|
||||
|
||||
def test_a_task_failure_crosses_the_boundary_as_a_structured_failure() -> None:
|
||||
outcome = ProcessExecutor().launch(_request(_boom)).wait(timeout=_START)
|
||||
|
||||
assert outcome is not None
|
||||
assert not outcome.ok
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.EXCEPTION
|
||||
assert "子进程里炸了" in outcome.failure.message
|
||||
assert outcome.failure.traceback, "子进程里的 traceback 必须带回来"
|
||||
|
||||
|
||||
def test_an_unpicklable_result_is_reported_as_a_result_problem() -> None:
|
||||
"""返回值过不去，报的是"产出接不住"，不是"任务出错了"——两者的修法完全不同。"""
|
||||
outcome = ProcessExecutor().launch(_request(_unpicklable_result)).wait(timeout=_START)
|
||||
|
||||
assert outcome is not None
|
||||
assert not outcome.ok
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.RESULT_INVALID
|
||||
assert outcome.failure.retryable is False, "同一个返回值重跑多少次都还是过不去"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 心跳：idle 超时靠它，而它必须真的跨过进程边界
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_heartbeats_cross_the_process_boundary() -> None:
|
||||
"""把父进程的时钟推远，再等子进程的一次 heartbeat 把它拉回来。
|
||||
|
||||
用假时钟是为了**确定性地**区分"子进程真的报了心跳"与"父进程自己刚记了一次"。
|
||||
"""
|
||||
clock = FakeClock()
|
||||
handle = ProcessExecutor(clock=clock).launch(_request(_beater))
|
||||
try:
|
||||
assert handle.wait(timeout=0.2) is None, "子进程不该这么快结束"
|
||||
clock.advance(300)
|
||||
assert handle.heartbeat.idle_seconds() >= 300
|
||||
|
||||
deadline = time.monotonic() + _START
|
||||
while handle.heartbeat.idle_seconds() >= 300 and time.monotonic() < deadline:
|
||||
time.sleep(0.02)
|
||||
assert handle.heartbeat.idle_seconds() == 0, "子进程的 checkpoint 应当刷新心跳"
|
||||
finally:
|
||||
handle.stop(graceful_timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_progress_reports_are_forwarded_to_the_parent() -> None:
|
||||
seen: list[Any] = []
|
||||
request = ExecutionRequest(
|
||||
task_id="t-progress",
|
||||
name="t-progress",
|
||||
target=_progress_reporter,
|
||||
on_progress=seen.append,
|
||||
)
|
||||
outcome = ProcessExecutor().launch(request).wait(timeout=_START)
|
||||
|
||||
assert outcome is not None and outcome.ok
|
||||
assert [item.done for item in seen] == [1, 2, 3]
|
||||
assert seen[-1].unit == "批"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 暂停：确认了才算，没确认就必须回滚
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_pause_suspends_the_child_and_resume_lets_it_finish() -> None:
|
||||
handle = ProcessExecutor().launch(_request(_bounded_beater))
|
||||
assert handle.pause(timeout=_START) is True, "子进程应当在下一次 checkpoint 停下"
|
||||
|
||||
# 若没真的暂停，这个任务 0.5 秒内就该跑完了。
|
||||
time.sleep(0.8)
|
||||
assert handle.wait(timeout=0) is None, "被暂停的任务不该继续推进"
|
||||
assert handle.running is True
|
||||
|
||||
handle.resume()
|
||||
outcome = handle.wait(timeout=_START)
|
||||
assert outcome is not None and outcome.ok
|
||||
assert outcome.value == "done"
|
||||
|
||||
|
||||
def test_pause_rolls_back_when_the_child_never_reaches_a_checkpoint() -> None:
|
||||
"""暂停请求超时后必须**撤销**。
|
||||
|
||||
不撤销的话，调用方已经按"暂停没生效"把状态回滚成 RUNNING，而子进程会在稍后的
|
||||
某个检查点自己停住：那是一个没人知道、也没人会恢复的挂起——比暂停失败更糟。
|
||||
"""
|
||||
handle = ProcessExecutor().launch(_request(_late_checkpointer))
|
||||
assert handle.wait(timeout=0.1) is None, "它还在那段不 checkpoint 的活里"
|
||||
assert handle.pause(timeout=0.2) is False, "窗口内它压根没到检查点"
|
||||
|
||||
outcome = handle.wait(timeout=_START)
|
||||
assert outcome is not None, "回滚之后它必须继续跑完，而不是停在下一个检查点上"
|
||||
assert outcome.ok, f"不该被自己撤销掉的暂停影响：{outcome.failure}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 停止：进程执行器存在的理由
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cooperative_stop_ends_the_task_without_killing_it() -> None:
|
||||
handle = ProcessExecutor().launch(_request(_beater))
|
||||
assert handle.wait(timeout=0.3) is None # 先让它跑起来
|
||||
|
||||
outcome = handle.stop(graceful_timeout=_WINDOW)
|
||||
|
||||
assert not outcome.ok
|
||||
assert outcome.abandoned is False, "它是自己收尾的，不是被放弃"
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.CANCELLED
|
||||
assert handle.running is False
|
||||
|
||||
|
||||
def test_a_task_that_never_checks_the_token_is_actually_killed() -> None:
|
||||
"""**这是进程执行器唯一不可替代的能力。** 线程只能放弃等待并记 LOST；
|
||||
子进程会真的被停掉，结论因此是已知的。"""
|
||||
handle = ProcessExecutor().launch(_request(_ignore_everything))
|
||||
pid = handle.pid
|
||||
assert pid is not None
|
||||
assert handle.wait(timeout=0.3) is None # 它连 checkpoint 都不调，只能强杀
|
||||
|
||||
outcome = handle.stop(graceful_timeout=0.3)
|
||||
|
||||
assert not outcome.ok
|
||||
assert outcome.abandoned is False, "任务确实停了，不是'不知道停没停'"
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.CANCELLED
|
||||
assert handle.running is False, "载体已经死了，不能谎报还在跑"
|
||||
assert handle.process is not None and handle.process.exitcode is not None
|
||||
|
||||
|
||||
def test_stop_is_also_delivered_through_the_handle_token() -> None:
|
||||
"""控制面有几条路径直接调 ``handle.token.request_stop()``，绕过了 ``stop()``。
|
||||
|
||||
令牌不转发的话子进程永远收不到那次停止——表现出来就是"停止按钮没反应"。
|
||||
"""
|
||||
handle = ProcessExecutor().launch(_request(_beater))
|
||||
assert handle.wait(timeout=0.3) is None
|
||||
|
||||
handle.token.request_stop("走令牌这条路")
|
||||
|
||||
outcome = handle.wait(timeout=_WINDOW)
|
||||
assert outcome is not None
|
||||
assert not outcome.ok
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.CANCELLED
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 崩溃隔离：父进程必须活着，并且如实说清楚发生了什么
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_an_abrupt_child_exit_is_reported_with_its_exit_code() -> None:
|
||||
outcome = ProcessExecutor().launch(_request(_abrupt_exit)).wait(timeout=_START)
|
||||
|
||||
assert outcome is not None
|
||||
assert not outcome.ok
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.NONZERO_EXIT
|
||||
assert outcome.failure.exit_code == 3, "退出码是唯一能解释'它为什么没了'的事实"
|
||||
assert "副作用" in outcome.failure.message, "结果未知必须说清楚，别让人放心重跑"
|
||||
|
||||
|
||||
def test_a_child_that_never_boots_is_reported_distinctly() -> None:
|
||||
"""载荷在子进程里反序列化失败：任务**从未开始**，与"跑一半没了"是两回事。"""
|
||||
handle = ProcessExecutor().launch(_request(_add, _CannotCrossTheBorder(), 1))
|
||||
outcome = handle.wait(timeout=_START)
|
||||
|
||||
assert outcome is not None
|
||||
assert not outcome.ok
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.NONZERO_EXIT
|
||||
assert "从未开始" in outcome.failure.message
|
||||
@@ -0,0 +1,83 @@
|
||||
"""公开 API 的表面测试。
|
||||
|
||||
``chongming_task`` 的 ``__init__`` 是**嵌入其他应用**的入口，因此它本身的正确性
|
||||
值得一条用例：漏导出某个名字、或者导出顺序导致循环导入，都会让集成方第一步就失败。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import chongming_task
|
||||
|
||||
|
||||
def test_version_is_exposed() -> None:
|
||||
assert chongming_task.__version__
|
||||
|
||||
|
||||
def test_all_has_no_duplicates() -> None:
|
||||
"""重复导出会让"到底哪个生效"变得不确定。
|
||||
|
||||
顺序**不在这里**断言：``__all__`` 的排序由 ruff 的 ``RUF022``（isort 风格，
|
||||
全大写常量优先）负责，那是 lint 的职责。写一条与 linter 规则不一致的断言，
|
||||
只会让两边互相打架。
|
||||
"""
|
||||
names = chongming_task.__all__
|
||||
assert len(names) == len(set(names))
|
||||
|
||||
|
||||
def test_every_exported_name_resolves() -> None:
|
||||
"""``__all__`` 里写了却取不到，等于给集成方埋了一个 ImportError。"""
|
||||
missing = [name for name in chongming_task.__all__ if not hasattr(chongming_task, name)]
|
||||
assert missing == []
|
||||
|
||||
|
||||
def test_primary_integration_surface_is_importable() -> None:
|
||||
"""这一段就是 README 里"嵌入你自己的应用"的最小示例。"""
|
||||
from chongming_task import ( # noqa: F401 - 只为验证可导入
|
||||
Controller,
|
||||
InMemoryTaskStore,
|
||||
SqliteTaskStore,
|
||||
TaskStatus,
|
||||
TaskStore,
|
||||
run_task,
|
||||
)
|
||||
|
||||
controller = Controller(max_parallel=2)
|
||||
assert controller.registry.names() == ()
|
||||
assert TaskStatus.PENDING.value == "pending"
|
||||
assert callable(run_task)
|
||||
|
||||
|
||||
def test_store_implementations_satisfy_the_protocol() -> None:
|
||||
"""存储实现必须满足契约——第三方后端也照这条自证。"""
|
||||
from chongming_task import InMemoryTaskStore, SqliteTaskStore, TaskStore
|
||||
|
||||
assert isinstance(InMemoryTaskStore(), TaskStore)
|
||||
|
||||
store = SqliteTaskStore(":memory:")
|
||||
try:
|
||||
assert isinstance(store, TaskStore)
|
||||
finally:
|
||||
# 不关掉连接会在解释器退出时触发 ResourceWarning；本项目的 pytest 配置
|
||||
# 把 warning 升级为 error，因此泄漏是会被抓住的。
|
||||
store.close()
|
||||
|
||||
|
||||
def test_controller_accepts_a_store() -> None:
|
||||
"""存储已经接线。
|
||||
|
||||
这条用例原先钉的是"尚未接线"——当 ``Controller(store=...)`` 真的开始工作时，
|
||||
它会自己失败并提示删掉。现在它钉的是接线本身。
|
||||
"""
|
||||
from chongming_task import Controller, InMemoryTaskStore
|
||||
|
||||
store = InMemoryTaskStore()
|
||||
controller = Controller(store=store, max_parallel=2)
|
||||
assert controller.store is store
|
||||
assert controller.persist_failures == ()
|
||||
assert controller.owner
|
||||
|
||||
|
||||
def test_private_helpers_are_not_exported() -> None:
|
||||
"""以下划线开头的实现细节不应出现在公开表面上。"""
|
||||
leaked = [name for name in chongming_task.__all__ if name.startswith("_")]
|
||||
assert leaked == []
|
||||
@@ -0,0 +1,404 @@
|
||||
"""单任务受控执行的测试。
|
||||
|
||||
覆盖三件用户最关心的事：**失败会怎样**（重试 / 超时 / 最终失败）、**我能不能干预**
|
||||
（停止 / 暂停 / 恢复），以及**结果是否诚实**（不配合的任务落 LOST 而不是"已停止"）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.errors import ConfigurationError
|
||||
from chongming_task.models import FailureKind
|
||||
from chongming_task.policies import FixedBackoff, RetryPolicy, Timeout
|
||||
from chongming_task.run import TaskRun, run_task
|
||||
from chongming_task.runtime.executors.inline import InlineExecutor
|
||||
from chongming_task.states import TaskStatus
|
||||
|
||||
_WINDOW = 10.0
|
||||
|
||||
|
||||
def _wait_for_status(run: TaskRun, status: TaskStatus, timeout: float = _WINDOW) -> bool:
|
||||
"""等状态就位。
|
||||
|
||||
比 ``sleep`` 可靠：``start()`` 是异步的，"任务已经开始执行"与"状态机已迁移到
|
||||
RUNNING"之间存在极短窗口。
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if run.status is status:
|
||||
return True
|
||||
time.sleep(0.01)
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 成功路径
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_successful_run_returns_value_and_succeeds() -> None:
|
||||
outcome = run_task(lambda ctx, a, b: a + b, 2, 3, task_id="t-ok", name="加法")
|
||||
assert outcome.ok
|
||||
assert outcome.status is TaskStatus.SUCCEEDED
|
||||
assert outcome.value == 5
|
||||
assert outcome.failure is None
|
||||
assert outcome.snapshot.ended_at is not None
|
||||
|
||||
|
||||
def test_transition_history_is_complete_and_ordered() -> None:
|
||||
"""状态历史只追加，且每条都带触发源——这是可审计性的基础。"""
|
||||
run = TaskRun(lambda ctx: 1, task_id="t-hist", name="demo")
|
||||
run.start()
|
||||
assert run.wait(timeout=_WINDOW).ok
|
||||
|
||||
assert [t.to_status for t in run.transitions] == [
|
||||
TaskStatus.QUEUED,
|
||||
TaskStatus.STARTING,
|
||||
TaskStatus.RUNNING,
|
||||
TaskStatus.SUCCEEDED,
|
||||
]
|
||||
assert all(t.trigger for t in run.transitions)
|
||||
assert run.transitions[-1].task_id == "t-hist"
|
||||
|
||||
|
||||
def test_exception_is_captured_with_traceback() -> None:
|
||||
def boom(ctx: Any) -> None:
|
||||
raise ValueError("炸了")
|
||||
|
||||
outcome = run_task(boom, task_id="t-boom")
|
||||
assert outcome.status is TaskStatus.FAILED
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.EXCEPTION
|
||||
assert outcome.failure.traceback is not None
|
||||
assert "boom" in outcome.failure.traceback
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 重试
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_retries_until_success_and_counts_attempts() -> None:
|
||||
attempts: list[int] = []
|
||||
|
||||
def flaky(ctx: Any, fail_times: int) -> str:
|
||||
attempts.append(1)
|
||||
if len(attempts) <= fail_times:
|
||||
raise RuntimeError("暂时性故障")
|
||||
return "ok"
|
||||
|
||||
outcome = run_task(
|
||||
flaky,
|
||||
2,
|
||||
retry=RetryPolicy(max_attempts=3, backoff=FixedBackoff(0.0)),
|
||||
task_id="t-flaky",
|
||||
)
|
||||
assert outcome.ok
|
||||
assert outcome.snapshot.attempt == 3
|
||||
assert len(attempts) == 3
|
||||
|
||||
|
||||
def test_retry_exhaustion_lands_failed() -> None:
|
||||
def always_fails(ctx: Any) -> None:
|
||||
raise RuntimeError("一直失败")
|
||||
|
||||
outcome = run_task(
|
||||
always_fails,
|
||||
retry=RetryPolicy(max_attempts=2, backoff=FixedBackoff(0.0)),
|
||||
task_id="t-always",
|
||||
)
|
||||
assert outcome.status is TaskStatus.FAILED
|
||||
assert outcome.snapshot.attempt == 2
|
||||
assert outcome.failure is not None
|
||||
|
||||
|
||||
def test_retry_wait_is_visible_in_history() -> None:
|
||||
seen: list[int] = []
|
||||
|
||||
def flaky(ctx: Any) -> str:
|
||||
if not seen:
|
||||
seen.append(1)
|
||||
raise RuntimeError("第一次失败")
|
||||
return "ok"
|
||||
|
||||
run = TaskRun(
|
||||
flaky,
|
||||
retry=RetryPolicy(max_attempts=2, backoff=FixedBackoff(0.0)),
|
||||
task_id="t-wait",
|
||||
)
|
||||
run.start()
|
||||
assert run.wait(timeout=_WINDOW).ok
|
||||
assert TaskStatus.RETRY_WAIT in [t.to_status for t in run.transitions]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 超时
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_per_attempt_timeout_stops_a_slow_task() -> None:
|
||||
def slow(ctx: Any) -> str:
|
||||
for _ in range(1_000_000):
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.01)
|
||||
return "永远跑不完"
|
||||
|
||||
outcome = run_task(slow, timeout=Timeout(per_attempt=0.3), task_id="t-slow")
|
||||
assert outcome.status is TaskStatus.FAILED
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.TIMEOUT_ATTEMPT
|
||||
|
||||
|
||||
def test_idle_timeout_is_skipped_while_paused() -> None:
|
||||
"""被暂停的任务不推进心跳，若照常判 idle 超时，暂停就会被误判成卡死。"""
|
||||
started = threading.Event()
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
started.set()
|
||||
for _ in range(1_000_000):
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.02)
|
||||
return "done"
|
||||
|
||||
run = TaskRun(
|
||||
work,
|
||||
timeout=Timeout(idle=0.2),
|
||||
task_id="t-idle-paused",
|
||||
graceful_stop=2.0,
|
||||
)
|
||||
run.start()
|
||||
assert started.wait(_WINDOW)
|
||||
assert _wait_for_status(run, TaskStatus.RUNNING)
|
||||
assert run.pause(timeout=_WINDOW) is True
|
||||
|
||||
# 暂停期间超过 idle 阈值，但不应被判超时。
|
||||
time.sleep(0.5)
|
||||
assert run.status is TaskStatus.PAUSED
|
||||
|
||||
run.resume()
|
||||
run.stop(reason="测试结束")
|
||||
assert run.wait(timeout=_WINDOW).status is TaskStatus.STOPPED
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 停止
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_stop_lands_stopped_and_is_auditable() -> None:
|
||||
started = threading.Event()
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
started.set()
|
||||
for _ in range(1_000_000):
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.005)
|
||||
return "done"
|
||||
|
||||
run = TaskRun(work, task_id="t-stop")
|
||||
run.start()
|
||||
assert started.wait(_WINDOW)
|
||||
assert _wait_for_status(run, TaskStatus.RUNNING)
|
||||
|
||||
run.stop(reason="发布窗口")
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
assert outcome.status is TaskStatus.STOPPED
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.CANCELLED
|
||||
assert outcome.failure.retryable is False, "取消不是失败，不能重试"
|
||||
|
||||
stopping = [t for t in run.transitions if t.to_status is TaskStatus.STOPPING]
|
||||
assert stopping, "应当能看到 STOPPING 这个中间态"
|
||||
assert stopping[0].reason == "发布窗口"
|
||||
|
||||
|
||||
def test_uncooperative_task_is_reported_as_lost_not_stopped() -> None:
|
||||
"""线程停不掉时结果**未知**：任务可能仍在后台产生副作用。
|
||||
|
||||
记成"已停止"会让运维误判资源已释放，因此必须落 LOST。
|
||||
"""
|
||||
|
||||
def stubborn(ctx: Any) -> str:
|
||||
for _ in range(500):
|
||||
time.sleep(0.01)
|
||||
return "ignored"
|
||||
|
||||
run = TaskRun(stubborn, task_id="t-stubborn", graceful_stop=0.2)
|
||||
run.start()
|
||||
time.sleep(0.3)
|
||||
run.stop(reason="不配合")
|
||||
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
assert outcome.status is TaskStatus.LOST
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.LOST
|
||||
|
||||
|
||||
def test_stop_before_start_lands_stopped_without_burning_an_attempt() -> None:
|
||||
"""都还没跑就要求停止，不该白白消耗一次尝试。"""
|
||||
run = TaskRun(lambda ctx: "ok", task_id="t-pre-stop")
|
||||
run.stop(reason="还没跑就不要了")
|
||||
run.start()
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
assert outcome.status is TaskStatus.STOPPED
|
||||
assert outcome.snapshot.attempt == 0
|
||||
assert all(t.to_status is not TaskStatus.RUNNING for t in run.transitions)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 暂停与恢复
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_pause_freezes_progress_and_resume_continues() -> None:
|
||||
started = threading.Event()
|
||||
progress: list[int] = []
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
started.set()
|
||||
for index in range(1_000_000):
|
||||
ctx.checkpoint()
|
||||
progress.append(index)
|
||||
time.sleep(0.005)
|
||||
return "done"
|
||||
|
||||
run = TaskRun(work, task_id="t-pause")
|
||||
run.start()
|
||||
assert started.wait(_WINDOW)
|
||||
assert _wait_for_status(run, TaskStatus.RUNNING)
|
||||
|
||||
assert run.pause(timeout=_WINDOW) is True
|
||||
assert run.status is TaskStatus.PAUSED
|
||||
frozen = len(progress)
|
||||
time.sleep(0.15)
|
||||
assert len(progress) == frozen, "暂停生效后不应继续推进"
|
||||
|
||||
run.resume()
|
||||
assert run.status is TaskStatus.RUNNING, "就地暂停的任务恢复后回到 RUNNING"
|
||||
time.sleep(0.15)
|
||||
assert len(progress) > frozen, "恢复后应继续推进"
|
||||
|
||||
run.stop()
|
||||
assert run.wait(timeout=_WINDOW).status is TaskStatus.STOPPED
|
||||
|
||||
|
||||
def test_pause_returns_false_when_task_never_checkpoints() -> None:
|
||||
"""暂停没生效就必须如实回滚为 RUNNING，不能假装已暂停。"""
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
for _ in range(500):
|
||||
time.sleep(0.01)
|
||||
return "ignored"
|
||||
|
||||
run = TaskRun(work, task_id="t-nopause", graceful_stop=0.2)
|
||||
run.start()
|
||||
assert _wait_for_status(run, TaskStatus.RUNNING)
|
||||
|
||||
assert run.pause(timeout=0.2) is False
|
||||
assert run.status is TaskStatus.RUNNING
|
||||
assert any(t.to_status is TaskStatus.PAUSING for t in run.transitions)
|
||||
assert any(
|
||||
t.to_status is TaskStatus.RUNNING and t.trigger == "runtime:pause_aborted"
|
||||
for t in run.transitions
|
||||
), "回滚应当留下 PAUSE_ABORTED 记录"
|
||||
|
||||
run.stop()
|
||||
run.wait(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_pause_on_a_finished_run_is_rejected() -> None:
|
||||
run = TaskRun(lambda ctx: 1, task_id="t-pause-late")
|
||||
run.start()
|
||||
assert run.wait(timeout=_WINDOW).ok
|
||||
assert run.pause(timeout=0.01) is False
|
||||
assert run.status is TaskStatus.SUCCEEDED
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# wait 的契约
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_wait_timeout_raises_and_does_not_change_state() -> None:
|
||||
"""把"我没等到"当成"任务失败"是最常见的误用，因此这里只抛异常。"""
|
||||
release = threading.Event()
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
release.wait(_WINDOW)
|
||||
return "ok"
|
||||
|
||||
run = TaskRun(work, task_id="t-wait")
|
||||
run.start()
|
||||
with pytest.raises(TimeoutError, match="状态未被改变"):
|
||||
run.wait(timeout=0.2)
|
||||
assert run.status in (TaskStatus.STARTING, TaskStatus.RUNNING)
|
||||
|
||||
release.set()
|
||||
assert run.wait(timeout=_WINDOW).ok
|
||||
|
||||
|
||||
def test_wait_before_start_is_an_error() -> None:
|
||||
run = TaskRun(lambda ctx: 1, task_id="t-not-started")
|
||||
with pytest.raises(RuntimeError, match="start"):
|
||||
run.wait()
|
||||
|
||||
|
||||
def test_starting_twice_is_an_error() -> None:
|
||||
run = TaskRun(lambda ctx: 1, task_id="t-twice")
|
||||
run.start()
|
||||
run.wait(timeout=_WINDOW)
|
||||
with pytest.raises(RuntimeError, match="已经启动过"):
|
||||
run.start()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 执行器可替换
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_executor_is_pluggable() -> None:
|
||||
run = TaskRun(
|
||||
lambda ctx: "inline",
|
||||
task_id="t-inline",
|
||||
executor=InlineExecutor(),
|
||||
)
|
||||
run.start()
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
assert outcome.ok
|
||||
assert outcome.value == "inline"
|
||||
assert outcome.snapshot.executor == "inline"
|
||||
|
||||
|
||||
class _RefusingExecutor:
|
||||
"""拉不起来任何任务，只会抛错——进程执行器的"目标不可 pickle"就长这样。"""
|
||||
|
||||
name = "refusing"
|
||||
|
||||
def launch(self, request: Any) -> Any:
|
||||
del request
|
||||
raise ConfigurationError("目标不可 pickle：这类任务请改用线程执行器")
|
||||
|
||||
|
||||
def test_a_launch_failure_is_recorded_as_the_tasks_own_failure() -> None:
|
||||
"""执行器拉不起来时，必须如实记在**任务**上，而不是让异常逃进监督线程。
|
||||
|
||||
逃进去的后果有两条：任务先停在 ``starting``，最后由兜底说成"监督循环内部错误"
|
||||
——那等于把一个任务侧的问题指成框架的 bug；而调用方拿到的诊断里也看不到真正
|
||||
该改的东西。
|
||||
"""
|
||||
outcome = run_task(
|
||||
lambda ctx: "never",
|
||||
task_id="t-refused",
|
||||
executor=_RefusingExecutor(),
|
||||
)
|
||||
|
||||
assert outcome.status is TaskStatus.FAILED
|
||||
assert outcome.failure is not None
|
||||
assert "不可 pickle" in outcome.failure.message
|
||||
assert "监督循环内部错误" not in outcome.failure.message, "别把任务侧的问题说成框架的"
|
||||
assert outcome.failure.retryable is False, "配置类错误重试多少次都是同一个结果"
|
||||
@@ -0,0 +1,413 @@
|
||||
"""状态机行为测试。
|
||||
|
||||
覆盖三类东西：
|
||||
|
||||
1. 转移表被**真正强制执行**，而不是只写在文档里；
|
||||
2. 控制动词语义（stop / pause / resume / retry / reset / force）与设计文档一致；
|
||||
3. 审计记录与版本号的不变量。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.core.state_machine import StateMachine, Trigger
|
||||
from chongming_task.errors import AttemptsExhausted, InvalidTransition
|
||||
from chongming_task.models import Failure, FailureKind, Progress, RequestedAction
|
||||
from chongming_task.states import TaskStatus
|
||||
from chongming_task.testing import FakeClock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clock() -> FakeClock:
|
||||
return FakeClock()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def machine(clock: FakeClock) -> StateMachine:
|
||||
return StateMachine.create("t1", "抽取", max_attempts=3, clock=clock)
|
||||
|
||||
|
||||
def _run_to_running(machine: StateMachine, executor: str = "inline") -> None:
|
||||
machine.mark_queued()
|
||||
machine.mark_starting(executor=executor)
|
||||
machine.mark_running()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 创建与转移表强制
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_create_starts_pending(machine: StateMachine, clock: FakeClock) -> None:
|
||||
assert machine.status is TaskStatus.PENDING
|
||||
assert machine.snapshot.created_at == clock.now
|
||||
assert machine.snapshot.attempt == 0
|
||||
assert machine.history == ()
|
||||
|
||||
|
||||
def test_illegal_transition_raises_and_leaves_state_untouched(
|
||||
machine: StateMachine,
|
||||
) -> None:
|
||||
before = machine.snapshot
|
||||
with pytest.raises(InvalidTransition):
|
||||
machine.mark_running()
|
||||
assert machine.snapshot is before
|
||||
assert machine.history == ()
|
||||
|
||||
|
||||
def test_invalid_transition_message_lists_allowed_targets(machine: StateMachine) -> None:
|
||||
with pytest.raises(InvalidTransition) as info:
|
||||
machine.mark_succeeded()
|
||||
assert info.value.current is TaskStatus.PENDING
|
||||
assert info.value.requested is TaskStatus.SUCCEEDED
|
||||
assert "非法状态转移" in str(info.value)
|
||||
assert "queued" in str(info.value)
|
||||
|
||||
|
||||
def test_happy_path_pending_to_succeeded(machine: StateMachine, clock: FakeClock) -> None:
|
||||
_run_to_running(machine, executor="thread")
|
||||
record = machine.mark_succeeded(outcome={"rows": 10})
|
||||
|
||||
assert record.from_status is TaskStatus.RUNNING
|
||||
assert record.to_status is TaskStatus.SUCCEEDED
|
||||
assert machine.snapshot.outcome == {"rows": 10}
|
||||
assert machine.snapshot.ended_at == clock.now
|
||||
assert machine.snapshot.attempt == 1
|
||||
assert machine.snapshot.validate() == ()
|
||||
assert [t.to_status for t in machine.history] == [
|
||||
TaskStatus.QUEUED,
|
||||
TaskStatus.STARTING,
|
||||
TaskStatus.RUNNING,
|
||||
TaskStatus.SUCCEEDED,
|
||||
]
|
||||
|
||||
|
||||
def test_status_version_increases_by_one_per_transition(machine: StateMachine) -> None:
|
||||
_run_to_running(machine)
|
||||
assert [t.version for t in machine.history] == [1, 2, 3]
|
||||
assert machine.snapshot.status_version == 3
|
||||
|
||||
|
||||
def test_terminal_state_cannot_transition_at_all(machine: StateMachine) -> None:
|
||||
_run_to_running(machine)
|
||||
machine.mark_succeeded()
|
||||
for action in (
|
||||
machine.mark_queued,
|
||||
machine.mark_blocked,
|
||||
machine.mark_running,
|
||||
machine.mark_cancelled,
|
||||
machine.mark_lost,
|
||||
machine.mark_skipped,
|
||||
):
|
||||
with pytest.raises(InvalidTransition):
|
||||
action()
|
||||
assert machine.status is TaskStatus.SUCCEEDED
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 高频旁路更新
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_heartbeat_and_progress_do_not_touch_status_version(
|
||||
machine: StateMachine, clock: FakeClock
|
||||
) -> None:
|
||||
"""心跳是高频写入；若参与乐观并发，会与状态写入持续冲突。"""
|
||||
machine.mark_queued()
|
||||
before = machine.snapshot.status_version
|
||||
clock.advance(5)
|
||||
machine.heartbeat()
|
||||
machine.update_progress(Progress(done=1, total=4))
|
||||
assert machine.snapshot.status_version == before
|
||||
assert machine.snapshot.last_heartbeat_at == clock.now
|
||||
assert machine.snapshot.progress is not None
|
||||
assert machine.snapshot.progress.done == 1
|
||||
|
||||
|
||||
def test_transitions_use_injected_clock(machine: StateMachine, clock: FakeClock) -> None:
|
||||
machine.mark_queued()
|
||||
queued_at = machine.snapshot.queued_at
|
||||
clock.advance(30)
|
||||
machine.mark_starting(executor="inline")
|
||||
assert machine.history[-1].at == clock.now
|
||||
assert queued_at != machine.history[-1].at
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 重试预算
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_attempt_is_consumed_only_when_starting(machine: StateMachine) -> None:
|
||||
"""预算只在"真正要跑"的那一刻消费。"""
|
||||
machine.mark_queued()
|
||||
assert machine.snapshot.attempt == 0
|
||||
machine.mark_starting(executor="inline")
|
||||
assert machine.snapshot.attempt == 1
|
||||
|
||||
|
||||
def test_retry_revives_from_failed_and_keeps_history(clock: FakeClock) -> None:
|
||||
machine = StateMachine.create("t1", "x", max_attempts=3, clock=clock)
|
||||
_run_to_running(machine)
|
||||
machine.mark_failed(Failure(kind=FailureKind.EXCEPTION, message="boom"))
|
||||
|
||||
record = machine.retry()
|
||||
assert record.trigger == Trigger.RETRY_REQUESTED
|
||||
assert machine.status is TaskStatus.QUEUED
|
||||
assert machine.snapshot.failure is None
|
||||
assert machine.snapshot.ended_at is None
|
||||
assert machine.snapshot.attempt == 1 # 历史不被改写
|
||||
machine.mark_starting(executor="inline")
|
||||
assert machine.snapshot.attempt == 2
|
||||
|
||||
|
||||
def test_retry_rejected_when_budget_exhausted(clock: FakeClock) -> None:
|
||||
machine = StateMachine.create("t1", "x", max_attempts=1, clock=clock)
|
||||
_run_to_running(machine)
|
||||
machine.mark_failed(Failure(kind=FailureKind.EXCEPTION, message="boom"))
|
||||
with pytest.raises(AttemptsExhausted):
|
||||
machine.retry()
|
||||
|
||||
|
||||
def test_forced_retry_raises_the_budget(clock: FakeClock) -> None:
|
||||
"""force=True 是"我知道它已失败够多次了，再试一次"。"""
|
||||
machine = StateMachine.create("t1", "x", max_attempts=1, clock=clock)
|
||||
_run_to_running(machine)
|
||||
machine.mark_failed(Failure(kind=FailureKind.EXCEPTION, message="boom"))
|
||||
|
||||
machine.retry(force=True)
|
||||
assert machine.snapshot.max_attempts == 2
|
||||
machine.mark_starting(executor="inline")
|
||||
assert machine.snapshot.attempt == 2
|
||||
|
||||
|
||||
def test_retry_rejected_from_non_revivable_terminal(clock: FakeClock) -> None:
|
||||
"""SUCCEEDED / SKIPPED 不可复活：前者重跑无意义，后者是策略决定的结果。"""
|
||||
machine = StateMachine.create("t1", "x", clock=clock)
|
||||
_run_to_running(machine)
|
||||
machine.mark_succeeded()
|
||||
assert not machine.can_retry
|
||||
with pytest.raises(InvalidTransition):
|
||||
machine.retry()
|
||||
|
||||
|
||||
def test_retry_is_not_the_path_for_retry_wait(clock: FakeClock) -> None:
|
||||
"""RETRY_WAIT 已安排好重试，不应再被 retry() 接受，否则会双倍消费预算。"""
|
||||
machine = StateMachine.create("t1", "x", max_attempts=3, clock=clock)
|
||||
_run_to_running(machine)
|
||||
machine.mark_retry_wait(Failure(kind=FailureKind.EXCEPTION, message="boom"))
|
||||
with pytest.raises(InvalidTransition):
|
||||
machine.retry()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 停止语义
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_stop_on_unstarted_task_lands_stopped_without_buffer(
|
||||
machine: StateMachine,
|
||||
) -> None:
|
||||
"""还没跑起来的任务应当立即落 STOPPED；显示"停止中"是虚假信息。"""
|
||||
record = machine.request_stop(actor="ops")
|
||||
assert record.to_status is TaskStatus.STOPPED
|
||||
assert machine.snapshot.requested_action is None
|
||||
assert machine.snapshot.is_terminal
|
||||
|
||||
|
||||
def test_stop_on_running_task_goes_through_stopping(machine: StateMachine) -> None:
|
||||
_run_to_running(machine)
|
||||
record = machine.request_stop(actor="ops", reason="发布窗口")
|
||||
assert record.to_status is TaskStatus.STOPPING
|
||||
assert machine.snapshot.requested_action is RequestedAction.STOP
|
||||
assert not machine.snapshot.is_terminal
|
||||
|
||||
final = machine.mark_stopped()
|
||||
assert final.to_status is TaskStatus.STOPPED
|
||||
assert machine.snapshot.requested_action is None
|
||||
assert machine.snapshot.validate() == ()
|
||||
|
||||
|
||||
def test_forced_stop_uses_distinct_trigger(machine: StateMachine) -> None:
|
||||
_run_to_running(machine, executor="thread")
|
||||
machine.request_stop(actor="ops")
|
||||
record = machine.mark_stopped(forced=True, reason="优雅窗口超时")
|
||||
assert record.trigger == Trigger.STOP_FORCED
|
||||
assert not record.is_override
|
||||
|
||||
|
||||
def test_task_may_still_succeed_after_stop_requested(machine: StateMachine) -> None:
|
||||
"""停止请求发出后任务恰好跑完，是合法结果，不能判为失败。"""
|
||||
_run_to_running(machine)
|
||||
machine.request_stop(actor="ops")
|
||||
record = machine.mark_succeeded()
|
||||
assert record.to_status is TaskStatus.SUCCEEDED
|
||||
|
||||
|
||||
def test_stop_capability_is_false_on_terminal(machine: StateMachine) -> None:
|
||||
_run_to_running(machine)
|
||||
machine.mark_succeeded()
|
||||
assert not machine.can_stop
|
||||
with pytest.raises(InvalidTransition):
|
||||
machine.request_stop(actor="ops")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 暂停语义
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_pause_on_unstarted_task_lands_paused_directly(machine: StateMachine) -> None:
|
||||
"""排队中的任务本就静止，pause() 应立即落 PAUSED，不该显示"暂停中"。"""
|
||||
machine.mark_queued()
|
||||
record = machine.request_pause(actor="ops")
|
||||
assert record.to_status is TaskStatus.PAUSED
|
||||
assert machine.status is TaskStatus.PAUSED
|
||||
assert machine.snapshot.requested_action is None
|
||||
assert machine.can_resume
|
||||
|
||||
|
||||
def test_pause_while_pausing_is_not_short_circuited(machine: StateMachine) -> None:
|
||||
"""已在 PAUSING 的任务不能被"加急"成 PAUSED：它还没到达检查点。"""
|
||||
_run_to_running(machine)
|
||||
machine.request_pause()
|
||||
assert machine.status is TaskStatus.PAUSING
|
||||
with pytest.raises(InvalidTransition):
|
||||
machine.request_pause()
|
||||
assert machine.status is TaskStatus.PAUSING
|
||||
|
||||
|
||||
def test_starting_always_records_started_at(machine: StateMachine) -> None:
|
||||
"""任何到达 RUNNING 的路径都必须带有 started_at（含暂停撤销路径）。"""
|
||||
machine.mark_queued()
|
||||
machine.mark_starting(executor="inline")
|
||||
assert machine.snapshot.started_at is not None
|
||||
machine.request_pause()
|
||||
machine.abort_pause(reason="未到达检查点")
|
||||
assert machine.status is TaskStatus.RUNNING
|
||||
assert machine.snapshot.validate() == ()
|
||||
|
||||
|
||||
def test_pause_and_resume_round_trip(machine: StateMachine) -> None:
|
||||
_run_to_running(machine)
|
||||
machine.request_pause(actor="ops")
|
||||
assert machine.snapshot.requested_action is RequestedAction.PAUSE
|
||||
|
||||
machine.mark_paused()
|
||||
assert machine.status is TaskStatus.PAUSED
|
||||
|
||||
machine.resume()
|
||||
assert machine.status is TaskStatus.RUNNING
|
||||
assert machine.snapshot.requested_action is None
|
||||
|
||||
|
||||
def test_inplace_resume_returns_to_running_not_queued(machine: StateMachine) -> None:
|
||||
"""就地暂停的任务始终占着执行槽，调用栈还在内存里，恢复就是回到 RUNNING。
|
||||
|
||||
我们没有办法把任意的 Python 调用栈换出内存再换回来，所以"暂停后重新排队"
|
||||
对就地暂停并不成立——那会导致任务状态与实际执行情况脱节。
|
||||
"""
|
||||
_run_to_running(machine)
|
||||
machine.request_pause()
|
||||
machine.mark_paused()
|
||||
machine.resume()
|
||||
assert machine.status is TaskStatus.RUNNING
|
||||
assert machine.snapshot.queued_at is None or machine.snapshot.started_at is not None
|
||||
|
||||
|
||||
def test_resume_of_a_never_started_pause_returns_to_queued(machine: StateMachine) -> None:
|
||||
"""排队期间被暂停的任务从未启动过，恢复必须重新排队。"""
|
||||
machine.mark_queued()
|
||||
assert machine.request_pause().to_status is TaskStatus.PAUSED
|
||||
assert machine.snapshot.attempt == 0
|
||||
|
||||
machine.resume()
|
||||
assert machine.status is TaskStatus.QUEUED
|
||||
|
||||
|
||||
def test_resume_of_a_retry_wait_pause_returns_to_queued(clock: FakeClock) -> None:
|
||||
machine = StateMachine.create("t1", "x", max_attempts=3, clock=clock)
|
||||
_run_to_running(machine)
|
||||
machine.mark_retry_wait(Failure(kind=FailureKind.EXCEPTION, message="boom"))
|
||||
|
||||
assert machine.request_pause().to_status is TaskStatus.PAUSED
|
||||
machine.resume()
|
||||
assert machine.status is TaskStatus.QUEUED
|
||||
|
||||
|
||||
def test_aborted_pause_reverts_to_running(machine: StateMachine) -> None:
|
||||
"""暂停没能生效时必须如实回滚，而不是假装已暂停、让用户以为资源已释放。"""
|
||||
_run_to_running(machine, executor="thread")
|
||||
machine.request_pause()
|
||||
record = machine.abort_pause(reason="任务未调用 checkpoint")
|
||||
assert record.to_status is TaskStatus.RUNNING
|
||||
assert record.trigger == Trigger.PAUSE_ABORTED
|
||||
assert machine.snapshot.requested_action is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 人工干预
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_force_transition_requires_audit_fields(machine: StateMachine) -> None:
|
||||
with pytest.raises(ValueError, match="actor"):
|
||||
machine.force_transition(TaskStatus.SUCCEEDED, actor=" ", reason="ok")
|
||||
with pytest.raises(ValueError, match="reason"):
|
||||
machine.force_transition(TaskStatus.SUCCEEDED, actor="ops", reason="")
|
||||
|
||||
|
||||
def test_force_transition_bypasses_table_and_marks_override(machine: StateMachine) -> None:
|
||||
"""PENDING -> SUCCEEDED 在转移表中非法，人工改写必须能通过，但要留痕。"""
|
||||
record = machine.force_transition(
|
||||
TaskStatus.SUCCEEDED, actor="ops@example", reason="已人工核验产物"
|
||||
)
|
||||
assert record.to_status is TaskStatus.SUCCEEDED
|
||||
assert record.is_override
|
||||
assert record.actor == "ops@example"
|
||||
assert machine.snapshot.overridden
|
||||
assert machine.snapshot.validate() == ()
|
||||
|
||||
|
||||
def test_reset_returns_to_pending_and_clears_attempt_state(clock: FakeClock) -> None:
|
||||
machine = StateMachine.create("t1", "x", max_attempts=2, clock=clock)
|
||||
_run_to_running(machine)
|
||||
machine.mark_failed(Failure(kind=FailureKind.EXCEPTION, message="boom"))
|
||||
|
||||
machine.reset(actor="ops", reason="重新跑一遍")
|
||||
snapshot = machine.snapshot
|
||||
assert snapshot.status is TaskStatus.PENDING
|
||||
assert snapshot.attempt == 0
|
||||
assert snapshot.failure is None
|
||||
assert snapshot.ended_at is None
|
||||
assert snapshot.started_at is None
|
||||
assert not snapshot.overridden
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 能力预检（供控制面提前置灰按钮）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_capability_prechecks_match_actions(machine: StateMachine) -> None:
|
||||
assert machine.can_stop
|
||||
assert not machine.can_pause
|
||||
assert not machine.can_resume
|
||||
|
||||
_run_to_running(machine)
|
||||
assert machine.can_stop
|
||||
assert machine.can_pause
|
||||
assert not machine.can_resume
|
||||
|
||||
machine.request_pause()
|
||||
machine.mark_paused()
|
||||
assert machine.can_resume
|
||||
assert machine.can_stop
|
||||
|
||||
|
||||
def test_lost_is_terminal_but_revivable(machine: StateMachine) -> None:
|
||||
_run_to_running(machine, executor="thread")
|
||||
machine.mark_lost(reason="心跳超时")
|
||||
assert machine.snapshot.is_terminal
|
||||
assert machine.can_retry
|
||||
@@ -0,0 +1,135 @@
|
||||
"""状态表的结构性不变量。
|
||||
|
||||
这些测试只约束 :mod:`chongming_task.states` 的数据本身，不依赖状态机实现。
|
||||
它们的作用是：让"转移表是唯一真相"成为可执行的断言，而不是文档里的承诺。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.states import (
|
||||
NON_TERMINAL,
|
||||
REVIVABLE,
|
||||
TERMINAL,
|
||||
TRANSITIONS,
|
||||
TaskStatus,
|
||||
allowed_from,
|
||||
can_transition,
|
||||
is_terminal,
|
||||
)
|
||||
|
||||
|
||||
def test_transition_table_is_exhaustive() -> None:
|
||||
"""每个规范状态都必须有表项，否则状态机会在运行期 KeyError。"""
|
||||
assert set(TRANSITIONS) == set(TaskStatus)
|
||||
|
||||
|
||||
def test_alias_does_not_add_a_table_key() -> None:
|
||||
"""``FINISHED`` 只是别名，不应在表中占据独立条目。"""
|
||||
assert len(TRANSITIONS) == len(set(TaskStatus))
|
||||
|
||||
|
||||
def test_terminal_set_is_derived_from_the_table() -> None:
|
||||
"""终态必须由表推导，避免"表"与"集合"两处维护而不一致。"""
|
||||
derived = frozenset(status for status, nxt in TRANSITIONS.items() if not nxt)
|
||||
assert derived == TERMINAL
|
||||
|
||||
|
||||
def test_terminal_and_non_terminal_partition_the_enum() -> None:
|
||||
partition = TERMINAL | NON_TERMINAL
|
||||
assert not (TERMINAL & NON_TERMINAL)
|
||||
assert partition == set(TaskStatus)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", sorted(TERMINAL))
|
||||
def test_terminal_states_have_no_outgoing_transitions(status: TaskStatus) -> None:
|
||||
assert allowed_from(status) == frozenset()
|
||||
for target in TaskStatus:
|
||||
assert not can_transition(status, target)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", sorted(NON_TERMINAL))
|
||||
def test_non_terminal_states_have_outgoing_transitions(status: TaskStatus) -> None:
|
||||
"""非终态若无出边，任务会永久卡死。"""
|
||||
assert allowed_from(status), f"{status} 必须有至少一条出边"
|
||||
|
||||
|
||||
def test_no_self_loops() -> None:
|
||||
"""自环会掩盖状态机缺陷，并让 status_version 无意义地增长。"""
|
||||
for source, targets in TRANSITIONS.items():
|
||||
assert source not in targets, f"{source} 存在自环"
|
||||
|
||||
|
||||
def test_all_targets_are_known_statuses() -> None:
|
||||
known = set(TaskStatus)
|
||||
for targets in TRANSITIONS.values():
|
||||
assert targets <= known
|
||||
|
||||
|
||||
@pytest.mark.parametrize("start", sorted(NON_TERMINAL))
|
||||
def test_every_non_terminal_state_can_reach_a_terminal_state(start: TaskStatus) -> None:
|
||||
"""不存在只能内部互跳、永远收敛不了的死角。"""
|
||||
seen: set[TaskStatus] = set()
|
||||
stack: list[TaskStatus] = [start]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if current in TERMINAL:
|
||||
return
|
||||
if current in seen:
|
||||
continue
|
||||
seen.add(current)
|
||||
stack.extend(allowed_from(current))
|
||||
pytest.fail(f"{start} 无法到达任何终态")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 以下用例锁定设计决策本身，防止后续重构悄悄改变语义
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_paused_can_resume_either_in_place_or_back_to_the_queue() -> None:
|
||||
"""恢复目标取决于暂停发生在哪一层：
|
||||
|
||||
* 就地暂停（经 ``PAUSING``）→ ``RUNNING``，槽位从未释放；
|
||||
* 未启动即暂停 → ``QUEUED``，重新排队。
|
||||
"""
|
||||
assert can_transition(TaskStatus.PAUSED, TaskStatus.RUNNING)
|
||||
assert can_transition(TaskStatus.PAUSED, TaskStatus.QUEUED)
|
||||
|
||||
|
||||
def test_stopping_may_still_succeed() -> None:
|
||||
"""停止请求发出后任务恰好自然结束，是合法结果，不能判为失败。"""
|
||||
assert can_transition(TaskStatus.STOPPING, TaskStatus.SUCCEEDED)
|
||||
|
||||
|
||||
def test_pending_cannot_jump_straight_to_running() -> None:
|
||||
"""PENDING 必须经过 QUEUED/STARTING，不允许绕过并发槽与执行器启动。"""
|
||||
assert not can_transition(TaskStatus.PENDING, TaskStatus.RUNNING)
|
||||
|
||||
|
||||
def test_running_cannot_return_to_queued_directly() -> None:
|
||||
"""RUNNING 回落 QUEUED 只能经由 PAUSED，否则会绕过暂停语义。"""
|
||||
assert not can_transition(TaskStatus.RUNNING, TaskStatus.QUEUED)
|
||||
|
||||
|
||||
def test_revivable_is_a_subset_of_terminal() -> None:
|
||||
assert REVIVABLE <= TERMINAL
|
||||
|
||||
|
||||
def test_success_and_skip_are_not_revivable() -> None:
|
||||
"""成功重跑没有意义；SKIPPED 是策略决定的结果，不应被 retry 复活。"""
|
||||
assert TaskStatus.SUCCEEDED not in REVIVABLE
|
||||
assert TaskStatus.SKIPPED not in REVIVABLE
|
||||
|
||||
|
||||
def test_legacy_finished_alias() -> None:
|
||||
assert TaskStatus.FINISHED is TaskStatus.SUCCEEDED
|
||||
assert "finished" not in {s.value for s in TaskStatus}
|
||||
|
||||
|
||||
def test_helpers_agree_with_the_table() -> None:
|
||||
assert is_terminal(TaskStatus.SUCCEEDED)
|
||||
assert not is_terminal(TaskStatus.RUNNING)
|
||||
assert can_transition(TaskStatus.RUNNING, TaskStatus.STOPPING)
|
||||
assert not can_transition(TaskStatus.SUCCEEDED, TaskStatus.RUNNING)
|
||||
@@ -0,0 +1,439 @@
|
||||
"""``@task`` 装饰器与 ``TaskContext`` 的能力测试。
|
||||
|
||||
三块：
|
||||
|
||||
1. **``@task``**——元数据从装饰器来；不产生全局状态（导入无副作用）；
|
||||
2. **进度**——进快照、进存储、**不碰版本号**；
|
||||
3. **上游数据**——管道节点之间真正能传值，且失败的上游不传 ``None``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.controller import Controller, TaskRegistry
|
||||
from chongming_task.decorators import get_task_spec, task
|
||||
from chongming_task.errors import ConfigurationError
|
||||
from chongming_task.pipeline import Pipeline
|
||||
from chongming_task.policies import RetryPolicy, Timeout
|
||||
from chongming_task.states import TaskStatus
|
||||
from chongming_task.stores.memory import InMemoryTaskStore
|
||||
|
||||
_WINDOW = 15.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. @task
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@task
|
||||
def bare(ctx: Any) -> str:
|
||||
"""裸装饰器。"""
|
||||
del ctx
|
||||
return "bare"
|
||||
|
||||
|
||||
@task(tags={"io", "nightly"}, retry=RetryPolicy(max_attempts=4), timeout=Timeout(per_attempt=9.0))
|
||||
def decorated(ctx: Any) -> str:
|
||||
"""带元数据的任务。
|
||||
|
||||
第二行不该出现在描述里。
|
||||
"""
|
||||
del ctx
|
||||
return "decorated"
|
||||
|
||||
|
||||
@task(name="renamed")
|
||||
def original_name(ctx: Any) -> str:
|
||||
del ctx
|
||||
return "renamed"
|
||||
|
||||
|
||||
def undecorated(ctx: Any) -> str:
|
||||
del ctx
|
||||
return "nope"
|
||||
|
||||
|
||||
def test_bare_decorator_works() -> None:
|
||||
"""``@task`` 不带括号也太自然，不支持会让人反复踩空。"""
|
||||
spec = get_task_spec(bare)
|
||||
assert spec is not None
|
||||
assert spec.name == "bare"
|
||||
assert spec.description == "裸装饰器。"
|
||||
|
||||
|
||||
def test_decorator_captures_metadata() -> None:
|
||||
spec = get_task_spec(decorated)
|
||||
assert spec is not None
|
||||
assert spec.tags == frozenset({"io", "nightly"})
|
||||
assert spec.retry is not None
|
||||
assert spec.retry.max_attempts == 4
|
||||
assert spec.timeout is not None
|
||||
assert spec.timeout.per_attempt == 9.0
|
||||
|
||||
|
||||
def test_description_defaults_to_the_first_docstring_line() -> None:
|
||||
spec = get_task_spec(decorated)
|
||||
assert spec is not None
|
||||
assert spec.description == "带元数据的任务。", "只取第一行"
|
||||
assert "第二行" not in spec.description
|
||||
|
||||
|
||||
def test_name_override() -> None:
|
||||
spec = get_task_spec(original_name)
|
||||
assert spec is not None
|
||||
assert spec.name == "renamed"
|
||||
|
||||
|
||||
def test_undecorated_function_has_no_spec() -> None:
|
||||
assert get_task_spec(undecorated) is None
|
||||
|
||||
|
||||
def test_decorating_does_not_create_global_state() -> None:
|
||||
"""规格只挂在函数对象上。
|
||||
|
||||
全局注册表意味着 **import 即注册**，而 import 可能发生在任何地方（包括测试收集
|
||||
阶段），那种隐式行为极难排查。
|
||||
"""
|
||||
registry = TaskRegistry()
|
||||
assert registry.names() == ()
|
||||
assert get_task_spec(bare) is not None, "规格依然取得到，但没有登记进任何注册表"
|
||||
|
||||
|
||||
def test_registry_registers_decorated_functions() -> None:
|
||||
registry = TaskRegistry()
|
||||
registration = registry.register_decorated(decorated)
|
||||
assert registration.name == "decorated"
|
||||
assert registration.tags == frozenset({"io", "nightly"})
|
||||
assert registration.retry.max_attempts == 4
|
||||
|
||||
|
||||
def test_registry_rejects_an_undecorated_function() -> None:
|
||||
registry = TaskRegistry()
|
||||
with pytest.raises(ConfigurationError, match="没有用 @task 装饰过"):
|
||||
registry.register_decorated(undecorated)
|
||||
|
||||
|
||||
def test_register_module_picks_up_every_decorated_function() -> None:
|
||||
import sys
|
||||
|
||||
module = sys.modules[__name__]
|
||||
registry = TaskRegistry()
|
||||
registered = registry.register_module(module)
|
||||
|
||||
names = {item.name for item in registered}
|
||||
assert {"bare", "decorated", "renamed"} <= names
|
||||
assert "undecorated" not in names
|
||||
|
||||
|
||||
def test_register_module_rejects_a_module_with_nothing_decorated() -> None:
|
||||
import json
|
||||
|
||||
registry = TaskRegistry()
|
||||
with pytest.raises(ConfigurationError, match="没有用 @task 装饰过的函数"):
|
||||
registry.register_module(json)
|
||||
|
||||
|
||||
def test_controller_registers_a_module_end_to_end() -> None:
|
||||
import sys
|
||||
|
||||
controller = Controller(max_parallel=2)
|
||||
controller.register_module(sys.modules[__name__])
|
||||
run = controller.submit("bare")
|
||||
outcome = controller.wait(run.snapshot.task_id, timeout=_WINDOW)
|
||||
assert outcome.ok
|
||||
assert outcome.value == "bare"
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_decorated_retry_actually_applies() -> None:
|
||||
"""装饰器上写的重试次数必须真的生效，而不只是躺在元数据里。"""
|
||||
store = InMemoryTaskStore()
|
||||
controller = Controller(store=store, max_parallel=2)
|
||||
registry = TaskRegistry()
|
||||
registry.register(
|
||||
"flaky",
|
||||
_flaky,
|
||||
retry=RetryPolicy(max_attempts=3, backoff=_no_backoff()),
|
||||
)
|
||||
controller.register("flaky", _flaky, retry=RetryPolicy(max_attempts=3, backoff=_no_backoff()))
|
||||
run = controller.submit("flaky")
|
||||
outcome = controller.wait(run.snapshot.task_id, timeout=_WINDOW)
|
||||
assert outcome.ok, "第三次应成功"
|
||||
snapshot = store.get(run.snapshot.task_id)
|
||||
assert snapshot is not None
|
||||
assert snapshot.attempt == 3
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. TaskContext：进度与日志
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_progress_reaches_the_snapshot_without_bumping_the_version() -> None:
|
||||
"""进度若参与乐观并发，会与状态写入持续冲突。"""
|
||||
controller = Controller(max_parallel=2)
|
||||
controller.register("reporter", _reporter)
|
||||
run = controller.submit("reporter")
|
||||
outcome = controller.wait(run.snapshot.task_id, timeout=_WINDOW)
|
||||
assert outcome.ok
|
||||
|
||||
snapshot = outcome.snapshot
|
||||
assert snapshot.progress is not None
|
||||
assert snapshot.progress.done == 3
|
||||
assert snapshot.progress.total == 3
|
||||
assert snapshot.progress.unit == "rows"
|
||||
# 状态迁移只有 4 次（queued/starting/running/succeeded），进度不该挤进去。
|
||||
assert snapshot.status_version == 4
|
||||
|
||||
|
||||
def test_progress_is_persisted_and_visible_in_queries() -> None:
|
||||
"""进度要经 touch 旁路落到存储，否则看板的 PROGRESS 列永远是空的。"""
|
||||
store = InMemoryTaskStore()
|
||||
controller = Controller(store=store, max_parallel=2, command_poll_interval=0.05)
|
||||
controller.register("slow_reporter", _slow_reporter)
|
||||
run = controller.submit("slow_reporter")
|
||||
|
||||
assert _wait_until(
|
||||
lambda: (store.get(run.snapshot.task_id) or _stub()).progress is not None,
|
||||
timeout=_WINDOW,
|
||||
)
|
||||
persisted = store.get(run.snapshot.task_id)
|
||||
assert persisted is not None
|
||||
assert persisted.progress is not None
|
||||
assert persisted.progress.done >= 1
|
||||
|
||||
controller.stop(run.snapshot.task_id)
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_progress_on_an_unsupported_context_is_silently_dropped() -> None:
|
||||
"""没有回写通道时静默丢弃，而不是抛异常。
|
||||
|
||||
直接构造上下文（不经过执行器）是测试与工具里的常见用法，不该因此报错。
|
||||
"""
|
||||
from chongming_task.runtime.cancellation import CancellationToken
|
||||
from chongming_task.runtime.context import RuntimeContext
|
||||
from chongming_task.runtime.heartbeat import Heartbeat
|
||||
|
||||
context = RuntimeContext(
|
||||
task_id="t",
|
||||
name="t",
|
||||
attempt=1,
|
||||
token=CancellationToken(),
|
||||
heartbeat=Heartbeat(),
|
||||
)
|
||||
context.progress(1, total=2)
|
||||
context.log("hello")
|
||||
assert list(context.logs) == ["hello"]
|
||||
|
||||
|
||||
def test_log_records_into_the_buffer_and_the_stdlib_logger(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
# caplog 默认只捕获 WARNING 及以上，而 INFO 才是我们最常用的级别。
|
||||
caplog.set_level(logging.INFO, logger="chongming_task.task")
|
||||
|
||||
controller = Controller(max_parallel=2)
|
||||
controller.register("talker", _talker)
|
||||
run = controller.submit("talker")
|
||||
outcome = controller.wait(run.snapshot.task_id, timeout=_WINDOW)
|
||||
assert outcome.ok
|
||||
|
||||
assert outcome.value == "done"
|
||||
messages = [record.message for record in caplog.records]
|
||||
assert "hello" in messages, "INFO 日志应当走到标准库 logging"
|
||||
assert "careful" in messages
|
||||
|
||||
record = next(r for r in caplog.records if r.message == "hello")
|
||||
assert record.task_id == run.snapshot.task_id
|
||||
assert record.task_name == "talker"
|
||||
assert record.attempt == 1
|
||||
|
||||
|
||||
def test_task_context_is_the_runtime_context() -> None:
|
||||
"""公开名字与内部名字是**同一个类**。
|
||||
|
||||
任务作者只面对一个 ``ctx``；两个类会立刻带来"我该用哪个"的困惑。
|
||||
"""
|
||||
from chongming_task import RuntimeContext, TaskContext
|
||||
|
||||
assert TaskContext is RuntimeContext
|
||||
|
||||
|
||||
def test_checkpoint_still_works_alongside_progress() -> None:
|
||||
controller = Controller(max_parallel=2)
|
||||
controller.register("stopme", _stopme)
|
||||
run = controller.submit("stopme")
|
||||
assert _wait_until(lambda: run.status is TaskStatus.RUNNING)
|
||||
|
||||
controller.stop(run.snapshot.task_id, reason="收工")
|
||||
outcome = controller.wait(run.snapshot.task_id, timeout=_WINDOW)
|
||||
assert outcome.status is TaskStatus.STOPPED
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. 上游数据（管道传值）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_downstream_receives_upstream_values() -> None:
|
||||
controller = Controller(max_parallel=4)
|
||||
controller.register("produce", _produce)
|
||||
controller.register("consume", _consume)
|
||||
|
||||
pipeline = Pipeline("data")
|
||||
source = pipeline.node("produce", node_id="source")
|
||||
pipeline.node("consume", node_id="sink", depends_on=[source])
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="d1")
|
||||
outcome = run.wait(timeout=_WINDOW)
|
||||
assert outcome.status.value == "succeeded"
|
||||
assert outcome.nodes["sink"] is TaskStatus.SUCCEEDED
|
||||
|
||||
snapshot = controller.get("d1:sink")
|
||||
assert snapshot is not None
|
||||
assert snapshot.outcome == {"value": 42}, "下游应当拿到上游的返回值 42"
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_a_failed_upstream_contributes_no_value() -> None:
|
||||
"""失败的上游不塞 ``None``：那会让下游以为"上游返回了 None"。"""
|
||||
controller = Controller(max_parallel=4)
|
||||
controller.register("boom", _boom)
|
||||
controller.register("inspect", _inspect)
|
||||
|
||||
pipeline = Pipeline("missing")
|
||||
bad = pipeline.node("boom", node_id="bad")
|
||||
pipeline.node("inspect", node_id="probe", depends_on=[bad], join=_all_settled())
|
||||
|
||||
run = controller.submit_pipeline(pipeline, pipeline_run_id="d2")
|
||||
run.wait(timeout=_WINDOW)
|
||||
|
||||
snapshot = controller.get("d2:probe")
|
||||
assert snapshot is not None
|
||||
assert snapshot.outcome == {"value": []}, "上游没有值，ctx.upstream 必须是空的"
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_input_can_be_read_with_a_default() -> None:
|
||||
controller = Controller()
|
||||
controller.register("use_default", _use_default)
|
||||
run = controller.submit("use_default")
|
||||
outcome = controller.wait(run.snapshot.task_id, timeout=_WINDOW)
|
||||
assert outcome.value == "fallback"
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
def test_inputs_cannot_be_injected_after_the_task_started() -> None:
|
||||
controller = Controller(max_parallel=2)
|
||||
controller.register("stopme", _stopme)
|
||||
run = controller.submit("stopme")
|
||||
assert _wait_until(lambda: run.status is TaskStatus.RUNNING)
|
||||
|
||||
with pytest.raises(RuntimeError, match="已经开始执行"):
|
||||
run.set_inputs({"x": 1})
|
||||
|
||||
controller.stop(run.snapshot.task_id)
|
||||
controller.wait_all(timeout=_WINDOW)
|
||||
controller.close(timeout=_WINDOW)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 任务体
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _stub() -> Any:
|
||||
from chongming_task.models import TaskSnapshot
|
||||
|
||||
return TaskSnapshot(task_id="stub", name="stub")
|
||||
|
||||
|
||||
def _no_backoff() -> Any:
|
||||
from chongming_task.policies import FixedBackoff
|
||||
|
||||
return FixedBackoff(0.0)
|
||||
|
||||
|
||||
def _flaky(ctx: Any) -> str:
|
||||
state = getattr(_flaky, "_count", 0) + 1
|
||||
_flaky._count = state # type: ignore[attr-defined]
|
||||
if state < 3:
|
||||
raise RuntimeError(f"第 {state} 次失败")
|
||||
return "ok"
|
||||
|
||||
|
||||
def _reporter(ctx: Any) -> str:
|
||||
for index in range(1, 4):
|
||||
ctx.checkpoint()
|
||||
ctx.progress(index, total=3, unit="rows")
|
||||
return "reported"
|
||||
|
||||
|
||||
def _slow_reporter(ctx: Any) -> str:
|
||||
for index in range(1, 200):
|
||||
ctx.checkpoint()
|
||||
ctx.progress(index, total=200, unit="rows")
|
||||
time.sleep(0.01)
|
||||
return "done"
|
||||
|
||||
|
||||
def _talker(ctx: Any) -> str:
|
||||
ctx.log("hello")
|
||||
ctx.warning("careful")
|
||||
return "done"
|
||||
|
||||
|
||||
def _stopme(ctx: Any) -> str:
|
||||
for _ in range(1_000_000):
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.005)
|
||||
return "never"
|
||||
|
||||
|
||||
def _produce(ctx: Any) -> int:
|
||||
del ctx
|
||||
return 42
|
||||
|
||||
|
||||
def _consume(ctx: Any) -> int:
|
||||
value = ctx.input("source")
|
||||
assert value == 42, f"预期 42，收到 {value!r}"
|
||||
return int(value)
|
||||
|
||||
|
||||
def _inspect(ctx: Any) -> list[str]:
|
||||
return sorted(ctx.upstream)
|
||||
|
||||
|
||||
def _use_default(ctx: Any) -> str:
|
||||
return str(ctx.input("nobody", "fallback"))
|
||||
|
||||
|
||||
def _boom(ctx: Any) -> str:
|
||||
del ctx
|
||||
raise RuntimeError("故意失败")
|
||||
|
||||
|
||||
def _all_settled() -> Any:
|
||||
from chongming_task.policies import JoinMode
|
||||
|
||||
return JoinMode.ALL_SETTLED
|
||||
|
||||
|
||||
def _wait_until(predicate: Any, timeout: float = _WINDOW) -> bool:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return True
|
||||
time.sleep(0.01)
|
||||
return False
|
||||
@@ -0,0 +1,410 @@
|
||||
"""线程执行器测试。
|
||||
|
||||
这些用例全部使用**真实线程与真实时间**：执行器的全部价值就在于线程能不能被
|
||||
停下、能不能被暂停、放弃之后进程还能不能退出。用假的线程模型测不出这些。
|
||||
|
||||
为避免偶发失败，协作路径给足超时（5 秒），而"不配合"的用例刻意把任务做成
|
||||
有限时长（约 5 秒后自行结束），这样即使被放弃，残留的守护线程也不会长期占用资源。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.models import FailureKind
|
||||
from chongming_task.runtime.executors.base import ExecutionRequest
|
||||
from chongming_task.runtime.executors.thread import ThreadExecutor
|
||||
from chongming_task.testing import FakeClock
|
||||
|
||||
_COOPERATIVE_WINDOW = 5.0
|
||||
_ABANDON_WINDOW = 0.2
|
||||
|
||||
|
||||
def _request(task_id: str, target: Any, *args: Any) -> ExecutionRequest:
|
||||
return ExecutionRequest(task_id=task_id, name=task_id, target=target, args=args)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 上下文注入契约
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_context_is_injected_with_the_handle_token_and_metadata() -> None:
|
||||
"""令牌必须是句柄那一个，否则控制面会停错对象。
|
||||
|
||||
这也是"令牌由执行器注入、而非被任务闭包捕获"的直接体现——令牌每次尝试才诞生，
|
||||
闭包捕获到的只可能是上一次尝试留下的那一个。
|
||||
"""
|
||||
seen: dict[str, Any] = {}
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
seen["token"] = ctx.token
|
||||
seen["task_id"] = ctx.task_id
|
||||
seen["name"] = ctx.name
|
||||
seen["attempt"] = ctx.attempt
|
||||
return "ok"
|
||||
|
||||
handle = ThreadExecutor().launch(
|
||||
ExecutionRequest(task_id="t-ctx", name="抽取", target=work, attempt=2)
|
||||
)
|
||||
assert handle.wait(timeout=_COOPERATIVE_WINDOW) is not None
|
||||
assert seen["token"] is handle.token
|
||||
assert seen["task_id"] == "t-ctx"
|
||||
assert seen["name"] == "抽取"
|
||||
assert seen["attempt"] == 2
|
||||
|
||||
|
||||
def test_arguments_are_forwarded_after_the_context() -> None:
|
||||
def work(ctx: Any, left: int, right: int, *, scale: int = 1) -> int:
|
||||
del ctx
|
||||
return (left + right) * scale
|
||||
|
||||
handle = ThreadExecutor().launch(
|
||||
ExecutionRequest(
|
||||
task_id="t-args",
|
||||
name="t-args",
|
||||
target=work,
|
||||
args=(2, 3),
|
||||
kwargs={"scale": 10},
|
||||
)
|
||||
)
|
||||
outcome = handle.wait(timeout=_COOPERATIVE_WINDOW)
|
||||
assert outcome is not None
|
||||
assert outcome.value == 50
|
||||
|
||||
|
||||
def test_checkpoint_refreshes_the_heartbeat() -> None:
|
||||
"""checkpoint 就是"我还在推进"的信号，心跳绑在它上面是有意的。"""
|
||||
clock = FakeClock()
|
||||
observed: list[float] = []
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
for _ in range(3):
|
||||
clock.advance(30)
|
||||
observed.append(ctx.heartbeat.idle_seconds())
|
||||
ctx.checkpoint()
|
||||
return "ok"
|
||||
|
||||
handle = ThreadExecutor(clock=clock).launch(_request("t-beat", work))
|
||||
outcome = handle.wait(timeout=_COOPERATIVE_WINDOW)
|
||||
assert outcome is not None
|
||||
assert outcome.ok
|
||||
assert observed == [30.0, 30.0, 30.0], "每次 checkpoint 都应把静默时长清零"
|
||||
assert handle.heartbeat.idle_seconds() == 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# async 任务
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_async_target_is_actually_awaited() -> None:
|
||||
"""``async def`` 任务必须真的被执行完，而不是把协程对象当返回值。
|
||||
|
||||
后者曾经是静默的：没有存储时它看起来"成功"了，返回值却是个 ``coroutine``；
|
||||
有存储时才会以"返回值不可序列化"失败。
|
||||
"""
|
||||
|
||||
async def work(ctx: Any) -> str:
|
||||
await asyncio.sleep(0)
|
||||
return "done"
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-async", work))
|
||||
outcome = handle.wait(timeout=_COOPERATIVE_WINDOW)
|
||||
assert outcome is not None
|
||||
assert outcome.ok
|
||||
assert outcome.value == "done"
|
||||
|
||||
|
||||
def test_async_target_cooperates_with_stop() -> None:
|
||||
"""协作式停止对 async 任务同样有效：``await ctx.acheckpoint()``。"""
|
||||
|
||||
async def work(ctx: Any) -> str:
|
||||
for _ in range(10_000):
|
||||
await ctx.acheckpoint()
|
||||
await asyncio.sleep(0.005)
|
||||
return "never"
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-async-stop", work))
|
||||
outcome = handle.stop(graceful_timeout=_COOPERATIVE_WINDOW)
|
||||
assert not outcome.ok, "停止请求之后不该报成功"
|
||||
assert not outcome.abandoned, "它会到达检查点，应当优雅收尾而不是被放弃"
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.CANCELLED
|
||||
|
||||
|
||||
def test_async_target_failure_is_reported_across_the_loop() -> None:
|
||||
"""协程里抛的异常照常转成结构化失败，不能被事件循环吞掉。"""
|
||||
|
||||
async def work(ctx: Any) -> str:
|
||||
await asyncio.sleep(0)
|
||||
raise ValueError("协程里炸了")
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-async-boom", work))
|
||||
outcome = handle.wait(timeout=_COOPERATIVE_WINDOW)
|
||||
assert outcome is not None
|
||||
assert not outcome.ok
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.EXCEPTION
|
||||
assert "协程里炸了" in outcome.failure.message
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 正常路径
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_launch_returns_immediately_without_waiting_for_the_task() -> None:
|
||||
"""调度器契约：launch 必须立刻返回，否则调度循环会被单个任务卡住。"""
|
||||
release = threading.Event()
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
del ctx
|
||||
release.wait(5)
|
||||
return "done"
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-async-launch", work))
|
||||
try:
|
||||
assert handle.running is True
|
||||
assert handle.wait(timeout=0.05) is None, "任务尚未结束，wait 应返回 None"
|
||||
finally:
|
||||
release.set()
|
||||
outcome = handle.wait(timeout=_COOPERATIVE_WINDOW)
|
||||
assert outcome is not None
|
||||
assert outcome.ok
|
||||
assert outcome.value == "done"
|
||||
|
||||
|
||||
def test_successful_task_returns_its_value() -> None:
|
||||
handle = ThreadExecutor().launch(_request("t-ok", lambda ctx: 6 * 7))
|
||||
outcome = handle.wait(timeout=_COOPERATIVE_WINDOW)
|
||||
assert outcome is not None
|
||||
assert outcome.ok
|
||||
assert outcome.value == 42
|
||||
assert handle.running is False
|
||||
|
||||
|
||||
def test_failing_task_reports_exception_with_traceback() -> None:
|
||||
def boom(ctx: Any) -> None:
|
||||
del ctx
|
||||
raise ValueError("坏掉了")
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-boom", boom))
|
||||
outcome = handle.wait(timeout=_COOPERATIVE_WINDOW)
|
||||
assert outcome is not None
|
||||
assert not outcome.ok
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.EXCEPTION
|
||||
assert "ValueError" in outcome.failure.message
|
||||
assert outcome.failure.traceback is not None
|
||||
assert "boom" in outcome.failure.traceback
|
||||
assert outcome.failure.retryable is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 协作式停止
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_stop_completes_cooperatively_when_task_checks_the_token() -> None:
|
||||
started = threading.Event()
|
||||
steps: list[int] = []
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
started.set()
|
||||
for index in range(100_000):
|
||||
ctx.checkpoint()
|
||||
steps.append(index)
|
||||
time.sleep(0.001)
|
||||
return "ran to completion"
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-coop", work))
|
||||
assert started.wait(_COOPERATIVE_WINDOW)
|
||||
|
||||
outcome = handle.stop(graceful_timeout=_COOPERATIVE_WINDOW, reason="发布窗口")
|
||||
assert outcome.abandoned is False
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.CANCELLED
|
||||
assert outcome.failure.retryable is False, "取消不是失败，绝不能被重试"
|
||||
assert handle.running is False
|
||||
assert len(steps) < 100_000, "任务应当在收到停止请求后提前退出"
|
||||
|
||||
|
||||
def test_cancellation_is_not_swallowed_by_a_broad_except_in_the_task_body() -> None:
|
||||
"""这正是 ``TaskCancelled`` 继承 ``BaseException`` 的理由。
|
||||
|
||||
若它继承 ``Exception``，下面这种极常见的写法会把停止信号吃掉，任务永远停不掉。
|
||||
"""
|
||||
started = threading.Event()
|
||||
swallowed: list[BaseException] = []
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
started.set()
|
||||
try:
|
||||
while True:
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.001)
|
||||
except Exception as exc: # 用户代码里最常见的写法
|
||||
swallowed.append(exc)
|
||||
return "signal was swallowed"
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-swallow", work))
|
||||
assert started.wait(_COOPERATIVE_WINDOW)
|
||||
|
||||
outcome = handle.stop(graceful_timeout=_COOPERATIVE_WINDOW)
|
||||
assert swallowed == [], "取消信号被 except Exception 吞掉了"
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.CANCELLED
|
||||
|
||||
|
||||
def test_stop_is_idempotent() -> None:
|
||||
def work(ctx: Any) -> str:
|
||||
del ctx
|
||||
return "quick"
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-idem", work))
|
||||
first = handle.stop(graceful_timeout=_COOPERATIVE_WINDOW)
|
||||
second = handle.stop(graceful_timeout=_COOPERATIVE_WINDOW)
|
||||
assert first.ok
|
||||
assert second.ok
|
||||
assert handle.running is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 协作式暂停（D6 定案的语义，端到端验证）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_pause_suspends_a_running_task_and_resume_continues_it() -> None:
|
||||
started = threading.Event()
|
||||
progress: list[int] = []
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
started.set()
|
||||
for index in range(100_000):
|
||||
ctx.checkpoint()
|
||||
progress.append(index)
|
||||
time.sleep(0.001)
|
||||
return "ran to completion"
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-pause", work))
|
||||
assert started.wait(_COOPERATIVE_WINDOW)
|
||||
|
||||
assert handle.pause(timeout=_COOPERATIVE_WINDOW) is True
|
||||
frozen = len(progress)
|
||||
assert frozen > 0, "暂停前任务应当已经推进过"
|
||||
time.sleep(0.05)
|
||||
assert len(progress) == frozen, "暂停生效后任务不应继续推进"
|
||||
|
||||
handle.resume()
|
||||
time.sleep(0.05)
|
||||
assert len(progress) > frozen, "恢复后任务应继续推进"
|
||||
|
||||
outcome = handle.stop(graceful_timeout=_COOPERATIVE_WINDOW)
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.CANCELLED
|
||||
|
||||
|
||||
def test_pause_reports_false_when_task_never_reaches_a_checkpoint() -> None:
|
||||
"""这正是"暂停没生效必须回滚为 RUNNING"的依据：如实返回 False。"""
|
||||
started = threading.Event()
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
del ctx
|
||||
started.set()
|
||||
for _ in range(500):
|
||||
time.sleep(0.01)
|
||||
return "ignored"
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-nopause", work))
|
||||
assert started.wait(_COOPERATIVE_WINDOW)
|
||||
assert handle.pause(timeout=0.2) is False
|
||||
handle.stop(graceful_timeout=_ABANDON_WINDOW)
|
||||
|
||||
|
||||
def test_stop_wakes_a_paused_task() -> None:
|
||||
"""暂停中的任务必须能被停止，否则暂停就成了不可撤销的挂起。"""
|
||||
started = threading.Event()
|
||||
|
||||
def work(ctx: Any) -> str:
|
||||
started.set()
|
||||
for _ in range(100_000):
|
||||
ctx.checkpoint()
|
||||
time.sleep(0.001)
|
||||
return "ran to completion"
|
||||
|
||||
handle = ThreadExecutor().launch(_request("t-pause-stop", work))
|
||||
assert started.wait(_COOPERATIVE_WINDOW)
|
||||
assert handle.pause(timeout=_COOPERATIVE_WINDOW) is True
|
||||
|
||||
outcome = handle.stop(graceful_timeout=_COOPERATIVE_WINDOW, reason="运维介入")
|
||||
assert outcome.abandoned is False
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.CANCELLED
|
||||
assert "运维介入" in outcome.failure.message
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 无法停止：放弃
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _uncooperative(ctx: Any) -> str:
|
||||
"""刻意不检查令牌；约 5 秒后自行结束，避免残留线程长期占用资源。"""
|
||||
del ctx
|
||||
for _ in range(500):
|
||||
time.sleep(0.01)
|
||||
return "ignored the stop request"
|
||||
|
||||
|
||||
def test_uncooperative_task_is_abandoned_and_reported_as_lost() -> None:
|
||||
"""线程无法强杀：必须如实上报，不能假装已停止。"""
|
||||
handle = ThreadExecutor().launch(_request("t-stubborn", _uncooperative))
|
||||
outcome = handle.stop(graceful_timeout=_ABANDON_WINDOW)
|
||||
assert outcome.abandoned is True
|
||||
assert outcome.ok is False
|
||||
assert outcome.failure is not None
|
||||
assert outcome.failure.kind is FailureKind.LOST
|
||||
assert "无法强制终止" in outcome.failure.message
|
||||
|
||||
|
||||
def test_abandoned_task_still_reports_running() -> None:
|
||||
"""它确实还在跑。谎报为 False 会让上层以为资源已经释放。"""
|
||||
handle = ThreadExecutor().launch(_request("t-still-running", _uncooperative))
|
||||
handle.stop(graceful_timeout=_ABANDON_WINDOW)
|
||||
assert handle.running is True
|
||||
|
||||
|
||||
def test_abandoned_outcome_is_sticky() -> None:
|
||||
"""已记为 LOST 的结论不应被后台线程后来的返回悄悄改写，否则审计自相矛盾。"""
|
||||
handle = ThreadExecutor().launch(_request("t-sticky", _uncooperative))
|
||||
first = handle.stop(graceful_timeout=_ABANDON_WINDOW)
|
||||
assert handle.wait(timeout=_ABANDON_WINDOW) is first
|
||||
assert handle.stop(graceful_timeout=_ABANDON_WINDOW) is first
|
||||
|
||||
|
||||
def test_worker_threads_are_daemon_so_an_abandoned_task_cannot_block_shutdown() -> None:
|
||||
"""``ThreadPoolExecutor`` 的工作线程会在解释器退出时被 join，一个不响应取消
|
||||
的任务会让整个进程无法退出。这里用守护线程正是为了避开这一点。"""
|
||||
handle = ThreadExecutor().launch(_request("t-daemon", _uncooperative))
|
||||
try:
|
||||
assert handle.thread.daemon is True
|
||||
assert handle.thread.name == "task-t-daemon"
|
||||
finally:
|
||||
handle.stop(graceful_timeout=_ABANDON_WINDOW)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("graceful_timeout", [0.0, 0.05])
|
||||
def test_zero_or_tiny_graceful_window_still_yields_a_conclusion(
|
||||
graceful_timeout: float,
|
||||
) -> None:
|
||||
"""优雅窗口再短也必须给出结论，不能抛异常或永远阻塞。"""
|
||||
handle = ThreadExecutor().launch(_request(f"t-window-{graceful_timeout}", _uncooperative))
|
||||
outcome = handle.stop(graceful_timeout=graceful_timeout)
|
||||
assert outcome.abandoned is True
|
||||
assert outcome.failure is not None
|
||||
@@ -0,0 +1,154 @@
|
||||
"""心跳与超时判定的测试。
|
||||
|
||||
全部基于 ``FakeClock``：不 sleep、不依赖真实时间，因此既快又不会有偶发失败。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from chongming_task.models import FailureKind, TimeoutScope
|
||||
from chongming_task.policies import Timeout
|
||||
from chongming_task.runtime.heartbeat import Heartbeat
|
||||
from chongming_task.runtime.timeouts import (
|
||||
Deadline,
|
||||
is_expired,
|
||||
next_deadline,
|
||||
remaining_seconds,
|
||||
)
|
||||
from chongming_task.testing import FakeClock
|
||||
|
||||
_T0 = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 心跳
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_heartbeat_starts_at_construction() -> None:
|
||||
clock = FakeClock(_T0)
|
||||
beat = Heartbeat(clock)
|
||||
assert beat.last_beat == _T0
|
||||
assert beat.idle_seconds() == 0.0
|
||||
|
||||
|
||||
def test_heartbeat_advances_only_on_beat() -> None:
|
||||
"""心跳由 checkpoint 驱动，不额外起线程；没 beat 就不该自己变新。"""
|
||||
clock = FakeClock(_T0)
|
||||
beat = Heartbeat(clock)
|
||||
clock.advance(30)
|
||||
assert beat.idle_seconds() == 30.0
|
||||
beat.beat()
|
||||
assert beat.last_beat == clock.now
|
||||
assert beat.idle_seconds() == 0.0
|
||||
|
||||
|
||||
def test_heartbeat_staleness_boundary_is_inclusive() -> None:
|
||||
clock = FakeClock(_T0)
|
||||
beat = Heartbeat(clock)
|
||||
clock.advance(59.999)
|
||||
assert not beat.is_stale(60)
|
||||
clock.advance(0.001)
|
||||
assert beat.is_stale(60)
|
||||
|
||||
|
||||
def test_heartbeat_rejects_non_positive_timeout() -> None:
|
||||
beat = Heartbeat(FakeClock(_T0))
|
||||
with pytest.raises(ValueError, match="必须为正数"):
|
||||
beat.is_stale(0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 超时判定
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_no_timeout_configured_yields_none() -> None:
|
||||
assert next_deadline(Timeout()) is None
|
||||
|
||||
|
||||
def test_start_scope_applies_only_before_started() -> None:
|
||||
timeout = Timeout(start=30)
|
||||
pending = next_deadline(timeout, submitted_at=_T0)
|
||||
assert pending is not None
|
||||
assert pending.scope is TimeoutScope.START
|
||||
assert pending.at == _T0 + timedelta(seconds=30)
|
||||
|
||||
# 一旦开始执行，start 超时即失效。
|
||||
assert next_deadline(timeout, submitted_at=_T0, started_at=_T0) is None
|
||||
|
||||
|
||||
def test_start_scope_needs_submitted_at() -> None:
|
||||
assert next_deadline(Timeout(start=30)) is None
|
||||
|
||||
|
||||
def test_attempt_scope_measures_from_started_at() -> None:
|
||||
deadline = next_deadline(Timeout(per_attempt=600), started_at=_T0)
|
||||
assert deadline is not None
|
||||
assert deadline.scope is TimeoutScope.ATTEMPT
|
||||
assert deadline.at == _T0 + timedelta(seconds=600)
|
||||
|
||||
|
||||
def test_idle_scope_follows_the_latest_heartbeat() -> None:
|
||||
"""心跳推进后，idle 截止点必须随之顺延——否则长任务会被误杀。"""
|
||||
timeout = Timeout(idle=60)
|
||||
first = next_deadline(timeout, started_at=_T0, last_beat_at=_T0)
|
||||
assert first is not None
|
||||
assert first.at == _T0 + timedelta(seconds=60)
|
||||
|
||||
later = _T0 + timedelta(seconds=45)
|
||||
second = next_deadline(timeout, started_at=_T0, last_beat_at=later)
|
||||
assert second is not None
|
||||
assert second.at == later + timedelta(seconds=60)
|
||||
|
||||
|
||||
def test_idle_scope_requires_the_task_to_have_started() -> None:
|
||||
"""任务还没跑起来就谈不上"卡住"，否则排队时长会被误判成静默超时。"""
|
||||
assert next_deadline(Timeout(idle=60), last_beat_at=_T0) is None
|
||||
|
||||
|
||||
def test_before_start_only_the_start_scope_applies() -> None:
|
||||
timeout = Timeout(start=10, per_attempt=600, idle=60)
|
||||
deadline = next_deadline(timeout, submitted_at=_T0, last_beat_at=_T0)
|
||||
assert deadline is not None
|
||||
assert deadline.scope is TimeoutScope.START
|
||||
assert deadline.at == _T0 + timedelta(seconds=10)
|
||||
|
||||
|
||||
def test_after_start_attempt_and_idle_compete_and_earliest_wins() -> None:
|
||||
"""开始执行后只剩两档竞争：宽松的一档不能掩盖已经到期的一档。"""
|
||||
timeout = Timeout(per_attempt=600, idle=60)
|
||||
|
||||
fresh_beat = next_deadline(timeout, started_at=_T0, last_beat_at=_T0)
|
||||
assert fresh_beat is not None
|
||||
assert fresh_beat.scope is TimeoutScope.IDLE
|
||||
|
||||
# 心跳落后很久时，attempt 反而先到期。
|
||||
late_beat = _T0 + timedelta(seconds=590)
|
||||
stale_beat = next_deadline(timeout, started_at=_T0, last_beat_at=late_beat)
|
||||
assert stale_beat is not None
|
||||
assert stale_beat.scope is TimeoutScope.ATTEMPT
|
||||
assert stale_beat.at == _T0 + timedelta(seconds=600)
|
||||
|
||||
|
||||
def test_deadline_maps_to_failure_kind() -> None:
|
||||
"""超时是失败原因，不是生命周期状态；映射必须稳定。"""
|
||||
assert Deadline(TimeoutScope.START, _T0).kind is FailureKind.TIMEOUT_START
|
||||
assert Deadline(TimeoutScope.ATTEMPT, _T0).kind is FailureKind.TIMEOUT_ATTEMPT
|
||||
assert Deadline(TimeoutScope.IDLE, _T0).kind is FailureKind.TIMEOUT_IDLE
|
||||
|
||||
|
||||
def test_is_expired_boundary_is_inclusive() -> None:
|
||||
deadline = Deadline(TimeoutScope.ATTEMPT, _T0)
|
||||
assert not is_expired(deadline, now=_T0 - timedelta(microseconds=1))
|
||||
assert is_expired(deadline, now=_T0)
|
||||
|
||||
|
||||
def test_remaining_seconds_goes_negative_after_expiry() -> None:
|
||||
"""调度器要把负数当成"立刻到期"，而不是交给 sleep——sleep(-1) 会抛异常。"""
|
||||
deadline = Deadline(TimeoutScope.ATTEMPT, _T0)
|
||||
assert remaining_seconds(deadline, now=_T0 - timedelta(seconds=5)) == 5.0
|
||||
assert remaining_seconds(deadline, now=_T0 + timedelta(seconds=5)) == -5.0
|
||||
Reference in New Issue
Block a user