679 lines
26 KiB
Python
679 lines
26 KiB
Python
|
"""任务管理门面（Controller / Registry）的测试。
|
|||
|
|
|
|||
|
|
重点覆盖三件事：**白名单**（只能启动已注册的任务）、**并发额度**（超限排队而不是
|
|||
|
|
无限起线程）、以及**收尾的诚实性**（排队中的任务被取消时不应真的执行任务体）。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import time
|
|||
|
|
from typing import Any
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
|
|||
|
|
from chongming_task.controller import Controller, StatusCounts, TaskRegistry, render_table
|
|||
|
|
from chongming_task.errors import ConfigurationError
|
|||
|
|
from chongming_task.policies import BucketPolicy, ConcurrencyPolicy
|
|||
|
|
from chongming_task.run import TaskRun
|
|||
|
|
from chongming_task.states import TaskStatus
|
|||
|
|
|
|||
|
|
_WINDOW = 10.0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _wait_until(predicate: Any, timeout: float = _WINDOW) -> bool:
|
|||
|
|
deadline = time.monotonic() + timeout
|
|||
|
|
while time.monotonic() < deadline:
|
|||
|
|
if predicate():
|
|||
|
|
return True
|
|||
|
|
time.sleep(0.01)
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _wait_for_status(run: TaskRun, status: TaskStatus, timeout: float = _WINDOW) -> bool:
|
|||
|
|
return _wait_until(lambda: run.status is status, timeout)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# --- 注册用任务（模块级，便于复用） ---
|
|||
|
|
|
|||
|
|
|
|||
|
|
def add(ctx: Any, left: int, right: int) -> int:
|
|||
|
|
"""两数相加。"""
|
|||
|
|
del ctx
|
|||
|
|
return left + right
|
|||
|
|
|
|||
|
|
|
|||
|
|
def slow_loop(ctx: Any) -> str:
|
|||
|
|
"""持续小步推进直到被停止。"""
|
|||
|
|
for _ in range(1_000_000):
|
|||
|
|
ctx.checkpoint()
|
|||
|
|
time.sleep(0.005)
|
|||
|
|
return "done"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def never_runs(ctx: Any) -> str:
|
|||
|
|
"""排队期间被取消时，这个函数体不应该被执行。"""
|
|||
|
|
marker.append("ran")
|
|||
|
|
return "ran"
|
|||
|
|
|
|||
|
|
|
|||
|
|
marker: list[str] = []
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# 注册表：白名单
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_registry_rejects_duplicate_names() -> None:
|
|||
|
|
"""静默覆盖会让"我改了实现，跑的还是旧的"这类问题极难发现。"""
|
|||
|
|
registry = TaskRegistry()
|
|||
|
|
registry.register("add", add)
|
|||
|
|
with pytest.raises(ConfigurationError, match="已注册"):
|
|||
|
|
registry.register("add", add)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_registry_requires_a_callable() -> None:
|
|||
|
|
with pytest.raises(ConfigurationError, match="可调用"):
|
|||
|
|
TaskRegistry().register("bad", "not-callable") # type: ignore[arg-type]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_registry_rejects_blank_name() -> None:
|
|||
|
|
with pytest.raises(ConfigurationError, match="不能为空"):
|
|||
|
|
TaskRegistry().register(" ", add)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_registry_picks_up_the_docstring_as_description() -> None:
|
|||
|
|
registration = TaskRegistry().register("add", add)
|
|||
|
|
assert registration.description == "两数相加。"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_submitting_an_unregistered_task_is_rejected() -> None:
|
|||
|
|
"""安全底线：入口只接受**名字**，绝不接受代码。
|
|||
|
|
|
|||
|
|
一个能启动任意可调用对象的入口等于任意代码执行——这是将来接 CLI 还是
|
|||
|
|
HTTP 都不能破的规则。
|
|||
|
|
"""
|
|||
|
|
controller = Controller()
|
|||
|
|
with pytest.raises(ConfigurationError, match="未注册"):
|
|||
|
|
controller.submit("nope")
|
|||
|
|
assert controller.registry.names() == ()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_registry_lists_registered_names() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
assert controller.registry.names() == ("add", "slow")
|
|||
|
|
assert len(controller.registry) == 2
|
|||
|
|
assert "add" in controller.registry
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# 提交、查询、统计
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_submit_runs_and_reports_snapshot() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
run = controller.submit("add", 2, 3, task_id="t1", label="加法")
|
|||
|
|
assert isinstance(run, TaskRun)
|
|||
|
|
|
|||
|
|
outcome = controller.wait("t1", timeout=_WINDOW)
|
|||
|
|
assert outcome.ok
|
|||
|
|
assert outcome.value == 5
|
|||
|
|
|
|||
|
|
snapshot = controller.get("t1")
|
|||
|
|
assert snapshot is not None
|
|||
|
|
assert snapshot.task_id == "t1"
|
|||
|
|
assert snapshot.name == "加法"
|
|||
|
|
assert snapshot.status is TaskStatus.SUCCEEDED
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_get_returns_none_for_unknown_task() -> None:
|
|||
|
|
assert Controller().get("nope") is None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_stats_counts_by_status() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
for index in range(3):
|
|||
|
|
controller.submit("add", index, 1, task_id=f"t{index}")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
stats = controller.stats()
|
|||
|
|
assert isinstance(stats, StatusCounts)
|
|||
|
|
assert stats.total == 3
|
|||
|
|
assert stats.of(TaskStatus.SUCCEEDED) == 3
|
|||
|
|
assert stats.finished == 3
|
|||
|
|
assert stats.unfinished == 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_list_filters_by_status_and_name_prefix() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
controller.submit("add", 1, 1, task_id="a1", label="抽取-甲")
|
|||
|
|
controller.submit("add", 1, 1, task_id="a2", label="抽取-乙")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
assert len(controller.list_tasks()) == 2
|
|||
|
|
assert len(controller.list_tasks(name_prefix="抽取")) == 2
|
|||
|
|
assert controller.list_tasks(name_prefix="加载") == []
|
|||
|
|
assert len(controller.list_tasks(status=TaskStatus.SUCCEEDED)) == 2
|
|||
|
|
assert controller.list_tasks(status=TaskStatus.FAILED) == []
|
|||
|
|
assert len(controller.list_tasks(limit=1)) == 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_list_filters_by_tag() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add, tags={"io"})
|
|||
|
|
controller.submit("add", 1, 1, task_id="a1")
|
|||
|
|
controller.submit("add", 1, 1, task_id="a2", tags={"cpu"})
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
assert len(controller.list_tasks(tag="io")) == 2, "注册时的标签应当被继承"
|
|||
|
|
assert [s.task_id for s in controller.list_tasks(tag="cpu")] == ["a2"]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_duplicate_task_id_is_rejected() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
controller.submit("add", 1, 1, task_id="dup")
|
|||
|
|
with pytest.raises(ConfigurationError, match="已存在"):
|
|||
|
|
controller.submit("add", 1, 1, task_id="dup")
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# 并发额度
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_excess_submissions_queue_instead_of_starting() -> None:
|
|||
|
|
"""超限的任务状态是 ``queued``（已被受理、等待执行槽），不是 ``pending``。"""
|
|||
|
|
controller = Controller(max_parallel=1)
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
|
|||
|
|
first = controller.submit("slow", task_id="t1")
|
|||
|
|
assert _wait_for_status(first, TaskStatus.RUNNING)
|
|||
|
|
|
|||
|
|
second = controller.submit("slow", task_id="t2")
|
|||
|
|
third = controller.submit("slow", task_id="t3")
|
|||
|
|
|
|||
|
|
assert second.status is TaskStatus.QUEUED
|
|||
|
|
assert third.status is TaskStatus.QUEUED
|
|||
|
|
assert not second.started, "排队中的任务不应被启动"
|
|||
|
|
assert controller.active_count == 1
|
|||
|
|
assert controller.queued_count == 2
|
|||
|
|
|
|||
|
|
controller.stop_all(reason="测试结束")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
assert controller.queued_count == 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_queued_tasks_start_as_slots_free_up() -> None:
|
|||
|
|
controller = Controller(max_parallel=2)
|
|||
|
|
controller.register("add", add)
|
|||
|
|
runs = [controller.submit("add", index, 1, task_id=f"t{index}") for index in range(4)]
|
|||
|
|
|
|||
|
|
outcomes = controller.wait_all(timeout=_WINDOW)
|
|||
|
|
assert len(outcomes) == 4
|
|||
|
|
assert all(outcome.ok for outcome in outcomes)
|
|||
|
|
assert sorted(outcome.value for outcome in outcomes) == [1, 2, 3, 4]
|
|||
|
|
assert controller.active_count == 0
|
|||
|
|
assert all(run.status is TaskStatus.SUCCEEDED for run in runs)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_max_parallel_must_be_positive() -> None:
|
|||
|
|
with pytest.raises(ConfigurationError, match="max_parallel"):
|
|||
|
|
Controller(max_parallel=0)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# 具名资源信号量与优先级（ConcurrencyPolicy）
|
|||
|
|
#
|
|||
|
|
# 这三个字段曾经只是"声明了、校验了、文档里写着支持"——调度器从不读它，于是
|
|||
|
|
# ``limits`` 与 ``priority`` 设了等于没设，而且**不报任何错**。下面几条把它们钉住。
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_named_resource_limit_caps_only_the_tagged_tasks() -> None:
|
|||
|
|
"""``limits={"gpu": 1}`` 只约束带 ``gpu`` 标签的任务，不是给全局加一道闸。"""
|
|||
|
|
controller = Controller(concurrency=ConcurrencyPolicy(limits={"gpu": 1}))
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
|
|||
|
|
first = controller.submit("slow", task_id="gpu-1", tags=["gpu"])
|
|||
|
|
assert _wait_for_status(first, TaskStatus.RUNNING)
|
|||
|
|
second = controller.submit("slow", task_id="gpu-2", tags=["gpu"])
|
|||
|
|
assert second.status is TaskStatus.QUEUED, "gpu 名额已满，第二个必须排队"
|
|||
|
|
|
|||
|
|
# 不带该标签的任务不受它约束——具名信号量不是全局额度。
|
|||
|
|
plain = controller.submit("slow", task_id="plain")
|
|||
|
|
assert _wait_for_status(plain, TaskStatus.RUNNING)
|
|||
|
|
assert controller.active_count == 2
|
|||
|
|
|
|||
|
|
controller.stop_all(reason="测试结束")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_a_task_needs_room_in_every_limited_tag_it_carries() -> None:
|
|||
|
|
"""多标签任务要**每个**受限期资源都还有余量才放行。"""
|
|||
|
|
controller = Controller(concurrency=ConcurrencyPolicy(limits={"gpu": 1, "db": 1}))
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
|
|||
|
|
gpu = controller.submit("slow", task_id="gpu-holder", tags=["gpu"])
|
|||
|
|
assert _wait_for_status(gpu, TaskStatus.RUNNING)
|
|||
|
|
db = controller.submit("slow", task_id="db-holder", tags=["db"])
|
|||
|
|
assert _wait_for_status(db, TaskStatus.RUNNING)
|
|||
|
|
|
|||
|
|
both = controller.submit("slow", task_id="needs-both", tags=["gpu", "db"])
|
|||
|
|
assert both.status is TaskStatus.QUEUED, "两个资源都满了，带两个标签的任务更不能放行"
|
|||
|
|
|
|||
|
|
# 只空出一个：db 还占着，所以它依旧不能启动。等额度记账真的落下来再断言，
|
|||
|
|
# 否则这句话可能在调度器还没醒的时候"通过"，什么都没证明。
|
|||
|
|
assert controller.stop("gpu-holder") is True
|
|||
|
|
controller.wait("gpu-holder", timeout=_WINDOW)
|
|||
|
|
assert _wait_until(lambda: controller.active_count == 1), "额度应当已归还"
|
|||
|
|
assert both.status is TaskStatus.QUEUED, "gpu 空了但 db 还占着，不能放行"
|
|||
|
|
|
|||
|
|
# 两个都空出来才轮到它。
|
|||
|
|
assert controller.stop("db-holder") is True
|
|||
|
|
controller.wait("db-holder", timeout=_WINDOW)
|
|||
|
|
assert _wait_for_status(both, TaskStatus.RUNNING)
|
|||
|
|
|
|||
|
|
controller.stop_all(reason="测试结束")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_tags_without_a_limit_do_not_throttle_anything() -> None:
|
|||
|
|
"""没写进 ``limits`` 的标签不受约束（常见误解：以为打个标签就会被限流）。"""
|
|||
|
|
controller = Controller(concurrency=ConcurrencyPolicy(limits={"gpu": 1}))
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
|
|||
|
|
first = controller.submit("slow", task_id="io-1", tags=["io"])
|
|||
|
|
second = controller.submit("slow", task_id="io-2", tags=["io"])
|
|||
|
|
assert _wait_for_status(first, TaskStatus.RUNNING)
|
|||
|
|
assert _wait_for_status(second, TaskStatus.RUNNING)
|
|||
|
|
|
|||
|
|
controller.stop_all(reason="测试结束")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_priority_picks_the_highest_first() -> None:
|
|||
|
|
"""都能启动时先取优先级最高的；数值越大越优先。"""
|
|||
|
|
controller = Controller(max_parallel=1)
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
|
|||
|
|
blocker = controller.submit("slow", task_id="blocker")
|
|||
|
|
assert _wait_for_status(blocker, TaskStatus.RUNNING)
|
|||
|
|
low = controller.submit("slow", task_id="low", priority=1)
|
|||
|
|
high = controller.submit("slow", task_id="high", priority=5)
|
|||
|
|
|
|||
|
|
assert controller.stop("blocker") is True
|
|||
|
|
controller.wait("blocker", timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
assert _wait_for_status(high, TaskStatus.RUNNING), "高优先级的应当先被取走"
|
|||
|
|
assert low.status is TaskStatus.QUEUED, "低优先级的不该插到前面"
|
|||
|
|
|
|||
|
|
controller.stop_all(reason="测试结束")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_equal_priority_keeps_submission_order() -> None:
|
|||
|
|
"""同优先级仍然先进先出——优先级机制不该打乱时间序。"""
|
|||
|
|
controller = Controller(max_parallel=1)
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
|
|||
|
|
blocker = controller.submit("slow", task_id="blocker")
|
|||
|
|
assert _wait_for_status(blocker, TaskStatus.RUNNING)
|
|||
|
|
first_queued = controller.submit("slow", task_id="first")
|
|||
|
|
second_queued = controller.submit("slow", task_id="second")
|
|||
|
|
|
|||
|
|
assert controller.stop("blocker") is True
|
|||
|
|
controller.wait("blocker", timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
assert _wait_for_status(first_queued, TaskStatus.RUNNING), "先提交的应当先跑"
|
|||
|
|
assert second_queued.status is TaskStatus.QUEUED
|
|||
|
|
|
|||
|
|
controller.stop_all(reason="测试结束")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_policy_default_priority_applies_to_submissions() -> None:
|
|||
|
|
"""``ConcurrencyPolicy.priority`` 是 ``submit`` 不给 ``priority`` 时的默认值。"""
|
|||
|
|
controller = Controller(concurrency=ConcurrencyPolicy(max_parallel=1, priority=7))
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
|
|||
|
|
blocker = controller.submit("slow", task_id="blocker")
|
|||
|
|
assert _wait_for_status(blocker, TaskStatus.RUNNING)
|
|||
|
|
assert blocker.priority == 7, "没给 priority 时应当取策略里的默认值"
|
|||
|
|
|
|||
|
|
explicit_low = controller.submit("slow", task_id="low", priority=1)
|
|||
|
|
assert explicit_low.priority == 1, "显式给的优先级必须能覆盖默认值"
|
|||
|
|
default_high = controller.submit("slow", task_id="high")
|
|||
|
|
assert default_high.priority == 7
|
|||
|
|
|
|||
|
|
assert controller.stop("blocker") is True
|
|||
|
|
controller.wait("blocker", timeout=_WINDOW)
|
|||
|
|
assert _wait_for_status(default_high, TaskStatus.RUNNING)
|
|||
|
|
assert explicit_low.status is TaskStatus.QUEUED
|
|||
|
|
|
|||
|
|
controller.stop_all(reason="测试结束")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_concurrency_policy_max_parallel_is_honoured() -> None:
|
|||
|
|
"""策略里的 ``max_parallel`` 与 ``Controller(max_parallel=...)`` 等价。"""
|
|||
|
|
controller = Controller(concurrency=ConcurrencyPolicy(max_parallel=1))
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
|
|||
|
|
first = controller.submit("slow", task_id="t1")
|
|||
|
|
assert _wait_for_status(first, TaskStatus.RUNNING)
|
|||
|
|
second = controller.submit("slow", task_id="t2")
|
|||
|
|
assert second.status is TaskStatus.QUEUED
|
|||
|
|
assert controller.active_count == 1
|
|||
|
|
|
|||
|
|
controller.stop_all(reason="测试结束")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_policy_and_max_parallel_are_mutually_exclusive() -> None:
|
|||
|
|
"""两份配置只能给一份：谁说了算必须明确，不能靠"后者覆盖前者"这种猜法。"""
|
|||
|
|
with pytest.raises(ConfigurationError, match="只能给一个"):
|
|||
|
|
Controller(concurrency=ConcurrencyPolicy(max_parallel=2), max_parallel=1)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_policy_rejects_a_non_positive_resource_limit() -> None:
|
|||
|
|
with pytest.raises(ValueError, match="并发上限"):
|
|||
|
|
ConcurrencyPolicy(limits={"gpu": 0})
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_a_non_integer_priority_is_rejected_at_submit_time() -> None:
|
|||
|
|
"""非整数优先级必须在提交时拦下。
|
|||
|
|
|
|||
|
|
放过去的话，它会在**派发线程**里抛 ``TypeError``——线程直接死掉，此后所有排队
|
|||
|
|
任务都不再启动，而没有任何报错指向真正的原因。
|
|||
|
|
"""
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
with pytest.raises(ConfigurationError, match="priority"):
|
|||
|
|
controller.submit("add", 1, 2, task_id="t1", priority="high") # type: ignore[arg-type]
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# 控制
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_stop_routes_to_the_right_task() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
victim = controller.submit("slow", task_id="victim")
|
|||
|
|
bystander = controller.submit("slow", task_id="bystander")
|
|||
|
|
assert _wait_for_status(victim, TaskStatus.RUNNING)
|
|||
|
|
assert _wait_for_status(bystander, TaskStatus.RUNNING)
|
|||
|
|
|
|||
|
|
assert controller.stop("victim", reason="发布窗口") is True
|
|||
|
|
outcome = controller.wait("victim", timeout=_WINDOW)
|
|||
|
|
assert outcome.status is TaskStatus.STOPPED
|
|||
|
|
assert bystander.status is TaskStatus.RUNNING, "不该误伤别的任务"
|
|||
|
|
|
|||
|
|
controller.stop_all()
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_stop_returns_false_for_unknown_task() -> None:
|
|||
|
|
assert Controller().stop("nope") is False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_pause_and_resume_route_through_the_controller() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
run = controller.submit("slow", task_id="p1")
|
|||
|
|
assert _wait_for_status(run, TaskStatus.RUNNING)
|
|||
|
|
|
|||
|
|
assert controller.pause("p1", timeout=_WINDOW) is True
|
|||
|
|
assert run.status is TaskStatus.PAUSED
|
|||
|
|
|
|||
|
|
assert controller.resume("p1") is True
|
|||
|
|
assert run.status is TaskStatus.RUNNING
|
|||
|
|
|
|||
|
|
controller.stop_all()
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_stop_all_reports_how_many_were_affected() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
for index in range(3):
|
|||
|
|
controller.submit("slow", task_id=f"t{index}")
|
|||
|
|
assert _wait_until(lambda: controller.active_count == 3)
|
|||
|
|
|
|||
|
|
assert controller.stop_all(reason="批量收工") == 3
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
assert controller.stop_all() == 0, "已无未落终态的任务"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_submit_after_close_is_rejected() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
controller.close(timeout=_WINDOW)
|
|||
|
|
with pytest.raises(ConfigurationError, match="已关闭"):
|
|||
|
|
controller.submit("add", 1, 1)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# 收尾
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_close_does_not_execute_queued_task_bodies() -> None:
|
|||
|
|
"""排队中的任务被取消时不该真的跑起来——否则"停止"只是把副作用推迟了。"""
|
|||
|
|
marker.clear()
|
|||
|
|
controller = Controller(max_parallel=1)
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
controller.register("never", never_runs)
|
|||
|
|
|
|||
|
|
controller.submit("slow", task_id="t-block")
|
|||
|
|
assert _wait_until(lambda: controller.active_count == 1)
|
|||
|
|
controller.submit("never", task_id="t-queued")
|
|||
|
|
assert controller.queued_count == 1
|
|||
|
|
|
|||
|
|
outcomes = controller.close(reason="收工", timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
assert marker == [], "排队中的任务被启动了，任务体被执行"
|
|||
|
|
statuses = {outcome.snapshot.task_id: outcome.status for outcome in outcomes}
|
|||
|
|
assert statuses["t-queued"] is TaskStatus.STOPPED
|
|||
|
|
assert statuses["t-block"] is TaskStatus.STOPPED
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_prune_drops_oldest_terminal_records() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
for index in range(5):
|
|||
|
|
controller.submit("add", index, 1, task_id=f"t{index}")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
assert len(controller.list_tasks()) == 5
|
|||
|
|
|
|||
|
|
assert controller.prune(keep_terminal=2) == 3
|
|||
|
|
assert len(controller.list_tasks()) == 2
|
|||
|
|
assert controller.prune(keep_terminal=2) == 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# 渲染
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_render_table_aligns_chinese_names_by_display_width() -> None:
|
|||
|
|
"""中日韩全角字符占两列；不按显示宽度对齐，中文任务名会把整张表挤歪。"""
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
controller.submit("add", 1, 1, task_id="a1", label="抽取")
|
|||
|
|
controller.submit("add", 1, 1, task_id="a2", label="a-much-longer-name")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
table = render_table(controller.list_tasks())
|
|||
|
|
lines = table.splitlines()
|
|||
|
|
|
|||
|
|
assert lines[0].startswith("TASK_ID")
|
|||
|
|
assert set(lines[1]) <= {"-", " "}, "表头下方应当是分隔线"
|
|||
|
|
widths = {len(line.rstrip()) for line in lines}
|
|||
|
|
assert len(widths) > 1, "各行列宽应当随内容变化，而不是靠不齐"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_render_table_reports_failure_kind() -> None:
|
|||
|
|
def boom(ctx: Any) -> None:
|
|||
|
|
raise RuntimeError("炸了")
|
|||
|
|
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("boom", boom)
|
|||
|
|
controller.submit("boom", task_id="b1")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
table = render_table(controller.list_tasks())
|
|||
|
|
assert "exception" in table
|
|||
|
|
assert "failed" in table
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# 分桶
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _two_keys_in_different_buckets(policy: BucketPolicy) -> tuple[str, str]:
|
|||
|
|
"""找两个确实落在不同桶里的键。
|
|||
|
|
|
|||
|
|
硬编码两个键是不行的：它们有约 1/64 的概率撞进同一个桶，那会让"分桶隔离"
|
|||
|
|
的用例变成一句空话，而且只在特定版本上偶发失败。
|
|||
|
|
"""
|
|||
|
|
first = "tenant-0"
|
|||
|
|
first_bucket = policy.bucket_for(first)
|
|||
|
|
for index in range(1, 500):
|
|||
|
|
candidate = f"tenant-{index}"
|
|||
|
|
if policy.bucket_for(candidate) != first_bucket:
|
|||
|
|
return first, candidate
|
|||
|
|
raise AssertionError("在 500 个键里找不到第二个桶，分桶策略有问题")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_bucket_is_derived_from_the_key_and_recorded() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
run = controller.submit("add", 1, 1, task_id="a1", bucket_key="tenant-42")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
snapshot = run.snapshot
|
|||
|
|
assert snapshot.bucket == controller.bucket_policy.bucket_for("tenant-42")
|
|||
|
|
assert snapshot.bucket_key == "tenant-42", "原始键必须留存，否则改桶数时无法迁移"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_same_key_lands_in_the_same_bucket() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
for index in range(3):
|
|||
|
|
controller.submit("add", index, 1, task_id=f"t{index}", bucket_key="tenant-9")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
buckets = {snapshot.bucket for snapshot in controller.list_tasks()}
|
|||
|
|
assert len(buckets) == 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_task_without_bucket_key_is_unbucketed() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
controller.submit("add", 1, 1, task_id="a1")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
assert controller.get("a1").bucket is None # type: ignore[union-attr]
|
|||
|
|
assert controller.bucket_counts() == {None: 1}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_per_bucket_limit_isolates_buckets_from_each_other() -> None:
|
|||
|
|
"""一个桶把额度占满不应饿死其他桶——这是分桶在**本地**就有的价值。
|
|||
|
|
|
|||
|
|
若调度器直接取队首而不是扫描可运行者，一个积压的桶会阻塞所有桶，把分桶要解决
|
|||
|
|
的问题又原样引入一遍。
|
|||
|
|
"""
|
|||
|
|
controller = Controller(max_parallel=10, max_parallel_per_bucket=1)
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
hot, cold = _two_keys_in_different_buckets(controller.bucket_policy)
|
|||
|
|
|
|||
|
|
first = controller.submit("slow", task_id="a1", bucket_key=hot)
|
|||
|
|
second = controller.submit("slow", task_id="a2", bucket_key=hot)
|
|||
|
|
other = controller.submit("slow", task_id="b1", bucket_key=cold)
|
|||
|
|
|
|||
|
|
assert _wait_for_status(first, TaskStatus.RUNNING)
|
|||
|
|
assert _wait_for_status(other, TaskStatus.RUNNING), "别的桶被积压的桶挡住了"
|
|||
|
|
assert second.status is TaskStatus.QUEUED, "同桶超出额度应当排队"
|
|||
|
|
assert controller.active_in_bucket(controller.bucket_policy.bucket_for(hot)) == 1
|
|||
|
|
|
|||
|
|
controller.stop_all()
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_per_bucket_limit_still_respects_the_global_limit() -> None:
|
|||
|
|
controller = Controller(max_parallel=1, max_parallel_per_bucket=5)
|
|||
|
|
controller.register("slow", slow_loop)
|
|||
|
|
hot, cold = _two_keys_in_different_buckets(controller.bucket_policy)
|
|||
|
|
|
|||
|
|
first = controller.submit("slow", task_id="a1", bucket_key=hot)
|
|||
|
|
second = controller.submit("slow", task_id="b1", bucket_key=cold)
|
|||
|
|
assert _wait_for_status(first, TaskStatus.RUNNING)
|
|||
|
|
assert second.status is TaskStatus.QUEUED, "总额度才是硬上限"
|
|||
|
|
|
|||
|
|
controller.stop_all()
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_per_bucket_limit_must_be_positive() -> None:
|
|||
|
|
with pytest.raises(ConfigurationError, match="max_parallel_per_bucket"):
|
|||
|
|
Controller(max_parallel_per_bucket=0)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_list_tasks_filters_by_bucket_and_bucket_set() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
hot, cold = _two_keys_in_different_buckets(controller.bucket_policy)
|
|||
|
|
controller.submit("add", 1, 1, task_id="a1", bucket_key=hot)
|
|||
|
|
controller.submit("add", 1, 1, task_id="b1", bucket_key=cold)
|
|||
|
|
controller.submit("add", 1, 1, task_id="c1")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
hot_bucket = controller.bucket_policy.bucket_for(hot)
|
|||
|
|
cold_bucket = controller.bucket_policy.bucket_for(cold)
|
|||
|
|
|
|||
|
|
assert [s.task_id for s in controller.list_tasks(bucket=hot_bucket)] == ["a1"]
|
|||
|
|
assert [s.task_id for s in controller.list_tasks(buckets=[hot_bucket, cold_bucket])] == [
|
|||
|
|
"a1",
|
|||
|
|
"b1",
|
|||
|
|
]
|
|||
|
|
assert controller.list_tasks(bucket=9999) == []
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_bucket_counts_summarises_each_bucket() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
hot, cold = _two_keys_in_different_buckets(controller.bucket_policy)
|
|||
|
|
controller.submit("add", 1, 1, task_id="a1", bucket_key=hot)
|
|||
|
|
controller.submit("add", 1, 1, task_id="a2", bucket_key=hot)
|
|||
|
|
controller.submit("add", 1, 1, task_id="b1", bucket_key=cold)
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
counts = controller.bucket_counts()
|
|||
|
|
assert counts[controller.bucket_policy.bucket_for(hot)] == 2
|
|||
|
|
assert counts[controller.bucket_policy.bucket_for(cold)] == 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_render_table_shows_the_bucket_column() -> None:
|
|||
|
|
controller = Controller()
|
|||
|
|
controller.register("add", add)
|
|||
|
|
controller.submit("add", 1, 1, task_id="a1", bucket_key="tenant-42")
|
|||
|
|
controller.submit("add", 1, 1, task_id="a2")
|
|||
|
|
controller.wait_all(timeout=_WINDOW)
|
|||
|
|
|
|||
|
|
table = render_table(controller.list_tasks())
|
|||
|
|
lines = table.splitlines()
|
|||
|
|
assert "BUCKET" in lines[0]
|
|||
|
|
expected = str(controller.bucket_policy.bucket_for("tenant-42"))
|
|||
|
|
assert expected in lines[2]
|
|||
|
|
assert lines[3].split()[2] == "-", "未分桶的任务应当显示 -"
|