chakcy
f4de43c743
单任务受控执行（15 态状态机 + 审计轨迹）、DAG 管道、Controller 门面。
执行器三个：线程（默认，含 async def 任务）、内联（同步 / 测试）、进程
（spawn 子进程：硬性停止、绕开 GIL、崩溃隔离）。
调度：并发额度 + 每桶限流 + 具名资源信号量（按标签）+ 优先级排队。
持久层：TaskStore 契约 + 内存 / SQLite / Postgres 实现，契约测试随包发布。
跨进程与多节点：写穿 / 读穿、指令通道与租约、owned_buckets 划界、reconcile。
CLI、12 个可直接运行的示例、docs/ 下 5 份文档。
门禁：ruff / ruff format / mypy(strict) 干净；
pytest 563 passed（44 skipped 是需要真实 Postgres 的那批）；
带一台可连的 Postgres 时 607 passed。
155 lines
5.4 KiB
Python
155 lines
5.4 KiB
Python
"""心跳与超时判定的测试。
|
|
|
|
全部基于 ``FakeClock``：不 sleep、不依赖真实时间，因此既快又不会有偶发失败。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
|
|
from chongming_task.models import FailureKind, TimeoutScope
|
|
from chongming_task.policies import Timeout
|
|
from chongming_task.runtime.heartbeat import Heartbeat
|
|
from chongming_task.runtime.timeouts import (
|
|
Deadline,
|
|
is_expired,
|
|
next_deadline,
|
|
remaining_seconds,
|
|
)
|
|
from chongming_task.testing import FakeClock
|
|
|
|
_T0 = datetime(2026, 1, 1, tzinfo=UTC)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 心跳
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_heartbeat_starts_at_construction() -> None:
|
|
clock = FakeClock(_T0)
|
|
beat = Heartbeat(clock)
|
|
assert beat.last_beat == _T0
|
|
assert beat.idle_seconds() == 0.0
|
|
|
|
|
|
def test_heartbeat_advances_only_on_beat() -> None:
|
|
"""心跳由 checkpoint 驱动，不额外起线程；没 beat 就不该自己变新。"""
|
|
clock = FakeClock(_T0)
|
|
beat = Heartbeat(clock)
|
|
clock.advance(30)
|
|
assert beat.idle_seconds() == 30.0
|
|
beat.beat()
|
|
assert beat.last_beat == clock.now
|
|
assert beat.idle_seconds() == 0.0
|
|
|
|
|
|
def test_heartbeat_staleness_boundary_is_inclusive() -> None:
|
|
clock = FakeClock(_T0)
|
|
beat = Heartbeat(clock)
|
|
clock.advance(59.999)
|
|
assert not beat.is_stale(60)
|
|
clock.advance(0.001)
|
|
assert beat.is_stale(60)
|
|
|
|
|
|
def test_heartbeat_rejects_non_positive_timeout() -> None:
|
|
beat = Heartbeat(FakeClock(_T0))
|
|
with pytest.raises(ValueError, match="必须为正数"):
|
|
beat.is_stale(0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 超时判定
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_no_timeout_configured_yields_none() -> None:
|
|
assert next_deadline(Timeout()) is None
|
|
|
|
|
|
def test_start_scope_applies_only_before_started() -> None:
|
|
timeout = Timeout(start=30)
|
|
pending = next_deadline(timeout, submitted_at=_T0)
|
|
assert pending is not None
|
|
assert pending.scope is TimeoutScope.START
|
|
assert pending.at == _T0 + timedelta(seconds=30)
|
|
|
|
# 一旦开始执行，start 超时即失效。
|
|
assert next_deadline(timeout, submitted_at=_T0, started_at=_T0) is None
|
|
|
|
|
|
def test_start_scope_needs_submitted_at() -> None:
|
|
assert next_deadline(Timeout(start=30)) is None
|
|
|
|
|
|
def test_attempt_scope_measures_from_started_at() -> None:
|
|
deadline = next_deadline(Timeout(per_attempt=600), started_at=_T0)
|
|
assert deadline is not None
|
|
assert deadline.scope is TimeoutScope.ATTEMPT
|
|
assert deadline.at == _T0 + timedelta(seconds=600)
|
|
|
|
|
|
def test_idle_scope_follows_the_latest_heartbeat() -> None:
|
|
"""心跳推进后，idle 截止点必须随之顺延——否则长任务会被误杀。"""
|
|
timeout = Timeout(idle=60)
|
|
first = next_deadline(timeout, started_at=_T0, last_beat_at=_T0)
|
|
assert first is not None
|
|
assert first.at == _T0 + timedelta(seconds=60)
|
|
|
|
later = _T0 + timedelta(seconds=45)
|
|
second = next_deadline(timeout, started_at=_T0, last_beat_at=later)
|
|
assert second is not None
|
|
assert second.at == later + timedelta(seconds=60)
|
|
|
|
|
|
def test_idle_scope_requires_the_task_to_have_started() -> None:
|
|
"""任务还没跑起来就谈不上"卡住"，否则排队时长会被误判成静默超时。"""
|
|
assert next_deadline(Timeout(idle=60), last_beat_at=_T0) is None
|
|
|
|
|
|
def test_before_start_only_the_start_scope_applies() -> None:
|
|
timeout = Timeout(start=10, per_attempt=600, idle=60)
|
|
deadline = next_deadline(timeout, submitted_at=_T0, last_beat_at=_T0)
|
|
assert deadline is not None
|
|
assert deadline.scope is TimeoutScope.START
|
|
assert deadline.at == _T0 + timedelta(seconds=10)
|
|
|
|
|
|
def test_after_start_attempt_and_idle_compete_and_earliest_wins() -> None:
|
|
"""开始执行后只剩两档竞争：宽松的一档不能掩盖已经到期的一档。"""
|
|
timeout = Timeout(per_attempt=600, idle=60)
|
|
|
|
fresh_beat = next_deadline(timeout, started_at=_T0, last_beat_at=_T0)
|
|
assert fresh_beat is not None
|
|
assert fresh_beat.scope is TimeoutScope.IDLE
|
|
|
|
# 心跳落后很久时，attempt 反而先到期。
|
|
late_beat = _T0 + timedelta(seconds=590)
|
|
stale_beat = next_deadline(timeout, started_at=_T0, last_beat_at=late_beat)
|
|
assert stale_beat is not None
|
|
assert stale_beat.scope is TimeoutScope.ATTEMPT
|
|
assert stale_beat.at == _T0 + timedelta(seconds=600)
|
|
|
|
|
|
def test_deadline_maps_to_failure_kind() -> None:
|
|
"""超时是失败原因，不是生命周期状态；映射必须稳定。"""
|
|
assert Deadline(TimeoutScope.START, _T0).kind is FailureKind.TIMEOUT_START
|
|
assert Deadline(TimeoutScope.ATTEMPT, _T0).kind is FailureKind.TIMEOUT_ATTEMPT
|
|
assert Deadline(TimeoutScope.IDLE, _T0).kind is FailureKind.TIMEOUT_IDLE
|
|
|
|
|
|
def test_is_expired_boundary_is_inclusive() -> None:
|
|
deadline = Deadline(TimeoutScope.ATTEMPT, _T0)
|
|
assert not is_expired(deadline, now=_T0 - timedelta(microseconds=1))
|
|
assert is_expired(deadline, now=_T0)
|
|
|
|
|
|
def test_remaining_seconds_goes_negative_after_expiry() -> None:
|
|
"""调度器要把负数当成"立刻到期"，而不是交给 sleep——sleep(-1) 会抛异常。"""
|
|
deadline = Deadline(TimeoutScope.ATTEMPT, _T0)
|
|
assert remaining_seconds(deadline, now=_T0 - timedelta(seconds=5)) == 5.0
|
|
assert remaining_seconds(deadline, now=_T0 + timedelta(seconds=5)) == -5.0
|