493 lines
23 KiB
Python
493 lines
23 KiB
Python
|
"""示例 11：最佳实践 —— 一份可以直接照抄的多节点任务服务模板。
|
|||
|
|
|
|||
|
|
示例 01–10 各自演示一个机制；这一个把它们收拢成**一个完整、可运行、可运维的
|
|||
|
|
最小形态**，回答集成时的四个必答题：
|
|||
|
|
|
|||
|
|
① 桶怎么划（含节点划分）—— 业务键稳定映射到桶；桶按连续区间分给节点；
|
|||
|
|
一处计算，代码与 CLI 共用。
|
|||
|
|
② 谁跑谁的任务 —— 每个节点只接受自己桶里的任务；配错桶在**提交那一刻**
|
|||
|
|
报错，而不是产出一个永远没有节点执行的任务。
|
|||
|
|
③ 进程启停做什么 —— 启动先 ``reconcile()``（只清理自己桶里的僵尸）；
|
|||
|
|
退出用 ``close(close_store=True)`` 把连接关干净。
|
|||
|
|
④ 怎么观测与控制 —— 所有节点共用一套存储：一个控制台看全部；
|
|||
|
|
跨节点停止走指令通道（写指令 -> 持有者轮询取走）。
|
|||
|
|
|
|||
|
|
文件按"照抄顺序"分四部分：
|
|||
|
|
|
|||
|
|
# 第一部分 部署参数与桶划分
|
|||
|
|
# 第二部分 任务定义（业务代码）
|
|||
|
|
# 第三部分 接线：build_node()（唯一集成点）
|
|||
|
|
# 第四部分 演示：启动 -> 提交 -> 观测 -> 控制 -> 收尾
|
|||
|
|
|
|||
|
|
运行::
|
|||
|
|
|
|||
|
|
uv run python examples/11_best_practice.py
|
|||
|
|
|
|||
|
|
本示例在一个进程里模拟"两个节点 + 一个运维控制台"，是为了**一条命令跑通全流程**；
|
|||
|
|
生产上它们是各自独立的进程（同一份代码，只有 NODE_INDEX 不同），见文件末尾的部署说明。
|
|||
|
|
注意：SQLite 文件存储要求所有节点在同一台机器上；**跨机部署**把 ``build_node()`` 里的
|
|||
|
|
``SqliteTaskStore(deployment.db_path)`` 换成 ``PostgresTaskStore(dsn)``
|
|||
|
|
（``pip install "chongming-task[postgres]"``）即可，业务代码一行都不用改。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import logging
|
|||
|
|
import os
|
|||
|
|
import sys
|
|||
|
|
import tempfile
|
|||
|
|
import time
|
|||
|
|
from collections.abc import Sequence
|
|||
|
|
from dataclasses import dataclass
|
|||
|
|
from datetime import timedelta
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
from chongming_task import (
|
|||
|
|
BucketPolicy,
|
|||
|
|
ConcurrencyPolicy,
|
|||
|
|
ConfigurationError,
|
|||
|
|
Controller,
|
|||
|
|
ExponentialBackoff,
|
|||
|
|
RetryPolicy,
|
|||
|
|
SqliteTaskStore,
|
|||
|
|
TaskContext,
|
|||
|
|
TaskSnapshot,
|
|||
|
|
TaskStatus,
|
|||
|
|
render_table,
|
|||
|
|
task,
|
|||
|
|
utcnow,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
_log = logging.getLogger("best_practice")
|
|||
|
|
|
|||
|
|
# =============================================================================
|
|||
|
|
# 第一部分：部署参数与桶划分（照抄进你的配置模块）
|
|||
|
|
# =============================================================================
|
|||
|
|
|
|||
|
|
#: 桶数。默认 64；这里取 16 让演示输出可读。
|
|||
|
|
#: 它是**部署范围的约定**，不是调优旋钮：所有节点与所有运维命令都必须用同一个数。
|
|||
|
|
#: 经验值：节点数的 4~16 倍（给节点增减留出重新划分的空间）。
|
|||
|
|
NUM_BUCKETS = 16
|
|||
|
|
|
|||
|
|
#: 本示例部署的节点数。生产上每个节点是一个独立进程。
|
|||
|
|
NODE_COUNT = 2
|
|||
|
|
|
|||
|
|
|
|||
|
|
def buckets_for_node(node_index: int, node_count: int, num_buckets: int = NUM_BUCKETS) -> range:
|
|||
|
|
"""把桶**连续**分给各节点：第 ``node_index`` 个节点负责 ``[start, stop)`` 区间。
|
|||
|
|
|
|||
|
|
为什么连续划分，而不是取模交错（``bucket % node_count``）：
|
|||
|
|
|
|||
|
|
* 人可读可核对：``--owned-buckets 0-7`` 与代码里的 ``range(0, 8)`` 是同一件事；
|
|||
|
|
* 负载不偏：桶号本身就是 blake2b 的均匀散列，等长区间的期望负载相同；
|
|||
|
|
* 迁移简单：节点增减时整段区间搬运，不用逐桶列举。
|
|||
|
|
|
|||
|
|
除不尽时余数归靠前的节点，保证**不重不漏**——漏桶是静默积压、重桶是重复执行，
|
|||
|
|
两者都不报错，所以划分必须在一处算出来、处处引用。
|
|||
|
|
"""
|
|||
|
|
if node_count < 1:
|
|||
|
|
raise ValueError(f"node_count 至少为 1，收到 {node_count}")
|
|||
|
|
if num_buckets < node_count:
|
|||
|
|
raise ValueError(
|
|||
|
|
f"桶数（{num_buckets}）必须不小于节点数（{node_count}）："
|
|||
|
|
"否则会有节点分不到桶。经验上桶数取节点数的 4~16 倍。"
|
|||
|
|
)
|
|||
|
|
if not 0 <= node_index < node_count:
|
|||
|
|
raise ValueError(f"node_index 必须在 [0, {node_count}) 内，收到 {node_index}")
|
|||
|
|
size, remainder = divmod(num_buckets, node_count)
|
|||
|
|
start = node_index * size + min(node_index, remainder)
|
|||
|
|
length = size + (1 if node_index < remainder else 0)
|
|||
|
|
return range(start, start + length)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass(frozen=True)
|
|||
|
|
class Deployment:
|
|||
|
|
"""一个节点进程的部署参数。生产环境从环境变量 / 配置中心来，不要写死在代码里。"""
|
|||
|
|
|
|||
|
|
node_index: int
|
|||
|
|
node_count: int
|
|||
|
|
db_path: Path
|
|||
|
|
num_buckets: int = NUM_BUCKETS
|
|||
|
|
|
|||
|
|
@classmethod
|
|||
|
|
def from_env(cls) -> Deployment:
|
|||
|
|
"""从环境变量读取部署参数（systemd / 容器编排注入）。"""
|
|||
|
|
return cls(
|
|||
|
|
node_index=int(os.environ.get("TASK_NODE_INDEX", "0")),
|
|||
|
|
node_count=int(os.environ.get("TASK_NODE_COUNT", "1")),
|
|||
|
|
db_path=Path(os.environ.get("TASK_DB", "tasks.db")),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def owner(self) -> str:
|
|||
|
|
"""写进存储的节点身份。排障时能回答"是谁在管这个任务"。"""
|
|||
|
|
return f"node-{self.node_index}"
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def owned_buckets(self) -> range:
|
|||
|
|
"""本节点负责的桶。三个地方必须用同一份结果：Controller 参数、CLI 参数、日志。"""
|
|||
|
|
return buckets_for_node(self.node_index, self.node_count, self.num_buckets)
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def cli_buckets(self) -> str:
|
|||
|
|
"""同一个划分的 CLI 写法（``--owned-buckets 0-7``），供运维直接复制。"""
|
|||
|
|
owned = self.owned_buckets
|
|||
|
|
return f"{owned.start}-{owned.stop - 1}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def owner_of(bucket: int, deployments: Sequence[Deployment]) -> Deployment:
|
|||
|
|
"""桶号反查节点。查不到意味着划分有空洞或配置不一致——直接炸，不静默。"""
|
|||
|
|
for deployment in deployments:
|
|||
|
|
if bucket in deployment.owned_buckets:
|
|||
|
|
return deployment
|
|||
|
|
raise AssertionError(
|
|||
|
|
f"没有任何节点负责桶 {bucket}：请检查 num_buckets / node_count 是否各处一致"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# =============================================================================
|
|||
|
|
# 第二部分：任务定义（业务代码：只依赖 ctx 约定，照抄进你的 tasks 模块）
|
|||
|
|
# =============================================================================
|
|||
|
|
|
|||
|
|
|
|||
|
|
@task(
|
|||
|
|
tags={"sync", "db"},
|
|||
|
|
retry=RetryPolicy(max_attempts=2, backoff=ExponentialBackoff(base=0.05, factor=2)),
|
|||
|
|
)
|
|||
|
|
def sync_tenant(ctx: TaskContext, tenant: str, batches: int = 24) -> dict[str, object]:
|
|||
|
|
"""同步一个租户的数据。
|
|||
|
|
|
|||
|
|
标签 ``db`` 不只是检索用的：它还是**具名资源信号量**的键——每个节点到库里的
|
|||
|
|
并发连接数是有限的，靠它限流比调小全局并发更准（见 ``build_node``）。
|
|||
|
|
|
|||
|
|
长耗时循环里必须周期性调用 ``ctx.checkpoint()``——停止、暂停、心跳都在那里生效；
|
|||
|
|
不调用它的任务既停不掉，也会被 idle 超时判定为疑似卡死。
|
|||
|
|
"""
|
|||
|
|
ctx.log(f"{tenant}：开始同步（{batches} 批）")
|
|||
|
|
done = 0
|
|||
|
|
for index in range(1, batches + 1):
|
|||
|
|
if ctx.is_stop_requested:
|
|||
|
|
# 优雅收尾：正常 return -> 终态 succeeded，返回值保留（"请收工"语义）。
|
|||
|
|
# 若希望"停止即中止"，去掉这个检查、让 checkpoint() 抛 TaskCancelled
|
|||
|
|
# -> 终态 stopped、无返回值。两种都如实上报，见示例 09。
|
|||
|
|
break
|
|||
|
|
ctx.checkpoint()
|
|||
|
|
time.sleep(0.05)
|
|||
|
|
done = index
|
|||
|
|
ctx.progress(done, total=batches, unit="批", message=tenant)
|
|||
|
|
ctx.log(f"{tenant}：本次同步 {done}/{batches} 批")
|
|||
|
|
return {"tenant": tenant, "batches": done}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# =============================================================================
|
|||
|
|
# 第三部分：接线 —— build_node()（唯一集成点，应用启动时调用一次）
|
|||
|
|
# =============================================================================
|
|||
|
|
|
|||
|
|
|
|||
|
|
def register(controller: Controller) -> None:
|
|||
|
|
"""把任务定义登记进白名单。
|
|||
|
|
|
|||
|
|
两种用法共用同一份定义：应用嵌入时由 :func:`build_node` 调用；
|
|||
|
|
CLI 用 ``--tasks 本模块`` 时，由 CLI 调用（``register(controller)``
|
|||
|
|
正是 CLI 的约定函数名）。
|
|||
|
|
"""
|
|||
|
|
controller.register_all(sync_tenant)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_node(deployment: Deployment) -> Controller:
|
|||
|
|
"""建好一个节点：存储 + 身份 + 桶归属 + 并发上限 + 任务定义 + 启动恢复。
|
|||
|
|
|
|||
|
|
这是应用**唯一**需要接进本库的地方：你的服务（Web / 定时器 / 消费者）启动时
|
|||
|
|
调用一次，把返回的 ``Controller`` 持有到进程结束；退出前 ``close(close_store=True)``。
|
|||
|
|
"""
|
|||
|
|
controller = Controller(
|
|||
|
|
store=SqliteTaskStore(deployment.db_path),
|
|||
|
|
owner=deployment.owner,
|
|||
|
|
bucket_policy=BucketPolicy(num_buckets=deployment.num_buckets),
|
|||
|
|
# 桶归属 = 划分的落地处：本节点只接受、只清理这些桶里的任务。
|
|||
|
|
owned_buckets=deployment.owned_buckets,
|
|||
|
|
# 并发与资源约束集中在这一处（ConcurrencyPolicy）：
|
|||
|
|
# max_parallel=4 —— 本进程同时最多执行 4 个；
|
|||
|
|
# limits={"db": 3} —— 其中最多 3 个可以同时压数据库（带 db 标签的任务），
|
|||
|
|
# 保护的是"本节点到库里的连接数"这份预算；
|
|||
|
|
# priority=... —— 默认优先级，submit(priority=...) 可逐条覆盖。
|
|||
|
|
concurrency=ConcurrencyPolicy(max_parallel=4, limits={"db": 3}),
|
|||
|
|
max_parallel_per_bucket=1, # 同一个租户（同桶）串行，避免自己挤自己
|
|||
|
|
command_poll_interval=0.1, # 仅为让演示输出得快；生产用默认 0.5s 即可
|
|||
|
|
)
|
|||
|
|
register(controller)
|
|||
|
|
# 启动恢复：把上次进程留下的、占着资源却没人管的僵尸任务如实标为 lost。
|
|||
|
|
# 只看**自己负责的桶**——别的节点的任务由它自己的 reconcile 处理。
|
|||
|
|
report = controller.reconcile()
|
|||
|
|
owned = deployment.owned_buckets
|
|||
|
|
_log.info(
|
|||
|
|
"[%s] 启动：负责桶 %d-%d；reconcile：%d 个判失联，%d 个待重新提交",
|
|||
|
|
deployment.owner,
|
|||
|
|
owned.start,
|
|||
|
|
owned.stop - 1,
|
|||
|
|
len(report.lost),
|
|||
|
|
len(report.adoptable),
|
|||
|
|
)
|
|||
|
|
return controller
|
|||
|
|
|
|||
|
|
|
|||
|
|
# =============================================================================
|
|||
|
|
# 第四部分：演示（启动 -> 提交 -> 观测 -> 控制 -> 收尾；这一段不用照抄）
|
|||
|
|
# =============================================================================
|
|||
|
|
|
|||
|
|
#: 演示用的租户（业务键）。生产上它来自你的业务清单。
|
|||
|
|
TENANTS = ("tenant-acme", "tenant-umbrella", "tenant-globex", "tenant-hooli")
|
|||
|
|
|
|||
|
|
#: 给"跨节点停止"演示留足运行窗口：80 批 × 0.05s ≈ 4 秒。
|
|||
|
|
SLOW_BATCHES = {"tenant-globex": 80}
|
|||
|
|
|
|||
|
|
#: 模拟"上一个进程崩溃"遗留的僵尸任务的业务键（bucket 7，归 node-0）。
|
|||
|
|
ZOMBIE_KEY = "tenant-vandelay"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _plant_zombie(db_path: Path, policy: BucketPolicy, key: str) -> str:
|
|||
|
|
"""往存储里放一条"崩溃进程留下的"任务：RUNNING 且租约已过期。
|
|||
|
|
|
|||
|
|
真实场景里这是崩溃的进程自己留下的（写穿 + 租约续不上）；这里手工造一条，
|
|||
|
|
让 reconcile 有东西可清、让"如实标记而不是假装恢复"看得见。
|
|||
|
|
"""
|
|||
|
|
now = utcnow()
|
|||
|
|
task_id = f"sync-{key}"
|
|||
|
|
snapshot = TaskSnapshot(
|
|||
|
|
task_id=task_id,
|
|||
|
|
name=f"同步-{key}",
|
|||
|
|
bucket=policy.bucket_for(key),
|
|||
|
|
bucket_key=key,
|
|||
|
|
status=TaskStatus.RUNNING,
|
|||
|
|
status_version=1,
|
|||
|
|
attempt=1,
|
|||
|
|
max_attempts=1,
|
|||
|
|
created_at=now - timedelta(minutes=12),
|
|||
|
|
queued_at=now - timedelta(minutes=12),
|
|||
|
|
started_at=now - timedelta(minutes=11),
|
|||
|
|
last_heartbeat_at=now - timedelta(minutes=11),
|
|||
|
|
executor="thread",
|
|||
|
|
lease_owner="node-0@dead-process",
|
|||
|
|
lease_until=now - timedelta(minutes=6),
|
|||
|
|
)
|
|||
|
|
store = SqliteTaskStore(db_path)
|
|||
|
|
try:
|
|||
|
|
store.create(snapshot)
|
|||
|
|
finally:
|
|||
|
|
store.close()
|
|||
|
|
return task_id
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _print_plan(deployments: Sequence[Deployment], policy: BucketPolicy, db_path: Path) -> None:
|
|||
|
|
"""打印部署计划、桶划分与路由表——集成前先把"谁负责什么"看清楚。"""
|
|||
|
|
print("=" * 72)
|
|||
|
|
print(f"部署：{NUM_BUCKETS} 个桶，{NODE_COUNT} 个节点，共享一套存储（{db_path.name}）")
|
|||
|
|
print("=" * 72)
|
|||
|
|
print(" 节点 负责桶 等价的 CLI 参数")
|
|||
|
|
print(" -------- ---------- -----------------------------------")
|
|||
|
|
for deployment in deployments:
|
|||
|
|
owned = deployment.owned_buckets
|
|||
|
|
spec = f"--owned-buckets {deployment.cli_buckets}"
|
|||
|
|
print(f" {deployment.owner:<8} {owned.start:>2}-{owned.stop - 1:<8} {spec}")
|
|||
|
|
print(" 注意：--num-buckets 也必须一致，否则同一个键会落到不同的桶，且无任何报错。")
|
|||
|
|
|
|||
|
|
print("\n 路由表（blake2b 稳定哈希：同一个键在任何进程、任何机器上都落到同一个桶）：")
|
|||
|
|
for tenant in TENANTS:
|
|||
|
|
bucket = policy.bucket_for(tenant)
|
|||
|
|
print(f" {tenant:<18} -> 桶 {bucket:>2} -> {owner_of(bucket, deployments).owner}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main() -> int:
|
|||
|
|
# 输出到 stdout（默认是 stderr）：这样日志与 print 的顺序一致，
|
|||
|
|
# 管道重定向时也不会看到"日志跑到前面去了"。
|
|||
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s", stream=sys.stdout)
|
|||
|
|
|
|||
|
|
with tempfile.TemporaryDirectory(prefix="chongming-best-practice-") as directory:
|
|||
|
|
db_path = Path(directory) / "tasks.db"
|
|||
|
|
deployments = [
|
|||
|
|
Deployment(node_index=index, node_count=NODE_COUNT, db_path=db_path)
|
|||
|
|
for index in range(NODE_COUNT)
|
|||
|
|
]
|
|||
|
|
policy = BucketPolicy(num_buckets=NUM_BUCKETS)
|
|||
|
|
|
|||
|
|
_print_plan(deployments, policy, db_path)
|
|||
|
|
|
|||
|
|
zombie_id = _plant_zombie(db_path, policy, ZOMBIE_KEY)
|
|||
|
|
zombie_bucket = policy.bucket_for(ZOMBIE_KEY)
|
|||
|
|
assert zombie_bucket in deployments[0].owned_buckets, "僵尸任务应落在 node-0 的桶里"
|
|||
|
|
print(
|
|||
|
|
f"\n[准备] 模拟上一个进程崩溃：存储里留有一条 RUNNING 且租约过期的任务 "
|
|||
|
|
f"{zombie_id}（桶 {zombie_bucket}，归 node-0）"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- 启动
|
|||
|
|
print("\n" + "=" * 72)
|
|||
|
|
print("一、启动：每个节点先 reconcile()，只清理**自己桶里**的僵尸任务")
|
|||
|
|
print("=" * 72)
|
|||
|
|
nodes = [build_node(deployment) for deployment in deployments]
|
|||
|
|
# 运维控制台：一个不持有任务的 Controller，只查询与控制（不声明 owned_buckets）。
|
|||
|
|
console = Controller(store=SqliteTaskStore(db_path), owner="ops-console")
|
|||
|
|
|
|||
|
|
zombie = console.get(zombie_id)
|
|||
|
|
assert zombie is not None and zombie.status is TaskStatus.LOST
|
|||
|
|
print(
|
|||
|
|
f" 控制台看到 {zombie_id} 现在是 {zombie.status.value}"
|
|||
|
|
"（被如实标记为丢失，而不是永远显示 running；它不会被自动重跑——"
|
|||
|
|
"存储里只有状态、没有配方）"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- 提交
|
|||
|
|
print("\n" + "=" * 72)
|
|||
|
|
print("二、触发：按桶把任务交给拥有它的节点（配错桶 = 永远不会有人执行）")
|
|||
|
|
print("=" * 72)
|
|||
|
|
for tenant in TENANTS:
|
|||
|
|
bucket = policy.bucket_for(tenant)
|
|||
|
|
node = nodes[owner_of(bucket, deployments).node_index]
|
|||
|
|
run = node.submit(
|
|||
|
|
"sync_tenant",
|
|||
|
|
tenant,
|
|||
|
|
task_id=f"sync-{tenant}",
|
|||
|
|
label=f"同步-{tenant}",
|
|||
|
|
bucket_key=tenant,
|
|||
|
|
batches=SLOW_BATCHES.get(tenant, 24),
|
|||
|
|
)
|
|||
|
|
print(f" {tenant:<18} 桶 {bucket:>2} -> {node.owner} 提交 {run.snapshot.task_id}")
|
|||
|
|
|
|||
|
|
foreign_tenant = next(
|
|||
|
|
tenant
|
|||
|
|
for tenant in TENANTS
|
|||
|
|
if owner_of(policy.bucket_for(tenant), deployments).node_index != 0
|
|||
|
|
)
|
|||
|
|
foreign_bucket = policy.bucket_for(foreign_tenant)
|
|||
|
|
foreign_owner = owner_of(foreign_bucket, deployments).owner
|
|||
|
|
print(
|
|||
|
|
f"\n >>> 反例：把 {foreign_tenant}（桶 {foreign_bucket}，归 {foreign_owner}）"
|
|||
|
|
"提交给 node-0："
|
|||
|
|
)
|
|||
|
|
try:
|
|||
|
|
nodes[0].submit(
|
|||
|
|
"sync_tenant",
|
|||
|
|
foreign_tenant,
|
|||
|
|
task_id="sync-misrouted",
|
|||
|
|
bucket_key=foreign_tenant,
|
|||
|
|
)
|
|||
|
|
except ConfigurationError as exc:
|
|||
|
|
print(f" ✅ 提交那一刻就拦住了：{exc}")
|
|||
|
|
else:
|
|||
|
|
raise AssertionError("提交给错误的节点居然成功了——桶归属校验失效")
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- 观测
|
|||
|
|
time.sleep(0.3)
|
|||
|
|
print("\n" + "=" * 72)
|
|||
|
|
print("三、观测：一套存储 = 一个看板看全部；按桶过滤 = 只看我这一份")
|
|||
|
|
print("=" * 72)
|
|||
|
|
print("--- 全局视图（控制台不在任何节点上，也能看到全部任务）---")
|
|||
|
|
print(render_table(console.list_tasks()))
|
|||
|
|
|
|||
|
|
owned_0 = deployments[0].owned_buckets
|
|||
|
|
print(f"\n--- node-0 的视角（list_tasks(buckets={owned_0.start}-{owned_0.stop - 1})）---")
|
|||
|
|
print(render_table(nodes[0].list_tasks(buckets=owned_0)))
|
|||
|
|
|
|||
|
|
for node in nodes:
|
|||
|
|
print(f" {node.owner}：占用执行额度 {node.active_count}，排队 {node.queued_count}")
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- 控制
|
|||
|
|
target_id = f"sync-{foreign_tenant}"
|
|||
|
|
target = console.get(target_id)
|
|||
|
|
assert target is not None and not target.is_terminal, "演示假设：停止目标仍在运行"
|
|||
|
|
|
|||
|
|
print("\n" + "=" * 72)
|
|||
|
|
print("四、控制：跨节点停止（不能跨进程发信号 -> 写一条指令，由持有者取走）")
|
|||
|
|
print("=" * 72)
|
|||
|
|
print(
|
|||
|
|
f" ops-console 请求停止 {target_id}（持有者是 {foreign_owner}）："
|
|||
|
|
f"下发结果 {console.stop(target_id, reason='发布窗口')}"
|
|||
|
|
)
|
|||
|
|
outcome = nodes[1].wait(target_id, timeout=15)
|
|||
|
|
print(f" 终态：{outcome.status.value}；返回值：{outcome.value}")
|
|||
|
|
print(" 说明：任务体看到停止请求后优雅收尾 -> succeeded 且数值保留；")
|
|||
|
|
print(" 让 checkpoint() 抛 TaskCancelled 则为 stopped。两种都如实上报。")
|
|||
|
|
print("\n 审计轨迹（从共享存储读；谁按的、为什么，都可追溯）：")
|
|||
|
|
for record in console.store.history(target_id): # type: ignore[union-attr]
|
|||
|
|
reason = f" reason={record.reason}" if record.reason else ""
|
|||
|
|
print(
|
|||
|
|
f" {record.version:>2} {record.from_status.value:>9} -> "
|
|||
|
|
f"{record.to_status.value:<9} {record.trigger}{reason}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- 收尾
|
|||
|
|
for node in nodes:
|
|||
|
|
node.wait_all(timeout=30)
|
|||
|
|
|
|||
|
|
print("\n" + "=" * 72)
|
|||
|
|
print("五、收尾：校验一致性 -> close(close_store=True)")
|
|||
|
|
print("=" * 72)
|
|||
|
|
print(render_table(console.list_tasks()))
|
|||
|
|
stats = console.stats()
|
|||
|
|
print(f" 统计：共 {stats.total}，未结束 {stats.unfinished}，已结束 {stats.finished}")
|
|||
|
|
|
|||
|
|
for tenant in TENANTS:
|
|||
|
|
snapshot = console.get(f"sync-{tenant}")
|
|||
|
|
assert snapshot is not None and snapshot.status in (
|
|||
|
|
TaskStatus.SUCCEEDED,
|
|||
|
|
TaskStatus.STOPPED,
|
|||
|
|
), f"{tenant} 的终态不对：{snapshot.status if snapshot else '不存在'}"
|
|||
|
|
for node in nodes:
|
|||
|
|
assert node.persist_failures == (), f"{node.owner} 有写穿失败——存储不可信"
|
|||
|
|
assert node.desynced_tasks == {}, f"{node.owner} 有永久分叉的任务"
|
|||
|
|
print(" ✅ 结论与存储一致：没有写穿失败，没有永久分叉，僵尸任务如实落 lost")
|
|||
|
|
|
|||
|
|
for node in nodes:
|
|||
|
|
node.close(timeout=30, close_store=True)
|
|||
|
|
console.close(timeout=5, close_store=True)
|
|||
|
|
|
|||
|
|
_print_ops_notes()
|
|||
|
|
|
|||
|
|
print("\n全部通过 ✅")
|
|||
|
|
return 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _print_ops_notes() -> None:
|
|||
|
|
"""部署与运维说明。"""
|
|||
|
|
print(
|
|||
|
|
f"""
|
|||
|
|
生产部署：把它变成一个真正的服务
|
|||
|
|
────────────────────────────────────────────────────────────────────────
|
|||
|
|
# 每台机器 / 每个容器跑一个节点进程：同一份代码，只有 NODE_INDEX 不同
|
|||
|
|
TASK_NODE_INDEX=0 TASK_NODE_COUNT={NODE_COUNT} TASK_DB=/srv/tasks/tasks.db \\
|
|||
|
|
python -m your_service.node
|
|||
|
|
TASK_NODE_INDEX=1 TASK_NODE_COUNT={NODE_COUNT} TASK_DB=/srv/tasks/tasks.db \\
|
|||
|
|
python -m your_service.node
|
|||
|
|
|
|||
|
|
# 运维（任意一台机器；--num-buckets 与代码里的 NUM_BUCKETS 必须一致）
|
|||
|
|
uv run chongming-task --db /srv/tasks/tasks.db --num-buckets {NUM_BUCKETS} \\
|
|||
|
|
ps --all
|
|||
|
|
uv run chongming-task --db /srv/tasks/tasks.db --num-buckets {NUM_BUCKETS} \\
|
|||
|
|
--owned-buckets 0-7 ps --all --mine
|
|||
|
|
uv run chongming-task --db /srv/tasks/tasks.db --num-buckets {NUM_BUCKETS} \\
|
|||
|
|
--owned-buckets 0-7 reconcile --dry-run
|
|||
|
|
uv run chongming-task --db /srv/tasks/tasks.db --num-buckets {NUM_BUCKETS} \\
|
|||
|
|
stop sync-tenant-globex --reason "发布窗口"
|
|||
|
|
|
|||
|
|
# 本模块也能直接当 CLI 任务模块用（register(controller) 约定）
|
|||
|
|
PYTHONPATH=examples uv run chongming-task --db ./demo.db --num-buckets {NUM_BUCKETS} \\
|
|||
|
|
--owned-buckets 0-7 run --tasks 11_best_practice sync_tenant \\
|
|||
|
|
--arg tenant=tenant-acme --task-id sync-tenant-acme --bucket-key tenant-acme
|
|||
|
|
|
|||
|
|
集成三步（照抄本文件的三个"部分"）
|
|||
|
|
────────────────────────────────────────────────────────────────────────
|
|||
|
|
# 1. 第一部分 + 第二部分 -> 你的配置模块与任务模块；
|
|||
|
|
# 2. 应用启动时 build_node(Deployment.from_env())，退出前 close(close_store=True)；
|
|||
|
|
# 3. 触发处 submit(任务名, ..., bucket_key=业务键)。
|
|||
|
|
# 两条硬约束：任务只能提交给拥有它那个桶的节点；bucket_key 必须给
|
|||
|
|
# （限定了 owned_buckets 时不给会被当场拒绝——没有归属桶的任务没人认领）。
|
|||
|
|
#
|
|||
|
|
# 跨机部署：SQLite 文件不能跨机共享，把第三部分的 store 换成
|
|||
|
|
# PostgresTaskStore("postgresql://user:pw@db.internal:5432/chongming")
|
|||
|
|
# （pip install "chongming-task[postgres]"）即可，其余部分不用改。""".rstrip()
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
sys.exit(main())
|