657 lines
23 KiB
Python
657 lines
23 KiB
Python
"""Scheduler poll loop against real file stores (T08)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, cast
|
|
|
|
from tests.scheduling.controlled import (
|
|
DictDeployments,
|
|
ScriptedDispatcher,
|
|
fixture_environment,
|
|
)
|
|
from wf_artifacts.runs.models import StoredRunStatus
|
|
from wf_artifacts.runs.store import FileRunStore
|
|
from wf_scheduling.calendar import OneShotSource
|
|
from wf_scheduling.models import Schedule
|
|
from wf_scheduling.ownership import SchedulerOwnership
|
|
from wf_scheduling.poll import SCAN_CAP, Scheduler
|
|
from wf_scheduling.prepare import SchedulePreparer
|
|
from wf_scheduling.store import FileScheduleStore
|
|
|
|
UTC_TZ = UTC
|
|
|
|
|
|
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0, s: int = 0) -> datetime:
|
|
return datetime(y, mo, d, h, mi, s, tzinfo=UTC_TZ)
|
|
|
|
|
|
class PeriodicSource:
|
|
def __init__(self, period: timedelta, start: datetime) -> None:
|
|
self.period = period
|
|
self.start = start
|
|
self.next_calls = 0
|
|
self.prev_calls = 0
|
|
|
|
def next_after(self, instant: datetime) -> datetime | None:
|
|
self.next_calls += 1
|
|
if instant < self.start:
|
|
return self.start
|
|
n = (instant - self.start) // self.period + 1
|
|
return self.start + n * self.period
|
|
|
|
def prev_before(self, instant: datetime) -> datetime | None:
|
|
self.prev_calls += 1
|
|
if instant <= self.start:
|
|
return None
|
|
n = (instant - self.start - timedelta(microseconds=1)) // self.period
|
|
return self.start + n * self.period
|
|
|
|
|
|
def _sched_model(sid: str, start_hint: str = "cron", **kw: Any) -> Schedule:
|
|
now = ts(2026, 9, 8, 12, 0)
|
|
base: dict[str, Any] = {
|
|
"id": sid,
|
|
"deployment_id": "dep-1",
|
|
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
|
|
"input_bindings": [],
|
|
"created_at": now.isoformat(),
|
|
"updated_at": now.isoformat(),
|
|
}
|
|
base.update(kw)
|
|
return Schedule.model_validate(base)
|
|
|
|
|
|
def _harness(
|
|
tmp_path: Path,
|
|
*,
|
|
capacity: int = 4,
|
|
script: dict | None = None,
|
|
deployments: dict[str, dict] | None = None,
|
|
) -> tuple[Scheduler, FileScheduleStore, FileRunStore, dict]:
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sources: dict = {}
|
|
preparer = SchedulePreparer(
|
|
DictDeployments(
|
|
deployments
|
|
if deployments is not None
|
|
else {"dep-1": {"rev": 1, "required": []}}
|
|
),
|
|
fixture_environment,
|
|
)
|
|
sched = Scheduler(
|
|
schedule_store=sched_store,
|
|
run_store=run_store,
|
|
sources=sources,
|
|
capacity=capacity,
|
|
preparer=preparer,
|
|
dispatcher=ScriptedDispatcher(script),
|
|
ownership=SchedulerOwnership(tmp_path, owner="test").acquire(),
|
|
)
|
|
return sched, sched_store, run_store, sources
|
|
|
|
|
|
def _add(
|
|
sched: Scheduler,
|
|
store: FileScheduleStore,
|
|
sources: dict,
|
|
sid: str,
|
|
src: Any,
|
|
consumed: datetime,
|
|
**kw: Any,
|
|
) -> Any:
|
|
model = _sched_model(sid, **kw)
|
|
store.create_schedule(model)
|
|
store.save_consumed(sid, consumed)
|
|
sources[sid] = src
|
|
return store.get_schedule(sid)
|
|
|
|
|
|
def _history(store: FileScheduleStore, sid: str) -> list[dict]:
|
|
page = store.list_occurrences(sid, limit=100)
|
|
return page["occurrences"] # type: ignore[return-value]
|
|
|
|
|
|
def test_overlap_skip_blocks_and_late_drops() -> None:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
sched, store, runs, sources = _harness(root, capacity=4, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"a",
|
|
PeriodicSource(timedelta(minutes=10), t0),
|
|
t0 - timedelta(minutes=10),
|
|
)
|
|
sched.poll(t0)
|
|
assert len(runs.list_runs()) == 1
|
|
sched.poll(t0 + timedelta(minutes=10))
|
|
kinds = [(r["kind"], r["resolved_at"]) for r in _history(store, "a")]
|
|
assert any(k == "skipped-overlap" and "12:10" in str(v) for k, v in kinds)
|
|
sched.poll(t0 + timedelta(minutes=30))
|
|
kinds = [(r["kind"], r["resolved_at"]) for r in _history(store, "a")]
|
|
assert any(k == "skipped-misfire" for k, _ in kinds)
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_latest_coalesces_to_one_candidate_and_no_double_admit() -> None:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
sched, store, runs, sources = _harness(root, capacity=0)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"h",
|
|
PeriodicSource(timedelta(hours=1), ts(2026, 9, 8, 9, 0)),
|
|
ts(2026, 9, 8, 9, 0),
|
|
misfire="latest",
|
|
)
|
|
sched.poll(ts(2026, 9, 8, 12, 20))
|
|
cand = store.get_candidate("h")
|
|
assert cand is not None and cand.intended_at == ts(2026, 9, 8, 12, 0)
|
|
assert [r for r in _history(store, "h") if r["kind"] == "admitted"] == []
|
|
# Capacity returns at 13:00 while 13:00 is also due: exactly one
|
|
# admission for 13:00, 12:00 superseded, never both.
|
|
sched.capacity = 4
|
|
cast(ScriptedDispatcher, sched.dispatcher).script = {"*": "complete"}
|
|
sched.poll(ts(2026, 9, 8, 13, 0))
|
|
admitted = [r for r in _history(store, "h") if r["kind"] == "admitted"]
|
|
assert len(admitted) == 1
|
|
assert datetime.fromisoformat(admitted[0]["resolved_at"]) == ts(
|
|
2026, 9, 8, 13, 0
|
|
)
|
|
assert store.get_candidate("h") is None
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_parallel_limits_and_interrupted_slots() -> None:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
sched, store, runs, sources = _harness(root, capacity=4, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"p",
|
|
PeriodicSource(timedelta(minutes=5), t0),
|
|
t0 - timedelta(minutes=5),
|
|
overlap="parallel",
|
|
max_active_runs=2,
|
|
)
|
|
sched.poll(t0)
|
|
assert sched._task_load() == 1
|
|
first = runs.list_runs()[0]
|
|
# A waiting interruption holds no task slot: production dispatch
|
|
# clears the executing mark when persisting the stopped result, so
|
|
# the simulated interruption clears it too.
|
|
runs.save_run(first.model_copy(update={"status": StoredRunStatus.INTERRUPTED}))
|
|
runs.clear_executing(first.id)
|
|
assert sched._task_load() == 0
|
|
sched.poll(t0 + timedelta(minutes=5))
|
|
assert len(runs.list_runs()) == 2
|
|
sched.poll(t0 + timedelta(minutes=10))
|
|
assert any(r["kind"] == "skipped-overlap" for r in _history(store, "p"))
|
|
# Lowering the limit never cancels; blocks new admission instead.
|
|
sched_model = store.get_schedule("p")
|
|
sched_model.max_active_runs = 1
|
|
store.save_schedule(sched_model)
|
|
second = [r for r in runs.list_runs() if r.id != first.id][0]
|
|
runs.save_run(second.model_copy(update={"status": StoredRunStatus.INTERRUPTED}))
|
|
runs.clear_executing(second.id)
|
|
sched.poll(t0 + timedelta(minutes=15))
|
|
assert any(
|
|
r["kind"] == "skipped-overlap" and "12:15" in str(r["resolved_at"])
|
|
for r in _history(store, "p")
|
|
)
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_pause_is_not_downtime() -> None:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
sched, store, runs, sources = _harness(
|
|
root, capacity=4, script={"*": "complete"}
|
|
)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"a",
|
|
PeriodicSource(timedelta(hours=1), ts(2026, 9, 8, 9, 0)),
|
|
ts(2026, 9, 8, 9, 0),
|
|
misfire="latest",
|
|
)
|
|
sched.poll(ts(2026, 9, 8, 9, 0))
|
|
model = store.get_schedule("a")
|
|
model.paused = True
|
|
store.save_schedule(model)
|
|
sched.poll(ts(2026, 9, 8, 10, 30))
|
|
assert store.get_candidate("a") is None
|
|
sched.resume_schedule("a", ts(2026, 9, 8, 12, 30))
|
|
sched.poll(ts(2026, 9, 8, 12, 30))
|
|
admitted = [
|
|
r["resolved_at"] for r in _history(store, "a") if r["kind"] == "admitted"
|
|
]
|
|
assert "2026-09-08T10:00:00+00:00" not in admitted
|
|
assert "2026-09-08T11:00:00+00:00" not in admitted
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_long_downtime_is_bounded(tmp_path: Path) -> None:
|
|
sched, store, runs, sources = _harness(
|
|
tmp_path, capacity=4, script={"*": "complete"}
|
|
)
|
|
src = PeriodicSource(timedelta(minutes=1), ts(2023, 9, 8, 12, 0))
|
|
_add(sched, store, sources, "m", src, ts(2023, 9, 8, 12, 0), misfire="latest")
|
|
sched.poll(ts(2026, 9, 8, 12, 0, 0))
|
|
assert src.next_calls + src.prev_calls <= SCAN_CAP + 2
|
|
admitted = [r for r in _history(store, "m") if r["kind"] == "admitted"]
|
|
assert len(admitted) == 1
|
|
# Latest-eligible <= now: a poll exactly at a due instant selects that
|
|
# instant (F3), not its exclusive predecessor.
|
|
assert datetime.fromisoformat(admitted[0]["resolved_at"]) == ts(2026, 9, 8, 12, 0)
|
|
assert (
|
|
len([r for r in _history(store, "m") if r["kind"] == "interval-summary"]) == 1
|
|
)
|
|
# A second poll at the same instant admits nothing more.
|
|
sched.poll(ts(2026, 9, 8, 12, 0, 0))
|
|
assert len([r for r in _history(store, "m") if r["kind"] == "admitted"]) == 1
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_fairness_slow_schedule_not_starved(tmp_path: Path) -> None:
|
|
sched, store, runs, sources = _harness(tmp_path, capacity=1, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"fast",
|
|
PeriodicSource(timedelta(minutes=1), t0),
|
|
t0 - timedelta(minutes=1),
|
|
)
|
|
_add(sched, store, sources, "slow", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
sched.poll(t0)
|
|
assert any(
|
|
r["kind"] == "admitted" and r["schedule_id"] == "fast"
|
|
for r in _history(store, "fast")
|
|
)
|
|
for run in runs.list_runs():
|
|
runs.save_run(run.model_copy(update={"status": StoredRunStatus.COMPLETED}))
|
|
runs.clear_executing(run.id)
|
|
sched.poll(t0 + timedelta(seconds=30))
|
|
assert any(r["kind"] == "admitted" for r in _history(store, "slow"))
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_capacity_wait_then_expire_for_skip(tmp_path: Path) -> None:
|
|
sched, store, runs, sources = _harness(tmp_path, capacity=0)
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(sched, store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
assert sched.poll(t0) == {"a": "admit:held-undecided"}
|
|
assert store.get_candidate("a") is None
|
|
sched.capacity = 1
|
|
cast(ScriptedDispatcher, sched.dispatcher).script = {"*": "complete"}
|
|
sched.poll(t0 + timedelta(seconds=30))
|
|
assert len([r for r in _history(store, "a") if r["kind"] == "admitted"]) == 1
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_poll_one_rereads_pause_before_deciding(tmp_path: Path) -> None:
|
|
sched, store, runs, sources = _harness(tmp_path, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(sched, store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
stale = store.get_schedule("a")
|
|
# Same-process administration lands after the tick listed schedules:
|
|
# the decision must observe the pause, not the stale snapshot.
|
|
live = store.get_schedule("a")
|
|
live.paused = True
|
|
store.save_schedule(live)
|
|
assert sched._poll_one(stale, t0 + timedelta(seconds=1)) == "paused"
|
|
assert runs.list_runs() == []
|
|
assert runs.list_admissions() == []
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_poll_one_uses_fresh_definition_after_edit(tmp_path: Path) -> None:
|
|
sched, store, runs, sources = _harness(tmp_path, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"a",
|
|
OneShotSource(t0),
|
|
t0 - timedelta(hours=1),
|
|
input_bindings=[
|
|
{"target": "team", "expression": {"kind": "literal", "value": "old"}}
|
|
],
|
|
)
|
|
stale = store.get_schedule("a")
|
|
live = store.get_schedule("a")
|
|
live.revision = 2
|
|
from wf_core.models.input_bindings import ScheduleInputBinding
|
|
|
|
live.input_bindings = [
|
|
ScheduleInputBinding.model_validate(
|
|
{"target": "team", "expression": {"kind": "literal", "value": "new"}}
|
|
)
|
|
]
|
|
store.save_schedule(live)
|
|
result = sched._poll_one(stale, t0 + timedelta(seconds=1))
|
|
assert result.startswith("admit:run-")
|
|
run_id = result.split(":", 1)[1]
|
|
admission = runs.get_admission(run_id)
|
|
assert admission.resolved_input["team"] == "new"
|
|
assert admission.schedule_revision == 2
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_manual_and_other_schedule_runs_are_overlap_independent(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
from wf_api.run_lifecycle import (
|
|
materialize_admitted_view,
|
|
persist_admission,
|
|
)
|
|
|
|
sched, store, runs, sources = _harness(tmp_path, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
# An active run of another schedule occupies only its own slot.
|
|
_add(sched, store, sources, "b", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
assert sched.poll(t0)["b"].startswith("admit:run-")
|
|
# A manual run (no schedule owner) participates in no overlap check.
|
|
manual_id = runs.allocate_run_id()
|
|
manual = persist_admission(
|
|
store=runs,
|
|
run_id=manual_id,
|
|
environment=fixture_environment(object()),
|
|
resolved_input={},
|
|
max_steps=None,
|
|
)
|
|
materialize_admitted_view(store=runs, admission=manual)
|
|
# A due one-shot admits despite both unrelated active runs.
|
|
_add(sched, store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
result = sched.poll(t0 + timedelta(seconds=1))
|
|
assert result["a"].startswith("admit:run-")
|
|
assert result["b"] == "exhausted"
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_manual_admission_does_not_consume_scheduler_capacity(tmp_path: Path) -> None:
|
|
"""Manual runs bypass scheduler capacity; scheduled work still dispatches."""
|
|
from wf_api.run_lifecycle import materialize_admitted_view, persist_admission
|
|
|
|
sched, store, runs, sources = _harness(tmp_path, capacity=1, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
manual = persist_admission(
|
|
store=runs,
|
|
run_id=runs.allocate_run_id(),
|
|
environment=fixture_environment(object()),
|
|
resolved_input={},
|
|
max_steps=None,
|
|
)
|
|
materialize_admitted_view(store=runs, admission=manual)
|
|
assert sched._task_load() == 0
|
|
|
|
_add(sched, store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
assert sched.poll(t0)["a"].startswith("admit:run-")
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_existing_admission_rebuilds_missing_view_and_pending_marker(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""A history tear after admission remains dispatchable on the next poll."""
|
|
|
|
class FailHistoryOnce(FileScheduleStore):
|
|
def __init__(self, root: Path) -> None:
|
|
super().__init__(root)
|
|
self.armed = True
|
|
|
|
def append_history(self, record: Any) -> None:
|
|
if self.armed:
|
|
self.armed = False
|
|
raise OSError("injected history failure")
|
|
super().append_history(record)
|
|
|
|
sched_store = FailHistoryOnce(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sources: dict[str, Any] = {}
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
sched = Scheduler(
|
|
schedule_store=sched_store,
|
|
run_store=run_store,
|
|
sources=sources,
|
|
capacity=1,
|
|
preparer=SchedulePreparer(
|
|
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
|
|
fixture_environment,
|
|
),
|
|
dispatcher=ScriptedDispatcher({"*": "complete"}),
|
|
ownership=SchedulerOwnership(tmp_path, owner="test").acquire(),
|
|
)
|
|
try:
|
|
_add(
|
|
sched, sched_store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1)
|
|
)
|
|
try:
|
|
sched.poll(t0)
|
|
raise AssertionError("history failure must propagate")
|
|
except OSError as exc:
|
|
assert "injected history failure" in str(exc)
|
|
[admission] = run_store.list_admissions()
|
|
assert run_store.list_runs() == []
|
|
assert not run_store.is_pending_dispatch(admission.id)
|
|
|
|
result = sched.poll(t0 + timedelta(seconds=1))
|
|
assert result["a"] == f"admit:{admission.id}"
|
|
assert run_store.get_run(admission.id).status.value == "admitted"
|
|
assert run_store.is_pending_dispatch(admission.id)
|
|
|
|
sched.poll(t0 + timedelta(seconds=2))
|
|
assert run_store.get_run(admission.id).status.value == "completed"
|
|
assert not run_store.is_pending_dispatch(admission.id)
|
|
finally:
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_oneshot_terminal_bookkeeping_is_idempotent_after_torn_write(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""A torn exhausted transition does not duplicate history or retain a candidate."""
|
|
|
|
class FailScheduleOnce(FileScheduleStore):
|
|
def __init__(self, root: Path) -> None:
|
|
super().__init__(root)
|
|
self.armed = False
|
|
|
|
def save_schedule(self, schedule: Any) -> None:
|
|
if self.armed:
|
|
self.armed = False
|
|
raise OSError("injected schedule failure")
|
|
super().save_schedule(schedule)
|
|
|
|
sched_store = FailScheduleOnce(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sources: dict[str, Any] = {}
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
sched = Scheduler(
|
|
schedule_store=sched_store,
|
|
run_store=run_store,
|
|
sources=sources,
|
|
capacity=1,
|
|
preparer=SchedulePreparer(
|
|
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
|
|
fixture_environment,
|
|
),
|
|
dispatcher=ScriptedDispatcher({"*": "complete"}),
|
|
ownership=SchedulerOwnership(tmp_path, owner="test").acquire(),
|
|
)
|
|
try:
|
|
_add(
|
|
sched, sched_store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1)
|
|
)
|
|
from wf_scheduling.models import PendingCandidate
|
|
|
|
sched_store.save_candidate(
|
|
PendingCandidate(schedule_id="a", intended_at=t0, revision=1),
|
|
schedule_id="a",
|
|
)
|
|
sched_store.armed = True
|
|
try:
|
|
sched.poll(t0 + timedelta(minutes=5))
|
|
raise AssertionError("schedule failure must propagate")
|
|
except OSError as exc:
|
|
assert "injected schedule failure" in str(exc)
|
|
assert (
|
|
len([r for r in _history(sched_store, "a") if r["kind"] == "exhausted"])
|
|
== 1
|
|
)
|
|
assert sched_store.get_candidate("a") is not None
|
|
|
|
assert sched.poll(t0 + timedelta(minutes=6))["a"] == "exhausted"
|
|
assert (
|
|
len([r for r in _history(sched_store, "a") if r["kind"] == "exhausted"])
|
|
== 1
|
|
)
|
|
assert sched_store.get_candidate("a") is None
|
|
finally:
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_corrupt_unattributed_view_isolated_from_healthy_schedules(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""A view without an admission fails closed without blocking siblings."""
|
|
from wf_artifacts import ResumeReadiness, WorkflowRunRecord
|
|
from wf_artifacts.runs.models import StoredRunStatus
|
|
|
|
sched, store, runs, sources = _harness(tmp_path, script={"*": "complete"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(sched, store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
_add(sched, store, sources, "b", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
corrupt_id = runs.allocate_run_id()
|
|
runs.save_run(
|
|
WorkflowRunRecord(
|
|
id=corrupt_id,
|
|
status=StoredRunStatus.ADMITTED,
|
|
resume_readiness=ResumeReadiness.NOT_APPLICABLE,
|
|
environment=fixture_environment(object()),
|
|
latest_checkpoint_id=None,
|
|
created_at=t0,
|
|
updated_at=t0,
|
|
)
|
|
)
|
|
|
|
results = sched.poll(t0)
|
|
|
|
assert results["a"].startswith("admit:run-")
|
|
assert results["b"].startswith("admit:run-")
|
|
assert runs.get_run(corrupt_id).status.value == "failed"
|
|
assert all(
|
|
runs.get_run(run.id).status.value == "completed"
|
|
for run in runs.list_runs()
|
|
if run.id != corrupt_id
|
|
)
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_record_resumed_stopped_result_is_idempotent_and_attributed(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Live resumed results record exactly once with schedule attribution (B3)."""
|
|
from tests.scheduling.controlled import fixture_environment
|
|
from wf_api.run_lifecycle import persist_admission
|
|
|
|
sched, store, runs, sources = _harness(tmp_path, script={"*": "interrupt"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(sched, store, sources, "a", OneShotSource(t0), t0 - timedelta(seconds=1))
|
|
assert sched.poll(t0)["a"].startswith("admit:run-")
|
|
[run] = runs.list_runs()
|
|
assert run.status.value == "interrupted"
|
|
assert len([r for r in _history(store, "a") if r["kind"] == "interrupted"]) == 1
|
|
|
|
now = t0 + timedelta(minutes=1)
|
|
resumed_ckpt = f"{run.id}.000002"
|
|
assert (
|
|
sched.record_resumed_stopped_result(
|
|
run.id,
|
|
status_value="completed",
|
|
checkpoint_id=resumed_ckpt,
|
|
now=now,
|
|
)
|
|
is True
|
|
)
|
|
# Repeats (poll retries, restart recovery) never duplicate the entry.
|
|
assert (
|
|
sched.record_resumed_stopped_result(
|
|
run.id,
|
|
status_value="completed",
|
|
checkpoint_id=resumed_ckpt,
|
|
now=now,
|
|
)
|
|
is False
|
|
)
|
|
completed = [r for r in _history(store, "a") if r["kind"] == "completed"]
|
|
assert len(completed) == 1
|
|
assert completed[0]["run_id"] == run.id
|
|
assert completed[0]["checkpoint_id"] == resumed_ckpt
|
|
assert completed[0]["revision"] == 1
|
|
assert completed[0]["reason"] == "resumed"
|
|
# A resumed re-interruption is a new result (new checkpoint), not a dup.
|
|
assert (
|
|
sched.record_resumed_stopped_result(
|
|
run.id,
|
|
status_value="interrupted",
|
|
checkpoint_id=f"{run.id}.000003",
|
|
now=now,
|
|
)
|
|
is True
|
|
)
|
|
assert len([r for r in _history(store, "a") if r["kind"] == "interrupted"]) == 2
|
|
# Unknown runs, manual runs, and bad statuses record nothing.
|
|
assert (
|
|
sched.record_resumed_stopped_result(
|
|
"run-999999",
|
|
status_value="completed",
|
|
checkpoint_id=None,
|
|
now=now,
|
|
)
|
|
is False
|
|
)
|
|
assert (
|
|
sched.record_resumed_stopped_result(
|
|
run.id, status_value="bogus", checkpoint_id=None, now=now
|
|
)
|
|
is False
|
|
)
|
|
manual = persist_admission(
|
|
store=runs,
|
|
run_id=runs.allocate_run_id(),
|
|
environment=fixture_environment(object()),
|
|
resolved_input={},
|
|
max_steps=None,
|
|
)
|
|
assert (
|
|
sched.record_resumed_stopped_result(
|
|
manual.id, status_value="completed", checkpoint_id=None, now=now
|
|
)
|
|
is False
|
|
)
|
|
sched.ownership.release()
|