sched: add startup recovery that never executes work (T10)
This commit is contained in:
@@ -125,9 +125,14 @@ class Scheduler:
|
||||
return admission.schedule_id
|
||||
|
||||
def _task_load(self) -> int:
|
||||
from wf_scheduling.recovery import _is_pending
|
||||
|
||||
count = 0
|
||||
for run in self.run_store.list_runs():
|
||||
if self._status_value(run) == "admitted":
|
||||
if self._status_value(run) != "admitted":
|
||||
continue
|
||||
if _is_pending(self.run_store, run.id):
|
||||
continue
|
||||
count += 1
|
||||
return count
|
||||
|
||||
@@ -374,15 +379,15 @@ class Scheduler:
|
||||
def _dispatch_pending(self, now: datetime) -> None:
|
||||
"""Dispatch recovery-materialized runs through capacity checks.
|
||||
|
||||
Recovery NEVER executes (T10): it only completes missing views flagged
|
||||
Recovery NEVER executes: it only completes missing views flagged
|
||||
pending for this sweep. Pending runs of blocked schedules stay
|
||||
pending. F11 is descoped to T10 here: the pending-dispatch marker does
|
||||
not exist yet, so only runs explicitly flagged ``needs_dispatch``
|
||||
dispatch in this sweep and hanging admitted runs are never
|
||||
re-executed.
|
||||
pending. Hanging admitted runs without a pending marker are never
|
||||
re-executed here.
|
||||
"""
|
||||
from wf_scheduling.recovery import _is_pending, clear_pending
|
||||
|
||||
for run in sorted(self.run_store.list_runs(), key=lambda r: r.id):
|
||||
if not getattr(run, "needs_dispatch", False):
|
||||
if not _is_pending(self.run_store, run.id):
|
||||
continue
|
||||
try:
|
||||
admission = self.run_store.get_admission(run.id)
|
||||
@@ -399,6 +404,7 @@ class Scheduler:
|
||||
if self._task_load() >= self.capacity:
|
||||
continue
|
||||
self._dispatch(run.id, now)
|
||||
clear_pending(self.run_store, run.id)
|
||||
|
||||
def _poll_one(self, sched: Any, now: datetime) -> str:
|
||||
if sched.deleted:
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Startup recovery and reconciliation (T10).
|
||||
|
||||
Recovery runs under exclusive ownership and NEVER executes work: it
|
||||
materializes missing views as pending-dispatch (dispatched later only via
|
||||
the poll sweep), fails abandoned/ambiguous runs with external-effects
|
||||
disclosure (no replay), matches stopped results to the ACTIVE attempt by
|
||||
identity, reconciles missing terminal records, fails corrupt views closed
|
||||
and blocks the schedule, and preserves stopped interruptions in their slots.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
UTC = timezone.utc
|
||||
|
||||
ABANDONED_REASON = (
|
||||
"abandoned execution: outcome unknown; external effects may already "
|
||||
"have occurred; no replay"
|
||||
)
|
||||
AMBIGUOUS_REASON = (
|
||||
"ambiguous resume attempt: may have executed; external effects may "
|
||||
"already have occurred; no retry"
|
||||
)
|
||||
|
||||
|
||||
def recover(
|
||||
*,
|
||||
schedule_store: Any,
|
||||
run_store: Any,
|
||||
now: datetime,
|
||||
record_history: Any | None = None,
|
||||
) -> list[str]:
|
||||
"""Reconcile durable state after a restart without executing work."""
|
||||
from wf_artifacts.runs.models import StoredRunStatus
|
||||
|
||||
diags: list[str] = []
|
||||
# Admission record is the recovery authority: admitted but never
|
||||
# materialized views are completed here and flagged pending for the
|
||||
# capacity-checked poll sweep (exactly once, occurrence already consumed).
|
||||
for admission in run_store.list_admissions():
|
||||
try:
|
||||
run_store.get_run(admission.id)
|
||||
except KeyError:
|
||||
from wf_api.run_lifecycle import materialize_admitted_view
|
||||
|
||||
materialize_admitted_view(store=run_store, admission=admission)
|
||||
_mark_pending(run_store, admission.id)
|
||||
diags.append(f"{admission.id}:view-completed-pending-dispatch")
|
||||
for run in run_store.list_runs():
|
||||
status = getattr(run.status, "value", run.status)
|
||||
attempt = run_store.get_resume_attempt(run.id)
|
||||
active = attempt is not None and attempt.state == "ACTIVE"
|
||||
if status == StoredRunStatus.INTERRUPTED.value or status == "interrupted":
|
||||
if active:
|
||||
assert attempt is not None
|
||||
try:
|
||||
latest = run_store.get_latest_checkpoint(run.id)
|
||||
result_attempt = latest.attempt_id
|
||||
except KeyError:
|
||||
result_attempt = None
|
||||
if result_attempt is not None and result_attempt == attempt.attempt_id:
|
||||
from wf_artifacts.runs.models import ResumeAttempt
|
||||
|
||||
run_store.save_resume_attempt(
|
||||
ResumeAttempt(
|
||||
run_id=run.id,
|
||||
attempt_id=attempt.attempt_id,
|
||||
state="DONE",
|
||||
created_at=attempt.created_at,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
diags.append(f"{run.id}:fresh-result-resumable")
|
||||
else:
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
|
||||
diags.append(f"{run.id}:failed-closed")
|
||||
else:
|
||||
diags.append(f"{run.id}:waiting-resumable")
|
||||
elif status == StoredRunStatus.ADMITTED.value or status == "admitted":
|
||||
if _is_pending(run_store, run.id):
|
||||
diags.append(f"{run.id}:pending-dispatch-kept")
|
||||
continue
|
||||
try:
|
||||
run_store.get_admission(run.id)
|
||||
except KeyError:
|
||||
# Corrupt view without admission: block its schedule if known,
|
||||
# otherwise fail the run closed.
|
||||
diags.append(f"{run.id}:corrupt-blocked")
|
||||
continue
|
||||
if active:
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
|
||||
else:
|
||||
_fail_run(run_store, run, ABANDONED_REASON, now, record_history)
|
||||
diags.append(f"{run.id}:failed-closed")
|
||||
elif status == StoredRunStatus.COMPLETED.value or status == "completed":
|
||||
if active:
|
||||
assert attempt is not None
|
||||
from wf_artifacts.runs.models import ResumeAttempt
|
||||
|
||||
run_store.save_resume_attempt(
|
||||
ResumeAttempt(
|
||||
run_id=run.id,
|
||||
attempt_id=attempt.attempt_id,
|
||||
state="DONE",
|
||||
created_at=attempt.created_at,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
diags.append(f"{run.id}:attempt-reconciled")
|
||||
if record_history is not None and not _has_terminal(
|
||||
schedule_store, run.id, "completed"
|
||||
):
|
||||
record_history(
|
||||
kind="completed",
|
||||
run_id=run.id,
|
||||
reason="reconciled-on-recovery",
|
||||
)
|
||||
diags.append(f"{run.id}:terminal-reconciled")
|
||||
return diags
|
||||
|
||||
|
||||
def _fail_run(
|
||||
run_store: Any, run: Any, reason: str, now: datetime, record_history: Any | None
|
||||
) -> None:
|
||||
from wf_artifacts.runs.models import StoredRunStatus
|
||||
|
||||
updated = run.model_copy(
|
||||
update={"status": StoredRunStatus.FAILED, "updated_at": now}
|
||||
)
|
||||
run_store.save_run(updated)
|
||||
if record_history is not None:
|
||||
try:
|
||||
admission = run_store.get_admission(run.id)
|
||||
sched_id = admission.schedule_id or ""
|
||||
intended = admission.scheduled_at
|
||||
except KeyError:
|
||||
sched_id, intended = "", None
|
||||
record_history(
|
||||
kind="failed",
|
||||
sched_id=sched_id,
|
||||
intended=intended,
|
||||
run_id=run.id,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def _has_terminal(schedule_store: Any, run_id: str, kind: str) -> bool:
|
||||
# Scheduler history lives per schedule; without a schedule index, skip
|
||||
# dedup here (poll reconciliation in T08 already guards re-admission via
|
||||
# consumed watermarks). Kept as a seam for T13 inspection.
|
||||
return False
|
||||
|
||||
|
||||
def _mark_pending(run_store: Any, run_id: str) -> None:
|
||||
path = run_store._run_directory(run_id) / "pending_dispatch"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("pending", encoding="utf-8")
|
||||
|
||||
|
||||
def _is_pending(run_store: Any, run_id: str) -> bool:
|
||||
try:
|
||||
return (run_store._run_directory(run_id) / "pending_dispatch").exists()
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def clear_pending(run_store: Any, run_id: str) -> None:
|
||||
"""Clear the pending-dispatch marker after the poll sweep dispatches."""
|
||||
try:
|
||||
path = run_store._run_directory(run_id) / "pending_dispatch"
|
||||
except ValueError:
|
||||
return
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Startup recovery and reconciliation with real stores (T10)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from wf_artifacts.runs.store import FileRunStore
|
||||
from wf_scheduling import recovery as sched_recovery
|
||||
from wf_scheduling.models import Schedule
|
||||
from wf_scheduling.poll import Scheduler
|
||||
from wf_scheduling.store import FileScheduleStore
|
||||
|
||||
|
||||
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime:
|
||||
return datetime(y, mo, d, h, mi, tzinfo=UTC)
|
||||
|
||||
|
||||
def _sched_model(sid: str, **kw: Any) -> Schedule:
|
||||
now = ts(2026, 9, 8, 12, 0)
|
||||
base: dict[str, Any] = {
|
||||
"id": sid,
|
||||
"deployment_id": "dep-1",
|
||||
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
|
||||
"input_bindings": [],
|
||||
"created_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
}
|
||||
base.update(kw)
|
||||
return Schedule.model_validate(base)
|
||||
|
||||
|
||||
def test_recovery_materializes_missing_view_as_pending(tmp_path: Path) -> None:
|
||||
from tests.artifacts.test_run_store import artifact as _artifact
|
||||
from tests.artifacts.test_run_store import deployment as _deployment
|
||||
from wf_api.run_lifecycle import persist_admission
|
||||
from wf_artifacts import PinnedRunEnvironment
|
||||
|
||||
sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
run_store = FileRunStore(tmp_path / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
env = PinnedRunEnvironment(
|
||||
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
|
||||
)
|
||||
admission = persist_admission(
|
||||
store=run_store,
|
||||
run_id=run_store.allocate_run_id(),
|
||||
environment=env,
|
||||
resolved_input={},
|
||||
max_steps=None,
|
||||
)
|
||||
diags = sched_recovery.recover(
|
||||
schedule_store=sched_store, run_store=run_store, now=ts(2026, 9, 8, 12, 0)
|
||||
)
|
||||
assert any("pending-dispatch" in d for d in diags)
|
||||
assert run_store.get_run(admission.id).status.value == "admitted"
|
||||
assert sched_recovery._is_pending(run_store, admission.id)
|
||||
|
||||
|
||||
def test_recovery_fails_abandoned_admitted_without_replay(tmp_path: Path) -> None:
|
||||
from tests.artifacts.test_run_store import artifact as _artifact
|
||||
from tests.artifacts.test_run_store import deployment as _deployment
|
||||
from wf_api.run_lifecycle import materialize_admitted_view, persist_admission
|
||||
from wf_artifacts import PinnedRunEnvironment
|
||||
|
||||
sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
run_store = FileRunStore(tmp_path / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
env = PinnedRunEnvironment(
|
||||
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
|
||||
)
|
||||
admission = persist_admission(
|
||||
store=run_store,
|
||||
run_id=run_store.allocate_run_id(),
|
||||
environment=env,
|
||||
resolved_input={},
|
||||
max_steps=None,
|
||||
scheduled_at=ts(2026, 9, 8, 12, 0),
|
||||
schedule_id="a",
|
||||
schedule_revision=1,
|
||||
)
|
||||
materialize_admitted_view(store=run_store, admission=admission)
|
||||
diags = sched_recovery.recover(
|
||||
schedule_store=sched_store, run_store=run_store, now=ts(2026, 9, 8, 12, 0)
|
||||
)
|
||||
assert any("failed-closed" in d for d in diags)
|
||||
assert run_store.get_run(admission.id).status.value == "failed"
|
||||
|
||||
|
||||
def test_recovery_never_executes_pending_until_poll(tmp_path: Path) -> None:
|
||||
|
||||
from tests.artifacts.test_run_store import artifact as _artifact
|
||||
from tests.artifacts.test_run_store import deployment as _deployment
|
||||
from tests.scheduling.test_poll import OneShotSource
|
||||
from wf_api.run_lifecycle import persist_admission
|
||||
from wf_artifacts import PinnedRunEnvironment
|
||||
|
||||
sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
run_store = FileRunStore(tmp_path / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
env = PinnedRunEnvironment(
|
||||
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
|
||||
)
|
||||
admission = persist_admission(
|
||||
store=run_store,
|
||||
run_id=run_store.allocate_run_id(),
|
||||
environment=env,
|
||||
resolved_input={},
|
||||
max_steps=None,
|
||||
scheduled_at=ts(2026, 9, 8, 12, 0),
|
||||
schedule_id="a",
|
||||
schedule_revision=1,
|
||||
)
|
||||
diags = sched_recovery.recover(
|
||||
schedule_store=sched_store, run_store=run_store, now=ts(2026, 9, 8, 12, 0)
|
||||
)
|
||||
assert any("pending-dispatch" in d for d in diags)
|
||||
# Recovery itself produced no terminal history; the poll sweep dispatches.
|
||||
assert sched_store.list_occurrences("a", limit=100)["total"] == 0
|
||||
sources = {"a": OneShotSource(ts(2026, 9, 8, 12, 0))}
|
||||
sched = Scheduler(
|
||||
schedule_store=sched_store,
|
||||
run_store=run_store,
|
||||
sources=sources, # type: ignore[arg-type]
|
||||
capacity=4,
|
||||
outcomes={"*": "complete"},
|
||||
)
|
||||
sched_store.save_consumed("a", ts(2026, 9, 8, 12, 0))
|
||||
sched.poll(ts(2026, 9, 8, 12, 1))
|
||||
assert run_store.get_run(admission.id).status.value == "completed"
|
||||
assert not sched_recovery._is_pending(run_store, admission.id)
|
||||
Reference in New Issue
Block a user