"""Torn checkpoint/summary reconciliation: the checkpoint wins (R4/F2). ``persist_stopped_run`` writes the checkpoint before the summary; a crash in between leaves a stale summary. Recovery inspects and validates the durable stopped checkpoint before deciding from the summary, reconciling status, checkpoint pointer, readiness, and attempt state together — for first executions and resumes, matching and mismatching attempts, without fabricating execution output. """ from __future__ import annotations from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, cast from tests.artifacts.test_run_store import artifact as _artifact from tests.artifacts.test_run_store import deployment as _deployment from tests.scheduling.controlled import ( DictDeployments, WorkflowDispatcher, fixture_environment, ) from wf_api.run_lifecycle import ( load_stored_run, materialize_admitted_view, persist_admission, persist_stopped_run, restore_interrupted_run, ) from wf_artifacts import PinnedRunEnvironment from wf_artifacts.runs.models import ResumeAttempt from wf_artifacts.runs.store import FileRunStore from wf_core import ( END, Edge, InterruptNode, RunState, RunStatus, SchemaRef, StateSchema, Workflow, load_run_state, resume_workflow, ) from wf_scheduling import recovery as sched_recovery from wf_scheduling.models import Schedule from wf_scheduling.ownership import SchedulerOwnership from wf_scheduling.poll import Scheduler from wf_scheduling.prepare import SchedulePreparer from wf_scheduling.store import FileScheduleStore def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime: return datetime(y, mo, d, h, mi, tzinfo=UTC) def _sched_model(sid: str, **kw: Any) -> Schedule: now = ts(2026, 9, 8, 12, 0) base: dict[str, Any] = { "id": sid, "deployment_id": "dep-1", "trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"}, "input_bindings": [], "created_at": now.isoformat(), "updated_at": now.isoformat(), } base.update(kw) return Schedule.model_validate(base) def _env() -> PinnedRunEnvironment: return PinnedRunEnvironment( deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[] ) class FailSaveRunOnce(FileRunStore): """Real store that drops exactly one summary write (torn boundary).""" def __init__(self, root: Path) -> None: super().__init__(root) self.armed = False def save_run(self, run: Any) -> None: if self.armed: self.armed = False raise OSError("injected save_run failure") super().save_run(run) def _recover( sched_store: FileScheduleStore, run_store: FileRunStore, now: datetime ) -> list[str]: ownership = SchedulerOwnership(sched_store.root.parent, owner="test").acquire() try: return sched_recovery.recover( schedule_store=sched_store, run_store=run_store, now=now, ownership=ownership, ) finally: ownership.release() def _admit_view( run_store: FileRunStore, intended: datetime, *, schedule_id: str = "a", ) -> str: run_id = run_store.allocate_run_id() admission = persist_admission( store=run_store, run_id=run_id, environment=_env(), resolved_input={}, max_steps=None, scheduled_at=intended, schedule_id=schedule_id, schedule_revision=1, ) materialize_admitted_view(store=run_store, admission=admission) return run_id def _torn_persist( run_store: FailSaveRunOnce, run_id: str, status: RunStatus, *, attempt_id: int | None = None, ) -> None: record = run_store.get_run(run_id) run_store.armed = True try: persist_stopped_run( store=run_store, environment=record.environment, run=RunState( workflow_name="sched", status=status, workflow_input={}, state={} ), run_id=run_id, attempt_id=attempt_id, ) raise AssertionError("save_run failure must propagate") except OSError as exc: assert "injected save_run failure" in str(exc) def _entries(store: FileScheduleStore, sid: str, kind: str) -> list[dict[str, Any]]: page = store.list_occurrences(sid, limit=100) return [ r for r in cast(list[dict[str, Any]], page["occurrences"]) if r["kind"] == kind ] def _double_interrupt_workflow() -> Workflow: schema = SchemaRef(type="object", properties={}) return Workflow( name="ask_twice", input_schema=schema, state_schema=StateSchema.from_field_map({}), output_schema=schema, outcomes=["ok"], start="ask1", node_defs=[], nodes=[ InterruptNode(id="ask1", type="interrupt", kind="approval"), InterruptNode(id="ask2", type="interrupt", kind="approval"), ], edges=[ Edge.model_validate({"from": "ask1", "outcome": "submitted", "to": "ask2"}), Edge.model_validate({"from": "ask2", "outcome": "submitted", "to": END}), ], ) def test_torn_first_interrupt_reconciles_not_abandons(tmp_path: Path) -> None: sched_store = FileScheduleStore(tmp_path / "sched") run_store = FailSaveRunOnce(tmp_path / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_view(run_store, intended) _torn_persist(run_store, run_id, RunStatus.INTERRUPTED) assert run_store.get_run(run_id).status.value == "admitted" diags = _recover(sched_store, run_store, intended) assert any("summary-reconciled" in d for d in diags) assert any("waiting-resumable" in d for d in diags) assert not any("failed-closed" in d for d in diags) record = run_store.get_run(run_id) assert record.status.value == "interrupted" assert record.latest_checkpoint_id == f"{run_id}.000001" assert record.resume_readiness.value == "ready" assert len(_entries(sched_store, "a", "interrupted")) == 1 # Inspection and resume eligibility agree with the reconciled result. _, state = load_stored_run(run_store, run_id) assert state.status is RunStatus.INTERRUPTED settled, resumed = restore_interrupted_run(run_store, run_id) assert settled.resume_readiness.value == "ready" assert resumed.status is RunStatus.INTERRUPTED def test_torn_first_completed_reconciles(tmp_path: Path) -> None: sched_store = FileScheduleStore(tmp_path / "sched") run_store = FailSaveRunOnce(tmp_path / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_view(run_store, intended) _torn_persist(run_store, run_id, RunStatus.COMPLETED) diags = _recover(sched_store, run_store, intended) record = run_store.get_run(run_id) assert record.status.value == "completed" assert record.latest_checkpoint_id == f"{run_id}.000001" assert record.resume_readiness.value == "not_applicable" assert any("summary-reconciled" in d for d in diags) assert len(_entries(sched_store, "a", "completed")) == 1 def test_torn_first_failed_reconciles(tmp_path: Path) -> None: sched_store = FileScheduleStore(tmp_path / "sched") run_store = FailSaveRunOnce(tmp_path / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_view(run_store, intended) _torn_persist(run_store, run_id, RunStatus.FAILED) diags = _recover(sched_store, run_store, intended) record = run_store.get_run(run_id) assert record.status.value == "failed" assert record.latest_checkpoint_id == f"{run_id}.000001" assert any("summary-reconciled" in d for d in diags) assert len(_entries(sched_store, "a", "failed")) == 1 def test_torn_resume_completed_matches_active_attempt(tmp_path: Path) -> None: sched_store = FileScheduleStore(tmp_path / "sched") run_store = FailSaveRunOnce(tmp_path / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_view(run_store, intended) record = run_store.get_run(run_id) persist_stopped_run( store=run_store, environment=record.environment, run=RunState( workflow_name="sched", status=RunStatus.INTERRUPTED, workflow_input={}, state={}, ), run_id=run_id, ) attempt_id = run_store.allocate_resume_attempt_id() run_store.save_resume_attempt( ResumeAttempt( run_id=run_id, attempt_id=attempt_id, state="ACTIVE", created_at=intended, updated_at=intended, ) ) _torn_persist(run_store, run_id, RunStatus.COMPLETED, attempt_id=attempt_id) diags = _recover(sched_store, run_store, intended) # The fresh COMPLETED checkpoint wins over the stale INTERRUPTED summary: # reconciled as completed, never as fresh/resumable. record = run_store.get_run(run_id) assert record.status.value == "completed" assert record.latest_checkpoint_id == f"{run_id}.000002" assert run_store.get_resume_attempt(run_id).state == "DONE" # type: ignore[union-attr] assert any("summary-reconciled" in d for d in diags) assert not any("fresh-result-resumable" in d for d in diags) assert len(_entries(sched_store, "a", "completed")) == 1 def test_torn_resume_interrupted_matches_active_attempt(tmp_path: Path) -> None: sched_store = FileScheduleStore(tmp_path / "sched") run_store = FailSaveRunOnce(tmp_path / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_view(run_store, intended) record = run_store.get_run(run_id) persist_stopped_run( store=run_store, environment=record.environment, run=RunState( workflow_name="sched", status=RunStatus.INTERRUPTED, workflow_input={}, state={}, ), run_id=run_id, ) attempt_id = run_store.allocate_resume_attempt_id() run_store.save_resume_attempt( ResumeAttempt( run_id=run_id, attempt_id=attempt_id, state="ACTIVE", created_at=intended, updated_at=intended, ) ) _torn_persist(run_store, run_id, RunStatus.INTERRUPTED, attempt_id=attempt_id) diags = _recover(sched_store, run_store, intended) record = run_store.get_run(run_id) assert record.status.value == "interrupted" assert record.latest_checkpoint_id == f"{run_id}.000002" assert record.resume_readiness.value == "ready" assert run_store.get_resume_attempt(run_id).state == "DONE" # type: ignore[union-attr] assert any("fresh-result-resumable" in d for d in diags) assert len(_entries(sched_store, "a", "interrupted")) == 1 def test_stale_checkpoint_never_clears_active_attempt(tmp_path: Path) -> None: sched_store = FileScheduleStore(tmp_path / "sched") run_store = FileRunStore(tmp_path / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_view(run_store, intended) record = run_store.get_run(run_id) persist_stopped_run( store=run_store, environment=record.environment, run=RunState( workflow_name="sched", status=RunStatus.INTERRUPTED, workflow_input={}, state={}, ), run_id=run_id, attempt_id=3, ) run_store.save_resume_attempt( ResumeAttempt( run_id=run_id, attempt_id=5, state="ACTIVE", created_at=intended, updated_at=intended, ) ) diags = _recover(sched_store, run_store, intended) assert any("failed-closed" in d for d in diags) assert run_store.get_run(run_id).status.value == "failed" assert run_store.get_resume_attempt(run_id).state == "ACTIVE" # type: ignore[union-attr] def test_double_recovery_after_reconcile_is_stable(tmp_path: Path) -> None: sched_store = FileScheduleStore(tmp_path / "sched") run_store = FailSaveRunOnce(tmp_path / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_view(run_store, intended) _torn_persist(run_store, run_id, RunStatus.INTERRUPTED) first = _recover(sched_store, run_store, intended) assert any("summary-reconciled" in d for d in first) before = run_store.get_run(run_id) second = _recover(sched_store, run_store, intended + timedelta(minutes=1)) after = run_store.get_run(run_id) assert not any("summary-reconciled" in d for d in second) assert not any("terminal-reconciled" in d for d in second) assert after == before assert len(_entries(sched_store, "a", "interrupted")) == 1 def test_real_workflow_interrupt_resume_interrupt_with_torn_resume( tmp_path: Path, ) -> None: workflow = _double_interrupt_workflow() sched_store = FileScheduleStore(tmp_path / "sched") run_store = FailSaveRunOnce(tmp_path / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) ownership = SchedulerOwnership(tmp_path, owner="test").acquire() try: sched = Scheduler( schedule_store=sched_store, run_store=run_store, sources={}, capacity=4, preparer=SchedulePreparer( DictDeployments({"dep-1": {"rev": 1, "required": []}}), fixture_environment, ), dispatcher=WorkflowDispatcher(workflow, {}), ownership=ownership, ) sched_store.save_consumed("a", intended - timedelta(hours=1)) from wf_scheduling.calendar import OneShotSource sched.sources["a"] = OneShotSource(intended) result = sched.poll(intended) run_id = next(iter(run_store.list_runs())).id assert result == {"a": f"admit:{run_id}"} assert run_store.get_run(run_id).status.value == "interrupted" ownership.release() # Resume through the real runtime to a second durable interruption, # then tear the summary write. fresh = FileRunStore(tmp_path / "runs") checkpoint = fresh.get_latest_checkpoint(run_id) st1 = load_run_state(checkpoint.state.model_dump(mode="json")) st2 = resume_workflow(workflow, st1, {}, resume_payload={}) assert st2.status is RunStatus.INTERRUPTED attempt_id = fresh.allocate_resume_attempt_id() fresh.save_resume_attempt( ResumeAttempt( run_id=run_id, attempt_id=attempt_id, state="ACTIVE", created_at=intended, updated_at=intended, ) ) armed = FailSaveRunOnce(tmp_path / "runs") armed.armed = True record = armed.get_run(run_id) try: persist_stopped_run( store=armed, environment=record.environment, run=st2, run_id=run_id, attempt_id=attempt_id, ) raise AssertionError("torn resume persist must raise") except OSError as exc: assert "injected save_run failure" in str(exc) diags = _recover(sched_store, FileRunStore(tmp_path / "runs"), intended) assert any("summary-reconciled" in d for d in diags) assert any("fresh-result-resumable" in d for d in diags) settled = FileRunStore(tmp_path / "runs").get_run(run_id) assert settled.status.value == "interrupted" assert settled.latest_checkpoint_id == f"{run_id}.000002" assert settled.resume_readiness.value == "ready" _, state = load_stored_run(FileRunStore(tmp_path / "runs"), run_id) assert state.status is RunStatus.INTERRUPTED finally: ownership.release() def test_corrupt_checkpoint_fails_closed(tmp_path: Path) -> None: sched_store = FileScheduleStore(tmp_path / "sched") run_store = FileRunStore(tmp_path / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_view(run_store, intended) checkpoint_dir = tmp_path / "runs" / run_id / "checkpoints" checkpoint_dir.mkdir(parents=True, exist_ok=True) (checkpoint_dir / "000001.json").write_text("not json", encoding="utf-8") diags = _recover(sched_store, run_store, intended) assert any("failed-closed" in d for d in diags) assert run_store.get_run(run_id).status.value == "failed"