"""Durable recovery failure is stable across repeated recovery (R4 item 2). An older checkpoint cannot supersede a durable abandonment decision: once recovery fails a run, repeating recovery must leave status, readiness, diagnostics, and history stable. Only a genuinely newer durable stopped result may repair a torn summary. Failed recovery carries not-applicable resume readiness. """ from __future__ import annotations from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, cast from tests.artifacts.test_run_store import artifact as _artifact from tests.artifacts.test_run_store import deployment as _deployment from wf_api.run_lifecycle import ( load_stored_run, materialize_admitted_view, persist_admission, persist_stopped_run, restore_interrupted_run, ) from wf_artifacts import PinnedRunEnvironment from wf_artifacts.runs.models import ResumeAttempt from wf_artifacts.runs.store import FileRunStore from wf_core import RunState, RunStatus from wf_scheduling import recovery as sched_recovery from wf_scheduling.models import Schedule from wf_scheduling.ownership import SchedulerOwnership from wf_scheduling.store import FileScheduleStore def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime: return datetime(y, mo, d, h, mi, tzinfo=UTC) def _sched_model(sid: str, **kw: Any) -> Schedule: now = ts(2026, 9, 8, 12, 0) base: dict[str, Any] = { "id": sid, "deployment_id": "dep-1", "trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"}, "input_bindings": [], "created_at": now.isoformat(), "updated_at": now.isoformat(), } base.update(kw) return Schedule.model_validate(base) def _env() -> PinnedRunEnvironment: return PinnedRunEnvironment( deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[] ) def _recover( sched_store: FileScheduleStore, run_store: FileRunStore, now: datetime ) -> list[str]: ownership = SchedulerOwnership(sched_store.root.parent, owner="test").acquire() try: return sched_recovery.recover( schedule_store=sched_store, run_store=run_store, now=now, ownership=ownership, ) finally: ownership.release() def _stopped( run_store: FileRunStore, run_id: str, status: RunStatus, *, attempt_id: int | None = None, ) -> Any: record = run_store.get_run(run_id) return persist_stopped_run( store=run_store, environment=record.environment, run=RunState(workflow_name="sched", status=status, workflow_input={}, state={}), run_id=run_id, attempt_id=attempt_id, ) def _admit_interrupted( run_store: FileRunStore, intended: datetime, *, attempt_id: int | None = None ) -> str: run_id = run_store.allocate_run_id() admission = persist_admission( store=run_store, run_id=run_id, environment=_env(), resolved_input={}, max_steps=None, scheduled_at=intended, schedule_id="a", schedule_revision=1, ) materialize_admitted_view(store=run_store, admission=admission) _stopped(run_store, run_id, RunStatus.INTERRUPTED, attempt_id=attempt_id) return run_id def _mark_active( run_store: FileRunStore, run_id: str, attempt_id: int, now: datetime ) -> None: run_store.save_resume_attempt( ResumeAttempt( run_id=run_id, attempt_id=attempt_id, state="ACTIVE", created_at=now, updated_at=now, ) ) def _entries(store: FileScheduleStore, sid: str, kind: str) -> list[dict[str, Any]]: page = store.list_occurrences(sid, limit=100) return [ r for r in cast(list[dict[str, Any]], page["occurrences"]) if r["kind"] == kind ] def test_abandonment_decision_is_stable_across_recovery() -> None: import tempfile with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) sched_store = FileScheduleStore(root / "sched") run_store = FileRunStore(root / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_interrupted(run_store, intended) _mark_active(run_store, run_id, 9, intended) first = _recover(sched_store, run_store, intended) assert any("failed-closed" in d for d in first) record = run_store.get_run(run_id) assert record.status.value == "failed" assert record.resume_readiness.value == "not_applicable" assert len(record.diagnostics) == 1 # Fresh store objects across the restart boundary: everything stable. sched_store2 = FileScheduleStore(root / "sched") run_store2 = FileRunStore(root / "runs") second = _recover(sched_store2, run_store2, intended + timedelta(minutes=1)) assert not any(run_id in d for d in second) again = run_store2.get_run(run_id) assert again.status.value == "failed" assert again.resume_readiness.value == "not_applicable" assert len(again.diagnostics) == 1 assert _entries(sched_store2, "a", "failed") == _entries( sched_store, "a", "failed" ) assert len(_entries(sched_store2, "a", "failed")) == 1 def test_failed_readiness_and_inspection_agree() -> None: import tempfile with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) sched_store = FileScheduleStore(root / "sched") run_store = FileRunStore(root / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_interrupted(run_store, intended) _mark_active(run_store, run_id, 9, intended) _recover(sched_store, run_store, intended) record, _ = load_stored_run(run_store, run_id) assert record.resume_readiness.value == "not_applicable" try: restore_interrupted_run(run_store, run_id) raise AssertionError("failed run must not restore as interrupted") except ValueError: pass def test_genuinely_newer_result_repairs_after_decision() -> None: import tempfile with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) sched_store = FileScheduleStore(root / "sched") run_store = FileRunStore(root / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_interrupted(run_store, intended) _mark_active(run_store, run_id, 9, intended) _recover(sched_store, run_store, intended) assert run_store.get_run(run_id).status.value == "failed" # A genuinely newer stopped result under a new matching attempt: the # newer checkpoint repairs the summary and completes the attempt. _mark_active(run_store, run_id, 10, intended) _stopped(run_store, run_id, RunStatus.INTERRUPTED, attempt_id=10) diags = _recover(sched_store, run_store, intended + timedelta(minutes=1)) assert any("fresh-result-resumable" in d for d in diags) record = run_store.get_run(run_id) assert record.status.value == "interrupted" assert record.resume_readiness.value == "ready" assert run_store.get_resume_attempt(run_id).state == "DONE" # type: ignore[union-attr] interrupted = _entries(sched_store, "a", "interrupted") assert {e["checkpoint_id"] for e in interrupted} == {f"{run_id}.000002"} def test_completed_mismatch_decision_is_stable() -> None: import tempfile with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) sched_store = FileScheduleStore(root / "sched") run_store = FileRunStore(root / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = run_store.allocate_run_id() admission = persist_admission( store=run_store, run_id=run_id, environment=_env(), resolved_input={}, max_steps=None, scheduled_at=intended, schedule_id="a", schedule_revision=1, ) materialize_admitted_view(store=run_store, admission=admission) _stopped(run_store, run_id, RunStatus.COMPLETED, attempt_id=3) _mark_active(run_store, run_id, 5, intended) _recover(sched_store, run_store, intended) assert run_store.get_run(run_id).status.value == "failed" second = _recover( FileScheduleStore(root / "sched"), FileRunStore(root / "runs"), intended + timedelta(minutes=1), ) assert not any(run_id in d for d in second) assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1 def test_legacy_failed_run_gains_history_without_refail() -> None: import tempfile with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) sched_store = FileScheduleStore(root / "sched") run_store = FileRunStore(root / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_interrupted(run_store, intended) record = run_store.get_run(run_id) from wf_artifacts.runs.models import StoredRunStatus run_store.save_run(record.model_copy(update={"status": StoredRunStatus.FAILED})) assert _entries(sched_store, "a", "failed") == [] diags = _recover(sched_store, run_store, intended) assert any("terminal-reconciled" in d for d in diags) assert not any("failed-closed" in d for d in diags) assert run_store.get_run(run_id).diagnostics == [] assert len(_entries(sched_store, "a", "failed")) == 1 def test_crash_between_decision_writes_recovers_history_once() -> None: import tempfile with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) class FailFailedHistoryOnce(FileScheduleStore): def __init__(self, root: Path) -> None: super().__init__(root) self.armed = False def append_history(self, record: Any) -> None: if self.armed and getattr(record, "kind", None) == "failed": self.armed = False raise OSError("injected failed-history failure") super().append_history(record) sched_store = FailFailedHistoryOnce(root / "sched") run_store = FileRunStore(root / "runs") sched_store.create_schedule(_sched_model("a")) intended = ts(2026, 9, 8, 12, 0) run_id = _admit_interrupted(run_store, intended) _mark_active(run_store, run_id, 9, intended) sched_store.armed = True import pytest with pytest.raises(OSError, match="injected failed-history failure"): _recover(sched_store, run_store, intended) record = run_store.get_run(run_id) assert record.status.value == "failed" assert len(record.diagnostics) == 1 assert _entries(sched_store, "a", "failed") == [] # The decision (status + reason) survived; only history is missing. second = _recover( FileScheduleStore(root / "sched"), FileRunStore(root / "runs"), intended + timedelta(minutes=1), ) assert not any("failed-closed" in d for d in second) assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1 assert len(_entries(FileScheduleStore(root / "sched"), "a", "failed")) == 1