sched: bind ownership to store composition; stable recovery failure; schema-checked prepare; guarded settle (R4 wave 2)

2026-09-08 18:36:09 +07:00
parent b9d8eb9d16
commit 3f1d5d158b
16 changed files with 1299 additions and 60 deletions
@@ -0,0 +1,307 @@
+"""Durable recovery failure is stable across repeated recovery (R4 item 2).
+
+An older checkpoint cannot supersede a durable abandonment decision: once
+recovery fails a run, repeating recovery must leave status, readiness,
+diagnostics, and history stable. Only a genuinely newer durable stopped
+result may repair a torn summary. Failed recovery carries not-applicable
+resume readiness.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any, cast
+
+from tests.artifacts.test_run_store import artifact as _artifact
+from tests.artifacts.test_run_store import deployment as _deployment
+from wf_api.run_lifecycle import (
+    load_stored_run,
+    materialize_admitted_view,
+    persist_admission,
+    persist_stopped_run,
+    restore_interrupted_run,
+)
+from wf_artifacts import PinnedRunEnvironment
+from wf_artifacts.runs.models import ResumeAttempt
+from wf_artifacts.runs.store import FileRunStore
+from wf_core import RunState, RunStatus
+from wf_scheduling import recovery as sched_recovery
+from wf_scheduling.models import Schedule
+from wf_scheduling.ownership import SchedulerOwnership
+from wf_scheduling.store import FileScheduleStore
+
+
+def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime:
+    return datetime(y, mo, d, h, mi, tzinfo=UTC)
+
+
+def _sched_model(sid: str, **kw: Any) -> Schedule:
+    now = ts(2026, 9, 8, 12, 0)
+    base: dict[str, Any] = {
+        "id": sid,
+        "deployment_id": "dep-1",
+        "trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
+        "input_bindings": [],
+        "created_at": now.isoformat(),
+        "updated_at": now.isoformat(),
+    }
+    base.update(kw)
+    return Schedule.model_validate(base)
+
+
+def _env() -> PinnedRunEnvironment:
+    return PinnedRunEnvironment(
+        deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
+    )
+
+
+def _recover(
+    sched_store: FileScheduleStore, run_store: FileRunStore, now: datetime
+) -> list[str]:
+    ownership = SchedulerOwnership(sched_store.root.parent, owner="test").acquire()
+    try:
+        return sched_recovery.recover(
+            schedule_store=sched_store,
+            run_store=run_store,
+            now=now,
+            ownership=ownership,
+        )
+    finally:
+        ownership.release()
+
+
+def _stopped(
+    run_store: FileRunStore,
+    run_id: str,
+    status: RunStatus,
+    *,
+    attempt_id: int | None = None,
+) -> Any:
+    record = run_store.get_run(run_id)
+    return persist_stopped_run(
+        store=run_store,
+        environment=record.environment,
+        run=RunState(workflow_name="sched", status=status, workflow_input={}, state={}),
+        run_id=run_id,
+        attempt_id=attempt_id,
+    )
+
+
+def _admit_interrupted(
+    run_store: FileRunStore, intended: datetime, *, attempt_id: int | None = None
+) -> str:
+    run_id = run_store.allocate_run_id()
+    admission = persist_admission(
+        store=run_store,
+        run_id=run_id,
+        environment=_env(),
+        resolved_input={},
+        max_steps=None,
+        scheduled_at=intended,
+        schedule_id="a",
+        schedule_revision=1,
+    )
+    materialize_admitted_view(store=run_store, admission=admission)
+    _stopped(run_store, run_id, RunStatus.INTERRUPTED, attempt_id=attempt_id)
+    return run_id
+
+
+def _mark_active(
+    run_store: FileRunStore, run_id: str, attempt_id: int, now: datetime
+) -> None:
+    run_store.save_resume_attempt(
+        ResumeAttempt(
+            run_id=run_id,
+            attempt_id=attempt_id,
+            state="ACTIVE",
+            created_at=now,
+            updated_at=now,
+        )
+    )
+
+
+def _entries(store: FileScheduleStore, sid: str, kind: str) -> list[dict[str, Any]]:
+    page = store.list_occurrences(sid, limit=100)
+    return [
+        r for r in cast(list[dict[str, Any]], page["occurrences"]) if r["kind"] == kind
+    ]
+
+
+def test_abandonment_decision_is_stable_across_recovery() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        sched_store = FileScheduleStore(root / "sched")
+        run_store = FileRunStore(root / "runs")
+        sched_store.create_schedule(_sched_model("a"))
+        intended = ts(2026, 9, 8, 12, 0)
+        run_id = _admit_interrupted(run_store, intended)
+        _mark_active(run_store, run_id, 9, intended)
+        first = _recover(sched_store, run_store, intended)
+        assert any("failed-closed" in d for d in first)
+        record = run_store.get_run(run_id)
+        assert record.status.value == "failed"
+        assert record.resume_readiness.value == "not_applicable"
+        assert len(record.diagnostics) == 1
+        # Fresh store objects across the restart boundary: everything stable.
+        sched_store2 = FileScheduleStore(root / "sched")
+        run_store2 = FileRunStore(root / "runs")
+        second = _recover(sched_store2, run_store2, intended + timedelta(minutes=1))
+        assert not any(run_id in d for d in second)
+        again = run_store2.get_run(run_id)
+        assert again.status.value == "failed"
+        assert again.resume_readiness.value == "not_applicable"
+        assert len(again.diagnostics) == 1
+        assert _entries(sched_store2, "a", "failed") == _entries(
+            sched_store, "a", "failed"
+        )
+        assert len(_entries(sched_store2, "a", "failed")) == 1
+
+
+def test_failed_readiness_and_inspection_agree() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        sched_store = FileScheduleStore(root / "sched")
+        run_store = FileRunStore(root / "runs")
+        sched_store.create_schedule(_sched_model("a"))
+        intended = ts(2026, 9, 8, 12, 0)
+        run_id = _admit_interrupted(run_store, intended)
+        _mark_active(run_store, run_id, 9, intended)
+        _recover(sched_store, run_store, intended)
+        record, _ = load_stored_run(run_store, run_id)
+        assert record.resume_readiness.value == "not_applicable"
+        try:
+            restore_interrupted_run(run_store, run_id)
+            raise AssertionError("failed run must not restore as interrupted")
+        except ValueError:
+            pass
+
+
+def test_genuinely_newer_result_repairs_after_decision() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        sched_store = FileScheduleStore(root / "sched")
+        run_store = FileRunStore(root / "runs")
+        sched_store.create_schedule(_sched_model("a"))
+        intended = ts(2026, 9, 8, 12, 0)
+        run_id = _admit_interrupted(run_store, intended)
+        _mark_active(run_store, run_id, 9, intended)
+        _recover(sched_store, run_store, intended)
+        assert run_store.get_run(run_id).status.value == "failed"
+        # A genuinely newer stopped result under a new matching attempt: the
+        # newer checkpoint repairs the summary and completes the attempt.
+        _mark_active(run_store, run_id, 10, intended)
+        _stopped(run_store, run_id, RunStatus.INTERRUPTED, attempt_id=10)
+        diags = _recover(sched_store, run_store, intended + timedelta(minutes=1))
+        assert any("fresh-result-resumable" in d for d in diags)
+        record = run_store.get_run(run_id)
+        assert record.status.value == "interrupted"
+        assert record.resume_readiness.value == "ready"
+        assert run_store.get_resume_attempt(run_id).state == "DONE"  # type: ignore[union-attr]
+        interrupted = _entries(sched_store, "a", "interrupted")
+        assert {e["checkpoint_id"] for e in interrupted} == {f"{run_id}.000002"}
+
+
+def test_completed_mismatch_decision_is_stable() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        sched_store = FileScheduleStore(root / "sched")
+        run_store = FileRunStore(root / "runs")
+        sched_store.create_schedule(_sched_model("a"))
+        intended = ts(2026, 9, 8, 12, 0)
+        run_id = run_store.allocate_run_id()
+        admission = persist_admission(
+            store=run_store,
+            run_id=run_id,
+            environment=_env(),
+            resolved_input={},
+            max_steps=None,
+            scheduled_at=intended,
+            schedule_id="a",
+            schedule_revision=1,
+        )
+        materialize_admitted_view(store=run_store, admission=admission)
+        _stopped(run_store, run_id, RunStatus.COMPLETED, attempt_id=3)
+        _mark_active(run_store, run_id, 5, intended)
+        _recover(sched_store, run_store, intended)
+        assert run_store.get_run(run_id).status.value == "failed"
+        second = _recover(
+            FileScheduleStore(root / "sched"),
+            FileRunStore(root / "runs"),
+            intended + timedelta(minutes=1),
+        )
+        assert not any(run_id in d for d in second)
+        assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1
+
+
+def test_legacy_failed_run_gains_history_without_refail() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        sched_store = FileScheduleStore(root / "sched")
+        run_store = FileRunStore(root / "runs")
+        sched_store.create_schedule(_sched_model("a"))
+        intended = ts(2026, 9, 8, 12, 0)
+        run_id = _admit_interrupted(run_store, intended)
+        record = run_store.get_run(run_id)
+        from wf_artifacts.runs.models import StoredRunStatus
+
+        run_store.save_run(record.model_copy(update={"status": StoredRunStatus.FAILED}))
+        assert _entries(sched_store, "a", "failed") == []
+        diags = _recover(sched_store, run_store, intended)
+        assert any("terminal-reconciled" in d for d in diags)
+        assert not any("failed-closed" in d for d in diags)
+        assert run_store.get_run(run_id).diagnostics == []
+        assert len(_entries(sched_store, "a", "failed")) == 1
+
+
+def test_crash_between_decision_writes_recovers_history_once() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+
+        class FailFailedHistoryOnce(FileScheduleStore):
+            def __init__(self, root: Path) -> None:
+                super().__init__(root)
+                self.armed = False
+
+            def append_history(self, record: Any) -> None:
+                if self.armed and getattr(record, "kind", None) == "failed":
+                    self.armed = False
+                    raise OSError("injected failed-history failure")
+                super().append_history(record)
+
+        sched_store = FailFailedHistoryOnce(root / "sched")
+        run_store = FileRunStore(root / "runs")
+        sched_store.create_schedule(_sched_model("a"))
+        intended = ts(2026, 9, 8, 12, 0)
+        run_id = _admit_interrupted(run_store, intended)
+        _mark_active(run_store, run_id, 9, intended)
+        sched_store.armed = True
+        import pytest
+
+        with pytest.raises(OSError, match="injected failed-history failure"):
+            _recover(sched_store, run_store, intended)
+        record = run_store.get_run(run_id)
+        assert record.status.value == "failed"
+        assert len(record.diagnostics) == 1
+        assert _entries(sched_store, "a", "failed") == []
+        # The decision (status + reason) survived; only history is missing.
+        second = _recover(
+            FileScheduleStore(root / "sched"),
+            FileRunStore(root / "runs"),
+            intended + timedelta(minutes=1),
+        )
+        assert not any("failed-closed" in d for d in second)
+        assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1
+        assert len(_entries(FileScheduleStore(root / "sched"), "a", "failed")) == 1