sched: add durable resume-attempt marker with store-backed identities (T09)

src/wf_api/run_lifecycle.py

@@ -144,8 +144,13 @@ def persist_stopped_run(
     environment: PinnedRunEnvironment,
     run: RunState,
     run_id: str | None = None,
+    attempt_id: int | None = None,
 ) -> WorkflowRunRecord:
-    """Persist one externally visible stopped state and its typed checkpoint."""
+    """Persist one externally visible stopped state and its typed checkpoint.
+
+    ``attempt_id`` echoes the store-backed resume-attempt identity that
+    produced this result so recovery can match results to active attempts.
+    """
     if run.status not in {
         RunStatus.INTERRUPTED,
         RunStatus.COMPLETED,
@@ -187,6 +192,7 @@ def persist_stopped_run(
         sequence=sequence,
         reason=CheckpointReason(status.value),
         state=PersistedRunState.model_validate(dump_run_state(run)),
+        attempt_id=attempt_id,
         created_at=now,
     )
     record = WorkflowRunRecord(

src/wf_api/runs.py

@@ -216,6 +216,34 @@ class WorkflowRunApi:
                 steps_executed=stopped_run.steps_executed,
                 steps_remaining=stopped_run.steps_remaining,
             )
+        # Durable resume-attempt marker: persist ACTIVE with a store-backed
+        # attempt identity before re-executing. A crash during resume leaves
+        # the ACTIVE marker so recovery fails closed instead of presenting
+        # the old checkpoint as safe to retry. Every stopped result echoes
+        # the attempt identity back for matching.
+        from datetime import UTC as _UTC
+        from datetime import datetime as _datetime
+
+        from wf_artifacts.runs.models import ResumeAttempt
+
+        store = self._run_store()
+        existing_attempt = store.get_resume_attempt(run_id)
+        if existing_attempt is not None and existing_attempt.state == "ACTIVE":
+            raise ValueError(
+                f"workflow run {run_id!r} has an ambiguous active resume attempt; "
+                "recovery must fail it closed before retry"
+            )
+        attempt_id = store.allocate_resume_attempt_id()
+        now_marker = _datetime.now(_UTC)
+        store.save_resume_attempt(
+            ResumeAttempt(
+                run_id=run_id,
+                attempt_id=attempt_id,
+                state="ACTIVE",
+                created_at=now_marker,
+                updated_at=now_marker,
+            )
+        )
         plan = raw_plan_from_artifact(environment.root_artifact)
         tree = saved_subgraph_tree_from_snapshots(environment.child_artifacts)
         run = await self.context.runtime.resume_workflow_from_plan(
@@ -227,11 +255,25 @@ class WorkflowRunApi:
             artifact=environment.root_artifact,
             saved_subgraph_tree=tree,
         )
+        # Granular completion: stopped persist, attempt-clear, and history
+        # are separate persists with fault boundaries between each pair. A
+        # resumed run may interrupt again (durable re-interruption).
         next_record = persist_stopped_run(
-            store=self._run_store(),
+            store=store,
             environment=environment,
             run=run,
             run_id=run_id,
+            attempt_id=attempt_id,
+        )
+        cleared_at = _datetime.now(_UTC)
+        store.save_resume_attempt(
+            ResumeAttempt(
+                run_id=run_id,
+                attempt_id=attempt_id,
+                state="DONE",
+                created_at=now_marker,
+                updated_at=cleared_at,
+            )
         )
         return _run_payload(
             deployment=environment.deployment,

src/wf_artifacts/__init__.py

@@ -45,6 +45,7 @@ from .runs import (
     CheckpointReason,
     FileRunStore,
     PinnedRunEnvironment,
+    ResumeAttempt,
     ResumeReadiness,
     RunAdmission,
     RunCheckpoint,
@@ -73,6 +74,7 @@ __all__ = [
     "PinnedRunEnvironment",
     "RequiredCapability",
     "ResumeReadiness",
+    "ResumeAttempt",
     "RunAdmission",
     "RunCheckpoint",
     "RunStore",

src/wf_artifacts/runs/__init__.py

@@ -1,6 +1,7 @@
 from .models import (
     CheckpointReason,
     PinnedRunEnvironment,
+    ResumeAttempt,
     ResumeReadiness,
     RunAdmission,
     RunCheckpoint,
@@ -15,6 +16,7 @@ __all__ = [
     "CheckpointReason",
     "FileRunStore",
     "PinnedRunEnvironment",
+    "ResumeAttempt",
     "ResumeReadiness",
     "RunAdmission",
     "RunCheckpoint",

src/wf_artifacts/runs/models.py

@@ -127,4 +127,25 @@ class RunCheckpoint(BaseModel):
     sequence: int = Field(ge=1)
     reason: CheckpointReason
     state: PersistedRunState | VersionedCheckpointState
+    attempt_id: int | None = Field(
+        default=None,
+        ge=1,
+        description=(
+            "Store-backed resume-attempt identity that produced this result. "
+            "Recovery matches result to the active attempt: fresh results are "
+            "resumable, stale results fail closed without retry."
+        ),
+    )
     created_at: datetime
+
+
+class ResumeAttempt(BaseModel):
+    """Durable resume-attempt marker persisted before re-execution."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    run_id: str = Field(pattern=RUN_ID_PATTERN)
+    attempt_id: int = Field(ge=1)
+    state: Literal["ACTIVE", "DONE"]
+    created_at: datetime
+    updated_at: datetime

src/wf_artifacts/runs/store.py

@@ -4,7 +4,13 @@ import json
 from pathlib import Path
 from threading import RLock
 
-from .models import RunAdmission, RunCheckpoint, WorkflowRunRecord, ensure_run_id
+from .models import (
+    ResumeAttempt,
+    RunAdmission,
+    RunCheckpoint,
+    WorkflowRunRecord,
+    ensure_run_id,
+)
 
 
 class RunStore:
@@ -40,6 +46,15 @@ class RunStore:
     def allocate_run_id(self) -> str:
         raise NotImplementedError
 
+    def allocate_resume_attempt_id(self) -> int:
+        raise NotImplementedError
+
+    def save_resume_attempt(self, attempt: ResumeAttempt) -> None:
+        raise NotImplementedError
+
+    def get_resume_attempt(self, run_id: str) -> ResumeAttempt | None:
+        raise NotImplementedError
+
 
 class FileRunStore(RunStore):
     """JSON file-backed admitted- and stopped-run store for local dev/tests.
@@ -138,6 +153,39 @@ class FileRunStore(RunStore):
             self._write_json(seq_path, {"seq": seq})
             return f"run-{seq:06d}"
 
+    def allocate_resume_attempt_id(self) -> int:
+        """Allocate a store-backed resume-attempt identity (no reuse)."""
+        with self._lock:
+            seq_path = self.runs_dir / "_resume_attempt_seq.json"
+            seq = 0
+            if seq_path.exists():
+                try:
+                    raw = json.loads(seq_path.read_text(encoding="utf-8"))
+                    seq_value = raw.get("seq", 0) if isinstance(raw, dict) else None
+                    if not isinstance(seq_value, int) or seq_value < 0:
+                        raise ValueError(
+                            f"corrupt attempt sequence at {seq_path}: {raw!r}"
+                        )
+                    seq = seq_value
+                except (ValueError, AttributeError, TypeError) as exc:
+                    raise ValueError(f"corrupt attempt sequence at {seq_path}") from exc
+            seq += 1
+            self._write_json(seq_path, {"seq": seq})
+            return seq
+
+    def save_resume_attempt(self, attempt: ResumeAttempt) -> None:
+        with self._lock:
+            self._write_json(
+                self._resume_attempt_path(attempt.run_id),
+                attempt.model_dump(mode="json"),
+            )
+
+    def get_resume_attempt(self, run_id: str) -> ResumeAttempt | None:
+        path = self._resume_attempt_path(run_id)
+        if not path.exists():
+            return None
+        return ResumeAttempt.model_validate_json(path.read_text(encoding="utf-8"))
+
     def _write_json(self, path: Path, payload: object) -> None:
         path.parent.mkdir(parents=True, exist_ok=True)
         temp_path = path.with_suffix(".json.tmp")
@@ -158,5 +206,8 @@ class FileRunStore(RunStore):
     def _admission_path(self, run_id: str) -> Path:
         return self._run_directory(run_id) / "admission.json"
 
+    def _resume_attempt_path(self, run_id: str) -> Path:
+        return self._run_directory(run_id) / "resume_attempt.json"
+
     def _checkpoint_path(self, run_id: str, sequence: int) -> Path:
         return self._run_directory(run_id) / "checkpoints" / f"{sequence:06d}.json"