sched: checkpoint-first reconcile across torn checkpoint/summary writes (F2)

src/wf_scheduling/recovery.py

@@ -20,9 +20,11 @@ clear → executor → stopped persist → executing clear → history):
 - any admitted run with the executing mark: the executor may already have
   produced external effects; failed (ACTIVE attempt: ambiguous, else
   abandoned) with external-effects disclosure, never redispatched.
-- admitted + view with neither mark and no ACTIVE attempt: legacy or
-  manually cleared state whose outcome is unprovable; failed closed
-  without replay (current code never produces this shape on crash).
+- admitted + view with neither mark and no ACTIVE attempt: a durable
+  stopped checkpoint reconciles the summary first (checkpoint-first rule);
+  only when no checkpoint exists is the outcome unprovable, and the run is
+  failed closed without replay (current code never produces this shape on
+  crash).
 - stopped summary + executing/pending marks: completion-window leftovers
   (stopped result persisted, marker clearing lost); markers are cleared,
   the stopped status stands, nothing is re-executed. Checkpoint-vs-summary
@@ -94,6 +96,21 @@ def recover(
         status = getattr(run.status, "value", run.status)
         attempt = run_store.get_resume_attempt(run.id)
         active = attempt is not None and attempt.state == "ACTIVE"
+        # Checkpoint-first reconcile across the torn
+        # save_checkpoint/save_run boundary: a durable stopped checkpoint
+        # wins over a stale summary (status, checkpoint pointer, and
+        # readiness are rewritten together from the checkpoint's own
+        # persisted fields; nothing is fabricated). Attempt matching below
+        # still distinguishes fresh results from stale ones.
+        try:
+            changed, status = _reconcile_summary_from_checkpoint(run_store, run, now)
+        except (ValueError, OSError) as exc:
+            _fail_run(run_store, run, f"corrupt checkpoint: {exc}", now, history)
+            diags.append(f"{run.id}:failed-closed")
+            continue
+        if changed:
+            run = run_store.get_run(run.id)
+            diags.append(f"{run.id}:summary-reconciled")
         if status in (
             StoredRunStatus.INTERRUPTED.value,
             StoredRunStatus.COMPLETED.value,
@@ -217,6 +234,49 @@ def recover(
     return diags
 
 
+def _reconcile_summary_from_checkpoint(
+    run_store: Any, run: Any, now: datetime
+) -> tuple[bool, str]:
+    """Rewrite a stale summary from its durable stopped checkpoint.
+
+    Returns ``(changed, status)``. Only the checkpoint's own persisted
+    fields (status, checkpoint id, readiness derived exactly as
+    ``persist_stopped_run`` derives it) are copied; creation time,
+    environment, and diagnostics are preserved. A run without checkpoints
+    is untouched (an admitted summary with no checkpoint is genuinely
+    undispatched-or-unknown, handled by the marker rules).
+    """
+    from wf_artifacts.runs.models import ResumeReadiness, StoredRunStatus
+
+    try:
+        latest = run_store.get_latest_checkpoint(run.id)
+    except KeyError:
+        return False, getattr(run.status, "value", run.status)
+    expected = StoredRunStatus(latest.reason.value)
+    readiness = (
+        ResumeReadiness.READY
+        if expected is StoredRunStatus.INTERRUPTED
+        else ResumeReadiness.NOT_APPLICABLE
+    )
+    if (
+        run.status == expected
+        and run.latest_checkpoint_id == latest.id
+        and run.resume_readiness == readiness
+    ):
+        return False, getattr(run.status, "value", run.status)
+    run_store.save_run(
+        run.model_copy(
+            update={
+                "status": expected,
+                "latest_checkpoint_id": latest.id,
+                "resume_readiness": readiness,
+                "updated_at": now,
+            }
+        )
+    )
+    return True, expected.value
+
+
 def _attribution(
     run_store: Any, run_id: str
 ) -> tuple[str | None, datetime | None, int | None]: