sched: persist executing transition before executor; abandon on crash (F1)

src/wf_scheduling/recovery.py

@@ -6,6 +6,29 @@ the poll sweep), fails abandoned/ambiguous runs with external-effects
 disclosure (no replay), matches stopped results to the ACTIVE attempt by
 identity, reconciles missing terminal records, fails corrupt views closed
 and blocks the schedule, and preserves stopped interruptions in their slots.
+
+Restart behavior at each marker/admission/view write boundary (admission
+persist → pending mark → view materialize → executing mark → pending
+clear → executor → stopped persist → executing clear → history):
+
+- admission only: view is materialized and flagged pending-dispatch; the
+  occurrence was already consumed, so the later poll dispatches exactly
+  once through capacity checks.
+- admission + pending, no view: same as above (the view write was lost).
+- admission + view + pending, no executing mark: provably undispatched
+  (the executor is unreachable without the executing mark); kept pending.
+- any admitted run with the executing mark: the executor may already have
+  produced external effects; failed (ACTIVE attempt: ambiguous, else
+  abandoned) with external-effects disclosure, never redispatched.
+- admitted + view with neither mark and no ACTIVE attempt: legacy or
+  manually cleared state whose outcome is unprovable; failed closed
+  without replay (current code never produces this shape on crash).
+- stopped summary + executing/pending marks: completion-window leftovers
+  (stopped result persisted, marker clearing lost); markers are cleared,
+  the stopped status stands, nothing is re-executed. Checkpoint-vs-summary
+  reconciliation across the torn ``save_checkpoint``/``save_run`` boundary
+  is owned by the F2 reconcile step, which runs before marker handling.
+- stopped summary, no marks: existing terminal/attempt reconciliation.
 """
 
 from __future__ import annotations
@@ -25,6 +48,10 @@ AMBIGUOUS_REASON = (
     "ambiguous resume attempt: may have executed; external effects may "
     "already have occurred; no retry"
 )
+CORRUPT_PENDING_REASON = (
+    "corrupt pending marker: no admission owns this run; the marker cannot "
+    "be trusted for dispatch"
+)
 
 
 def recover(
@@ -60,6 +87,20 @@ def recover(
         status = getattr(run.status, "value", run.status)
         attempt = run_store.get_resume_attempt(run.id)
         active = attempt is not None and attempt.state == "ACTIVE"
+        if status in (
+            StoredRunStatus.INTERRUPTED.value,
+            StoredRunStatus.COMPLETED.value,
+            StoredRunStatus.FAILED.value,
+            "interrupted",
+            "completed",
+            "failed",
+        ) and (is_executing(run_store, run.id) or _is_pending(run_store, run.id)):
+            # Completion-window leftovers: the stopped result was persisted
+            # but marker clearing was lost. The stopped status stands;
+            # nothing is re-executed.
+            clear_executing(run_store, run.id)
+            clear_pending(run_store, run.id)
+            diags.append(f"{run.id}:completion-window-cleared")
         if status == StoredRunStatus.INTERRUPTED.value or status == "interrupted":
             if active:
                 assert attempt is not None
@@ -87,6 +128,18 @@ def recover(
             else:
                 diags.append(f"{run.id}:waiting-resumable")
         elif status == StoredRunStatus.ADMITTED.value or status == "admitted":
+            if is_executing(run_store, run.id):
+                # The executor may already have produced external effects:
+                # abandon, never retry. Both markers are cleared so a later
+                # recovery does not re-fail the now-terminal run.
+                if active:
+                    _fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
+                else:
+                    _fail_run(run_store, run, ABANDONED_REASON, now, record_history)
+                clear_pending(run_store, run.id)
+                clear_executing(run_store, run.id)
+                diags.append(f"{run.id}:failed-closed")
+                continue
             if _is_pending(run_store, run.id):
                 diags.append(f"{run.id}:pending-dispatch-kept")
                 continue
@@ -178,6 +231,15 @@ def _mark_pending(run_store: Any, run_id: str) -> None:
     run_store.mark_pending_dispatch(run_id)
 
 
+def is_executing(run_store: Any, run_id: str) -> bool:
+    return bool(run_store.is_executing(run_id))
+
+
+def clear_executing(run_store: Any, run_id: str) -> None:
+    """Clear the executing mark after a stopped result is durably persisted."""
+    run_store.clear_executing(run_id)
+
+
 def _is_pending(run_store: Any, run_id: str) -> bool:
     return bool(run_store.is_pending_dispatch(run_id))