sched: persist executing transition before executor; abandon on crash (F1)

src/wf_artifacts/runs/store.py

@@ -64,6 +64,15 @@ class RunStore:
     def is_pending_dispatch(self, run_id: str) -> bool:
         raise NotImplementedError
 
+    def mark_executing(self, run_id: str) -> None:
+        raise NotImplementedError
+
+    def clear_executing(self, run_id: str) -> None:
+        raise NotImplementedError
+
+    def is_executing(self, run_id: str) -> bool:
+        raise NotImplementedError
+
 
 class FileRunStore(RunStore):
     """JSON file-backed admitted- and stopped-run store for local dev/tests.
@@ -215,6 +224,30 @@ class FileRunStore(RunStore):
         except ValueError:
             return False
 
+    def mark_executing(self, run_id: str) -> None:
+        """Persist the undispatched→executing transition before the executor.
+
+        After this write, a crash must abandon the run (failed, no replay):
+        the executor may already have produced external effects.
+        """
+        with self._lock:
+            self._write_json(self._executing_path(run_id), {"executing": True})
+
+    def clear_executing(self, run_id: str) -> None:
+        with self._lock:
+            try:
+                path = self._executing_path(run_id)
+            except ValueError:
+                return
+            if path.exists():
+                path.unlink()
+
+    def is_executing(self, run_id: str) -> bool:
+        try:
+            return self._executing_path(run_id).exists()
+        except ValueError:
+            return False
+
     def _write_json(self, path: Path, payload: object) -> None:
         path.parent.mkdir(parents=True, exist_ok=True)
         temp_path = path.with_suffix(".json.tmp")
@@ -241,5 +274,8 @@ class FileRunStore(RunStore):
     def _pending_path(self, run_id: str) -> Path:
         return self._run_directory(run_id) / "pending_dispatch.json"
 
+    def _executing_path(self, run_id: str) -> Path:
+        return self._run_directory(run_id) / "executing.json"
+
     def _checkpoint_path(self, run_id: str, sequence: int) -> Path:
         return self._run_directory(run_id) / "checkpoints" / f"{sequence:06d}.json"

src/wf_scheduling/poll.py

@@ -300,38 +300,31 @@ class Scheduler:
         self.schedule_store.save_consumed(sched.id, intended)
         from wf_api.run_lifecycle import materialize_admitted_view
 
-        # Dispatch mark precedes the view so a crash after admission but
-        # before/during first dispatch stays pending (not abandoned). Cleared
-        # after dispatch returns regardless of outcome (hang still dispatched).
         self.run_store.mark_pending_dispatch(run_id)
         materialize_admitted_view(store=self.run_store, admission=admission)
-        try:
-            self._dispatch(run_id, now)
-        finally:
-            self.run_store.clear_pending_dispatch(run_id)
+        self._execute_guarded(run_id, now)
         return run_id
 
-    def _dispatch(self, run_id: str, now: datetime) -> None:
-        """Persist one stopped result for an admitted run via the dispatcher.
+    def _execute_guarded(self, run_id: str, now: datetime) -> None:
+        """Dispatch one provably undispatched run behind a durable transition.
 
-        The dispatcher returns a genuine stopped :class:`wf_core.RunState`
-        (or :class:`StillRunning` when the outcome is unknown); persistence
-        goes through the shared ``wf_api.run_lifecycle`` boundary so
-        scheduler dispatches share the manual-run torn-write protocol.
+        The undispatched→executing mark is persisted BEFORE the executor is
+        invoked, and the pending marker is cleared with it. Clearing is
+        explicit after a stopped result is durably persisted — there is
+        deliberately no ``finally``: a terminated process must leave the
+        executing mark so recovery abandons the run instead of retrying it.
         """
         from wf_api.run_lifecycle import persist_stopped_run
         from wf_artifacts.runs.models import StoredRunStatus
 
-        try:
-            self.run_store.get_run(run_id)
-        except KeyError as exc:
-            raise BlockedSchedule(f"dispatch missing run view: {run_id!r}") from exc
         try:
             admission = self.run_store.get_admission(run_id)
         except KeyError as exc:
             raise BlockedSchedule(f"dispatch missing admission: {run_id!r}") from exc
         if admission.schedule_id is None:
             raise BlockedSchedule(f"dispatch missing schedule owner: {run_id!r}")
+        self.run_store.mark_executing(run_id)
+        self.run_store.clear_pending_dispatch(run_id)
         result = self.dispatcher.dispatch(admission=admission, now=now)
         if isinstance(result, StillRunning):
             return
@@ -341,6 +334,52 @@ class Scheduler:
             run=result.result,
             run_id=run_id,
         )
+        self.run_store.clear_executing(run_id)
+        kind = {
+            StoredRunStatus.COMPLETED: "completed",
+            StoredRunStatus.INTERRUPTED: "interrupted",
+            StoredRunStatus.FAILED: "failed",
+        }[stopped.status]
+        self._record(
+            kind=kind,  # type: ignore[arg-type]
+            sched_id=admission.schedule_id,
+            intended=_admission_intended(self.run_store, run_id),
+            run_id=run_id,
+            now=now,
+            started_at=now,
+        )
+
+    def record_stopped_execution(self, run_id: str, result: Any, now: datetime) -> None:
+        """Settle a hanging execution with its late stopped result.
+
+        Async-completion seam for dispatchers that returned
+        :class:`StillRunning`: the executor later produced a genuine stopped
+        :class:`wf_core.RunState`. Persists through the shared lifecycle
+        boundary, clears the executing mark, and records terminal history.
+        Never re-invokes the dispatcher.
+        """
+        from wf_api.run_lifecycle import persist_stopped_run
+        from wf_artifacts.runs.models import StoredRunStatus
+
+        try:
+            admission = self.run_store.get_admission(run_id)
+        except KeyError as exc:
+            raise BlockedSchedule(f"settle missing admission: {run_id!r}") from exc
+        if admission.schedule_id is None:
+            raise BlockedSchedule(f"settle missing schedule owner: {run_id!r}")
+        try:
+            record = self.run_store.get_run(run_id)
+        except KeyError as exc:
+            raise BlockedSchedule(f"settle missing run view: {run_id!r}") from exc
+        if self._status_value(record) != "admitted":
+            raise BlockedSchedule(f"settle non-admitted run: {run_id!r}")
+        stopped = persist_stopped_run(
+            store=self.run_store,
+            environment=admission.environment,
+            run=result,
+            run_id=run_id,
+        )
+        self.run_store.clear_executing(run_id)
         kind = {
             StoredRunStatus.COMPLETED: "completed",
             StoredRunStatus.INTERRUPTED: "interrupted",
@@ -380,20 +419,56 @@ class Scheduler:
         """Dispatch recovery-materialized runs through capacity checks.
 
         Recovery NEVER executes: it only completes missing views flagged
-        pending for this sweep. Pending runs of blocked schedules stay
-        pending. Hanging admitted runs without a pending marker are never
-        re-executed here.
+        pending for this sweep. Each pending marker is validated before it
+        is trusted: markers without an admission (or without a schedule
+        owner) fail the run closed instead of dispatching; markers on
+        stopped runs are stale and cleared without redispatch; unknown
+        owners and blocked schedules are left untouched. Hanging admitted
+        runs without a pending marker are never re-executed here.
         """
-        from wf_scheduling.recovery import _is_pending, clear_pending
+        from wf_scheduling.recovery import (
+            CORRUPT_PENDING_REASON,
+            _fail_run,
+            clear_executing,
+            clear_pending,
+            is_executing,
+        )
+        from wf_scheduling.recovery import _is_pending as _pending
 
         for run in sorted(self.run_store.list_runs(), key=lambda r: r.id):
-            if not _is_pending(self.run_store, run.id):
+            if not _pending(self.run_store, run.id):
                 continue
             try:
                 admission = self.run_store.get_admission(run.id)
             except KeyError:
+                _fail_run(
+                    self.run_store,
+                    run,
+                    CORRUPT_PENDING_REASON,
+                    now,
+                    None,
+                )
+                clear_pending(self.run_store, run.id)
+                clear_executing(self.run_store, run.id)
                 continue
             if admission.schedule_id is None:
+                _fail_run(
+                    self.run_store,
+                    run,
+                    CORRUPT_PENDING_REASON,
+                    now,
+                    None,
+                )
+                clear_pending(self.run_store, run.id)
+                clear_executing(self.run_store, run.id)
+                continue
+            if self._status_value(run) != "admitted":
+                # Stale marker on a stopped run: clear it, never redispatch.
+                # Terminal-history reconciliation is owned by recovery (which
+                # dedups); the sweep only removes the untrustworthy marker.
+                clear_pending(self.run_store, run.id)
+                if is_executing(self.run_store, run.id):
+                    clear_executing(self.run_store, run.id)
                 continue
             try:
                 sched = self.schedule_store.get_schedule(admission.schedule_id)
@@ -403,8 +478,7 @@ class Scheduler:
                 continue
             if self._task_load() >= self.capacity:
                 continue
-            self._dispatch(run.id, now)
-            clear_pending(self.run_store, run.id)
+            self._execute_guarded(run.id, now)
 
     def _poll_one(self, sched: Any, now: datetime) -> str:
         if sched.deleted:

src/wf_scheduling/recovery.py

@@ -6,6 +6,29 @@ the poll sweep), fails abandoned/ambiguous runs with external-effects
 disclosure (no replay), matches stopped results to the ACTIVE attempt by
 identity, reconciles missing terminal records, fails corrupt views closed
 and blocks the schedule, and preserves stopped interruptions in their slots.
+
+Restart behavior at each marker/admission/view write boundary (admission
+persist → pending mark → view materialize → executing mark → pending
+clear → executor → stopped persist → executing clear → history):
+
+- admission only: view is materialized and flagged pending-dispatch; the
+  occurrence was already consumed, so the later poll dispatches exactly
+  once through capacity checks.
+- admission + pending, no view: same as above (the view write was lost).
+- admission + view + pending, no executing mark: provably undispatched
+  (the executor is unreachable without the executing mark); kept pending.
+- any admitted run with the executing mark: the executor may already have
+  produced external effects; failed (ACTIVE attempt: ambiguous, else
+  abandoned) with external-effects disclosure, never redispatched.
+- admitted + view with neither mark and no ACTIVE attempt: legacy or
+  manually cleared state whose outcome is unprovable; failed closed
+  without replay (current code never produces this shape on crash).
+- stopped summary + executing/pending marks: completion-window leftovers
+  (stopped result persisted, marker clearing lost); markers are cleared,
+  the stopped status stands, nothing is re-executed. Checkpoint-vs-summary
+  reconciliation across the torn ``save_checkpoint``/``save_run`` boundary
+  is owned by the F2 reconcile step, which runs before marker handling.
+- stopped summary, no marks: existing terminal/attempt reconciliation.
 """
 
 from __future__ import annotations
@@ -25,6 +48,10 @@ AMBIGUOUS_REASON = (
     "ambiguous resume attempt: may have executed; external effects may "
     "already have occurred; no retry"
 )
+CORRUPT_PENDING_REASON = (
+    "corrupt pending marker: no admission owns this run; the marker cannot "
+    "be trusted for dispatch"
+)
 
 
 def recover(
@@ -60,6 +87,20 @@ def recover(
         status = getattr(run.status, "value", run.status)
         attempt = run_store.get_resume_attempt(run.id)
         active = attempt is not None and attempt.state == "ACTIVE"
+        if status in (
+            StoredRunStatus.INTERRUPTED.value,
+            StoredRunStatus.COMPLETED.value,
+            StoredRunStatus.FAILED.value,
+            "interrupted",
+            "completed",
+            "failed",
+        ) and (is_executing(run_store, run.id) or _is_pending(run_store, run.id)):
+            # Completion-window leftovers: the stopped result was persisted
+            # but marker clearing was lost. The stopped status stands;
+            # nothing is re-executed.
+            clear_executing(run_store, run.id)
+            clear_pending(run_store, run.id)
+            diags.append(f"{run.id}:completion-window-cleared")
         if status == StoredRunStatus.INTERRUPTED.value or status == "interrupted":
             if active:
                 assert attempt is not None
@@ -87,6 +128,18 @@ def recover(
             else:
                 diags.append(f"{run.id}:waiting-resumable")
         elif status == StoredRunStatus.ADMITTED.value or status == "admitted":
+            if is_executing(run_store, run.id):
+                # The executor may already have produced external effects:
+                # abandon, never retry. Both markers are cleared so a later
+                # recovery does not re-fail the now-terminal run.
+                if active:
+                    _fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
+                else:
+                    _fail_run(run_store, run, ABANDONED_REASON, now, record_history)
+                clear_pending(run_store, run.id)
+                clear_executing(run_store, run.id)
+                diags.append(f"{run.id}:failed-closed")
+                continue
             if _is_pending(run_store, run.id):
                 diags.append(f"{run.id}:pending-dispatch-kept")
                 continue
@@ -178,6 +231,15 @@ def _mark_pending(run_store: Any, run_id: str) -> None:
     run_store.mark_pending_dispatch(run_id)
 
 
+def is_executing(run_store: Any, run_id: str) -> bool:
+    return bool(run_store.is_executing(run_id))
+
+
+def clear_executing(run_store: Any, run_id: str) -> None:
+    """Clear the executing mark after a stopped result is durably persisted."""
+    run_store.clear_executing(run_id)
+
+
 def _is_pending(run_store: Any, run_id: str) -> bool:
     return bool(run_store.is_pending_dispatch(run_id))