sched: bind ownership to store composition; stable recovery failure; schema-checked prepare; guarded settle (R4 wave 2)

src/wf_scheduling/ownership.py

@@ -9,11 +9,18 @@ startup is rejected instead of running unprotected.
 
 Locking design (Windows-tested, per the store transaction boundary): one
 new module holding ``msvcrt.locking`` (Windows) / ``fcntl.flock`` (POSIX)
-on ``<store root>/scheduler.lock``. No ad-hoc per-run lock files.
+on ``<composition root>/scheduler.lock``. No ad-hoc per-run lock files.
+
+The lock is acquired on the composition root that contains the protected
+schedule and run stores. Entry-point guards do not trust the caller to
+have passed the right lock: they validate the held guard against the
+actual store roots (canonicalized for platform aliases) before any write
+or side effect.
 """
 
 from __future__ import annotations
 
+import os
 from pathlib import Path
 from typing import BinaryIO
 
@@ -26,8 +33,31 @@ class StartupRejected(Exception):
     """Scheduler startup rejected where locking is unsupported."""
 
 
+def canonical_store_path(path: Path | str) -> str:
+    """Canonicalize a store path for ownership-coverage comparison.
+
+    ``realpath`` resolves symlinks, junctions, ``..`` segments, and
+    relative paths; ``normcase`` folds case and separators on Windows
+    (identity on POSIX, which stays case-sensitive). Both sides of every
+    comparison pass through this function, so platform aliases of one
+    directory compare equal. Exotic aliases (subst drives, UNC-vs-mapped
+    paths, 8.3 short names where the OS preserves them) are out of scope:
+    operators should acquire the lock on the same canonical composition
+    root the stores live under.
+    """
+    return os.path.normcase(os.path.realpath(os.fspath(path)))
+
+
+def _is_within(candidate: str, base: str) -> bool:
+    """Whether canonical ``candidate`` lies beneath canonical ``base``."""
+    if candidate == base:
+        return True
+    prefix = base if base.endswith(os.sep) else base + os.sep
+    return candidate.startswith(prefix)
+
+
 class SchedulerOwnership:
-    """Held exclusive ownership of a schedule store root."""
+    """Held exclusive ownership of a store composition root."""
 
     def __init__(self, root: Path, *, owner: str) -> None:
         self.root = root
@@ -49,6 +79,27 @@ class SchedulerOwnership:
         """
         return self._locked
 
+    def canonical_root(self) -> str:
+        """Return the canonical composition root this lock was acquired on."""
+        return canonical_store_path(self.root)
+
+    def covers(self, *store_roots: Path | str | None) -> bool:
+        """Whether this held lock protects the given store roots.
+
+        Coverage means each root canonicalizes to the lock's composition
+        root or beneath it. An unrelated (even held) lock, a released lock,
+        and rootless stores are never covered: guards reject those before
+        any write or side effect instead of trusting the caller's promise.
+        """
+        if not self._locked or not store_roots:
+            return False
+        base = self.canonical_root()
+        return all(
+            store_root is not None
+            and _is_within(canonical_store_path(store_root), base)
+            for store_root in store_roots
+        )
+
     def acquire(self) -> SchedulerOwnership:
         """Acquire the held lock non-blockingly or raise SecondOwnerError."""
         self.lock_path.parent.mkdir(parents=True, exist_ok=True)

src/wf_scheduling/poll.py

@@ -139,13 +139,20 @@ class Scheduler:
     def _require_ownership(self) -> None:
         """Reject schedule mutation/dispatch without proven live ownership.
 
-        Runs before any store write or dispatcher side effect: without a
-        held lock this process cannot prove exclusive ownership, so polling
+        Runs before any store write or dispatcher side effect: the held
+        lock must cover the actual schedule and run store composition, not
+        merely be held on some unrelated directory. Without covering
+        ownership this process cannot prove exclusive ownership, so polling
         or administering schedules would risk double admission.
         """
-        if self.ownership is None or not self.ownership.held:
+        ownership = self.ownership
+        if ownership is None or not ownership.covers(
+            getattr(self.schedule_store, "root", None),
+            getattr(self.run_store, "root", None),
+        ):
             raise SecondOwnerError(
-                "scheduler ownership is required before polling or mutating schedules"
+                "scheduler ownership must cover the schedule and run stores "
+                "before polling or mutating schedules"
             )
 
     # -- helpers ------------------------------------------------------
@@ -431,6 +438,12 @@ class Scheduler:
         :class:`wf_core.RunState`. Persists through the shared lifecycle
         boundary, clears the executing mark, and records terminal history.
         Never re-invokes the dispatcher.
+
+        An admitted status alone is not enough: pending, never-dispatched
+        work is also admitted. Settlement requires the durable executing
+        transition and refuses contradictory pending/executing state (a
+        crash between the transition writes owns that run now, not this
+        caller); it also refuses runs that already stopped.
         """
         self._require_ownership()
         from wf_api.run_lifecycle import persist_stopped_run
@@ -448,6 +461,12 @@ class Scheduler:
             raise BlockedSchedule(f"settle missing run view: {run_id!r}") from exc
         if self._status_value(record) != "admitted":
             raise BlockedSchedule(f"settle non-admitted run: {run_id!r}")
+        if self.run_store.is_pending_dispatch(run_id):
+            raise BlockedSchedule(
+                f"settle contradictory pending/executing run: {run_id!r}"
+            )
+        if not self.run_store.is_executing(run_id):
+            raise BlockedSchedule(f"settle without executing transition: {run_id!r}")
         stopped = persist_stopped_run(
             store=self.run_store,
             environment=admission.environment,

src/wf_scheduling/prepare.py

@@ -17,7 +17,9 @@ from datetime import datetime
 from typing import Any, Protocol
 
 from wf_artifacts.runs.models import PinnedRunEnvironment
+from wf_core.errors import WorkflowExecutionError
 from wf_core.runtime.input_sources import resolve_schedule_input_bindings
+from wf_core.runtime.ops.schemas import validate_payload_against_schema
 from wf_scheduling.occurrences import occurrence_id
 
 
@@ -63,7 +65,14 @@ class SchedulePreparer:
     Occurrence references resolve through the shared schedule-expression
     contract (never graph paths); target conflicts, over-budget trees, and
     invalid resolved input fail closed as preflight rejections before any
-    run identity is allocated.
+    run identity is allocated. The resolved object is additionally
+    validated against the pinned root artifact's input schema with the same
+    validator the runtime uses for fresh runs, so a schedule cannot admit
+    work the deployment contract would refuse.
+
+    The environment is built once per preparation: the deployment revision,
+    required inputs, and input schema all describe that single captured
+    snapshot, never independently resolved versions.
     """
 
     def __init__(
@@ -77,9 +86,12 @@ class SchedulePreparer:
     def prepare(
         self, *, sched: Any, intended: datetime, now: datetime
     ) -> PreparedInvocation | PreparationRejected:
+        environment = self._build_environment(sched)
         try:
-            revision = self._deployments.deployment_revision(sched.deployment_id)
-            required = list(self._deployments.required_inputs(sched.deployment_id))
+            revision = self._deployments.deployment_revision(environment.deployment.id)
+            required = list(
+                self._deployments.required_inputs(environment.deployment.id)
+            )
         except KeyError:
             return PreparationRejected(reason="deployment-deleted")
         occurrence = {
@@ -98,8 +110,16 @@ class SchedulePreparer:
         missing = [key for key in required if key not in resolved]
         if missing:
             return PreparationRejected(reason=f"missing-input:{missing}")
+        try:
+            validate_payload_against_schema(
+                environment.root_artifact.input_schema,
+                dict(resolved),
+                f"schedule {sched.id} input",
+            )
+        except WorkflowExecutionError as exc:
+            return PreparationRejected(reason=f"invalid-input:schema:{exc}")
         return PreparedInvocation(
-            environment=self._build_environment(sched),
+            environment=environment,
             resolved_input=dict(resolved),
             max_steps=getattr(sched, "max_steps", None),
             deployment_revision=revision,

src/wf_scheduling/recovery.py

@@ -31,6 +31,13 @@ clear → executor → stopped persist → executing clear → history):
   reconciliation across the torn ``save_checkpoint``/``save_run`` boundary
   is owned by the F2 reconcile step, which runs before marker handling.
 - stopped summary, no marks: existing terminal/attempt reconciliation.
+- failed summary: a durable recovery decision is never re-processed. When
+  the checkpoint pointer still matches the decided state, history is
+  ensured without touching status, readiness, or diagnostics (repeat
+  recovery is silent). Only a genuinely newer checkpoint (pointer
+  mismatch) may reopen the run via the reconcile path; an older
+  checkpoint can never supersede the abandonment. Failed runs carry
+  not-applicable resume readiness.
 """
 
 from __future__ import annotations
@@ -72,10 +79,14 @@ def recover(
     """Reconcile durable state after a restart without executing work."""
     from wf_artifacts.runs.models import StoredRunStatus
 
-    if ownership is None or not ownership.held:
+    if ownership is None or not ownership.covers(
+        getattr(schedule_store, "root", None),
+        getattr(run_store, "root", None),
+    ):
         raise SecondOwnerError(
-            "scheduler ownership is required before recovery: an unowned "
-            "recovery could abandon or redispatch another owner's work"
+            "scheduler ownership must cover the schedule and run stores "
+            "before recovery: an unowned recovery could abandon or "
+            "redispatch another owner's work"
         )
     if history is None:
         history = FileScheduleHistoryRecorder(schedule_store)
@@ -96,6 +107,27 @@ def recover(
         status = getattr(run.status, "value", run.status)
         attempt = run_store.get_resume_attempt(run.id)
         active = attempt is not None and attempt.state == "ACTIVE"
+        if status in (
+            StoredRunStatus.INTERRUPTED.value,
+            StoredRunStatus.COMPLETED.value,
+            StoredRunStatus.FAILED.value,
+            "interrupted",
+            "completed",
+            "failed",
+        ) and (is_executing(run_store, run.id) or _is_pending(run_store, run.id)):
+            # Completion-window leftovers: the stopped result was persisted
+            # but marker clearing was lost. The stopped status stands;
+            # nothing is re-executed.
+            clear_executing(run_store, run.id)
+            clear_pending(run_store, run.id)
+            diags.append(f"{run.id}:completion-window-cleared")
+        # Durable-decision stability: a FAILED summary for the current
+        # checkpoint state is never re-processed. Only a genuinely newer
+        # checkpoint (pointer mismatch) may reopen it via the reconcile
+        # path below. History is ensured without mutating the summary.
+        if status == StoredRunStatus.FAILED.value or status == "failed":
+            if _failed_decision_stable(run_store, run, history, now, diags):
+                continue
         # Checkpoint-first reconcile across the torn
         # save_checkpoint/save_run boundary: a durable stopped checkpoint
         # wins over a stale summary (status, checkpoint pointer, and
@@ -105,6 +137,25 @@ def recover(
         try:
             changed, status = _reconcile_summary_from_checkpoint(run_store, run, now)
         except (ValueError, OSError) as exc:
+            if _has_recovery_decision(run):
+                # A decided run stays decided even when checkpoint state
+                # corrupts afterwards: ensure history from the summary
+                # pointer, note the corruption, never re-fail.
+                sched_id, intended, revision = _attribution(run_store, run.id)
+                if _reconcile_terminal(
+                    history,
+                    sched_id,
+                    run.id,
+                    "failed",
+                    run.latest_checkpoint_id,
+                    intended,
+                    revision,
+                    _decision_reason(run) or "reconciled-on-recovery",
+                    now,
+                ):
+                    diags.append(f"{run.id}:terminal-reconciled")
+                diags.append(f"{run.id}:corrupt-checkpoint-noted")
+                continue
             _fail_run(run_store, run, f"corrupt checkpoint: {exc}", now, history)
             diags.append(f"{run.id}:failed-closed")
             continue
@@ -119,9 +170,8 @@ def recover(
             "completed",
             "failed",
         ) and (is_executing(run_store, run.id) or _is_pending(run_store, run.id)):
-            # Completion-window leftovers: the stopped result was persisted
-            # but marker clearing was lost. The stopped status stands;
-            # nothing is re-executed.
+            # A repaired summary may carry completion-window leftovers (the
+            # stopped result was persisted but marker clearing was lost).
             clear_executing(run_store, run.id)
             clear_pending(run_store, run.id)
             diags.append(f"{run.id}:completion-window-cleared")
@@ -277,6 +327,67 @@ def _reconcile_summary_from_checkpoint(
     return True, expected.value
 
 
+def _has_recovery_decision(run: Any) -> bool:
+    """Whether the summary carries a durable recovery-failure decision."""
+    return any(
+        getattr(item, "code", None) == "schedule-recovery" for item in run.diagnostics
+    )
+
+
+def _decision_reason(run: Any) -> str:
+    """Return the first durable recovery-failure reason, if any."""
+    for item in run.diagnostics:
+        if getattr(item, "code", None) == "schedule-recovery":
+            return str(getattr(item, "message", ""))
+    return ""
+
+
+def _failed_decision_stable(
+    run_store: Any,
+    run: Any,
+    history: HistoryRecorder,
+    now: datetime,
+    diags: list[str],
+) -> bool:
+    """Leave a decided FAILED summary untouched, ensuring history only.
+
+    Returns True when the run needs no further processing this pass: the
+    summary already reflects a recovery decision (or a legacy/external
+    failure) for the current checkpoint state, so re-running reconcile or
+    re-failing would only duplicate diagnostics and rewrite readiness. A
+    genuinely newer checkpoint (pointer mismatch) returns False so the
+    reconcile path may repair the summary. Unreadable checkpoint state
+    likewise returns False: the reconcile path fails closed for undecided
+    runs and keeps decided ones via its own branch.
+    """
+    try:
+        latest = run_store.get_latest_checkpoint(run.id)
+    except KeyError:
+        latest = None
+    except ValueError, OSError:
+        return False
+    pointer = run.latest_checkpoint_id
+    latest_id = latest.id if latest is not None else None
+    if latest_id != pointer:
+        # Genuinely newer (or torn-away) checkpoint state: not stable.
+        return False
+    sched_id, intended, revision = _attribution(run_store, run.id)
+    reason = _decision_reason(run) or "reconciled-on-recovery"
+    if _reconcile_terminal(
+        history,
+        sched_id,
+        run.id,
+        "failed",
+        pointer,
+        intended,
+        revision,
+        reason,
+        now,
+    ):
+        diags.append(f"{run.id}:terminal-reconciled")
+    return True
+
+
 def _attribution(
     run_store: Any, run_id: str
 ) -> tuple[str | None, datetime | None, int | None]:
@@ -366,12 +477,13 @@ def _fail_run(
 ) -> None:
     """Fail a run closed with its truthful reason durably preserved.
 
-    The reason is always written into the run summary diagnostics; when an
-    admission attributes the run to a schedule, a failed occurrence entry is
-    reconciled exactly once through the history recorder.
+    The reason is always written into the run summary diagnostics, and a
+    failed run is never resumable. When an admission attributes the run to
+    a schedule, a failed occurrence entry is reconciled exactly once
+    through the history recorder.
     """
     from wf_artifacts import DependencyDiagnostic, DiagnosticSeverity
-    from wf_artifacts.runs.models import StoredRunStatus
+    from wf_artifacts.runs.models import ResumeReadiness, StoredRunStatus
 
     diagnostic = DependencyDiagnostic(
         severity=DiagnosticSeverity.ERROR,
@@ -383,6 +495,7 @@ def _fail_run(
     updated = run.model_copy(
         update={
             "status": StoredRunStatus.FAILED,
+            "resume_readiness": ResumeReadiness.NOT_APPLICABLE,
             "updated_at": now,
             "diagnostics": [*run.diagnostics, diagnostic],
         }