sched: crash-safe admin ordering, create watermark, poll freshness (T13 fix)

src/wf_api/schedules.py

@@ -21,6 +21,7 @@ from wf_scheduling.history import FileScheduleHistoryRecorder, HistoryEntry
 from wf_scheduling.models import OccurrenceRecord, Schedule
 from wf_scheduling.occurrences import occurrence_id
 from wf_scheduling.prepare import PreparationRejected, SchedulePreparer
+from wf_scheduling.store import StaleScheduleRevisionError
 
 from .models import (
     JsonProjector,
@@ -207,6 +208,10 @@ class WorkflowScheduleApi:
         )
         self._validate_definition(schedule)
         stored = store.create_schedule(schedule)
+        # Creation never backfills time before the revision: the consumed
+        # watermark starts at creation so catch-up only ever covers
+        # downtime after this revision, never pre-creation instants.
+        store.save_consumed(schedule_id, now)
         return _PROJECT_SCHEDULE(stored.model_dump(mode="json"))
 
     async def get_schedule(self, *, schedule_id: str) -> ScheduleResult:
@@ -251,14 +256,24 @@ class WorkflowScheduleApi:
         ``expected_revision + 1`` and ``updated_at`` becomes now, while
         ``created_at`` and the admitted-run snapshots stay untouched (later
         edits only affect future admissions). Stale revisions raise
-        ``StaleScheduleRevisionError``. Side effects mirror
-        ``Scheduler.edit_schedule``: a held candidate is cleared with a
-        ``superseded``/``schedule-edit`` history row, and the consumed
-        watermark advances to at least now (no backfill).
+        ``StaleScheduleRevisionError`` before any write. Crash-safe
+        ordering: the candidate is cleared (with a ``superseded`` /
+        ``schedule-edit`` history row) and the consumed watermark advances
+        BEFORE the revision bump is persisted, so a crash can only leave
+        the edit unapplied (safe over-skip under the old revision, freely
+        retryable) and never a bumped revision that backfills pre-edit
+        instants on restart. A concurrent edit that commits first still
+        wins via the store's authoritative revision check; our already
+        applied watermark advance is a safe over-skip in that case too.
         """
         store = self._schedule_store()
         now = datetime.now(UTC)
         current = store.get_schedule(schedule_id)
+        if current.revision != expected_revision:
+            raise StaleScheduleRevisionError(
+                f"stale schedule revision for {schedule_id!r}: "
+                f"expected {expected_revision}, found {current.revision}"
+            )
         data = current.model_dump(mode="python")
         if deployment_id is not None:
             data["deployment_id"] = deployment_id
@@ -285,7 +300,11 @@ class WorkflowScheduleApi:
         self._artifact_store().get_deployment(data["deployment_id"])
         updated = Schedule.model_validate(data)
         self._validate_definition(updated)
-        stored = store.update_schedule(updated, expected_revision=expected_revision)
+        # Crash-safe ordering: discard the old revision's unadmitted work
+        # and advance the watermark BEFORE the revision bump below. A
+        # crash here leaves the edit unapplied under the old revision
+        # (retryable); the bumped revision can never observe pre-edit
+        # instants on restart.
         old_candidate = store.get_candidate(schedule_id)
         if old_candidate is not None:
             FileScheduleHistoryRecorder(store).record(
@@ -293,7 +312,7 @@ class WorkflowScheduleApi:
                     schedule_id=schedule_id,
                     kind="superseded",
                     resolved_at=old_candidate.intended_at,
-                    revision=stored.revision,
+                    revision=expected_revision + 1,
                     reason="schedule-edit",
                     created_at=now,
                 )
@@ -303,6 +322,7 @@ class WorkflowScheduleApi:
         store.save_consumed(
             schedule_id, max(consumed, now) if consumed is not None else now
         )
+        stored = store.update_schedule(updated, expected_revision=expected_revision)
         return _PROJECT_SCHEDULE(stored.model_dump(mode="json"))
 
     async def pause_schedule(self, *, schedule_id: str) -> ScheduleResult:
@@ -310,18 +330,24 @@ class WorkflowScheduleApi:
 
         Sets ``paused``, clears any held candidate, and advances the
         consumed watermark to at least now so the paused span is never
-        backfilled on resume. No history row is written.
+        backfilled on resume. No history row is written. Crash-safe
+        ordering: the candidate is cleared and the watermark advances
+        BEFORE the flag flip is persisted, so a crash can only leave the
+        schedule unpaused (retryable) and never a paused flag whose span
+        backfills on resume.
         """
         store = self._schedule_store()
         now = datetime.now(UTC)
-        schedule = store.get_schedule(schedule_id)
-        schedule.paused = True
-        store.save_schedule(schedule)
+        # Existence first: unknown ids raise KeyError before any write.
+        store.get_schedule(schedule_id)
         store.save_candidate(None, schedule_id=schedule_id)
         consumed = store.get_consumed(schedule_id)
         store.save_consumed(
             schedule_id, max(consumed, now) if consumed is not None else now
         )
+        schedule = store.get_schedule(schedule_id)
+        schedule.paused = True
+        store.save_schedule(schedule)
         return _PROJECT_SCHEDULE(schedule.model_dump(mode="json"))
 
     async def resume_schedule(self, *, schedule_id: str) -> ScheduleResult:
@@ -329,28 +355,37 @@ class WorkflowScheduleApi:
 
         Clears ``paused`` (resume selects the next future occurrence),
         clears any held candidate, and advances the consumed watermark to
-        at least now. No history row is written.
+        at least now. No history row is written. Crash-safe ordering like
+        pause: candidate and watermark first, flag flip last.
         """
         store = self._schedule_store()
         now = datetime.now(UTC)
-        schedule = store.get_schedule(schedule_id)
-        schedule.paused = False
-        store.save_schedule(schedule)
+        # Existence first: unknown ids raise KeyError before any write.
+        store.get_schedule(schedule_id)
+        store.save_candidate(None, schedule_id=schedule_id)
         consumed = store.get_consumed(schedule_id)
         store.save_consumed(
             schedule_id, max(consumed, now) if consumed is not None else now
         )
-        store.save_candidate(None, schedule_id=schedule_id)
+        schedule = store.get_schedule(schedule_id)
+        schedule.paused = False
+        store.save_schedule(schedule)
         return _PROJECT_SCHEDULE(schedule.model_dump(mode="json"))
 
     async def delete_schedule(self, *, schedule_id: str) -> ScheduleResult:
         """Soft-delete one schedule (mirror the poll-loop deleted branch).
 
-        Sets ``deleted`` and clears any held candidate. Runs and history
+        Clears any held candidate, then sets ``deleted``. Runs and history
         are untouched, the consumed watermark does not advance, and the id
-        stays reserved (re-creation is rejected by create).
+        stays reserved (re-creation is rejected by create). Either crash
+        half heals: an unclearable candidate on a deleted schedule is
+        dropped by the poll loop, and a cleared candidate on a live
+        schedule is rebuilt from the untouched watermark.
         """
         store = self._schedule_store()
+        # Existence first: unknown ids raise KeyError before any write.
+        store.get_schedule(schedule_id)
+        store.save_candidate(None, schedule_id=schedule_id)
         schedule = store.get_schedule(schedule_id)
         schedule.deleted = True
         store.save_schedule(schedule)