sched: typed history recorder with exactly-once terminal reconcile (F4)

src/wf_scheduling/poll.py

@@ -25,8 +25,12 @@ from wf_scheduling.calendar import (
     OneShotSource,
 )
 from wf_scheduling.dispatch import RunDispatcher, StillRunning
-from wf_scheduling.models import OccurrenceRecord, PendingCandidate
-from wf_scheduling.occurrences import occurrence_id
+from wf_scheduling.history import (
+    FileScheduleHistoryRecorder,
+    HistoryEntry,
+    HistoryRecorder,
+)
+from wf_scheduling.models import PendingCandidate
 from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
 from wf_scheduling.prepare import InvocationPreparer, PreparationRejected
 
@@ -116,6 +120,7 @@ class Scheduler:
         preparer: InvocationPreparer,
         dispatcher: RunDispatcher,
         ownership: SchedulerOwnership,
+        history: HistoryRecorder | None = None,
     ) -> None:
         self.schedule_store = schedule_store
         self.run_store = run_store
@@ -124,6 +129,11 @@ class Scheduler:
         self.preparer = preparer
         self.dispatcher = dispatcher
         self.ownership = ownership
+        self.history: HistoryRecorder = (
+            history
+            if history is not None
+            else FileScheduleHistoryRecorder(schedule_store)
+        )
         self._poll_cursor = 0
 
     def _require_ownership(self) -> None:
@@ -191,18 +201,12 @@ class Scheduler:
         now: datetime | None = None,
         admitted_at: datetime | None = None,
         started_at: datetime | None = None,
+        checkpoint_id: str | None = None,
     ) -> None:
         created = now if now is not None else datetime.now(UTC)
-        if intended is not None:
-            oid = occurrence_id(sched_id, intended)
-        elif interval is not None:
-            oid = f"{sched_id}|summary|{interval[0].isoformat()}|{interval[1].isoformat()}"
-        else:
-            oid = f"{sched_id}|summary|{created.isoformat()}"
-        self.schedule_store.append_history(
-            OccurrenceRecord(
+        self.history.record(
+            HistoryEntry(
                 schedule_id=sched_id,
-                occurrence_id=oid,
                 kind=kind,  # type: ignore[arg-type]
                 resolved_at=intended,
                 run_id=run_id,
@@ -210,6 +214,7 @@ class Scheduler:
                 reason=reason,
                 admitted_at=admitted_at,
                 started_at=started_at,
+                checkpoint_id=checkpoint_id,
                 interval_start=interval[0] if interval else None,
                 interval_end=interval[1] if interval else None,
                 interval_count=count,
@@ -217,6 +222,20 @@ class Scheduler:
             )
         )
 
+    def _existing_occurrence_run(self, sched_id: str, intended: datetime) -> str | None:
+        """Return the run already owning this occurrence, if any.
+
+        Identity is ``(schedule_id, resolved UTC instant)`` from the durable
+        admission record — the admission persist is the decision point, so
+        admissions are scanned rather than views (a crashed admission may
+        not have a view yet). Manual runs carry no scheduled instant and
+        never match.
+        """
+        for admission in self.run_store.list_admissions():
+            if admission.schedule_id == sched_id and admission.scheduled_at == intended:
+                return admission.id
+        return None
+
     def _admit(self, sched: Any, intended: datetime, now: datetime) -> str | None:
         if getattr(sched, "blocked_reason", None):
             raise BlockedSchedule(getattr(sched, "blocked_reason"))
@@ -232,6 +251,30 @@ class Scheduler:
                 sched.blocked_reason = reason
                 self.schedule_store.save_schedule(sched)
                 raise BlockedSchedule(reason) from None
+        existing = self._existing_occurrence_run(sched.id, intended)
+        if existing is not None:
+            # The occurrence already owns a run (crash between the admission
+            # persist and the watermark/history writes, or a lost watermark
+            # write): an occurrence is immutable and never replayed. Advance
+            # the watermark, reconcile a missing admitted entry exactly once,
+            # and return the owner without dispatching (the pending sweep
+            # owns dispatch).
+            cand = self.schedule_store.get_candidate(sched.id)
+            if cand is not None and cand.intended_at == intended:
+                self.schedule_store.save_candidate(None, schedule_id=sched.id)
+            _save_consumed_max(self.schedule_store, sched.id, intended)
+            if not self.history.has_terminal(sched.id, existing, "admitted", None):
+                self._record(
+                    kind="admitted",
+                    sched_id=sched.id,
+                    intended=intended,
+                    run_id=existing,
+                    reason=f"rev={sched.revision}",
+                    revision=sched.revision,
+                    now=now,
+                    admitted_at=now,
+                )
+            return existing
         prepared = self.preparer.prepare(sched=sched, intended=intended, now=now)
         if isinstance(prepared, PreparationRejected):
             self._record(
@@ -377,6 +420,7 @@ class Scheduler:
             run_id=run_id,
             now=now,
             started_at=now,
+            checkpoint_id=stopped.latest_checkpoint_id,
         )
 
     def record_stopped_execution(self, run_id: str, result: Any, now: datetime) -> None:
@@ -422,6 +466,7 @@ class Scheduler:
             run_id=run_id,
             now=now,
             started_at=now,
+            checkpoint_id=stopped.latest_checkpoint_id,
         )
 
     # -- polling --------------------------------------------------------
@@ -476,7 +521,7 @@ class Scheduler:
                     run,
                     CORRUPT_PENDING_REASON,
                     now,
-                    None,
+                    self.history,
                 )
                 clear_pending(self.run_store, run.id)
                 clear_executing(self.run_store, run.id)
@@ -487,7 +532,7 @@ class Scheduler:
                     run,
                     CORRUPT_PENDING_REASON,
                     now,
-                    None,
+                    self.history,
                 )
                 clear_pending(self.run_store, run.id)
                 clear_executing(self.run_store, run.id)