sched: select latest eligible occurrence <= now; watermarks never regress (F3)

src/wf_scheduling/poll.py

@@ -66,6 +66,36 @@ def _is_oneshot(source: OccurrenceSource) -> bool:
     return isinstance(source, OneShotSource)
 
 
+def _save_consumed_max(store: Any, sched_id: str, instant: datetime) -> None:
+    """Advance the consumed watermark without ever moving it backwards.
+
+    Admission decides instants out of order across retries (a held
+    candidate admitted after the watermark already advanced past it); a
+    backwards write would resurrect the intervening instants on restart.
+    """
+    existing = store.get_consumed(sched_id)
+    if existing is None or instant > existing:
+        store.save_consumed(sched_id, instant)
+
+
+def _latest_eligible(source: OccurrenceSource, now: datetime) -> datetime | None:
+    """Select the latest eligible occurrence ``<= now`` with bounded queries.
+
+    ``prev_before`` is exclusive, so a poll exactly at a due instant would
+    miss it. ``prev_before(now)`` is the greatest occurrence strictly before
+    ``now``; at most one occurrence (``now`` itself) can lie in between, so
+    a single bounded ``next_after`` probe closes the gap without any custom
+    calendar math.
+    """
+    latest = source.prev_before(now)
+    if latest is None:
+        return None
+    forward = source.next_after(latest)
+    if forward is not None and forward <= now:
+        latest = forward
+    return latest
+
+
 class Scheduler:
     """File-store scheduler core with injected clock and collaborators.
 
@@ -215,7 +245,7 @@ class Scheduler:
             cand = self.schedule_store.get_candidate(sched.id)
             if cand is not None and cand.intended_at == intended:
                 self.schedule_store.save_candidate(None, schedule_id=sched.id)
-            self.schedule_store.save_consumed(sched.id, intended)
+            _save_consumed_max(self.schedule_store, sched.id, intended)
             return None
         active = self._active(sched.id)
         if sched.overlap == "skip" and active:
@@ -229,7 +259,7 @@ class Scheduler:
             cand = self.schedule_store.get_candidate(sched.id)
             if cand is not None and cand.intended_at == intended:
                 self.schedule_store.save_candidate(None, schedule_id=sched.id)
-            self.schedule_store.save_consumed(sched.id, intended)
+            _save_consumed_max(self.schedule_store, sched.id, intended)
             return None
         if sched.overlap == "parallel" and len(active) >= sched.max_active_runs:
             self._record(
@@ -242,7 +272,7 @@ class Scheduler:
             cand = self.schedule_store.get_candidate(sched.id)
             if cand is not None and cand.intended_at == intended:
                 self.schedule_store.save_candidate(None, schedule_id=sched.id)
-            self.schedule_store.save_consumed(sched.id, intended)
+            _save_consumed_max(self.schedule_store, sched.id, intended)
             return None
         if self._task_load() >= self.capacity:
             if sched.misfire == "latest":
@@ -254,7 +284,7 @@ class Scheduler:
                     ),
                     schedule_id=sched.id,
                 )
-                self.schedule_store.save_consumed(sched.id, intended)
+                _save_consumed_max(self.schedule_store, sched.id, intended)
                 return "held"
             if (now - intended).total_seconds() > sched.lateness_allowance_s:
                 self._record(
@@ -264,7 +294,7 @@ class Scheduler:
                     reason="capacity-deadline",
                     revision=sched.revision,
                 )
-                self.schedule_store.save_consumed(sched.id, intended)
+                _save_consumed_max(self.schedule_store, sched.id, intended)
                 return None
             return "held-undecided"
         run_id = self.run_store.allocate_run_id()
@@ -297,7 +327,7 @@ class Scheduler:
         cand = self.schedule_store.get_candidate(sched.id)
         if cand is not None and cand.intended_at == intended:
             self.schedule_store.save_candidate(None, schedule_id=sched.id)
-        self.schedule_store.save_consumed(sched.id, intended)
+        _save_consumed_max(self.schedule_store, sched.id, intended)
         from wf_api.run_lifecycle import materialize_admitted_view
 
         self.run_store.mark_pending_dispatch(run_id)
@@ -557,7 +587,7 @@ class Scheduler:
             if sched.misfire == "latest":
                 from wf_scheduling.calendar import ScheduleExhaustedError
 
-                latest = src.prev_before(now)
+                latest = _latest_eligible(src, now)
                 if latest is None:
                     raise ScheduleExhaustedError(
                         f"latest-missed lookup exhausted for {sched.id!r}"