"""Poll loop: overlap, misfire, candidates, fairness, capacity (T08). Mirrors the reference state model (probes/deployment_scheduling_verify/ test_schedule_state_model.py) against real file stores. Calendar iteration uses ``OccurrenceSource`` (``next_after``/``prev_before`` only, never enumeration); latest-missed catch-up is one bounded ``prev_before`` query (F1). Overlap decisions precede capacity checks; terminal skips never reappear; ``latest`` retains at most one candidate. """ from __future__ import annotations from datetime import datetime, timezone from typing import Any, Protocol from wf_scheduling.calendar import CronSource, OneShotSource from wf_scheduling.models import OccurrenceRecord, PendingCandidate from wf_scheduling.occurrences import occurrence_id UTC = timezone.utc SCAN_CAP = 100 EPOCH = datetime(2020, 1, 1, tzinfo=UTC) class BlockedSchedule(ValueError): """A schedule is blocked by a corrupt record and fails closed.""" class InvalidScheduleDefinitionError(ValueError): """A schedule definition or its trigger source is invalid.""" class OccurrenceSource(Protocol): """Due-instant source over aware datetimes, UTC at the boundary.""" def next_after(self, instant: datetime) -> datetime | None: ... def prev_before(self, instant: datetime) -> datetime | None: ... def source_for_trigger(trigger: Any) -> OccurrenceSource: """Build a calendar source from a schedule trigger model.""" kind = trigger.kind if hasattr(trigger, "kind") else trigger.get("kind") if kind == "cron": expression = ( trigger.expression if hasattr(trigger, "expression") else trigger["expression"] ) zone = ( trigger.timezone if hasattr(trigger, "timezone") else trigger.get("timezone", "UTC") ) return CronSource(expression, zone) if kind == "oneshot": at = trigger.at if hasattr(trigger, "at") else trigger["at"] if isinstance(at, str): at = datetime.fromisoformat(at) return OneShotSource(at) raise ValueError(f"unknown trigger kind {kind!r}") def _is_oneshot(source: OccurrenceSource) -> bool: # Calendar and test-double one-shots share the class name and an ``at`` # instant; periodic sources carry a ``period`` instead. isinstance covers # the calendar type; the name check covers duck-typed test doubles. if isinstance(source, OneShotSource): return True return type(source).__name__ == "OneShotSource" and hasattr(source, "at") class Scheduler: """File-store scheduler core with injected clock and controlled dispatch. ``outcomes`` maps run ids (or ``"*"``) to ``complete`` | ``hang`` | ``interrupt`` for deterministic tests. Production dispatch (real runtime) is wired in T12; here ``hang`` keeps an admitted run occupying its schedule slot, ``interrupt`` marks it interrupted (schedule slot only), and ``complete`` marks it completed (releases overlap). """ def __init__( self, *, schedule_store: Any, run_store: Any, sources: dict[str, OccurrenceSource], capacity: int, outcomes: dict[str, str] | None = None, deployments: dict[str, dict] | None = None, ) -> None: self.schedule_store = schedule_store self.run_store = run_store self.sources = sources self.capacity = capacity self.outcomes = outcomes or {} self.deployments = deployments or {"dep-1": {"rev": 1, "required": []}} self._poll_cursor = 0 # -- helpers ------------------------------------------------------ @staticmethod def _status_value(run: Any) -> Any: status = getattr(run, "status", None) return getattr(status, "value", status) def _active(self, sched_id: str) -> list[Any]: active: list[Any] = [] for run in self.run_store.list_runs(): if self._status_value(run) not in ("admitted", "interrupted"): continue direct = getattr(run, "sched_id", None) if direct is not None: if direct == sched_id: active.append(run) continue if self._run_sched(run) == sched_id: active.append(run) return active def _run_sched(self, run: Any) -> str | None: try: admission = self.run_store.get_admission(run.id) except KeyError: return None return admission.schedule_id def _task_load(self) -> int: from wf_scheduling.recovery import _is_pending count = 0 for run in self.run_store.list_runs(): if self._status_value(run) != "admitted": continue if _is_pending(self.run_store, run.id): continue count += 1 return count def _record( self, *, kind: str, sched_id: str, intended: datetime | None = None, run_id: str | None = None, reason: str = "", interval: tuple[datetime, datetime] | None = None, count: int = 0, revision: int | None = None, now: datetime | None = None, admitted_at: datetime | None = None, started_at: datetime | None = None, ) -> None: created = now if now is not None else datetime.now(UTC) if intended is not None: oid = occurrence_id(sched_id, intended) elif interval is not None: oid = f"{sched_id}|summary|{interval[0].isoformat()}|{interval[1].isoformat()}" else: oid = f"{sched_id}|summary|{created.isoformat()}" self.schedule_store.append_history( OccurrenceRecord( schedule_id=sched_id, occurrence_id=oid, kind=kind, # type: ignore[arg-type] resolved_at=intended, run_id=run_id, revision=revision, reason=reason, admitted_at=admitted_at, started_at=started_at, interval_start=interval[0] if interval else None, interval_end=interval[1] if interval else None, interval_count=count, created_at=created, ) ) def _admit(self, sched: Any, intended: datetime, now: datetime) -> str | None: if getattr(sched, "blocked_reason", None): raise BlockedSchedule(getattr(sched, "blocked_reason")) if not sched.enabled or sched.deleted or sched.paused: return None # Fail closed before any new admission when a corrupt active view # exists for this schedule. for run in self._active(sched.id): try: self.run_store.get_admission(run.id) except KeyError: reason = f"corrupt run view without admission: {run.id}" sched.blocked_reason = reason self.schedule_store.save_schedule(sched) raise BlockedSchedule(reason) from None dep = self.deployments.get(sched.deployment_id) if dep is None: self._record( kind="preflight-rejected", sched_id=sched.id, intended=intended, reason="deployment-deleted", revision=sched.revision, now=now, ) cand = self.schedule_store.get_candidate(sched.id) if cand is not None and cand.intended_at == intended: self.schedule_store.save_candidate(None, schedule_id=sched.id) self.schedule_store.save_consumed(sched.id, intended) return None frozen = { "team": "eng", "report_time": intended.isoformat(), "sched": sched.id, "dep_rev": dep["rev"], } missing = [k for k in dep.get("required", []) if k not in frozen] if missing: self._record( kind="preflight-rejected", sched_id=sched.id, intended=intended, reason=f"missing-input:{missing}", revision=sched.revision, ) cand = self.schedule_store.get_candidate(sched.id) if cand is not None and cand.intended_at == intended: self.schedule_store.save_candidate(None, schedule_id=sched.id) self.schedule_store.save_consumed(sched.id, intended) return None active = self._active(sched.id) if sched.overlap == "skip" and active: self._record( kind="skipped-overlap", sched_id=sched.id, intended=intended, reason=f"active={[getattr(r, 'id', None) for r in active]}", revision=sched.revision, ) cand = self.schedule_store.get_candidate(sched.id) if cand is not None and cand.intended_at == intended: self.schedule_store.save_candidate(None, schedule_id=sched.id) self.schedule_store.save_consumed(sched.id, intended) return None if sched.overlap == "parallel" and len(active) >= sched.max_active_runs: self._record( kind="skipped-overlap", sched_id=sched.id, intended=intended, reason=f"max_active={sched.max_active_runs}", revision=sched.revision, ) cand = self.schedule_store.get_candidate(sched.id) if cand is not None and cand.intended_at == intended: self.schedule_store.save_candidate(None, schedule_id=sched.id) self.schedule_store.save_consumed(sched.id, intended) return None if self._task_load() >= self.capacity: if sched.misfire == "latest": self.schedule_store.save_candidate( PendingCandidate( schedule_id=sched.id, intended_at=intended, revision=sched.revision, ), schedule_id=sched.id, ) self.schedule_store.save_consumed(sched.id, intended) return "held" if (now - intended).total_seconds() > sched.lateness_allowance_s: self._record( kind="skipped-misfire", sched_id=sched.id, intended=intended, reason="capacity-deadline", revision=sched.revision, ) self.schedule_store.save_consumed(sched.id, intended) return None return "held-undecided" run_id = self.run_store.allocate_run_id() from wf_artifacts.runs.models import RunAdmission admission = RunAdmission( id=run_id, environment=_test_environment(sched), resolved_input=dict(frozen), max_steps=getattr(sched, "max_steps", None), scheduled_at=intended, schedule_id=sched.id, schedule_revision=sched.revision, deployment_revision=dep["rev"], created_at=now, ) self.run_store.save_admission(admission) self._record( kind="admitted", sched_id=sched.id, intended=intended, run_id=run_id, reason=f"rev={sched.revision}", revision=sched.revision, now=now, admitted_at=now, ) if _is_oneshot(self.sources.get(sched.id)): # type: ignore[arg-type] sched.exhausted = True self.schedule_store.save_schedule(sched) cand = self.schedule_store.get_candidate(sched.id) if cand is not None and cand.intended_at == intended: self.schedule_store.save_candidate(None, schedule_id=sched.id) self.schedule_store.save_consumed(sched.id, intended) from wf_api.run_lifecycle import materialize_admitted_view # Dispatch mark precedes the view so a crash after admission but # before/during first dispatch stays pending (not abandoned). Cleared # after dispatch returns regardless of outcome (hang still dispatched). try: self.run_store.mark_pending_dispatch(run_id) except AttributeError: pass materialize_admitted_view(store=self.run_store, admission=admission) try: self._dispatch(run_id, now) finally: try: self.run_store.clear_pending_dispatch(run_id) except AttributeError: pass return run_id def _dispatch(self, run_id: str, now: datetime) -> None: from wf_artifacts.runs.models import StoredRunStatus outcome = self.outcomes.get(run_id, self.outcomes.get("*", "complete")) if outcome not in ("complete", "hang", "interrupt"): raise ValueError(f"unknown controlled outcome {outcome!r} for {run_id!r}") try: record = self.run_store.get_run(run_id) except KeyError as exc: raise BlockedSchedule(f"dispatch missing run view: {run_id!r}") from exc sched_id = self._run_sched(record) if sched_id is None: raise BlockedSchedule(f"dispatch missing admission: {run_id!r}") if outcome == "hang": return if outcome == "interrupt": updated = record.model_copy( update={"status": StoredRunStatus.INTERRUPTED, "updated_at": now} ) self.run_store.save_run(updated) self._record( kind="interrupted", sched_id=sched_id, intended=_admission_intended(self.run_store, run_id), run_id=run_id, now=now, started_at=now, ) return updated = record.model_copy( update={"status": StoredRunStatus.COMPLETED, "updated_at": now} ) self.run_store.save_run(updated) self._record( kind="completed", sched_id=sched_id, intended=_admission_intended(self.run_store, run_id), run_id=run_id, now=now, started_at=now, ) # -- polling -------------------------------------------------------- def poll(self, now: datetime) -> dict[str, str]: self._dispatch_pending(now) schedules = self.schedule_store.list_schedules(include_deleted=True) ids = sorted(item.id for item in schedules) if not ids: return {} cursor = self.schedule_store.get_poll_cursor() start = cursor % len(ids) order = ids[start:] + ids[:start] self.schedule_store.save_poll_cursor(cursor + 1) results: dict[str, str] = {} by_id = {item.id: item for item in schedules} for sid in order: sched = by_id[sid] if getattr(sched, "blocked_reason", None): raise BlockedSchedule(getattr(sched, "blocked_reason")) results[sid] = self._poll_one(sched, now) return results def _dispatch_pending(self, now: datetime) -> None: """Dispatch recovery-materialized runs through capacity checks. Recovery NEVER executes: it only completes missing views flagged pending for this sweep. Pending runs of blocked schedules stay pending. Hanging admitted runs without a pending marker are never re-executed here. """ from wf_scheduling.recovery import _is_pending, clear_pending for run in sorted(self.run_store.list_runs(), key=lambda r: r.id): if not _is_pending(self.run_store, run.id): continue try: admission = self.run_store.get_admission(run.id) except KeyError: continue if admission.schedule_id is None: continue try: sched = self.schedule_store.get_schedule(admission.schedule_id) except KeyError: continue if getattr(sched, "blocked_reason", None): continue if self._task_load() >= self.capacity: continue self._dispatch(run.id, now) clear_pending(self.run_store, run.id) def _poll_one(self, sched: Any, now: datetime) -> str: if sched.deleted: if self.schedule_store.get_candidate(sched.id) is not None: self.schedule_store.save_candidate(None, schedule_id=sched.id) return "deleted" if not sched.enabled: if self.schedule_store.get_candidate(sched.id) is not None: self.schedule_store.save_candidate(None, schedule_id=sched.id) consumed = self.schedule_store.get_consumed(sched.id) or EPOCH self.schedule_store.save_consumed(sched.id, max(consumed, now)) return "disabled" if sched.exhausted: return "exhausted" if sched.paused: if self.schedule_store.get_candidate(sched.id) is not None: self.schedule_store.save_candidate(None, schedule_id=sched.id) consumed = self.schedule_store.get_consumed(sched.id) or EPOCH self.schedule_store.save_consumed(sched.id, max(consumed, now)) return "paused" # Fail closed on corrupt active views before calendar work (F5). for run in self.run_store.list_runs(): if self._status_value(run) not in ("admitted", "interrupted"): continue try: self.run_store.get_admission(run.id) except KeyError: reason = f"corrupt run view without admission: {run.id}" sched.blocked_reason = reason self.schedule_store.save_schedule(sched) raise BlockedSchedule(reason) from None try: src = self.sources[sched.id] except KeyError as exc: raise InvalidScheduleDefinitionError( f"no occurrence source for schedule {sched.id!r}" ) from exc consumed = self.schedule_store.get_consumed(sched.id) or EPOCH if consumed > now: return "clock-rollback-held" due: list[datetime] = [] cursor = consumed jumped = False while True: nxt = src.next_after(cursor) if nxt is None or nxt > now: if ( not due and _is_oneshot(src) and nxt is None and not sched.exhausted and sched.misfire == "skip" and getattr(src, "at", None) is not None and getattr(src, "at") <= now ): at = getattr(src, "at") self._record( kind="exhausted", sched_id=sched.id, intended=at, reason="oneshot-expired-skip", revision=sched.revision, ) sched.exhausted = True self.schedule_store.save_schedule(sched) self.schedule_store.save_consumed(sched.id, now) self.schedule_store.save_candidate(None, schedule_id=sched.id) return "exhausted" break due.append(nxt) cursor = nxt if len(due) >= SCAN_CAP: jumped = True break if jumped: if sched.misfire == "latest": from wf_scheduling.calendar import ScheduleExhaustedError latest = src.prev_before(now) if latest is None: raise ScheduleExhaustedError( f"latest-missed lookup exhausted for {sched.id!r}" ) old = self.schedule_store.get_candidate(sched.id) if old is not None and old.intended_at != latest: self._record( kind="superseded", sched_id=sched.id, intended=old.intended_at, reason=f"coalesced-into:{latest.isoformat()}", revision=sched.revision, now=now, ) self.schedule_store.save_candidate( PendingCandidate( schedule_id=sched.id, intended_at=latest, revision=sched.revision, ), schedule_id=sched.id, ) self._record( kind="interval-summary", sched_id=sched.id, reason="coalesced-missed-span", revision=sched.revision, interval=(consumed, now), count=-1, now=now, ) self.schedule_store.save_consumed(sched.id, now) held = self._admit_held_candidate(sched, now) if held is None: # Terminal skip inside admission (e.g. overlap) clears the # candidate; do not misreport it as held. return "skipped-overlap" return held self._record( kind="interval-summary", sched_id=sched.id, reason="skipped-missed-span", revision=sched.revision, interval=(consumed, now), count=-1, now=now, ) self.schedule_store.save_consumed(sched.id, now) return "span-skipped" last_result = "idle" for instant in due: stored_consumed = self.schedule_store.get_consumed(sched.id) or EPOCH if instant <= stored_consumed: continue age = (now - instant).total_seconds() if age <= sched.lateness_allowance_s: old = self.schedule_store.get_candidate(sched.id) if old is not None and old.intended_at != instant: self._record( kind="superseded", sched_id=sched.id, intended=old.intended_at, reason=f"admitted-newer:{instant.isoformat()}", revision=sched.revision, ) self.schedule_store.save_candidate(None, schedule_id=sched.id) result = self._admit(sched, instant, now) last_result = f"admit:{result}" elif sched.misfire == "latest": old = self.schedule_store.get_candidate(sched.id) if old is not None and old.intended_at != instant: self._record( kind="superseded", sched_id=sched.id, intended=old.intended_at, reason=f"coalesced-into:{instant.isoformat()}", revision=sched.revision, ) self.schedule_store.save_candidate( PendingCandidate( schedule_id=sched.id, intended_at=instant, revision=sched.revision, ), schedule_id=sched.id, ) self.schedule_store.save_consumed(sched.id, instant) held = self._admit_held_candidate(sched, now) if held is None: last_result = "skipped-overlap" elif held == "held-undecided": last_result = "candidate-held" else: last_result = held else: if _is_oneshot(src) and not sched.exhausted: self._record( kind="exhausted", sched_id=sched.id, intended=instant, reason="oneshot-expired-skip", revision=sched.revision, ) sched.exhausted = True self.schedule_store.save_schedule(sched) self.schedule_store.save_consumed(sched.id, instant) last_result = "exhausted" continue self._record( kind="skipped-misfire", sched_id=sched.id, intended=instant, reason=f"age={age:.0f}s", revision=sched.revision, ) self.schedule_store.save_consumed(sched.id, instant) last_result = "skipped-misfire" if last_result in ("idle",): cand = self.schedule_store.get_candidate(sched.id) if cand is not None: held = self._admit_held_candidate(sched, now) if held is None: last_result = "skipped-overlap" elif held == "held-undecided": last_result = "candidate-held" else: last_result = held return last_result def _admit_held_candidate(self, sched: Any, now: datetime) -> str | None: cand = self.schedule_store.get_candidate(sched.id) if cand is None or cand.revision != sched.revision: if cand is not None and cand.revision != sched.revision: self._record( kind="superseded", sched_id=sched.id, intended=cand.intended_at, reason="schedule-edit", revision=sched.revision, now=now, ) self.schedule_store.save_candidate(None, schedule_id=sched.id) return None return self._admit(sched, cand.intended_at, now) # -- administration --------------------------------------------------- def resume_schedule(self, sid: str, now: datetime) -> None: """Unpause: resume selects the next future occurrence.""" sched = self.schedule_store.get_schedule(sid) sched.paused = False self.schedule_store.save_schedule(sched) consumed = self.schedule_store.get_consumed(sid) or EPOCH self.schedule_store.save_consumed(sid, max(consumed, now)) self.schedule_store.save_candidate(None, schedule_id=sid) def edit_schedule(self, sid: str, now: datetime) -> None: """Definition edit: new revision, discard old candidates, no backfill.""" sched = self.schedule_store.get_schedule(sid) sched.revision += 1 sched.updated_at = now self.schedule_store.save_schedule(sched) old = self.schedule_store.get_candidate(sid) if old is not None: self._record( kind="superseded", sched_id=sid, intended=old.intended_at, reason="schedule-edit", revision=sched.revision, ) self.schedule_store.save_candidate(None, schedule_id=sid) consumed = self.schedule_store.get_consumed(sid) or EPOCH self.schedule_store.save_consumed(sid, max(consumed, now)) def _test_environment(sched: Any) -> Any: from wf_artifacts import PinnedRunEnvironment, WorkflowArtifact, WorkflowDeployment deployment = WorkflowDeployment( id=getattr(sched, "deployment_id", "dep-1"), artifact_id="wf-1", artifact_version=1, bindings=[], ) artifact = WorkflowArtifact( id="wf-1", version=1, title="Wf-1", input_schema={"type": "object", "properties": {}}, output_schema={"type": "object", "properties": {}}, outcomes=("ok",), plan={"name": "wf-1", "nodes": [], "edges": []}, ) return PinnedRunEnvironment( deployment=deployment, root_artifact=artifact, child_artifacts=[] ) def _admission_intended(run_store: Any, run_id: str) -> datetime | None: try: return run_store.get_admission(run_id).scheduled_at except KeyError: return None