sched: add poll loop with overlap, misfire, fairness, and capacity (T08)

This commit is contained in:
lda
2026-09-08 10:32:30 +07:00 Verified
parent 7ed759053a
commit 73cb08e5ca
2 changed files with 883 additions and 0 deletions
+590
View File
@@ -0,0 +1,590 @@
"""Poll loop: overlap, misfire, candidates, fairness, capacity (T08).
Mirrors the reference state model (probes/deployment_scheduling_verify/
test_schedule_state_model.py) against real file stores. Calendar iteration
uses ``OccurrenceSource`` (``next_after``/``prev_before`` only, never
enumeration); latest-missed catch-up is one bounded ``prev_before`` query
(F1). Overlap decisions precede capacity checks; terminal skips never
reappear; ``latest`` retains at most one candidate.
"""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Protocol
from wf_scheduling.calendar import CronSource, OneShotSource
from wf_scheduling.models import OccurrenceRecord, PendingCandidate
from wf_scheduling.occurrences import occurrence_id
UTC = timezone.utc
SCAN_CAP = 100
EPOCH = datetime(2020, 1, 1, tzinfo=UTC)
class OccurrenceSource(Protocol):
"""Due-instant source over aware datetimes, UTC at the boundary."""
def next_after(self, instant: datetime) -> datetime | None: ...
def prev_before(self, instant: datetime) -> datetime | None: ...
def source_for_trigger(trigger: Any) -> OccurrenceSource:
"""Build a calendar source from a schedule trigger model."""
kind = trigger.kind if hasattr(trigger, "kind") else trigger.get("kind")
if kind == "cron":
expression = (
trigger.expression
if hasattr(trigger, "expression")
else trigger["expression"]
)
zone = (
trigger.timezone
if hasattr(trigger, "timezone")
else trigger.get("timezone", "UTC")
)
return CronSource(expression, zone)
if kind == "oneshot":
at = trigger.at if hasattr(trigger, "at") else trigger["at"]
if isinstance(at, str):
at = datetime.fromisoformat(at)
return OneShotSource(at)
raise ValueError(f"unknown trigger kind {kind!r}")
def _is_oneshot(source: OccurrenceSource) -> bool:
return hasattr(source, "at") and not hasattr(source, "period")
class Scheduler:
"""File-store scheduler core with injected clock and controlled dispatch.
``outcomes`` maps run ids (or ``"*"``) to ``complete`` | ``hang`` |
``interrupt`` for deterministic tests. Production dispatch (real runtime)
is wired in T12; here ``hang`` keeps an admitted run occupying its
schedule slot, ``interrupt`` marks it interrupted (schedule slot only),
and ``complete`` marks it completed (releases overlap).
"""
def __init__(
self,
*,
schedule_store: Any,
run_store: Any,
sources: dict[str, OccurrenceSource],
capacity: int,
outcomes: dict[str, str] | None = None,
deployments: dict[str, dict] | None = None,
) -> None:
self.schedule_store = schedule_store
self.run_store = run_store
self.sources = sources
self.capacity = capacity
self.outcomes = outcomes or {}
self.deployments = deployments or {"dep-1": {"rev": 1, "required": []}}
self._poll_cursor = 0
# -- helpers ------------------------------------------------------
@staticmethod
def _status_value(run: Any) -> Any:
status = getattr(run, "status", None)
return getattr(status, "value", status)
def _active(self, sched_id: str) -> list[Any]:
active: list[Any] = []
for run in self.run_store.list_runs():
if self._status_value(run) not in ("admitted", "interrupted"):
continue
direct = getattr(run, "sched_id", None)
if direct is not None:
if direct == sched_id:
active.append(run)
continue
if self._run_sched(run) == sched_id:
active.append(run)
return active
def _run_sched(self, run: Any) -> str | None:
try:
admission = self.run_store.get_admission(run.id)
except KeyError:
return None
return admission.schedule_id
def _task_load(self) -> int:
count = 0
for run in self.run_store.list_runs():
if self._status_value(run) == "admitted":
count += 1
return count
def _record(
self,
*,
kind: str,
sched_id: str,
intended: datetime | None = None,
run_id: str | None = None,
reason: str = "",
interval: tuple[datetime, datetime] | None = None,
count: int = 0,
revision: int | None = None,
) -> None:
oid = (
occurrence_id(sched_id, intended)
if intended is not None
else f"{sched_id}|no-instant"
)
self.schedule_store.append_history(
OccurrenceRecord(
schedule_id=sched_id,
occurrence_id=oid,
kind=kind, # type: ignore[arg-type]
resolved_at=intended,
run_id=run_id,
revision=revision,
reason=reason,
interval_start=interval[0] if interval else None,
interval_end=interval[1] if interval else None,
interval_count=count,
created_at=datetime.now(UTC),
)
)
def _admit(self, sched: Any, intended: datetime, now: datetime) -> str | None:
if getattr(sched, "blocked_reason", None):
raise ValueError(getattr(sched, "blocked_reason"))
if not sched.enabled or sched.deleted or sched.paused:
return None
dep = self.deployments.get(sched.deployment_id)
if dep is None:
self._record(
kind="preflight-rejected",
sched_id=sched.id,
intended=intended,
reason="deployment-deleted",
revision=sched.revision,
)
self.schedule_store.save_consumed(sched.id, intended)
return None
frozen = {
"team": "eng",
"report_time": intended.isoformat(),
"sched": sched.id,
"dep_rev": dep["rev"],
}
missing = [k for k in dep.get("required", []) if k not in frozen]
if missing:
self._record(
kind="preflight-rejected",
sched_id=sched.id,
intended=intended,
reason=f"missing-input:{missing}",
revision=sched.revision,
)
cand = self.schedule_store.get_candidate(sched.id)
if cand is not None and cand.intended_at == intended:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
self.schedule_store.save_consumed(sched.id, intended)
return None
active = self._active(sched.id)
if sched.overlap == "skip" and active:
self._record(
kind="skipped-overlap",
sched_id=sched.id,
intended=intended,
reason=f"active={[getattr(r, 'id', None) for r in active]}",
revision=sched.revision,
)
cand = self.schedule_store.get_candidate(sched.id)
if cand is not None and cand.intended_at == intended:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
self.schedule_store.save_consumed(sched.id, intended)
return None
if sched.overlap == "parallel" and len(active) >= sched.max_active_runs:
self._record(
kind="skipped-overlap",
sched_id=sched.id,
intended=intended,
reason=f"max_active={sched.max_active_runs}",
revision=sched.revision,
)
cand = self.schedule_store.get_candidate(sched.id)
if cand is not None and cand.intended_at == intended:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
self.schedule_store.save_consumed(sched.id, intended)
return None
if self._task_load() >= self.capacity:
if sched.misfire == "latest":
self.schedule_store.save_candidate(
PendingCandidate(
schedule_id=sched.id,
intended_at=intended,
revision=sched.revision,
),
schedule_id=sched.id,
)
self.schedule_store.save_consumed(sched.id, intended)
return "held"
if (now - intended).total_seconds() > sched.lateness_allowance_s:
self._record(
kind="skipped-misfire",
sched_id=sched.id,
intended=intended,
reason="capacity-deadline",
revision=sched.revision,
)
self.schedule_store.save_consumed(sched.id, intended)
return None
return "held-undecided"
run_id = self.run_store.allocate_run_id()
from wf_artifacts.runs.models import RunAdmission
admission = RunAdmission(
id=run_id,
environment=_test_environment(sched),
resolved_input=dict(frozen),
max_steps=getattr(sched, "max_steps", None),
scheduled_at=intended,
schedule_id=sched.id,
schedule_revision=sched.revision,
deployment_revision=dep["rev"],
created_at=now,
)
self.run_store.save_admission(admission)
self._record(
kind="admitted",
sched_id=sched.id,
intended=intended,
run_id=run_id,
reason=f"rev={sched.revision}",
revision=sched.revision,
)
if _is_oneshot(self.sources.get(sched.id)): # type: ignore[arg-type]
sched.exhausted = True
self.schedule_store.save_schedule(sched)
cand = self.schedule_store.get_candidate(sched.id)
if cand is not None and cand.intended_at == intended:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
self.schedule_store.save_consumed(sched.id, intended)
from wf_api.run_lifecycle import materialize_admitted_view
materialize_admitted_view(store=self.run_store, admission=admission)
self._dispatch(run_id, now)
return run_id
def _dispatch(self, run_id: str, now: datetime) -> None:
from wf_artifacts.runs.models import StoredRunStatus
outcome = self.outcomes.get(run_id, self.outcomes.get("*", "complete"))
record = self.run_store.get_run(run_id)
if outcome == "hang":
return
if outcome == "interrupt":
updated = record.model_copy(
update={"status": StoredRunStatus.INTERRUPTED, "updated_at": now}
)
self.run_store.save_run(updated)
self._record(
kind="interrupted",
sched_id=self._run_sched(record) or "",
intended=_admission_intended(self.run_store, run_id),
run_id=run_id,
)
return
assert outcome == "complete"
updated = record.model_copy(
update={"status": StoredRunStatus.COMPLETED, "updated_at": now}
)
self.run_store.save_run(updated)
self._record(
kind="completed",
sched_id=self._run_sched(record) or "",
intended=_admission_intended(self.run_store, run_id),
run_id=run_id,
)
# -- polling --------------------------------------------------------
def poll(self, now: datetime) -> dict[str, str]:
self._dispatch_pending(now)
schedules = self.schedule_store.list_schedules(include_deleted=True)
ids = sorted(item.id for item in schedules)
if not ids:
return {}
start = self._poll_cursor % len(ids)
order = ids[start:] + ids[:start]
self._poll_cursor += 1
results: dict[str, str] = {}
by_id = {item.id: item for item in schedules}
for sid in order:
sched = by_id[sid]
if getattr(sched, "blocked_reason", None):
raise ValueError(getattr(sched, "blocked_reason"))
results[sid] = self._poll_one(sched, now)
return results
def _dispatch_pending(self, now: datetime) -> None:
"""Dispatch recovery-materialized runs through capacity checks.
Recovery NEVER executes (T10): it only completes missing views flagged
pending for this sweep. Pending runs of blocked schedules stay
pending. TODO(T10): introduce the pending-dispatch marker (F11) once
dispatch marks distinguish crash-after-dispatch from never-dispatched;
until then only runs explicitly flagged ``needs_dispatch`` dispatch
here so hanging admitted runs are never re-executed.
"""
for run in sorted(self.run_store.list_runs(), key=lambda r: r.id):
if not getattr(run, "needs_dispatch", False):
continue
try:
admission = self.run_store.get_admission(run.id)
except KeyError:
continue
if admission.schedule_id is None:
continue
try:
sched = self.schedule_store.get_schedule(admission.schedule_id)
except KeyError:
continue
if getattr(sched, "blocked_reason", None):
continue
if self._task_load() >= self.capacity:
continue
self._dispatch(run.id, now)
def _poll_one(self, sched: Any, now: datetime) -> str:
if sched.deleted:
if self.schedule_store.get_candidate(sched.id) is not None:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
return "deleted"
if not sched.enabled:
if self.schedule_store.get_candidate(sched.id) is not None:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
self.schedule_store.save_consumed(sched.id, max(consumed, now))
return "disabled"
if sched.exhausted:
return "exhausted"
if sched.paused:
if self.schedule_store.get_candidate(sched.id) is not None:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
self.schedule_store.save_consumed(sched.id, max(consumed, now))
return "paused"
src = self.sources[sched.id]
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
if consumed > now:
return "clock-rollback-held"
due: list[datetime] = []
cursor = consumed
jumped = False
while True:
nxt = src.next_after(cursor)
if nxt is None or nxt > now:
if (
not due
and _is_oneshot(src)
and nxt is None
and not sched.exhausted
and sched.misfire == "skip"
and getattr(src, "at", None) is not None
and getattr(src, "at") <= now
):
at = getattr(src, "at")
self._record(
kind="exhausted",
sched_id=sched.id,
intended=at,
reason="oneshot-expired-skip",
revision=sched.revision,
)
sched.exhausted = True
self.schedule_store.save_schedule(sched)
self.schedule_store.save_consumed(sched.id, now)
self.schedule_store.save_candidate(None, schedule_id=sched.id)
return "exhausted"
break
due.append(nxt)
cursor = nxt
if len(due) >= SCAN_CAP:
jumped = True
break
if jumped:
if sched.misfire == "latest":
latest = src.prev_before(now)
assert latest is not None
old = self.schedule_store.get_candidate(sched.id)
if old is not None and old.intended_at != latest:
self._record(
kind="superseded",
sched_id=sched.id,
intended=old.intended_at,
reason=f"coalesced-into:{latest.isoformat()}",
revision=sched.revision,
)
self.schedule_store.save_candidate(
PendingCandidate(
schedule_id=sched.id,
intended_at=latest,
revision=sched.revision,
),
schedule_id=sched.id,
)
self._record(
kind="interval-summary",
sched_id=sched.id,
reason="coalesced-missed-span",
revision=sched.revision,
)
self.schedule_store.save_consumed(sched.id, now)
return self._admit_held_candidate(sched, now) or "candidate-held"
self._record(
kind="interval-summary",
sched_id=sched.id,
reason="skipped-missed-span",
revision=sched.revision,
)
self.schedule_store.save_consumed(sched.id, now)
return "span-skipped"
last_result = "idle"
for instant in due:
stored_consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
if instant <= stored_consumed:
continue
age = (now - instant).total_seconds()
if age <= sched.lateness_allowance_s:
old = self.schedule_store.get_candidate(sched.id)
if old is not None and old.intended_at != instant:
self._record(
kind="superseded",
sched_id=sched.id,
intended=old.intended_at,
reason=f"admitted-newer:{instant.isoformat()}",
revision=sched.revision,
)
self.schedule_store.save_candidate(None, schedule_id=sched.id)
result = self._admit(sched, instant, now)
last_result = f"admit:{result}"
elif sched.misfire == "latest":
old = self.schedule_store.get_candidate(sched.id)
if old is not None and old.intended_at != instant:
self._record(
kind="superseded",
sched_id=sched.id,
intended=old.intended_at,
reason=f"coalesced-into:{instant.isoformat()}",
revision=sched.revision,
)
self.schedule_store.save_candidate(
PendingCandidate(
schedule_id=sched.id,
intended_at=instant,
revision=sched.revision,
),
schedule_id=sched.id,
)
self.schedule_store.save_consumed(sched.id, instant)
last_result = self._admit_held_candidate(sched, now) or "candidate-held"
else:
if _is_oneshot(src) and not sched.exhausted:
self._record(
kind="exhausted",
sched_id=sched.id,
intended=instant,
reason="oneshot-expired-skip",
revision=sched.revision,
)
sched.exhausted = True
self.schedule_store.save_schedule(sched)
self.schedule_store.save_consumed(sched.id, instant)
last_result = "exhausted"
continue
self._record(
kind="skipped-misfire",
sched_id=sched.id,
intended=instant,
reason=f"age={age:.0f}s",
revision=sched.revision,
)
self.schedule_store.save_consumed(sched.id, instant)
last_result = "skipped-misfire"
if last_result in ("idle",):
cand = self.schedule_store.get_candidate(sched.id)
if cand is not None:
last_result = self._admit_held_candidate(sched, now) or "candidate-held"
return last_result
def _admit_held_candidate(self, sched: Any, now: datetime) -> str | None:
cand = self.schedule_store.get_candidate(sched.id)
if cand is None or cand.revision != sched.revision:
if cand is not None and cand.revision != sched.revision:
self._record(
kind="superseded",
sched_id=sched.id,
intended=cand.intended_at,
reason="schedule-edit",
revision=sched.revision,
)
self.schedule_store.save_candidate(None, schedule_id=sched.id)
return None
result = self._admit(sched, cand.intended_at, now)
return result if result != "held-undecided" else None
# -- administration ---------------------------------------------------
def resume_schedule(self, sid: str, now: datetime) -> None:
"""Unpause: resume selects the next future occurrence."""
sched = self.schedule_store.get_schedule(sid)
sched.paused = False
self.schedule_store.save_schedule(sched)
consumed = self.schedule_store.get_consumed(sid) or EPOCH
self.schedule_store.save_consumed(sid, max(consumed, now))
self.schedule_store.save_candidate(None, schedule_id=sid)
def edit_schedule(self, sid: str, now: datetime) -> None:
"""Definition edit: new revision, discard old candidates, no backfill."""
sched = self.schedule_store.get_schedule(sid)
sched.revision += 1
sched.updated_at = now
self.schedule_store.save_schedule(sched)
old = self.schedule_store.get_candidate(sid)
if old is not None:
self._record(
kind="superseded",
sched_id=sid,
intended=old.intended_at,
reason="schedule-edit",
revision=sched.revision,
)
self.schedule_store.save_candidate(None, schedule_id=sid)
consumed = self.schedule_store.get_consumed(sid) or EPOCH
self.schedule_store.save_consumed(sid, max(consumed, now))
def _test_environment(sched: Any) -> Any:
from wf_artifacts import PinnedRunEnvironment, WorkflowArtifact, WorkflowDeployment
deployment = WorkflowDeployment(
id=getattr(sched, "deployment_id", "dep-1"),
artifact_id="wf-1",
artifact_version=1,
bindings=[],
)
artifact = WorkflowArtifact(
id="wf-1",
version=1,
title="Wf-1",
input_schema={"type": "object", "properties": {}},
output_schema={"type": "object", "properties": {}},
outcomes=("ok",),
plan={"name": "wf-1", "nodes": [], "edges": []},
)
return PinnedRunEnvironment(
deployment=deployment, root_artifact=artifact, child_artifacts=[]
)
def _admission_intended(run_store: Any, run_id: str) -> datetime | None:
try:
return run_store.get_admission(run_id).scheduled_at
except KeyError:
return None