lda.chat/lda-wf
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sched: address R3 blockers on pagination, intervals, and fail-closed polling

Browse Source
This commit is contained in:
lda
2026-09-08 10:39:45 +07:00 Verified
parent 73cb08e5ca
commit be8b40e184
4 changed files with 233 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/wf_scheduling/poll.py
+119 -26
View File
@@ -22,6 +22,14 @@ SCAN_CAP = 100
EPOCH = datetime(2020, 1, 1, tzinfo=UTC)
class BlockedSchedule(ValueError):
"""A schedule is blocked by a corrupt record and fails closed."""
class InvalidScheduleDefinitionError(ValueError):
"""A schedule definition or its trigger source is invalid."""
class OccurrenceSource(Protocol):
"""Due-instant source over aware datetimes, UTC at the boundary."""
@@ -53,7 +61,12 @@ def source_for_trigger(trigger: Any) -> OccurrenceSource:
def _is_oneshot(source: OccurrenceSource) -> bool:
return hasattr(source, "at") and not hasattr(source, "period")
# Calendar and test-double one-shots share the class name and an ``at``
# instant; periodic sources carry a ``period`` instead. isinstance covers
# the calendar type; the name check covers duck-typed test doubles.
if isinstance(source, OneShotSource):
return True
return type(source).__name__ == "OneShotSource" and hasattr(source, "at")
class Scheduler:
@@ -129,12 +142,17 @@ class Scheduler:
interval: tuple[datetime, datetime] | None = None,
count: int = 0,
revision: int | None = None,
now: datetime | None = None,
admitted_at: datetime | None = None,
started_at: datetime | None = None,
) -> None:
oid = (
occurrence_id(sched_id, intended)
if intended is not None
else f"{sched_id}|no-instant"
)
created = now if now is not None else datetime.now(UTC)
if intended is not None:
oid = occurrence_id(sched_id, intended)
elif interval is not None:
oid = f"{sched_id}|summary|{interval[0].isoformat()}|{interval[1].isoformat()}"
else:
oid = f"{sched_id}|summary|{created.isoformat()}"
self.schedule_store.append_history(
OccurrenceRecord(
schedule_id=sched_id,
@@ -144,18 +162,30 @@ class Scheduler:
run_id=run_id,
revision=revision,
reason=reason,
admitted_at=admitted_at,
started_at=started_at,
interval_start=interval[0] if interval else None,
interval_end=interval[1] if interval else None,
interval_count=count,
created_at=datetime.now(UTC),
created_at=created,
)
)
def _admit(self, sched: Any, intended: datetime, now: datetime) -> str | None:
if getattr(sched, "blocked_reason", None):
raise ValueError(getattr(sched, "blocked_reason"))
raise BlockedSchedule(getattr(sched, "blocked_reason"))
if not sched.enabled or sched.deleted or sched.paused:
return None
# Fail closed before any new admission when a corrupt active view
# exists for this schedule.
for run in self._active(sched.id):
try:
self.run_store.get_admission(run.id)
except KeyError:
reason = f"corrupt run view without admission: {run.id}"
sched.blocked_reason = reason
self.schedule_store.save_schedule(sched)
raise BlockedSchedule(reason) from None
dep = self.deployments.get(sched.deployment_id)
if dep is None:
self._record(
@@ -164,7 +194,11 @@ class Scheduler:
intended=intended,
reason="deployment-deleted",
revision=sched.revision,
now=now,
)
cand = self.schedule_store.get_candidate(sched.id)
if cand is not None and cand.intended_at == intended:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
self.schedule_store.save_consumed(sched.id, intended)
return None
frozen = {
@@ -259,6 +293,8 @@ class Scheduler:
run_id=run_id,
reason=f"rev={sched.revision}",
revision=sched.revision,
now=now,
admitted_at=now,
)
if _is_oneshot(self.sources.get(sched.id)): # type: ignore[arg-type]
sched.exhausted = True
@@ -277,7 +313,15 @@ class Scheduler:
from wf_artifacts.runs.models import StoredRunStatus
outcome = self.outcomes.get(run_id, self.outcomes.get("*", "complete"))
record = self.run_store.get_run(run_id)
if outcome not in ("complete", "hang", "interrupt"):
raise ValueError(f"unknown controlled outcome {outcome!r} for {run_id!r}")
try:
record = self.run_store.get_run(run_id)
except KeyError as exc:
raise BlockedSchedule(f"dispatch missing run view: {run_id!r}") from exc
sched_id = self._run_sched(record)
if sched_id is None:
raise BlockedSchedule(f"dispatch missing admission: {run_id!r}")
if outcome == "hang":
return
if outcome == "interrupt":
@@ -287,21 +331,24 @@ class Scheduler:
self.run_store.save_run(updated)
self._record(
kind="interrupted",
sched_id=self._run_sched(record) or "",
sched_id=sched_id,
intended=_admission_intended(self.run_store, run_id),
run_id=run_id,
now=now,
started_at=now,
)
return
assert outcome == "complete"
updated = record.model_copy(
update={"status": StoredRunStatus.COMPLETED, "updated_at": now}
)
self.run_store.save_run(updated)
self._record(
kind="completed",
sched_id=self._run_sched(record) or "",
sched_id=sched_id,
intended=_admission_intended(self.run_store, run_id),
run_id=run_id,
now=now,
started_at=now,
)
# -- polling --------------------------------------------------------
@@ -311,15 +358,16 @@ class Scheduler:
ids = sorted(item.id for item in schedules)
if not ids:
return {}
start = self._poll_cursor % len(ids)
cursor = self.schedule_store.get_poll_cursor()
start = cursor % len(ids)
order = ids[start:] + ids[:start]
self._poll_cursor += 1
self.schedule_store.save_poll_cursor(cursor + 1)
results: dict[str, str] = {}
by_id = {item.id: item for item in schedules}
for sid in order:
sched = by_id[sid]
if getattr(sched, "blocked_reason", None):
raise ValueError(getattr(sched, "blocked_reason"))
raise BlockedSchedule(getattr(sched, "blocked_reason"))
results[sid] = self._poll_one(sched, now)
return results
@@ -328,10 +376,10 @@ class Scheduler:
Recovery NEVER executes (T10): it only completes missing views flagged
pending for this sweep. Pending runs of blocked schedules stay
pending. TODO(T10): introduce the pending-dispatch marker (F11) once
dispatch marks distinguish crash-after-dispatch from never-dispatched;
until then only runs explicitly flagged ``needs_dispatch`` dispatch
here so hanging admitted runs are never re-executed.
pending. F11 is descoped to T10 here: the pending-dispatch marker does
not exist yet, so only runs explicitly flagged ``needs_dispatch``
dispatch in this sweep and hanging admitted runs are never
re-executed.
"""
for run in sorted(self.run_store.list_runs(), key=lambda r: r.id):
if not getattr(run, "needs_dispatch", False):
@@ -371,7 +419,23 @@ class Scheduler:
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
self.schedule_store.save_consumed(sched.id, max(consumed, now))
return "paused"
src = self.sources[sched.id]
# Fail closed on corrupt active views before calendar work (F5).
for run in self.run_store.list_runs():
if self._status_value(run) not in ("admitted", "interrupted"):
continue
try:
self.run_store.get_admission(run.id)
except KeyError:
reason = f"corrupt run view without admission: {run.id}"
sched.blocked_reason = reason
self.schedule_store.save_schedule(sched)
raise BlockedSchedule(reason) from None
try:
src = self.sources[sched.id]
except KeyError as exc:
raise InvalidScheduleDefinitionError(
f"no occurrence source for schedule {sched.id!r}"
) from exc
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
if consumed > now:
return "clock-rollback-held"
@@ -411,8 +475,13 @@ class Scheduler:
break
if jumped:
if sched.misfire == "latest":
from wf_scheduling.calendar import ScheduleExhaustedError
latest = src.prev_before(now)
assert latest is not None
if latest is None:
raise ScheduleExhaustedError(
f"latest-missed lookup exhausted for {sched.id!r}"
)
old = self.schedule_store.get_candidate(sched.id)
if old is not None and old.intended_at != latest:
self._record(
@@ -421,6 +490,7 @@ class Scheduler:
intended=old.intended_at,
reason=f"coalesced-into:{latest.isoformat()}",
revision=sched.revision,
now=now,
)
self.schedule_store.save_candidate(
PendingCandidate(
@@ -435,14 +505,25 @@ class Scheduler:
sched_id=sched.id,
reason="coalesced-missed-span",
revision=sched.revision,
interval=(consumed, now),
count=-1,
now=now,
)
self.schedule_store.save_consumed(sched.id, now)
return self._admit_held_candidate(sched, now) or "candidate-held"
held = self._admit_held_candidate(sched, now)
if held is None:
# Terminal skip inside admission (e.g. overlap) clears the
# candidate; do not misreport it as held.
return "skipped-overlap"
return held
self._record(
kind="interval-summary",
sched_id=sched.id,
reason="skipped-missed-span",
revision=sched.revision,
interval=(consumed, now),
count=-1,
now=now,
)
self.schedule_store.save_consumed(sched.id, now)
return "span-skipped"
@@ -484,7 +565,13 @@ class Scheduler:
schedule_id=sched.id,
)
self.schedule_store.save_consumed(sched.id, instant)
last_result = self._admit_held_candidate(sched, now) or "candidate-held"
held = self._admit_held_candidate(sched, now)
if held is None:
last_result = "skipped-overlap"
elif held == "held-undecided":
last_result = "candidate-held"
else:
last_result = held
else:
if _is_oneshot(src) and not sched.exhausted:
self._record(
@@ -511,7 +598,13 @@ class Scheduler:
if last_result in ("idle",):
cand = self.schedule_store.get_candidate(sched.id)
if cand is not None:
last_result = self._admit_held_candidate(sched, now) or "candidate-held"
held = self._admit_held_candidate(sched, now)
if held is None:
last_result = "skipped-overlap"
elif held == "held-undecided":
last_result = "candidate-held"
else:
last_result = held
return last_result
def _admit_held_candidate(self, sched: Any, now: datetime) -> str | None:
@@ -524,11 +617,11 @@ class Scheduler:
intended=cand.intended_at,
reason="schedule-edit",
revision=sched.revision,
now=now,
)
self.schedule_store.save_candidate(None, schedule_id=sched.id)
return None
result = self._admit(sched, cand.intended_at, now)
return result if result != "held-undecided" else None
return self._admit(sched, cand.intended_at, now)
# -- administration ---------------------------------------------------
def resume_schedule(self, sid: str, now: datetime) -> None: