fix: close scheduler persistence and capacity gaps

This commit is contained in:
lda
2026-09-09 19:13:17 +07:00 Verified
parent 39993c9d5e
commit 787a3c433f
7 changed files with 296 additions and 13 deletions
+51 -11
View File
@@ -189,11 +189,44 @@ class Scheduler:
return None
return admission.schedule_id
def _fail_unattributed_active_runs(self, now: datetime) -> None:
"""Fail corrupt active views without assigning blame to a sibling.
A run view without an admission has no schedule identity and cannot
safely participate in overlap or capacity accounting. It is failed
closed in place, while healthy schedules continue their own poll.
"""
from wf_scheduling.recovery import (
CORRUPT_VIEW_REASON,
_fail_run,
clear_executing,
clear_pending,
)
for run in self.run_store.list_runs():
if self._status_value(run) not in ("admitted", "interrupted"):
continue
try:
self.run_store.get_admission(run.id)
except KeyError:
_fail_run(self.run_store, run, CORRUPT_VIEW_REASON, now, self.history)
clear_pending(self.run_store, run.id)
clear_executing(self.run_store, run.id)
def _task_load(self) -> int:
from wf_scheduling.recovery import _is_pending, is_executing
count = 0
for run in self.run_store.list_runs():
# Manual API runs share the run store but do not consume the
# scheduler's bounded dispatch capacity. An admission with no
# schedule owner is the durable discriminator for that path.
try:
admission = self.run_store.get_admission(run.id)
except KeyError:
continue
if admission.schedule_id is None:
continue
if self._status_value(run) != "admitted":
# A mid-resume scheduled run is interrupted but holds the
# durable executing mark: it occupies a live execution
@@ -255,6 +288,17 @@ class Scheduler:
"""
for admission in self.run_store.list_admissions():
if admission.schedule_id == sched_id and admission.scheduled_at == intended:
# Admission is the occurrence authority. If a later write
# failed before the view/pending marker, rebuild only this
# run so the next pending sweep can dispatch it; never admit
# the same occurrence again.
try:
self.run_store.get_run(admission.id)
except KeyError:
from wf_api.run_lifecycle import materialize_admitted_view
materialize_admitted_view(store=self.run_store, admission=admission)
self.run_store.mark_pending_dispatch(admission.id)
return admission.id
return None
@@ -555,6 +599,7 @@ class Scheduler:
# -- polling --------------------------------------------------------
def poll(self, now: datetime) -> dict[str, str]:
self._require_ownership()
self._fail_unattributed_active_runs(now)
self._dispatch_pending(now)
schedules = self.schedule_store.list_schedules(include_deleted=True)
ids = sorted(item.id for item in schedules)
@@ -666,6 +711,11 @@ class Scheduler:
self.schedule_store.save_consumed(sched.id, max(consumed, now))
return "disabled"
if sched.exhausted:
# Exhaustion is durable terminal state. A torn one-shot
# transition may have written the history/flag before its stale
# candidate was cleared; retrying must finish that cleanup.
if self.schedule_store.get_candidate(sched.id) is not None:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
return "exhausted"
if sched.paused:
if self.schedule_store.get_candidate(sched.id) is not None:
@@ -673,17 +723,6 @@ class Scheduler:
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
self.schedule_store.save_consumed(sched.id, max(consumed, now))
return "paused"
# Fail closed on corrupt active views before calendar work (F5).
for run in self.run_store.list_runs():
if self._status_value(run) not in ("admitted", "interrupted"):
continue
try:
self.run_store.get_admission(run.id)
except KeyError:
reason = f"corrupt run view without admission: {run.id}"
sched.blocked_reason = reason
self.schedule_store.save_schedule(sched)
raise BlockedSchedule(reason) from None
try:
src = self.sources[sched.id]
except KeyError as exc:
@@ -838,6 +877,7 @@ class Scheduler:
sched.exhausted = True
self.schedule_store.save_schedule(sched)
self.schedule_store.save_consumed(sched.id, instant)
self.schedule_store.save_candidate(None, schedule_id=sched.id)
last_result = "exhausted"
continue
self._record(