fix: close scheduler persistence and capacity gaps
This commit is contained in:
+51
-11
@@ -189,11 +189,44 @@ class Scheduler:
|
||||
return None
|
||||
return admission.schedule_id
|
||||
|
||||
def _fail_unattributed_active_runs(self, now: datetime) -> None:
|
||||
"""Fail corrupt active views without assigning blame to a sibling.
|
||||
|
||||
A run view without an admission has no schedule identity and cannot
|
||||
safely participate in overlap or capacity accounting. It is failed
|
||||
closed in place, while healthy schedules continue their own poll.
|
||||
"""
|
||||
from wf_scheduling.recovery import (
|
||||
CORRUPT_VIEW_REASON,
|
||||
_fail_run,
|
||||
clear_executing,
|
||||
clear_pending,
|
||||
)
|
||||
|
||||
for run in self.run_store.list_runs():
|
||||
if self._status_value(run) not in ("admitted", "interrupted"):
|
||||
continue
|
||||
try:
|
||||
self.run_store.get_admission(run.id)
|
||||
except KeyError:
|
||||
_fail_run(self.run_store, run, CORRUPT_VIEW_REASON, now, self.history)
|
||||
clear_pending(self.run_store, run.id)
|
||||
clear_executing(self.run_store, run.id)
|
||||
|
||||
def _task_load(self) -> int:
|
||||
from wf_scheduling.recovery import _is_pending, is_executing
|
||||
|
||||
count = 0
|
||||
for run in self.run_store.list_runs():
|
||||
# Manual API runs share the run store but do not consume the
|
||||
# scheduler's bounded dispatch capacity. An admission with no
|
||||
# schedule owner is the durable discriminator for that path.
|
||||
try:
|
||||
admission = self.run_store.get_admission(run.id)
|
||||
except KeyError:
|
||||
continue
|
||||
if admission.schedule_id is None:
|
||||
continue
|
||||
if self._status_value(run) != "admitted":
|
||||
# A mid-resume scheduled run is interrupted but holds the
|
||||
# durable executing mark: it occupies a live execution
|
||||
@@ -255,6 +288,17 @@ class Scheduler:
|
||||
"""
|
||||
for admission in self.run_store.list_admissions():
|
||||
if admission.schedule_id == sched_id and admission.scheduled_at == intended:
|
||||
# Admission is the occurrence authority. If a later write
|
||||
# failed before the view/pending marker, rebuild only this
|
||||
# run so the next pending sweep can dispatch it; never admit
|
||||
# the same occurrence again.
|
||||
try:
|
||||
self.run_store.get_run(admission.id)
|
||||
except KeyError:
|
||||
from wf_api.run_lifecycle import materialize_admitted_view
|
||||
|
||||
materialize_admitted_view(store=self.run_store, admission=admission)
|
||||
self.run_store.mark_pending_dispatch(admission.id)
|
||||
return admission.id
|
||||
return None
|
||||
|
||||
@@ -555,6 +599,7 @@ class Scheduler:
|
||||
# -- polling --------------------------------------------------------
|
||||
def poll(self, now: datetime) -> dict[str, str]:
|
||||
self._require_ownership()
|
||||
self._fail_unattributed_active_runs(now)
|
||||
self._dispatch_pending(now)
|
||||
schedules = self.schedule_store.list_schedules(include_deleted=True)
|
||||
ids = sorted(item.id for item in schedules)
|
||||
@@ -666,6 +711,11 @@ class Scheduler:
|
||||
self.schedule_store.save_consumed(sched.id, max(consumed, now))
|
||||
return "disabled"
|
||||
if sched.exhausted:
|
||||
# Exhaustion is durable terminal state. A torn one-shot
|
||||
# transition may have written the history/flag before its stale
|
||||
# candidate was cleared; retrying must finish that cleanup.
|
||||
if self.schedule_store.get_candidate(sched.id) is not None:
|
||||
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
||||
return "exhausted"
|
||||
if sched.paused:
|
||||
if self.schedule_store.get_candidate(sched.id) is not None:
|
||||
@@ -673,17 +723,6 @@ class Scheduler:
|
||||
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
|
||||
self.schedule_store.save_consumed(sched.id, max(consumed, now))
|
||||
return "paused"
|
||||
# Fail closed on corrupt active views before calendar work (F5).
|
||||
for run in self.run_store.list_runs():
|
||||
if self._status_value(run) not in ("admitted", "interrupted"):
|
||||
continue
|
||||
try:
|
||||
self.run_store.get_admission(run.id)
|
||||
except KeyError:
|
||||
reason = f"corrupt run view without admission: {run.id}"
|
||||
sched.blocked_reason = reason
|
||||
self.schedule_store.save_schedule(sched)
|
||||
raise BlockedSchedule(reason) from None
|
||||
try:
|
||||
src = self.sources[sched.id]
|
||||
except KeyError as exc:
|
||||
@@ -838,6 +877,7 @@ class Scheduler:
|
||||
sched.exhausted = True
|
||||
self.schedule_store.save_schedule(sched)
|
||||
self.schedule_store.save_consumed(sched.id, instant)
|
||||
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
||||
last_result = "exhausted"
|
||||
continue
|
||||
self._record(
|
||||
|
||||
Reference in New Issue
Block a user