fix: release admission transaction before dispatch
This commit is contained in:
+33
-15
@@ -16,6 +16,7 @@ contains no fixture input, fixture environments, or canned results.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Any, cast
|
from typing import Any, cast
|
||||||
|
|
||||||
@@ -49,6 +50,14 @@ class BlockedSchedule(ValueError):
|
|||||||
"""A schedule is blocked by a corrupt record and fails closed."""
|
"""A schedule is blocked by a corrupt record and fails closed."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _AdmissionOutcome:
|
||||||
|
"""Result of one atomic admission and any dispatch required afterward."""
|
||||||
|
|
||||||
|
result: str | None
|
||||||
|
dispatch_run_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
def source_for_trigger(trigger: Any) -> OccurrenceSource:
|
def source_for_trigger(trigger: Any) -> OccurrenceSource:
|
||||||
"""Build a calendar source from a schedule trigger model."""
|
"""Build a calendar source from a schedule trigger model."""
|
||||||
kind = trigger.kind if hasattr(trigger, "kind") else trigger.get("kind")
|
kind = trigger.kind if hasattr(trigger, "kind") else trigger.get("kind")
|
||||||
@@ -316,27 +325,37 @@ class Scheduler:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
def _admit(self, sched: Any, intended: datetime, now: datetime) -> str | None:
|
def _admit(self, sched: Any, intended: datetime, now: datetime) -> str | None:
|
||||||
"""Admit one occurrence under the schedule store's local transaction."""
|
"""Admit atomically, then dispatch a new run after the transaction.
|
||||||
|
|
||||||
|
The durable admission and all schedule-side markers must commit as one
|
||||||
|
compound transition. Registration can wait on the event loop, though,
|
||||||
|
so it deliberately happens after that transaction releases; otherwise
|
||||||
|
an event-loop admin mutation can wait on a worker that is waiting for
|
||||||
|
the event loop to register its task.
|
||||||
|
"""
|
||||||
with schedule_store_transaction(self.schedule_store):
|
with schedule_store_transaction(self.schedule_store):
|
||||||
return self._admit_locked(sched, intended, now)
|
outcome = self._admit_locked(sched, intended, now)
|
||||||
|
if outcome.dispatch_run_id is not None:
|
||||||
|
self._execute_guarded(outcome.dispatch_run_id, now)
|
||||||
|
return outcome.result
|
||||||
|
|
||||||
def _admit_locked(
|
def _admit_locked(
|
||||||
self, sched: Any, intended: datetime, now: datetime
|
self, sched: Any, intended: datetime, now: datetime
|
||||||
) -> str | None:
|
) -> _AdmissionOutcome:
|
||||||
# The poller's listing is only a fairness snapshot. Re-read under the
|
# The poller's listing is only a fairness snapshot. Re-read under the
|
||||||
# admission transaction so an edit committed before this point wins;
|
# admission transaction so an edit committed before this point wins;
|
||||||
# an edit after this point waits and affects a later occurrence.
|
# an edit after this point waits and affects a later occurrence.
|
||||||
try:
|
try:
|
||||||
current = self.schedule_store.get_schedule(sched.id)
|
current = self.schedule_store.get_schedule(sched.id)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return None
|
return _AdmissionOutcome(None)
|
||||||
if current.revision != sched.revision:
|
if current.revision != sched.revision:
|
||||||
return "schedule-changed"
|
return _AdmissionOutcome("schedule-changed")
|
||||||
sched = current
|
sched = current
|
||||||
if getattr(sched, "blocked_reason", None):
|
if getattr(sched, "blocked_reason", None):
|
||||||
raise BlockedSchedule(getattr(sched, "blocked_reason"))
|
raise BlockedSchedule(getattr(sched, "blocked_reason"))
|
||||||
if not sched.enabled or sched.deleted or sched.paused:
|
if not sched.enabled or sched.deleted or sched.paused:
|
||||||
return None
|
return _AdmissionOutcome(None)
|
||||||
# Fail closed before any new admission when a corrupt active view
|
# Fail closed before any new admission when a corrupt active view
|
||||||
# exists for this schedule.
|
# exists for this schedule.
|
||||||
for run in self._active(sched.id):
|
for run in self._active(sched.id):
|
||||||
@@ -370,7 +389,7 @@ class Scheduler:
|
|||||||
now=now,
|
now=now,
|
||||||
admitted_at=now,
|
admitted_at=now,
|
||||||
)
|
)
|
||||||
return existing
|
return _AdmissionOutcome(existing)
|
||||||
prepared = self.preparer.prepare(sched=sched, intended=intended, now=now)
|
prepared = self.preparer.prepare(sched=sched, intended=intended, now=now)
|
||||||
if isinstance(prepared, PreparationRejected):
|
if isinstance(prepared, PreparationRejected):
|
||||||
self._record(
|
self._record(
|
||||||
@@ -385,7 +404,7 @@ class Scheduler:
|
|||||||
if cand is not None and cand.intended_at == intended:
|
if cand is not None and cand.intended_at == intended:
|
||||||
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
||||||
_save_consumed_max(self.schedule_store, sched.id, intended)
|
_save_consumed_max(self.schedule_store, sched.id, intended)
|
||||||
return None
|
return _AdmissionOutcome(None)
|
||||||
active = self._active(sched.id)
|
active = self._active(sched.id)
|
||||||
if sched.overlap == "skip" and active:
|
if sched.overlap == "skip" and active:
|
||||||
self._record(
|
self._record(
|
||||||
@@ -399,7 +418,7 @@ class Scheduler:
|
|||||||
if cand is not None and cand.intended_at == intended:
|
if cand is not None and cand.intended_at == intended:
|
||||||
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
||||||
_save_consumed_max(self.schedule_store, sched.id, intended)
|
_save_consumed_max(self.schedule_store, sched.id, intended)
|
||||||
return None
|
return _AdmissionOutcome(None)
|
||||||
if sched.overlap == "parallel" and len(active) >= sched.max_active_runs:
|
if sched.overlap == "parallel" and len(active) >= sched.max_active_runs:
|
||||||
self._record(
|
self._record(
|
||||||
kind="skipped-overlap",
|
kind="skipped-overlap",
|
||||||
@@ -412,7 +431,7 @@ class Scheduler:
|
|||||||
if cand is not None and cand.intended_at == intended:
|
if cand is not None and cand.intended_at == intended:
|
||||||
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
||||||
_save_consumed_max(self.schedule_store, sched.id, intended)
|
_save_consumed_max(self.schedule_store, sched.id, intended)
|
||||||
return None
|
return _AdmissionOutcome(None)
|
||||||
if self._task_load() >= self.capacity:
|
if self._task_load() >= self.capacity:
|
||||||
if sched.misfire == "latest":
|
if sched.misfire == "latest":
|
||||||
self.schedule_store.save_candidate(
|
self.schedule_store.save_candidate(
|
||||||
@@ -424,7 +443,7 @@ class Scheduler:
|
|||||||
schedule_id=sched.id,
|
schedule_id=sched.id,
|
||||||
)
|
)
|
||||||
_save_consumed_max(self.schedule_store, sched.id, intended)
|
_save_consumed_max(self.schedule_store, sched.id, intended)
|
||||||
return "held"
|
return _AdmissionOutcome("held")
|
||||||
if (now - intended).total_seconds() > sched.lateness_allowance_s:
|
if (now - intended).total_seconds() > sched.lateness_allowance_s:
|
||||||
self._record(
|
self._record(
|
||||||
kind="skipped-misfire",
|
kind="skipped-misfire",
|
||||||
@@ -434,8 +453,8 @@ class Scheduler:
|
|||||||
revision=sched.revision,
|
revision=sched.revision,
|
||||||
)
|
)
|
||||||
_save_consumed_max(self.schedule_store, sched.id, intended)
|
_save_consumed_max(self.schedule_store, sched.id, intended)
|
||||||
return None
|
return _AdmissionOutcome(None)
|
||||||
return "held-undecided"
|
return _AdmissionOutcome("held-undecided")
|
||||||
run_id = self.run_store.allocate_run_id()
|
run_id = self.run_store.allocate_run_id()
|
||||||
from wf_api.run_lifecycle import persist_admission
|
from wf_api.run_lifecycle import persist_admission
|
||||||
|
|
||||||
@@ -471,8 +490,7 @@ class Scheduler:
|
|||||||
|
|
||||||
self.run_store.mark_pending_dispatch(run_id)
|
self.run_store.mark_pending_dispatch(run_id)
|
||||||
materialize_admitted_view(store=self.run_store, admission=admission)
|
materialize_admitted_view(store=self.run_store, admission=admission)
|
||||||
self._execute_guarded(run_id, now)
|
return _AdmissionOutcome(run_id, dispatch_run_id=run_id)
|
||||||
return run_id
|
|
||||||
|
|
||||||
def _execute_guarded(self, run_id: str, now: datetime) -> None:
|
def _execute_guarded(self, run_id: str, now: datetime) -> None:
|
||||||
"""Dispatch one provably undispatched run behind a durable transition.
|
"""Dispatch one provably undispatched run behind a durable transition.
|
||||||
|
|||||||
Reference in New Issue
Block a user