sched: require proven ownership at poll/admin/recovery entries (F5)
This commit is contained in:
@@ -39,6 +39,16 @@ class SchedulerOwnership:
|
||||
def lock_path(self) -> Path:
|
||||
return self.root / "scheduler.lock"
|
||||
|
||||
@property
|
||||
def held(self) -> bool:
|
||||
"""Whether this process holds the lock through this object.
|
||||
|
||||
Only a successful :meth:`acquire` sets this: a second process can
|
||||
never observe ``held`` while another owner holds the OS lock, so
|
||||
entry-point guards can treat it as proof of exclusive ownership.
|
||||
"""
|
||||
return self._locked
|
||||
|
||||
def acquire(self) -> SchedulerOwnership:
|
||||
"""Acquire the held lock non-blockingly or raise SecondOwnerError."""
|
||||
self.lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -27,6 +27,7 @@ from wf_scheduling.calendar import (
|
||||
from wf_scheduling.dispatch import RunDispatcher, StillRunning
|
||||
from wf_scheduling.models import OccurrenceRecord, PendingCandidate
|
||||
from wf_scheduling.occurrences import occurrence_id
|
||||
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
|
||||
from wf_scheduling.prepare import InvocationPreparer, PreparationRejected
|
||||
|
||||
UTC = timezone.utc
|
||||
@@ -84,6 +85,7 @@ class Scheduler:
|
||||
capacity: int,
|
||||
preparer: InvocationPreparer,
|
||||
dispatcher: RunDispatcher,
|
||||
ownership: SchedulerOwnership,
|
||||
) -> None:
|
||||
self.schedule_store = schedule_store
|
||||
self.run_store = run_store
|
||||
@@ -91,8 +93,21 @@ class Scheduler:
|
||||
self.capacity = capacity
|
||||
self.preparer = preparer
|
||||
self.dispatcher = dispatcher
|
||||
self.ownership = ownership
|
||||
self._poll_cursor = 0
|
||||
|
||||
def _require_ownership(self) -> None:
|
||||
"""Reject schedule mutation/dispatch without proven live ownership.
|
||||
|
||||
Runs before any store write or dispatcher side effect: without a
|
||||
held lock this process cannot prove exclusive ownership, so polling
|
||||
or administering schedules would risk double admission.
|
||||
"""
|
||||
if self.ownership is None or not self.ownership.held:
|
||||
raise SecondOwnerError(
|
||||
"scheduler ownership is required before polling or mutating schedules"
|
||||
)
|
||||
|
||||
# -- helpers ------------------------------------------------------
|
||||
@staticmethod
|
||||
def _status_value(run: Any) -> Any:
|
||||
@@ -342,6 +357,7 @@ class Scheduler:
|
||||
|
||||
# -- polling --------------------------------------------------------
|
||||
def poll(self, now: datetime) -> dict[str, str]:
|
||||
self._require_ownership()
|
||||
self._dispatch_pending(now)
|
||||
schedules = self.schedule_store.list_schedules(include_deleted=True)
|
||||
ids = sorted(item.id for item in schedules)
|
||||
@@ -616,6 +632,7 @@ class Scheduler:
|
||||
# -- administration ---------------------------------------------------
|
||||
def resume_schedule(self, sid: str, now: datetime) -> None:
|
||||
"""Unpause: resume selects the next future occurrence."""
|
||||
self._require_ownership()
|
||||
sched = self.schedule_store.get_schedule(sid)
|
||||
sched.paused = False
|
||||
self.schedule_store.save_schedule(sched)
|
||||
@@ -625,6 +642,7 @@ class Scheduler:
|
||||
|
||||
def edit_schedule(self, sid: str, now: datetime) -> None:
|
||||
"""Definition edit: new revision, discard old candidates, no backfill."""
|
||||
self._require_ownership()
|
||||
sched = self.schedule_store.get_schedule(sid)
|
||||
sched.revision += 1
|
||||
sched.updated_at = now
|
||||
|
||||
@@ -13,6 +13,8 @@ from __future__ import annotations
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
|
||||
|
||||
UTC = timezone.utc
|
||||
|
||||
ABANDONED_REASON = (
|
||||
@@ -30,11 +32,17 @@ def recover(
|
||||
schedule_store: Any,
|
||||
run_store: Any,
|
||||
now: datetime,
|
||||
ownership: SchedulerOwnership,
|
||||
record_history: Any | None = None,
|
||||
) -> list[str]:
|
||||
"""Reconcile durable state after a restart without executing work."""
|
||||
from wf_artifacts.runs.models import StoredRunStatus
|
||||
|
||||
if ownership is None or not ownership.held:
|
||||
raise SecondOwnerError(
|
||||
"scheduler ownership is required before recovery: an unowned "
|
||||
"recovery could abandon or redispatch another owner's work"
|
||||
)
|
||||
diags: list[str] = []
|
||||
# Admission record is the recovery authority: admitted but never
|
||||
# materialized views are completed here and flagged pending for the
|
||||
|
||||
Reference in New Issue
Block a user