sched: require proven ownership at poll/admin/recovery entries (F5)

This commit is contained in:
lda
2026-09-08 11:24:40 +07:00 Verified
parent 15bd472bdc
commit 993ed07fd3
7 changed files with 293 additions and 22 deletions
+10
View File
@@ -39,6 +39,16 @@ class SchedulerOwnership:
def lock_path(self) -> Path:
return self.root / "scheduler.lock"
@property
def held(self) -> bool:
"""Whether this process holds the lock through this object.
Only a successful :meth:`acquire` sets this: a second process can
never observe ``held`` while another owner holds the OS lock, so
entry-point guards can treat it as proof of exclusive ownership.
"""
return self._locked
def acquire(self) -> SchedulerOwnership:
"""Acquire the held lock non-blockingly or raise SecondOwnerError."""
self.lock_path.parent.mkdir(parents=True, exist_ok=True)
+18
View File
@@ -27,6 +27,7 @@ from wf_scheduling.calendar import (
from wf_scheduling.dispatch import RunDispatcher, StillRunning
from wf_scheduling.models import OccurrenceRecord, PendingCandidate
from wf_scheduling.occurrences import occurrence_id
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
from wf_scheduling.prepare import InvocationPreparer, PreparationRejected
UTC = timezone.utc
@@ -84,6 +85,7 @@ class Scheduler:
capacity: int,
preparer: InvocationPreparer,
dispatcher: RunDispatcher,
ownership: SchedulerOwnership,
) -> None:
self.schedule_store = schedule_store
self.run_store = run_store
@@ -91,8 +93,21 @@ class Scheduler:
self.capacity = capacity
self.preparer = preparer
self.dispatcher = dispatcher
self.ownership = ownership
self._poll_cursor = 0
def _require_ownership(self) -> None:
"""Reject schedule mutation/dispatch without proven live ownership.
Runs before any store write or dispatcher side effect: without a
held lock this process cannot prove exclusive ownership, so polling
or administering schedules would risk double admission.
"""
if self.ownership is None or not self.ownership.held:
raise SecondOwnerError(
"scheduler ownership is required before polling or mutating schedules"
)
# -- helpers ------------------------------------------------------
@staticmethod
def _status_value(run: Any) -> Any:
@@ -342,6 +357,7 @@ class Scheduler:
# -- polling --------------------------------------------------------
def poll(self, now: datetime) -> dict[str, str]:
self._require_ownership()
self._dispatch_pending(now)
schedules = self.schedule_store.list_schedules(include_deleted=True)
ids = sorted(item.id for item in schedules)
@@ -616,6 +632,7 @@ class Scheduler:
# -- administration ---------------------------------------------------
def resume_schedule(self, sid: str, now: datetime) -> None:
"""Unpause: resume selects the next future occurrence."""
self._require_ownership()
sched = self.schedule_store.get_schedule(sid)
sched.paused = False
self.schedule_store.save_schedule(sched)
@@ -625,6 +642,7 @@ class Scheduler:
def edit_schedule(self, sid: str, now: datetime) -> None:
"""Definition edit: new revision, discard old candidates, no backfill."""
self._require_ownership()
sched = self.schedule_store.get_schedule(sid)
sched.revision += 1
sched.updated_at = now
+8
View File
@@ -13,6 +13,8 @@ from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
UTC = timezone.utc
ABANDONED_REASON = (
@@ -30,11 +32,17 @@ def recover(
schedule_store: Any,
run_store: Any,
now: datetime,
ownership: SchedulerOwnership,
record_history: Any | None = None,
) -> list[str]:
"""Reconcile durable state after a restart without executing work."""
from wf_artifacts.runs.models import StoredRunStatus
if ownership is None or not ownership.held:
raise SecondOwnerError(
"scheduler ownership is required before recovery: an unowned "
"recovery could abandon or redispatch another owner's work"
)
diags: list[str] = []
# Admission record is the recovery authority: admitted but never
# materialized views are completed here and flagged pending for the