sched: bind ownership to store composition; stable recovery failure; schema-checked prepare; guarded settle (R4 wave 2)

This commit is contained in:
lda
2026-09-08 18:36:09 +07:00 Verified
parent b9d8eb9d16
commit 3f1d5d158b
16 changed files with 1299 additions and 60 deletions
+53 -2
View File
@@ -9,11 +9,18 @@ startup is rejected instead of running unprotected.
Locking design (Windows-tested, per the store transaction boundary): one Locking design (Windows-tested, per the store transaction boundary): one
new module holding ``msvcrt.locking`` (Windows) / ``fcntl.flock`` (POSIX) new module holding ``msvcrt.locking`` (Windows) / ``fcntl.flock`` (POSIX)
on ``<store root>/scheduler.lock``. No ad-hoc per-run lock files. on ``<composition root>/scheduler.lock``. No ad-hoc per-run lock files.
The lock is acquired on the composition root that contains the protected
schedule and run stores. Entry-point guards do not trust the caller to
have passed the right lock: they validate the held guard against the
actual store roots (canonicalized for platform aliases) before any write
or side effect.
""" """
from __future__ import annotations from __future__ import annotations
import os
from pathlib import Path from pathlib import Path
from typing import BinaryIO from typing import BinaryIO
@@ -26,8 +33,31 @@ class StartupRejected(Exception):
"""Scheduler startup rejected where locking is unsupported.""" """Scheduler startup rejected where locking is unsupported."""
def canonical_store_path(path: Path | str) -> str:
"""Canonicalize a store path for ownership-coverage comparison.
``realpath`` resolves symlinks, junctions, ``..`` segments, and
relative paths; ``normcase`` folds case and separators on Windows
(identity on POSIX, which stays case-sensitive). Both sides of every
comparison pass through this function, so platform aliases of one
directory compare equal. Exotic aliases (subst drives, UNC-vs-mapped
paths, 8.3 short names where the OS preserves them) are out of scope:
operators should acquire the lock on the same canonical composition
root the stores live under.
"""
return os.path.normcase(os.path.realpath(os.fspath(path)))
def _is_within(candidate: str, base: str) -> bool:
"""Whether canonical ``candidate`` lies beneath canonical ``base``."""
if candidate == base:
return True
prefix = base if base.endswith(os.sep) else base + os.sep
return candidate.startswith(prefix)
class SchedulerOwnership: class SchedulerOwnership:
"""Held exclusive ownership of a schedule store root.""" """Held exclusive ownership of a store composition root."""
def __init__(self, root: Path, *, owner: str) -> None: def __init__(self, root: Path, *, owner: str) -> None:
self.root = root self.root = root
@@ -49,6 +79,27 @@ class SchedulerOwnership:
""" """
return self._locked return self._locked
def canonical_root(self) -> str:
"""Return the canonical composition root this lock was acquired on."""
return canonical_store_path(self.root)
def covers(self, *store_roots: Path | str | None) -> bool:
"""Whether this held lock protects the given store roots.
Coverage means each root canonicalizes to the lock's composition
root or beneath it. An unrelated (even held) lock, a released lock,
and rootless stores are never covered: guards reject those before
any write or side effect instead of trusting the caller's promise.
"""
if not self._locked or not store_roots:
return False
base = self.canonical_root()
return all(
store_root is not None
and _is_within(canonical_store_path(store_root), base)
for store_root in store_roots
)
def acquire(self) -> SchedulerOwnership: def acquire(self) -> SchedulerOwnership:
"""Acquire the held lock non-blockingly or raise SecondOwnerError.""" """Acquire the held lock non-blockingly or raise SecondOwnerError."""
self.lock_path.parent.mkdir(parents=True, exist_ok=True) self.lock_path.parent.mkdir(parents=True, exist_ok=True)
+23 -4
View File
@@ -139,13 +139,20 @@ class Scheduler:
def _require_ownership(self) -> None: def _require_ownership(self) -> None:
"""Reject schedule mutation/dispatch without proven live ownership. """Reject schedule mutation/dispatch without proven live ownership.
Runs before any store write or dispatcher side effect: without a Runs before any store write or dispatcher side effect: the held
held lock this process cannot prove exclusive ownership, so polling lock must cover the actual schedule and run store composition, not
merely be held on some unrelated directory. Without covering
ownership this process cannot prove exclusive ownership, so polling
or administering schedules would risk double admission. or administering schedules would risk double admission.
""" """
if self.ownership is None or not self.ownership.held: ownership = self.ownership
if ownership is None or not ownership.covers(
getattr(self.schedule_store, "root", None),
getattr(self.run_store, "root", None),
):
raise SecondOwnerError( raise SecondOwnerError(
"scheduler ownership is required before polling or mutating schedules" "scheduler ownership must cover the schedule and run stores "
"before polling or mutating schedules"
) )
# -- helpers ------------------------------------------------------ # -- helpers ------------------------------------------------------
@@ -431,6 +438,12 @@ class Scheduler:
:class:`wf_core.RunState`. Persists through the shared lifecycle :class:`wf_core.RunState`. Persists through the shared lifecycle
boundary, clears the executing mark, and records terminal history. boundary, clears the executing mark, and records terminal history.
Never re-invokes the dispatcher. Never re-invokes the dispatcher.
An admitted status alone is not enough: pending, never-dispatched
work is also admitted. Settlement requires the durable executing
transition and refuses contradictory pending/executing state (a
crash between the transition writes owns that run now, not this
caller); it also refuses runs that already stopped.
""" """
self._require_ownership() self._require_ownership()
from wf_api.run_lifecycle import persist_stopped_run from wf_api.run_lifecycle import persist_stopped_run
@@ -448,6 +461,12 @@ class Scheduler:
raise BlockedSchedule(f"settle missing run view: {run_id!r}") from exc raise BlockedSchedule(f"settle missing run view: {run_id!r}") from exc
if self._status_value(record) != "admitted": if self._status_value(record) != "admitted":
raise BlockedSchedule(f"settle non-admitted run: {run_id!r}") raise BlockedSchedule(f"settle non-admitted run: {run_id!r}")
if self.run_store.is_pending_dispatch(run_id):
raise BlockedSchedule(
f"settle contradictory pending/executing run: {run_id!r}"
)
if not self.run_store.is_executing(run_id):
raise BlockedSchedule(f"settle without executing transition: {run_id!r}")
stopped = persist_stopped_run( stopped = persist_stopped_run(
store=self.run_store, store=self.run_store,
environment=admission.environment, environment=admission.environment,
+24 -4
View File
@@ -17,7 +17,9 @@ from datetime import datetime
from typing import Any, Protocol from typing import Any, Protocol
from wf_artifacts.runs.models import PinnedRunEnvironment from wf_artifacts.runs.models import PinnedRunEnvironment
from wf_core.errors import WorkflowExecutionError
from wf_core.runtime.input_sources import resolve_schedule_input_bindings from wf_core.runtime.input_sources import resolve_schedule_input_bindings
from wf_core.runtime.ops.schemas import validate_payload_against_schema
from wf_scheduling.occurrences import occurrence_id from wf_scheduling.occurrences import occurrence_id
@@ -63,7 +65,14 @@ class SchedulePreparer:
Occurrence references resolve through the shared schedule-expression Occurrence references resolve through the shared schedule-expression
contract (never graph paths); target conflicts, over-budget trees, and contract (never graph paths); target conflicts, over-budget trees, and
invalid resolved input fail closed as preflight rejections before any invalid resolved input fail closed as preflight rejections before any
run identity is allocated. run identity is allocated. The resolved object is additionally
validated against the pinned root artifact's input schema with the same
validator the runtime uses for fresh runs, so a schedule cannot admit
work the deployment contract would refuse.
The environment is built once per preparation: the deployment revision,
required inputs, and input schema all describe that single captured
snapshot, never independently resolved versions.
""" """
def __init__( def __init__(
@@ -77,9 +86,12 @@ class SchedulePreparer:
def prepare( def prepare(
self, *, sched: Any, intended: datetime, now: datetime self, *, sched: Any, intended: datetime, now: datetime
) -> PreparedInvocation | PreparationRejected: ) -> PreparedInvocation | PreparationRejected:
environment = self._build_environment(sched)
try: try:
revision = self._deployments.deployment_revision(sched.deployment_id) revision = self._deployments.deployment_revision(environment.deployment.id)
required = list(self._deployments.required_inputs(sched.deployment_id)) required = list(
self._deployments.required_inputs(environment.deployment.id)
)
except KeyError: except KeyError:
return PreparationRejected(reason="deployment-deleted") return PreparationRejected(reason="deployment-deleted")
occurrence = { occurrence = {
@@ -98,8 +110,16 @@ class SchedulePreparer:
missing = [key for key in required if key not in resolved] missing = [key for key in required if key not in resolved]
if missing: if missing:
return PreparationRejected(reason=f"missing-input:{missing}") return PreparationRejected(reason=f"missing-input:{missing}")
try:
validate_payload_against_schema(
environment.root_artifact.input_schema,
dict(resolved),
f"schedule {sched.id} input",
)
except WorkflowExecutionError as exc:
return PreparationRejected(reason=f"invalid-input:schema:{exc}")
return PreparedInvocation( return PreparedInvocation(
environment=self._build_environment(sched), environment=environment,
resolved_input=dict(resolved), resolved_input=dict(resolved),
max_steps=getattr(sched, "max_steps", None), max_steps=getattr(sched, "max_steps", None),
deployment_revision=revision, deployment_revision=revision,
+123 -10
View File
@@ -31,6 +31,13 @@ clear → executor → stopped persist → executing clear → history):
reconciliation across the torn ``save_checkpoint``/``save_run`` boundary reconciliation across the torn ``save_checkpoint``/``save_run`` boundary
is owned by the F2 reconcile step, which runs before marker handling. is owned by the F2 reconcile step, which runs before marker handling.
- stopped summary, no marks: existing terminal/attempt reconciliation. - stopped summary, no marks: existing terminal/attempt reconciliation.
- failed summary: a durable recovery decision is never re-processed. When
the checkpoint pointer still matches the decided state, history is
ensured without touching status, readiness, or diagnostics (repeat
recovery is silent). Only a genuinely newer checkpoint (pointer
mismatch) may reopen the run via the reconcile path; an older
checkpoint can never supersede the abandonment. Failed runs carry
not-applicable resume readiness.
""" """
from __future__ import annotations from __future__ import annotations
@@ -72,10 +79,14 @@ def recover(
"""Reconcile durable state after a restart without executing work.""" """Reconcile durable state after a restart without executing work."""
from wf_artifacts.runs.models import StoredRunStatus from wf_artifacts.runs.models import StoredRunStatus
if ownership is None or not ownership.held: if ownership is None or not ownership.covers(
getattr(schedule_store, "root", None),
getattr(run_store, "root", None),
):
raise SecondOwnerError( raise SecondOwnerError(
"scheduler ownership is required before recovery: an unowned " "scheduler ownership must cover the schedule and run stores "
"recovery could abandon or redispatch another owner's work" "before recovery: an unowned recovery could abandon or "
"redispatch another owner's work"
) )
if history is None: if history is None:
history = FileScheduleHistoryRecorder(schedule_store) history = FileScheduleHistoryRecorder(schedule_store)
@@ -96,6 +107,27 @@ def recover(
status = getattr(run.status, "value", run.status) status = getattr(run.status, "value", run.status)
attempt = run_store.get_resume_attempt(run.id) attempt = run_store.get_resume_attempt(run.id)
active = attempt is not None and attempt.state == "ACTIVE" active = attempt is not None and attempt.state == "ACTIVE"
if status in (
StoredRunStatus.INTERRUPTED.value,
StoredRunStatus.COMPLETED.value,
StoredRunStatus.FAILED.value,
"interrupted",
"completed",
"failed",
) and (is_executing(run_store, run.id) or _is_pending(run_store, run.id)):
# Completion-window leftovers: the stopped result was persisted
# but marker clearing was lost. The stopped status stands;
# nothing is re-executed.
clear_executing(run_store, run.id)
clear_pending(run_store, run.id)
diags.append(f"{run.id}:completion-window-cleared")
# Durable-decision stability: a FAILED summary for the current
# checkpoint state is never re-processed. Only a genuinely newer
# checkpoint (pointer mismatch) may reopen it via the reconcile
# path below. History is ensured without mutating the summary.
if status == StoredRunStatus.FAILED.value or status == "failed":
if _failed_decision_stable(run_store, run, history, now, diags):
continue
# Checkpoint-first reconcile across the torn # Checkpoint-first reconcile across the torn
# save_checkpoint/save_run boundary: a durable stopped checkpoint # save_checkpoint/save_run boundary: a durable stopped checkpoint
# wins over a stale summary (status, checkpoint pointer, and # wins over a stale summary (status, checkpoint pointer, and
@@ -105,6 +137,25 @@ def recover(
try: try:
changed, status = _reconcile_summary_from_checkpoint(run_store, run, now) changed, status = _reconcile_summary_from_checkpoint(run_store, run, now)
except (ValueError, OSError) as exc: except (ValueError, OSError) as exc:
if _has_recovery_decision(run):
# A decided run stays decided even when checkpoint state
# corrupts afterwards: ensure history from the summary
# pointer, note the corruption, never re-fail.
sched_id, intended, revision = _attribution(run_store, run.id)
if _reconcile_terminal(
history,
sched_id,
run.id,
"failed",
run.latest_checkpoint_id,
intended,
revision,
_decision_reason(run) or "reconciled-on-recovery",
now,
):
diags.append(f"{run.id}:terminal-reconciled")
diags.append(f"{run.id}:corrupt-checkpoint-noted")
continue
_fail_run(run_store, run, f"corrupt checkpoint: {exc}", now, history) _fail_run(run_store, run, f"corrupt checkpoint: {exc}", now, history)
diags.append(f"{run.id}:failed-closed") diags.append(f"{run.id}:failed-closed")
continue continue
@@ -119,9 +170,8 @@ def recover(
"completed", "completed",
"failed", "failed",
) and (is_executing(run_store, run.id) or _is_pending(run_store, run.id)): ) and (is_executing(run_store, run.id) or _is_pending(run_store, run.id)):
# Completion-window leftovers: the stopped result was persisted # A repaired summary may carry completion-window leftovers (the
# but marker clearing was lost. The stopped status stands; # stopped result was persisted but marker clearing was lost).
# nothing is re-executed.
clear_executing(run_store, run.id) clear_executing(run_store, run.id)
clear_pending(run_store, run.id) clear_pending(run_store, run.id)
diags.append(f"{run.id}:completion-window-cleared") diags.append(f"{run.id}:completion-window-cleared")
@@ -277,6 +327,67 @@ def _reconcile_summary_from_checkpoint(
return True, expected.value return True, expected.value
def _has_recovery_decision(run: Any) -> bool:
"""Whether the summary carries a durable recovery-failure decision."""
return any(
getattr(item, "code", None) == "schedule-recovery" for item in run.diagnostics
)
def _decision_reason(run: Any) -> str:
"""Return the first durable recovery-failure reason, if any."""
for item in run.diagnostics:
if getattr(item, "code", None) == "schedule-recovery":
return str(getattr(item, "message", ""))
return ""
def _failed_decision_stable(
run_store: Any,
run: Any,
history: HistoryRecorder,
now: datetime,
diags: list[str],
) -> bool:
"""Leave a decided FAILED summary untouched, ensuring history only.
Returns True when the run needs no further processing this pass: the
summary already reflects a recovery decision (or a legacy/external
failure) for the current checkpoint state, so re-running reconcile or
re-failing would only duplicate diagnostics and rewrite readiness. A
genuinely newer checkpoint (pointer mismatch) returns False so the
reconcile path may repair the summary. Unreadable checkpoint state
likewise returns False: the reconcile path fails closed for undecided
runs and keeps decided ones via its own branch.
"""
try:
latest = run_store.get_latest_checkpoint(run.id)
except KeyError:
latest = None
except ValueError, OSError:
return False
pointer = run.latest_checkpoint_id
latest_id = latest.id if latest is not None else None
if latest_id != pointer:
# Genuinely newer (or torn-away) checkpoint state: not stable.
return False
sched_id, intended, revision = _attribution(run_store, run.id)
reason = _decision_reason(run) or "reconciled-on-recovery"
if _reconcile_terminal(
history,
sched_id,
run.id,
"failed",
pointer,
intended,
revision,
reason,
now,
):
diags.append(f"{run.id}:terminal-reconciled")
return True
def _attribution( def _attribution(
run_store: Any, run_id: str run_store: Any, run_id: str
) -> tuple[str | None, datetime | None, int | None]: ) -> tuple[str | None, datetime | None, int | None]:
@@ -366,12 +477,13 @@ def _fail_run(
) -> None: ) -> None:
"""Fail a run closed with its truthful reason durably preserved. """Fail a run closed with its truthful reason durably preserved.
The reason is always written into the run summary diagnostics; when an The reason is always written into the run summary diagnostics, and a
admission attributes the run to a schedule, a failed occurrence entry is failed run is never resumable. When an admission attributes the run to
reconciled exactly once through the history recorder. a schedule, a failed occurrence entry is reconciled exactly once
through the history recorder.
""" """
from wf_artifacts import DependencyDiagnostic, DiagnosticSeverity from wf_artifacts import DependencyDiagnostic, DiagnosticSeverity
from wf_artifacts.runs.models import StoredRunStatus from wf_artifacts.runs.models import ResumeReadiness, StoredRunStatus
diagnostic = DependencyDiagnostic( diagnostic = DependencyDiagnostic(
severity=DiagnosticSeverity.ERROR, severity=DiagnosticSeverity.ERROR,
@@ -383,6 +495,7 @@ def _fail_run(
updated = run.model_copy( updated = run.model_copy(
update={ update={
"status": StoredRunStatus.FAILED, "status": StoredRunStatus.FAILED,
"resume_readiness": ResumeReadiness.NOT_APPLICABLE,
"updated_at": now, "updated_at": now,
"diagnostics": [*run.diagnostics, diagnostic], "diagnostics": [*run.diagnostics, diagnostic],
} }
+9 -9
View File
@@ -83,7 +83,7 @@ def _due(sched: Scheduler, store: FileScheduleStore, intended: datetime) -> None
def test_executing_is_set_before_executor_runs(tmp_path: Path) -> None: def test_executing_is_set_before_executor_runs(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
observed: dict[str, bool] = {} observed: dict[str, bool] = {}
@@ -128,7 +128,7 @@ def test_failure_before_transition_never_dispatches(tmp_path: Path) -> None:
sched_store = FileScheduleStore(tmp_path / "sched") sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FailMarkOnce(tmp_path / "runs") run_store = FailMarkOnce(tmp_path / "runs")
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched = Scheduler( sched = Scheduler(
schedule_store=sched_store, schedule_store=sched_store,
@@ -160,7 +160,7 @@ def test_failure_before_transition_never_dispatches(tmp_path: Path) -> None:
def test_capacity_shortage_keeps_pending_undispatched(tmp_path: Path) -> None: def test_capacity_shortage_keeps_pending_undispatched(tmp_path: Path) -> None:
calls: list[str] = [] calls: list[str] = []
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs = _harness(tmp_path, ownership, capacity=0) sched, store, runs = _harness(tmp_path, ownership, capacity=0)
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
@@ -185,7 +185,7 @@ def test_capacity_shortage_keeps_pending_undispatched(tmp_path: Path) -> None:
def test_pending_on_terminal_run_clears_without_redispatch(tmp_path: Path) -> None: def test_pending_on_terminal_run_clears_without_redispatch(tmp_path: Path) -> None:
calls: list[str] = [] calls: list[str] = []
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs = _harness(tmp_path, ownership) sched, store, runs = _harness(tmp_path, ownership)
@@ -230,7 +230,7 @@ def test_pending_without_admission_fails_closed(tmp_path: Path) -> None:
WorkflowRunRecord, WorkflowRunRecord,
) )
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"}) sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
@@ -260,7 +260,7 @@ def test_pending_without_admission_fails_closed(tmp_path: Path) -> None:
def test_executing_admitted_is_abandoned_by_recovery(tmp_path: Path) -> None: def test_executing_admitted_is_abandoned_by_recovery(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"}) sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
@@ -289,7 +289,7 @@ def test_executing_admitted_is_abandoned_by_recovery(tmp_path: Path) -> None:
def test_settle_hanging_run_persists_and_clears(tmp_path: Path) -> None: def test_settle_hanging_run_persists_and_clears(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"}) sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
@@ -350,7 +350,7 @@ def _child_poll_and_die(
capacity=4, capacity=4,
preparer=_Prep(_Dict({"dep-1": {"rev": 1, "required": []}}), _env), preparer=_Prep(_Dict({"dep-1": {"rev": 1, "required": []}}), _env),
dispatcher=_SD({"*": killer}), dispatcher=_SD({"*": killer}),
ownership=_Own(base / "sched", owner="child").acquire(), ownership=_Own(base, owner="child").acquire(),
) )
sched.poll(intended) sched.poll(intended)
@@ -376,7 +376,7 @@ def test_subprocess_death_after_side_effect_dispatches_once(tmp_path: Path) -> N
# Restart with fresh objects: the side effect count stays one and the run # Restart with fresh objects: the side effect count stays one and the run
# is abandoned, never redispatched. # is abandoned, never redispatched.
ownership = SchedulerOwnership(root / "sched", owner="parent").acquire() ownership = SchedulerOwnership(root, owner="parent").acquire()
try: try:
sched, store, runs = _harness(root, ownership, script={"*": "hang"}) sched, store, runs = _harness(root, ownership, script={"*": "hang"})
sched.sources["a"] = OneShotSource(intended) sched.sources["a"] = OneShotSource(intended)
+6 -6
View File
@@ -124,7 +124,7 @@ def _entries(store: FileScheduleStore, sid: str, kind: str) -> list[dict[str, An
def _recover( def _recover(
sched_store: FileScheduleStore, run_store: FileRunStore, now: datetime sched_store: FileScheduleStore, run_store: FileRunStore, now: datetime
) -> list[str]: ) -> list[str]:
ownership = SchedulerOwnership(sched_store.root, owner="test").acquire() ownership = SchedulerOwnership(sched_store.root.parent, owner="test").acquire()
try: try:
return sched_recovery.recover( return sched_recovery.recover(
schedule_store=sched_store, schedule_store=sched_store,
@@ -223,7 +223,7 @@ def test_history_write_failure_reconciles_once_on_recovery(tmp_path: Path) -> No
sched_store = FailCompletedHistoryOnce(tmp_path / "sched") sched_store = FailCompletedHistoryOnce(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs") run_store = FileRunStore(tmp_path / "runs")
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched = Scheduler( sched = Scheduler(
schedule_store=sched_store, schedule_store=sched_store,
@@ -271,7 +271,7 @@ def test_history_write_failure_reconciles_once_on_recovery(tmp_path: Path) -> No
def test_dispatch_and_recovery_share_entry_identity(tmp_path: Path) -> None: def test_dispatch_and_recovery_share_entry_identity(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched_store = FileScheduleStore(tmp_path / "sched") sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs") run_store = FileRunStore(tmp_path / "runs")
@@ -383,7 +383,7 @@ def test_admission_tear_returns_existing_run_without_dup(tmp_path: Path) -> None
) )
materialize_admitted_view(store=run_store, admission=admission) materialize_admitted_view(store=run_store, admission=admission)
sched_store.save_consumed("a", intended - timedelta(hours=1)) sched_store.save_consumed("a", intended - timedelta(hours=1))
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched = _owned_scheduler( sched = _owned_scheduler(
sched_store, run_store, ownership, script={"*": "hang"} sched_store, run_store, ownership, script={"*": "hang"}
@@ -426,7 +426,7 @@ def test_parallel_tear_never_double_admits(tmp_path: Path) -> None:
) )
materialize_admitted_view(store=run_store, admission=admission) materialize_admitted_view(store=run_store, admission=admission)
sched_store.save_consumed("a", intended - timedelta(hours=1)) sched_store.save_consumed("a", intended - timedelta(hours=1))
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched = _owned_scheduler( sched = _owned_scheduler(
sched_store, run_store, ownership, script={"*": "hang"} sched_store, run_store, ownership, script={"*": "hang"}
@@ -460,7 +460,7 @@ def test_consumed_write_failure_recovers_without_dup(tmp_path: Path) -> None:
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
sched_store.save_consumed("a", intended - timedelta(hours=1)) sched_store.save_consumed("a", intended - timedelta(hours=1))
sched_store.armed = True # arm only the poll's watermark write sched_store.armed = True # arm only the poll's watermark write
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched = _owned_scheduler( sched = _owned_scheduler(
sched_store, run_store, ownership, script={"*": "hang"} sched_store, run_store, ownership, script={"*": "hang"}
+8 -8
View File
@@ -98,7 +98,7 @@ def _admitted(store: FileScheduleStore, sid: str) -> list[dict[str, Any]]:
def test_exact_boundary_admits_latest_once(tmp_path: Path) -> None: def test_exact_boundary_admits_latest_once(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs, sources = _harness( sched, store, runs, sources = _harness(
tmp_path, ownership, script={"*": "hang"} tmp_path, ownership, script={"*": "hang"}
@@ -120,7 +120,7 @@ def test_exact_boundary_admits_latest_once(tmp_path: Path) -> None:
def test_second_poll_at_same_instant_admits_nothing(tmp_path: Path) -> None: def test_second_poll_at_same_instant_admits_nothing(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs, sources = _harness( sched, store, runs, sources = _harness(
tmp_path, ownership, script={"*": "hang"} tmp_path, ownership, script={"*": "hang"}
@@ -140,7 +140,7 @@ def test_second_poll_at_same_instant_admits_nothing(tmp_path: Path) -> None:
def test_between_ticks_admits_latest(tmp_path: Path) -> None: def test_between_ticks_admits_latest(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs, sources = _harness( sched, store, runs, sources = _harness(
tmp_path, ownership, script={"*": "hang"} tmp_path, ownership, script={"*": "hang"}
@@ -163,7 +163,7 @@ def test_between_ticks_admits_latest(tmp_path: Path) -> None:
def test_restart_after_catchup_admits_nothing(tmp_path: Path) -> None: def test_restart_after_catchup_admits_nothing(tmp_path: Path) -> None:
now = ts(2026, 9, 8, 12, 0) now = ts(2026, 9, 8, 12, 0)
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, _, sources = _harness(tmp_path, ownership, script={"*": "hang"}) sched, store, _, sources = _harness(tmp_path, ownership, script={"*": "hang"})
store.create_schedule( store.create_schedule(
@@ -174,7 +174,7 @@ def test_restart_after_catchup_admits_nothing(tmp_path: Path) -> None:
sched.poll(now) sched.poll(now)
finally: finally:
ownership.release() ownership.release()
ownership2 = SchedulerOwnership(tmp_path / "sched", owner="test2").acquire() ownership2 = SchedulerOwnership(tmp_path, owner="test2").acquire()
try: try:
sched2, store2, runs2, sources2 = _harness( sched2, store2, runs2, sources2 = _harness(
tmp_path, ownership2, script={"*": "hang"} tmp_path, ownership2, script={"*": "hang"}
@@ -188,7 +188,7 @@ def test_restart_after_catchup_admits_nothing(tmp_path: Path) -> None:
def test_long_catchup_stays_bounded(tmp_path: Path) -> None: def test_long_catchup_stays_bounded(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs, sources = _harness( sched, store, runs, sources = _harness(
tmp_path, ownership, script={"*": "complete"} tmp_path, ownership, script={"*": "complete"}
@@ -208,7 +208,7 @@ def test_long_catchup_stays_bounded(tmp_path: Path) -> None:
def test_skip_policy_exact_boundary_spans_without_admission( def test_skip_policy_exact_boundary_spans_without_admission(
tmp_path: Path, tmp_path: Path,
) -> None: ) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs, sources = _harness( sched, store, runs, sources = _harness(
tmp_path, ownership, script={"*": "hang"} tmp_path, ownership, script={"*": "hang"}
@@ -229,7 +229,7 @@ def test_skip_policy_exact_boundary_spans_without_admission(
def test_overlap_skip_exact_boundary_skips_once(tmp_path: Path) -> None: def test_overlap_skip_exact_boundary_skips_once(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs, sources = _harness( sched, store, runs, sources = _harness(
tmp_path, ownership, script={"*": "hang"} tmp_path, ownership, script={"*": "hang"}
+179
View File
@@ -0,0 +1,179 @@
"""Ownership is bound to the protected store composition (R4 binding).
A held lock on an unrelated directory must not authorize mutation of a
store composition it does not cover. The guard validates the held guard
against the actual schedule/run store roots (canonicalized for platform
aliases) instead of trusting the caller to have passed the right lock.
"""
from __future__ import annotations
import os
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import pytest
from tests.scheduling.controlled import (
DictDeployments,
ScriptedDispatcher,
fixture_environment,
)
from wf_artifacts.runs.store import FileRunStore
from wf_scheduling.calendar import OneShotSource
from wf_scheduling.dispatch import StillRunning
from wf_scheduling.models import Schedule
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
from wf_scheduling.poll import Scheduler
from wf_scheduling.prepare import SchedulePreparer
from wf_scheduling.store import FileScheduleStore
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime:
return datetime(y, mo, d, h, mi, tzinfo=UTC)
def _sched_model(sid: str, **kw: Any) -> Schedule:
now = ts(2026, 9, 8, 12, 0)
base: dict[str, Any] = {
"id": sid,
"deployment_id": "dep-1",
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
"input_bindings": [],
"created_at": now.isoformat(),
"updated_at": now.isoformat(),
}
base.update(kw)
return Schedule.model_validate(base)
def _scheduler(
sched_store: FileScheduleStore,
run_store: FileRunStore,
ownership: SchedulerOwnership,
*,
script: dict | None = None,
) -> Scheduler:
return Scheduler(
schedule_store=sched_store,
run_store=run_store,
sources={},
capacity=4,
preparer=SchedulePreparer(
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
fixture_environment,
),
dispatcher=ScriptedDispatcher(script),
ownership=ownership,
)
def _due(sched: Scheduler, store: FileScheduleStore, intended: datetime) -> None:
store.create_schedule(_sched_model("a"))
store.save_consumed("a", intended - timedelta(hours=1))
sched.sources["a"] = OneShotSource(intended)
def test_unrelated_held_lock_rejected_before_writes(tmp_path: Path) -> None:
calls: list[str] = []
def spy(admission: Any, now: datetime) -> Any:
calls.append(admission.id)
return StillRunning()
sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
ownership = SchedulerOwnership(tmp_path / "other", owner="unrelated").acquire()
try:
sched = _scheduler(sched_store, run_store, ownership)
intended = ts(2026, 9, 8, 12, 0)
_due(sched, sched_store, intended)
with pytest.raises(SecondOwnerError):
sched.poll(intended)
assert calls == []
assert run_store.list_runs() == []
assert run_store.list_admissions() == []
assert sched_store.list_occurrences("a", limit=100)["total"] == 0
finally:
ownership.release()
def test_two_lock_dirs_cannot_operate_same_stores(tmp_path: Path) -> None:
sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
intended = ts(2026, 9, 8, 12, 0)
covering = SchedulerOwnership(tmp_path, owner="covering").acquire()
other = SchedulerOwnership(tmp_path / "other", owner="other").acquire()
try:
first = _scheduler(sched_store, run_store, covering, script={"*": "hang"})
_due(first, sched_store, intended)
assert first.poll(intended)["a"].startswith("admit:run-")
# The other directory's lock acquires fine (different lock file) but
# must not authorize the same stores.
second = _scheduler(sched_store, run_store, other, script={"*": "hang"})
second.sources["a"] = OneShotSource(intended)
with pytest.raises(SecondOwnerError):
second.poll(intended + timedelta(minutes=1))
assert len(run_store.list_runs()) == 1
finally:
covering.release()
other.release()
def test_recover_with_unrelated_lock_rejected(tmp_path: Path) -> None:
from tests.artifacts.test_run_store import artifact as _artifact
from tests.artifacts.test_run_store import deployment as _deployment
from wf_api.run_lifecycle import materialize_admitted_view, persist_admission
from wf_artifacts import PinnedRunEnvironment
from wf_scheduling import recovery as sched_recovery
sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
sched_store.create_schedule(_sched_model("a"))
admission = persist_admission(
store=run_store,
run_id=run_store.allocate_run_id(),
environment=PinnedRunEnvironment(
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
),
resolved_input={},
max_steps=None,
)
materialize_admitted_view(store=run_store, admission=admission)
ownership = SchedulerOwnership(tmp_path / "other", owner="unrelated").acquire()
try:
with pytest.raises(SecondOwnerError):
sched_recovery.recover(
schedule_store=sched_store,
run_store=run_store,
now=ts(2026, 9, 8, 12, 0),
ownership=ownership,
)
assert run_store.get_run(admission.id).status.value == "admitted"
finally:
ownership.release()
def test_canonicalizer_handles_aliases(tmp_path: Path) -> None:
from wf_scheduling.ownership import canonical_store_path
base = tmp_path / "sched"
dotted = tmp_path / "sub" / ".." / "sched"
assert canonical_store_path(dotted) == canonical_store_path(base)
if os.name == "nt":
assert canonical_store_path("C:\\Temp\\SCHED") == canonical_store_path(
"c:/temp/sched"
)
else:
assert canonical_store_path(dotted) != canonical_store_path(tmp_path / "other")
def test_covers_rejects_empty_root_set(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
assert ownership.held
assert ownership.covers() is False
assert ownership.covers(tmp_path) is True
finally:
ownership.release()
+8 -8
View File
@@ -83,7 +83,7 @@ def test_poll_without_ownership_rejects_before_writes(tmp_path: Path) -> None:
calls.append(admission.id) calls.append(admission.id)
raise AssertionError("dispatcher must not run without ownership") raise AssertionError("dispatcher must not run without ownership")
ownership = SchedulerOwnership(tmp_path / "sched", owner="never-acquired") ownership = SchedulerOwnership(tmp_path, owner="never-acquired")
sched, store, runs = _scheduler(tmp_path, ownership) sched, store, runs = _scheduler(tmp_path, ownership)
sched.dispatcher = ScriptedDispatcher({"*": spy}) sched.dispatcher = ScriptedDispatcher({"*": spy})
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
@@ -98,7 +98,7 @@ def test_poll_without_ownership_rejects_before_writes(tmp_path: Path) -> None:
def test_poll_with_released_ownership_rejects(tmp_path: Path) -> None: def test_poll_with_released_ownership_rejects(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
sched, store, runs = _scheduler(tmp_path, ownership) sched, store, runs = _scheduler(tmp_path, ownership)
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
_due_setup(sched, store, intended) _due_setup(sched, store, intended)
@@ -133,7 +133,7 @@ def test_recover_without_ownership_rejects_before_writes(tmp_path: Path) -> None
schedule_revision=1, schedule_revision=1,
) )
materialize_admitted_view(store=run_store, admission=admission) materialize_admitted_view(store=run_store, admission=admission)
ownership = SchedulerOwnership(tmp_path / "sched", owner="never-acquired") ownership = SchedulerOwnership(tmp_path, owner="never-acquired")
with pytest.raises(SecondOwnerError): with pytest.raises(SecondOwnerError):
sched_recovery.recover( sched_recovery.recover(
schedule_store=sched_store, schedule_store=sched_store,
@@ -146,7 +146,7 @@ def test_recover_without_ownership_rejects_before_writes(tmp_path: Path) -> None
def test_held_ownership_allows_poll_and_recovery(tmp_path: Path) -> None: def test_held_ownership_allows_poll_and_recovery(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs = _scheduler(tmp_path, ownership, script={"*": "hang"}) sched, store, runs = _scheduler(tmp_path, ownership, script={"*": "hang"})
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
@@ -164,9 +164,9 @@ def test_held_ownership_allows_poll_and_recovery(tmp_path: Path) -> None:
def test_second_owner_cannot_acquire_for_poll(tmp_path: Path) -> None: def test_second_owner_cannot_acquire_for_poll(tmp_path: Path) -> None:
first = SchedulerOwnership(tmp_path / "sched", owner="first").acquire() first = SchedulerOwnership(tmp_path, owner="first").acquire()
try: try:
second = SchedulerOwnership(tmp_path / "sched", owner="second") second = SchedulerOwnership(tmp_path, owner="second")
with pytest.raises(SecondOwnerError): with pytest.raises(SecondOwnerError):
second.acquire() second.acquire()
assert not second.held assert not second.held
@@ -181,7 +181,7 @@ def test_second_owner_cannot_acquire_for_poll(tmp_path: Path) -> None:
def test_admin_mutations_require_ownership(tmp_path: Path) -> None: def test_admin_mutations_require_ownership(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test") ownership = SchedulerOwnership(tmp_path, owner="test")
sched, store, _ = _scheduler(tmp_path, ownership) sched, store, _ = _scheduler(tmp_path, ownership)
store.create_schedule(_sched_model("a", paused=True)) store.create_schedule(_sched_model("a", paused=True))
with pytest.raises(SecondOwnerError): with pytest.raises(SecondOwnerError):
@@ -193,7 +193,7 @@ def test_admin_mutations_require_ownership(tmp_path: Path) -> None:
def test_settle_requires_ownership(tmp_path: Path) -> None: def test_settle_requires_ownership(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched, store, runs = _scheduler(tmp_path, ownership, script={"*": "hang"}) sched, store, runs = _scheduler(tmp_path, ownership, script={"*": "hang"})
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
+1 -1
View File
@@ -88,7 +88,7 @@ def _harness(
capacity=capacity, capacity=capacity,
preparer=preparer, preparer=preparer,
dispatcher=ScriptedDispatcher(script), dispatcher=ScriptedDispatcher(script),
ownership=SchedulerOwnership(tmp_path / "sched", owner="test").acquire(), ownership=SchedulerOwnership(tmp_path, owner="test").acquire(),
) )
return sched, sched_store, run_store, sources return sched, sched_store, run_store, sources
+2 -2
View File
@@ -67,7 +67,7 @@ def _scheduler(
capacity=4, capacity=4,
preparer=preparer, preparer=preparer,
dispatcher=ScriptedDispatcher(script), dispatcher=ScriptedDispatcher(script),
ownership=SchedulerOwnership(tmp_path / "sched", owner="test").acquire(), ownership=SchedulerOwnership(tmp_path, owner="test").acquire(),
) )
return sched, sched_store, run_store return sched, sched_store, run_store
@@ -173,7 +173,7 @@ def test_missing_required_input_rejects_without_a_run(tmp_path: Path) -> None:
capacity=4, capacity=4,
preparer=preparer, preparer=preparer,
dispatcher=ScriptedDispatcher({"*": "hang"}), dispatcher=ScriptedDispatcher({"*": "hang"}),
ownership=SchedulerOwnership(tmp_path / "sched", owner="test").acquire(), ownership=SchedulerOwnership(tmp_path, owner="test").acquire(),
) )
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
sched_store.create_schedule(_sched_model("need")) sched_store.create_schedule(_sched_model("need"))
+298
View File
@@ -0,0 +1,298 @@
"""Resolved input validates against the pinned workflow schema (R4 item 3).
Preparation checks more than required key names: the resolved object must
satisfy the pinned root artifact's input schema (same snapshot that is
captured in the admission) before any run identity is allocated. Invalid
input rejects the occurrence without admission, run, or dispatch.
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, cast
from tests.artifacts.test_run_store import artifact as _artifact
from tests.artifacts.test_run_store import deployment as _deployment
from tests.scheduling.controlled import DictDeployments, ScriptedDispatcher
from wf_artifacts.runs.models import PinnedRunEnvironment
from wf_artifacts.runs.store import FileRunStore
from wf_scheduling.calendar import OneShotSource
from wf_scheduling.dispatch import StillRunning
from wf_scheduling.models import Schedule
from wf_scheduling.ownership import SchedulerOwnership
from wf_scheduling.poll import Scheduler
from wf_scheduling.prepare import SchedulePreparer
from wf_scheduling.store import FileScheduleStore
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime:
return datetime(y, mo, d, h, mi, tzinfo=UTC)
STRICT_SCHEMA: dict[str, Any] = {
"type": "object",
"properties": {
"count": {"type": "integer"},
"nested": {
"type": "object",
"properties": {"name": {"type": "string"}},
"required": ["name"],
"additionalProperties": False,
},
},
"required": ["count"],
"additionalProperties": False,
}
def _sched_model(sid: str, **kw: Any) -> Schedule:
now = ts(2026, 9, 8, 12, 0)
base: dict[str, Any] = {
"id": sid,
"deployment_id": "dep-1",
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
"input_bindings": [],
"created_at": now.isoformat(),
"updated_at": now.isoformat(),
}
base.update(kw)
return Schedule.model_validate(base)
def _strict_env(sched: Any) -> PinnedRunEnvironment:
artifact = _artifact().model_copy(
update={"input_schema": dict(STRICT_SCHEMA)},
)
deployment = _deployment().model_copy(update={"id": sched.deployment_id})
return PinnedRunEnvironment(
deployment=deployment, root_artifact=artifact, child_artifacts=[]
)
def _scheduler(
tmp_path: Path,
ownership: SchedulerOwnership,
bindings: list[dict[str, Any]],
*,
script: dict | None = None,
) -> tuple[Scheduler, FileScheduleStore, FileRunStore, list[str]]:
calls: list[str] = []
sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
sched = Scheduler(
schedule_store=sched_store,
run_store=run_store,
sources={},
capacity=4,
preparer=SchedulePreparer(
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
_strict_env,
),
dispatcher=ScriptedDispatcher(script),
ownership=ownership,
)
intended = ts(2026, 9, 8, 12, 0)
sched_store.create_schedule(_sched_model("s", input_bindings=bindings))
sched_store.save_consumed("s", intended - timedelta(hours=1))
sched.sources["s"] = OneShotSource(intended)
dispatches = cast(ScriptedDispatcher, sched.dispatcher)
outcome = (script or {}).get("*", "complete")
assert isinstance(outcome, str)
def spy(admission: Any, now: datetime) -> Any:
calls.append(admission.id)
if outcome == "hang":
return StillRunning()
from wf_scheduling.dispatch import Stopped
return Stopped(result=dispatches.finish(admission, outcome))
dispatches.script = {"*": spy}
return sched, sched_store, run_store, calls
def _rejected_entry(store: FileScheduleStore) -> dict[str, Any]:
page = store.list_occurrences("s", limit=100)
rows = cast(list[dict[str, Any]], page["occurrences"])
assert len(rows) == 1
assert rows[0]["kind"] == "preflight-rejected"
return rows[0]
def test_wrong_scalar_type_rejects_without_admission(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, store, runs, calls = _scheduler(
tmp_path,
ownership,
[
{
"target": "count",
"expression": {
"kind": "literal",
"value": "definitely not integer",
},
}
],
)
intended = ts(2026, 9, 8, 12, 0)
assert sched.poll(intended) == {"s": "admit:None"}
entry = _rejected_entry(store)
assert "invalid-input" in entry["reason"]
assert runs.list_runs() == []
assert runs.list_admissions() == []
assert calls == []
finally:
ownership.release()
def test_nested_constraint_violation_rejects(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, store, runs, calls = _scheduler(
tmp_path,
ownership,
[
{"target": "count", "expression": {"kind": "literal", "value": 3}},
{
"target": "nested",
"expression": {
"kind": "object",
"fields": {"name": {"kind": "literal", "value": 7}},
},
},
],
)
assert sched.poll(ts(2026, 9, 8, 12, 0)) == {"s": "admit:None"}
_rejected_entry(store)
assert runs.list_runs() == []
assert calls == []
finally:
ownership.release()
def test_disallowed_extra_property_rejects(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, store, runs, calls = _scheduler(
tmp_path,
ownership,
[
{"target": "count", "expression": {"kind": "literal", "value": 3}},
{
"target": "surprise",
"expression": {"kind": "literal", "value": "x"},
},
],
)
assert sched.poll(ts(2026, 9, 8, 12, 0)) == {"s": "admit:None"}
_rejected_entry(store)
assert runs.list_runs() == []
assert calls == []
finally:
ownership.release()
def test_valid_structured_input_admits(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, store, runs, calls = _scheduler(
tmp_path,
ownership,
[
{"target": "count", "expression": {"kind": "literal", "value": 3}},
{
"target": "nested",
"expression": {
"kind": "object",
"fields": {"name": {"kind": "literal", "value": "ok"}},
},
},
],
script={"*": "hang"},
)
intended = ts(2026, 9, 8, 12, 0)
result = sched.poll(intended)
assert result["s"].startswith("admit:run-")
run_id = result["s"].split(":", 1)[1]
assert runs.get_admission(run_id).resolved_input == {
"count": 3,
"nested": {"name": "ok"},
}
assert len(calls) == 1
finally:
ownership.release()
def test_changed_contract_rejects_before_admission(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, store, runs, calls = _scheduler(
tmp_path,
ownership,
[
{"target": "count", "expression": {"kind": "literal", "value": 3}},
],
)
# The pinned contract now additionally requires a token the
# schedule does not provide.
strict = dict(STRICT_SCHEMA)
strict["required"] = ["count", "token"]
artifact = _artifact().model_copy(update={"input_schema": strict})
def changed_env(sched: Any) -> PinnedRunEnvironment:
deployment = _deployment().model_copy(update={"id": sched.deployment_id})
return PinnedRunEnvironment(
deployment=deployment, root_artifact=artifact, child_artifacts=[]
)
sched.preparer = SchedulePreparer(
DictDeployments({"dep-1": {"rev": 2, "required": []}}), changed_env
)
assert sched.poll(ts(2026, 9, 8, 12, 0)) == {"s": "admit:None"}
_rejected_entry(store)
assert runs.list_runs() == []
assert runs.list_admissions() == []
assert calls == []
finally:
ownership.release()
def test_environment_built_once_per_preparation(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
builds: list[str] = []
def counting_env(sched: Any) -> PinnedRunEnvironment:
builds.append(sched.id)
return _strict_env(sched)
sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
sched = Scheduler(
schedule_store=sched_store,
run_store=run_store,
sources={},
capacity=4,
preparer=SchedulePreparer(
DictDeployments({"dep-1": {"rev": 1, "required": []}}), counting_env
),
dispatcher=ScriptedDispatcher({"*": "hang"}),
ownership=ownership,
)
intended = ts(2026, 9, 8, 12, 0)
sched_store.create_schedule(
_sched_model(
"s",
input_bindings=[
{"target": "count", "expression": {"kind": "literal", "value": 1}}
],
)
)
sched_store.save_consumed("s", intended - timedelta(hours=1))
sched.sources["s"] = OneShotSource(intended)
sched.poll(intended)
assert builds == ["s"]
finally:
ownership.release()
+4 -4
View File
@@ -58,7 +58,7 @@ def test_recovery_materializes_missing_view_as_pending(tmp_path: Path) -> None:
resolved_input={}, resolved_input={},
max_steps=None, max_steps=None,
) )
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
diags = sched_recovery.recover( diags = sched_recovery.recover(
schedule_store=sched_store, schedule_store=sched_store,
@@ -96,7 +96,7 @@ def test_recovery_fails_abandoned_admitted_without_replay(tmp_path: Path) -> Non
schedule_revision=1, schedule_revision=1,
) )
materialize_admitted_view(store=run_store, admission=admission) materialize_admitted_view(store=run_store, admission=admission)
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
diags = sched_recovery.recover( diags = sched_recovery.recover(
schedule_store=sched_store, schedule_store=sched_store,
@@ -133,7 +133,7 @@ def test_recovery_never_executes_pending_until_poll(tmp_path: Path) -> None:
schedule_id="a", schedule_id="a",
schedule_revision=1, schedule_revision=1,
) )
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
diags = sched_recovery.recover( diags = sched_recovery.recover(
schedule_store=sched_store, schedule_store=sched_store,
@@ -147,7 +147,7 @@ def test_recovery_never_executes_pending_until_poll(tmp_path: Path) -> None:
# Recovery itself produced no terminal history; the poll sweep dispatches. # Recovery itself produced no terminal history; the poll sweep dispatches.
assert sched_store.list_occurrences("a", limit=100)["total"] == 0 assert sched_store.list_occurrences("a", limit=100)["total"] == 0
sources = {"a": OneShotSource(ts(2026, 9, 8, 12, 0))} sources = {"a": OneShotSource(ts(2026, 9, 8, 12, 0))}
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched = Scheduler( sched = Scheduler(
schedule_store=sched_store, schedule_store=sched_store,
+307
View File
@@ -0,0 +1,307 @@
"""Durable recovery failure is stable across repeated recovery (R4 item 2).
An older checkpoint cannot supersede a durable abandonment decision: once
recovery fails a run, repeating recovery must leave status, readiness,
diagnostics, and history stable. Only a genuinely newer durable stopped
result may repair a torn summary. Failed recovery carries not-applicable
resume readiness.
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, cast
from tests.artifacts.test_run_store import artifact as _artifact
from tests.artifacts.test_run_store import deployment as _deployment
from wf_api.run_lifecycle import (
load_stored_run,
materialize_admitted_view,
persist_admission,
persist_stopped_run,
restore_interrupted_run,
)
from wf_artifacts import PinnedRunEnvironment
from wf_artifacts.runs.models import ResumeAttempt
from wf_artifacts.runs.store import FileRunStore
from wf_core import RunState, RunStatus
from wf_scheduling import recovery as sched_recovery
from wf_scheduling.models import Schedule
from wf_scheduling.ownership import SchedulerOwnership
from wf_scheduling.store import FileScheduleStore
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime:
return datetime(y, mo, d, h, mi, tzinfo=UTC)
def _sched_model(sid: str, **kw: Any) -> Schedule:
now = ts(2026, 9, 8, 12, 0)
base: dict[str, Any] = {
"id": sid,
"deployment_id": "dep-1",
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
"input_bindings": [],
"created_at": now.isoformat(),
"updated_at": now.isoformat(),
}
base.update(kw)
return Schedule.model_validate(base)
def _env() -> PinnedRunEnvironment:
return PinnedRunEnvironment(
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
)
def _recover(
sched_store: FileScheduleStore, run_store: FileRunStore, now: datetime
) -> list[str]:
ownership = SchedulerOwnership(sched_store.root.parent, owner="test").acquire()
try:
return sched_recovery.recover(
schedule_store=sched_store,
run_store=run_store,
now=now,
ownership=ownership,
)
finally:
ownership.release()
def _stopped(
run_store: FileRunStore,
run_id: str,
status: RunStatus,
*,
attempt_id: int | None = None,
) -> Any:
record = run_store.get_run(run_id)
return persist_stopped_run(
store=run_store,
environment=record.environment,
run=RunState(workflow_name="sched", status=status, workflow_input={}, state={}),
run_id=run_id,
attempt_id=attempt_id,
)
def _admit_interrupted(
run_store: FileRunStore, intended: datetime, *, attempt_id: int | None = None
) -> str:
run_id = run_store.allocate_run_id()
admission = persist_admission(
store=run_store,
run_id=run_id,
environment=_env(),
resolved_input={},
max_steps=None,
scheduled_at=intended,
schedule_id="a",
schedule_revision=1,
)
materialize_admitted_view(store=run_store, admission=admission)
_stopped(run_store, run_id, RunStatus.INTERRUPTED, attempt_id=attempt_id)
return run_id
def _mark_active(
run_store: FileRunStore, run_id: str, attempt_id: int, now: datetime
) -> None:
run_store.save_resume_attempt(
ResumeAttempt(
run_id=run_id,
attempt_id=attempt_id,
state="ACTIVE",
created_at=now,
updated_at=now,
)
)
def _entries(store: FileScheduleStore, sid: str, kind: str) -> list[dict[str, Any]]:
page = store.list_occurrences(sid, limit=100)
return [
r for r in cast(list[dict[str, Any]], page["occurrences"]) if r["kind"] == kind
]
def test_abandonment_decision_is_stable_across_recovery() -> None:
import tempfile
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended)
_mark_active(run_store, run_id, 9, intended)
first = _recover(sched_store, run_store, intended)
assert any("failed-closed" in d for d in first)
record = run_store.get_run(run_id)
assert record.status.value == "failed"
assert record.resume_readiness.value == "not_applicable"
assert len(record.diagnostics) == 1
# Fresh store objects across the restart boundary: everything stable.
sched_store2 = FileScheduleStore(root / "sched")
run_store2 = FileRunStore(root / "runs")
second = _recover(sched_store2, run_store2, intended + timedelta(minutes=1))
assert not any(run_id in d for d in second)
again = run_store2.get_run(run_id)
assert again.status.value == "failed"
assert again.resume_readiness.value == "not_applicable"
assert len(again.diagnostics) == 1
assert _entries(sched_store2, "a", "failed") == _entries(
sched_store, "a", "failed"
)
assert len(_entries(sched_store2, "a", "failed")) == 1
def test_failed_readiness_and_inspection_agree() -> None:
import tempfile
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended)
_mark_active(run_store, run_id, 9, intended)
_recover(sched_store, run_store, intended)
record, _ = load_stored_run(run_store, run_id)
assert record.resume_readiness.value == "not_applicable"
try:
restore_interrupted_run(run_store, run_id)
raise AssertionError("failed run must not restore as interrupted")
except ValueError:
pass
def test_genuinely_newer_result_repairs_after_decision() -> None:
import tempfile
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended)
_mark_active(run_store, run_id, 9, intended)
_recover(sched_store, run_store, intended)
assert run_store.get_run(run_id).status.value == "failed"
# A genuinely newer stopped result under a new matching attempt: the
# newer checkpoint repairs the summary and completes the attempt.
_mark_active(run_store, run_id, 10, intended)
_stopped(run_store, run_id, RunStatus.INTERRUPTED, attempt_id=10)
diags = _recover(sched_store, run_store, intended + timedelta(minutes=1))
assert any("fresh-result-resumable" in d for d in diags)
record = run_store.get_run(run_id)
assert record.status.value == "interrupted"
assert record.resume_readiness.value == "ready"
assert run_store.get_resume_attempt(run_id).state == "DONE" # type: ignore[union-attr]
interrupted = _entries(sched_store, "a", "interrupted")
assert {e["checkpoint_id"] for e in interrupted} == {f"{run_id}.000002"}
def test_completed_mismatch_decision_is_stable() -> None:
import tempfile
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0)
run_id = run_store.allocate_run_id()
admission = persist_admission(
store=run_store,
run_id=run_id,
environment=_env(),
resolved_input={},
max_steps=None,
scheduled_at=intended,
schedule_id="a",
schedule_revision=1,
)
materialize_admitted_view(store=run_store, admission=admission)
_stopped(run_store, run_id, RunStatus.COMPLETED, attempt_id=3)
_mark_active(run_store, run_id, 5, intended)
_recover(sched_store, run_store, intended)
assert run_store.get_run(run_id).status.value == "failed"
second = _recover(
FileScheduleStore(root / "sched"),
FileRunStore(root / "runs"),
intended + timedelta(minutes=1),
)
assert not any(run_id in d for d in second)
assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1
def test_legacy_failed_run_gains_history_without_refail() -> None:
import tempfile
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended)
record = run_store.get_run(run_id)
from wf_artifacts.runs.models import StoredRunStatus
run_store.save_run(record.model_copy(update={"status": StoredRunStatus.FAILED}))
assert _entries(sched_store, "a", "failed") == []
diags = _recover(sched_store, run_store, intended)
assert any("terminal-reconciled" in d for d in diags)
assert not any("failed-closed" in d for d in diags)
assert run_store.get_run(run_id).diagnostics == []
assert len(_entries(sched_store, "a", "failed")) == 1
def test_crash_between_decision_writes_recovers_history_once() -> None:
import tempfile
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
class FailFailedHistoryOnce(FileScheduleStore):
def __init__(self, root: Path) -> None:
super().__init__(root)
self.armed = False
def append_history(self, record: Any) -> None:
if self.armed and getattr(record, "kind", None) == "failed":
self.armed = False
raise OSError("injected failed-history failure")
super().append_history(record)
sched_store = FailFailedHistoryOnce(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended)
_mark_active(run_store, run_id, 9, intended)
sched_store.armed = True
import pytest
with pytest.raises(OSError, match="injected failed-history failure"):
_recover(sched_store, run_store, intended)
record = run_store.get_run(run_id)
assert record.status.value == "failed"
assert len(record.diagnostics) == 1
assert _entries(sched_store, "a", "failed") == []
# The decision (status + reason) survived; only history is missing.
second = _recover(
FileScheduleStore(root / "sched"),
FileRunStore(root / "runs"),
intended + timedelta(minutes=1),
)
assert not any("failed-closed" in d for d in second)
assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1
assert len(_entries(FileScheduleStore(root / "sched"), "a", "failed")) == 1
+252
View File
@@ -0,0 +1,252 @@
"""Late settlement requires a real executing transition (R4 item 4).
An admitted status also describes pending, never-dispatched work, so the
async-completion seam must demand the executing marker and refuse
contradictory pending/executing state before persisting a late result.
Settling a stopped run twice is rejected; ownership stays enforced.
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, cast
import pytest
from tests.scheduling.controlled import (
DictDeployments,
ScriptedDispatcher,
fixture_environment,
)
from tests.scheduling.controlled import ScriptedDispatcher as SD
from wf_artifacts.runs.store import FileRunStore
from wf_scheduling.calendar import OneShotSource
from wf_scheduling.models import Schedule
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
from wf_scheduling.poll import BlockedSchedule, Scheduler
from wf_scheduling.prepare import SchedulePreparer
from wf_scheduling.store import FileScheduleStore
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime:
return datetime(y, mo, d, h, mi, tzinfo=UTC)
def _sched_model(sid: str, **kw: Any) -> Schedule:
now = ts(2026, 9, 8, 12, 0)
base: dict[str, Any] = {
"id": sid,
"deployment_id": "dep-1",
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
"input_bindings": [],
"created_at": now.isoformat(),
"updated_at": now.isoformat(),
}
base.update(kw)
return Schedule.model_validate(base)
def _harness(
root: Path,
ownership: SchedulerOwnership,
*,
script: dict | None = None,
) -> tuple[Scheduler, FileScheduleStore, FileRunStore]:
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched = Scheduler(
schedule_store=sched_store,
run_store=run_store,
sources={},
capacity=4,
preparer=SchedulePreparer(
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
fixture_environment,
),
dispatcher=ScriptedDispatcher(script),
ownership=ownership,
)
return sched, sched_store, run_store
def _due(sched: Scheduler, store: FileScheduleStore, intended: datetime) -> None:
store.create_schedule(_sched_model("a"))
store.save_consumed("a", intended - timedelta(hours=1))
sched.sources["a"] = OneShotSource(intended)
def test_pending_undispatched_run_rejects_settlement(tmp_path: Path) -> None:
from tests.artifacts.test_run_store import artifact as _artifact
from tests.artifacts.test_run_store import deployment as _deployment
from wf_api.run_lifecycle import materialize_admitted_view, persist_admission
from wf_artifacts import PinnedRunEnvironment
from wf_core import RunState, RunStatus
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, _, runs = _harness(tmp_path, ownership, script={"*": "hang"})
admission = persist_admission(
store=runs,
run_id=runs.allocate_run_id(),
environment=PinnedRunEnvironment(
deployment=_deployment(),
root_artifact=_artifact(),
child_artifacts=[],
),
resolved_input={},
max_steps=None,
scheduled_at=ts(2026, 9, 8, 12, 0),
schedule_id="a",
schedule_revision=1,
)
materialize_admitted_view(store=runs, admission=admission)
runs.mark_pending_dispatch(admission.id)
state = RunState(
workflow_name="sched",
status=RunStatus.COMPLETED,
workflow_input={},
state={},
)
with pytest.raises(BlockedSchedule):
sched.record_stopped_execution(admission.id, state, ts(2026, 9, 8, 12, 0))
assert runs.get_run(admission.id).status.value == "admitted"
assert runs.is_pending_dispatch(admission.id)
assert not runs.is_executing(admission.id)
with pytest.raises(KeyError):
runs.get_latest_checkpoint(admission.id)
finally:
ownership.release()
def test_missing_executing_marker_rejects_settlement(tmp_path: Path) -> None:
from tests.artifacts.test_run_store import artifact as _artifact
from tests.artifacts.test_run_store import deployment as _deployment
from wf_api.run_lifecycle import materialize_admitted_view, persist_admission
from wf_artifacts import PinnedRunEnvironment
from wf_core import RunState, RunStatus
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, _, runs = _harness(tmp_path, ownership, script={"*": "hang"})
admission = persist_admission(
store=runs,
run_id=runs.allocate_run_id(),
environment=PinnedRunEnvironment(
deployment=_deployment(),
root_artifact=_artifact(),
child_artifacts=[],
),
resolved_input={},
max_steps=None,
scheduled_at=ts(2026, 9, 8, 12, 0),
schedule_id="a",
schedule_revision=1,
)
materialize_admitted_view(store=runs, admission=admission)
assert not runs.is_pending_dispatch(admission.id)
state = RunState(
workflow_name="sched",
status=RunStatus.COMPLETED,
workflow_input={},
state={},
)
with pytest.raises(BlockedSchedule):
sched.record_stopped_execution(admission.id, state, ts(2026, 9, 8, 12, 0))
assert runs.get_run(admission.id).status.value == "admitted"
with pytest.raises(KeyError):
runs.get_latest_checkpoint(admission.id)
finally:
ownership.release()
def test_contradictory_pending_and_executing_rejects_settlement(
tmp_path: Path,
) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
intended = ts(2026, 9, 8, 12, 0)
_due(sched, store, intended)
sched.poll(intended)
run_id = runs.list_runs()[0].id
assert runs.is_executing(run_id)
# Crash shape between mark_executing and pending-clear: both markers.
runs.mark_pending_dispatch(run_id)
dispatcher = cast(SD, sched.dispatcher)
admission = runs.get_admission(run_id)
with pytest.raises(BlockedSchedule):
sched.record_stopped_execution(
run_id, dispatcher.finish(admission, "complete"), intended
)
assert runs.get_run(run_id).status.value == "admitted"
assert runs.is_executing(run_id)
assert runs.is_pending_dispatch(run_id)
with pytest.raises(KeyError):
runs.get_latest_checkpoint(run_id)
finally:
ownership.release()
def test_genuine_execution_settles_successfully(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
intended = ts(2026, 9, 8, 12, 0)
_due(sched, store, intended)
sched.poll(intended)
run_id = runs.list_runs()[0].id
dispatcher = cast(SD, sched.dispatcher)
admission = runs.get_admission(run_id)
sched.record_stopped_execution(
run_id, dispatcher.finish(admission, "complete"), intended
)
assert runs.get_run(run_id).status.value == "completed"
assert not runs.is_executing(run_id)
finally:
ownership.release()
def test_duplicate_settlement_cannot_overwrite_stopped_result(
tmp_path: Path,
) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
intended = ts(2026, 9, 8, 12, 0)
_due(sched, store, intended)
sched.poll(intended)
run_id = runs.list_runs()[0].id
dispatcher = cast(SD, sched.dispatcher)
admission = runs.get_admission(run_id)
sched.record_stopped_execution(
run_id, dispatcher.finish(admission, "complete"), intended
)
with pytest.raises(BlockedSchedule):
sched.record_stopped_execution(
run_id, dispatcher.finish(admission, "interrupt"), intended
)
assert runs.get_run(run_id).status.value == "completed"
assert len(runs.list_checkpoints(run_id)) == 1
finally:
ownership.release()
def test_settle_without_ownership_rejected(tmp_path: Path) -> None:
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
intended = ts(2026, 9, 8, 12, 0)
_due(sched, store, intended)
sched.poll(intended)
run_id = runs.list_runs()[0].id
dispatcher = cast(SD, sched.dispatcher)
admission = runs.get_admission(run_id)
ownership.release()
with pytest.raises(SecondOwnerError):
sched.record_stopped_execution(
run_id, dispatcher.finish(admission, "complete"), intended
)
assert runs.get_run(run_id).status.value == "admitted"
finally:
ownership.release()
+2 -2
View File
@@ -92,7 +92,7 @@ class FailSaveRunOnce(FileRunStore):
def _recover( def _recover(
sched_store: FileScheduleStore, run_store: FileRunStore, now: datetime sched_store: FileScheduleStore, run_store: FileRunStore, now: datetime
) -> list[str]: ) -> list[str]:
ownership = SchedulerOwnership(sched_store.root, owner="test").acquire() ownership = SchedulerOwnership(sched_store.root.parent, owner="test").acquire()
try: try:
return sched_recovery.recover( return sched_recovery.recover(
schedule_store=sched_store, schedule_store=sched_store,
@@ -373,7 +373,7 @@ def test_real_workflow_interrupt_resume_interrupt_with_torn_resume(
run_store = FailSaveRunOnce(tmp_path / "runs") run_store = FailSaveRunOnce(tmp_path / "runs")
sched_store.create_schedule(_sched_model("a")) sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire() ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try: try:
sched = Scheduler( sched = Scheduler(
schedule_store=sched_store, schedule_store=sched_store,