sched: persist executing transition before executor; abandon on crash (F1)
This commit is contained in:
@@ -0,0 +1,396 @@
|
||||
"""Durable dispatch transition: executing is persisted BEFORE the executor (R4/F1).
|
||||
|
||||
Pending means provably undispatched and safe to dispatch. The scheduler
|
||||
persists the undispatched→executing transition before invoking the
|
||||
executor; a crash afterwards abandons the run (failed, no replay) instead
|
||||
of retrying. ``finally`` cannot be trusted across process termination, so
|
||||
marker clearing is explicit on the stopped path only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import multiprocessing as mp
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.scheduling.controlled import (
|
||||
DictDeployments,
|
||||
ScriptedDispatcher,
|
||||
fixture_environment,
|
||||
stopped_state,
|
||||
)
|
||||
from tests.scheduling.controlled import ScriptedDispatcher as SD
|
||||
from wf_artifacts.runs.store import FileRunStore
|
||||
from wf_scheduling import recovery as sched_recovery
|
||||
from wf_scheduling.calendar import OneShotSource
|
||||
from wf_scheduling.dispatch import StillRunning, Stopped
|
||||
from wf_scheduling.models import Schedule
|
||||
from wf_scheduling.ownership import SchedulerOwnership
|
||||
from wf_scheduling.poll import Scheduler
|
||||
from wf_scheduling.prepare import SchedulePreparer
|
||||
from wf_scheduling.store import FileScheduleStore
|
||||
|
||||
|
||||
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime:
|
||||
return datetime(y, mo, d, h, mi, tzinfo=UTC)
|
||||
|
||||
|
||||
def _sched_model(sid: str, **kw: Any) -> Schedule:
|
||||
now = ts(2026, 9, 8, 12, 0)
|
||||
base: dict[str, Any] = {
|
||||
"id": sid,
|
||||
"deployment_id": "dep-1",
|
||||
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
|
||||
"input_bindings": [],
|
||||
"created_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
}
|
||||
base.update(kw)
|
||||
return Schedule.model_validate(base)
|
||||
|
||||
|
||||
def _harness(
|
||||
root: Path,
|
||||
ownership: SchedulerOwnership,
|
||||
*,
|
||||
capacity: int = 4,
|
||||
script: dict | None = None,
|
||||
) -> tuple[Scheduler, FileScheduleStore, FileRunStore]:
|
||||
sched_store = FileScheduleStore(root / "sched")
|
||||
run_store = FileRunStore(root / "runs")
|
||||
sched = Scheduler(
|
||||
schedule_store=sched_store,
|
||||
run_store=run_store,
|
||||
sources={},
|
||||
capacity=capacity,
|
||||
preparer=SchedulePreparer(
|
||||
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
|
||||
fixture_environment,
|
||||
),
|
||||
dispatcher=ScriptedDispatcher(script),
|
||||
ownership=ownership,
|
||||
)
|
||||
return sched, sched_store, run_store
|
||||
|
||||
|
||||
def _due(sched: Scheduler, store: FileScheduleStore, intended: datetime) -> None:
|
||||
store.create_schedule(_sched_model("a"))
|
||||
store.save_consumed("a", intended - timedelta(hours=1))
|
||||
sched.sources["a"] = OneShotSource(intended)
|
||||
|
||||
|
||||
def test_executing_is_set_before_executor_runs(tmp_path: Path) -> None:
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
observed: dict[str, bool] = {}
|
||||
|
||||
def probe(admission: Any, now: datetime) -> Any:
|
||||
observed["executing_at_entry"] = FileRunStore(
|
||||
tmp_path / "runs"
|
||||
).is_executing(admission.id)
|
||||
observed["pending_at_entry"] = FileRunStore(
|
||||
tmp_path / "runs"
|
||||
).is_pending_dispatch(admission.id)
|
||||
return StillRunning()
|
||||
|
||||
sched, store, runs = _harness(tmp_path, ownership, script={"*": probe})
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
_due(sched, store, intended)
|
||||
sched.poll(intended)
|
||||
assert observed == {"executing_at_entry": True, "pending_at_entry": False}
|
||||
run_id = runs.list_runs()[0].id
|
||||
assert runs.is_executing(run_id)
|
||||
assert not runs.is_pending_dispatch(run_id)
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_failure_before_transition_never_dispatches(tmp_path: Path) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
class FailMarkOnce(FileRunStore):
|
||||
def __init__(self, root: Path) -> None:
|
||||
super().__init__(root)
|
||||
self.armed = True
|
||||
|
||||
def mark_executing(self, run_id: str) -> None:
|
||||
if self.armed:
|
||||
self.armed = False
|
||||
raise OSError("injected mark_executing failure")
|
||||
super().mark_executing(run_id)
|
||||
|
||||
def spy(admission: Any, now: datetime) -> Any:
|
||||
calls.append(admission.id)
|
||||
return StillRunning()
|
||||
|
||||
sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
run_store = FailMarkOnce(tmp_path / "runs")
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched = Scheduler(
|
||||
schedule_store=sched_store,
|
||||
run_store=run_store,
|
||||
sources={},
|
||||
capacity=4,
|
||||
preparer=SchedulePreparer(
|
||||
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
|
||||
fixture_environment,
|
||||
),
|
||||
dispatcher=ScriptedDispatcher({"*": spy}),
|
||||
ownership=ownership,
|
||||
)
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
_due(sched, sched_store, intended)
|
||||
with pytest.raises(OSError, match="injected mark_executing"):
|
||||
sched.poll(intended)
|
||||
# The executor never ran; the run stays provably undispatched.
|
||||
assert calls == []
|
||||
run_id = run_store.list_runs()[0].id
|
||||
assert run_store.is_pending_dispatch(run_id)
|
||||
assert not run_store.is_executing(run_id)
|
||||
# Next poll dispatches exactly once.
|
||||
sched.poll(intended + timedelta(minutes=1))
|
||||
assert calls == [run_id]
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_capacity_shortage_keeps_pending_undispatched(tmp_path: Path) -> None:
|
||||
calls: list[str] = []
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs = _harness(tmp_path, ownership, capacity=0)
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
_due(sched, store, intended)
|
||||
|
||||
def spy(admission: Any, now: datetime) -> Any:
|
||||
calls.append(admission.id)
|
||||
return StillRunning()
|
||||
|
||||
cast(SD, sched.dispatcher).script = {"*": spy}
|
||||
assert sched.poll(intended) == {"a": "admit:held-undecided"}
|
||||
assert calls == []
|
||||
# A held candidate is pending-but-unadmitted, not an admitted run.
|
||||
assert runs.list_runs() == []
|
||||
sched.capacity = 4
|
||||
result = sched.poll(intended + timedelta(minutes=1))
|
||||
assert result["a"].startswith("admit:run-")
|
||||
assert len(calls) == 1
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_pending_on_terminal_run_clears_without_redispatch(tmp_path: Path) -> None:
|
||||
calls: list[str] = []
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs = _harness(tmp_path, ownership)
|
||||
|
||||
def spy(admission: Any, now: datetime) -> Any:
|
||||
calls.append(admission.id)
|
||||
return Stopped(result=stopped_state(admission, "complete"))
|
||||
|
||||
cast(SD, sched.dispatcher).script = {"*": spy}
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
_due(sched, store, intended)
|
||||
sched.poll(intended)
|
||||
assert calls != []
|
||||
run_id = runs.list_runs()[0].id
|
||||
assert runs.get_run(run_id).status.value == "completed"
|
||||
# A stale pending marker on a terminal run must not redispatch.
|
||||
runs.mark_pending_dispatch(run_id)
|
||||
before = len(calls)
|
||||
sched.poll(intended + timedelta(minutes=5))
|
||||
assert len(calls) == before
|
||||
assert not runs.is_pending_dispatch(run_id)
|
||||
assert runs.get_run(run_id).status.value == "completed"
|
||||
page = store.list_occurrences("a", limit=100)
|
||||
completed = [
|
||||
r
|
||||
for r in cast(list[dict[str, Any]], page["occurrences"])
|
||||
if r["kind"] == "completed"
|
||||
]
|
||||
assert len(completed) == 1
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_pending_without_admission_fails_closed(tmp_path: Path) -> None:
|
||||
from datetime import datetime as _dt
|
||||
|
||||
from tests.artifacts.test_run_store import artifact as _artifact
|
||||
from tests.artifacts.test_run_store import deployment as _deployment
|
||||
from wf_artifacts import PinnedRunEnvironment
|
||||
from wf_artifacts.runs.models import (
|
||||
ResumeReadiness,
|
||||
StoredRunStatus,
|
||||
WorkflowRunRecord,
|
||||
)
|
||||
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
_due(sched, store, intended)
|
||||
now = _dt.now(UTC)
|
||||
runs.save_run(
|
||||
WorkflowRunRecord(
|
||||
id="run-999009",
|
||||
status=StoredRunStatus.ADMITTED,
|
||||
resume_readiness=ResumeReadiness.NOT_APPLICABLE,
|
||||
environment=PinnedRunEnvironment(
|
||||
deployment=_deployment(),
|
||||
root_artifact=_artifact(),
|
||||
child_artifacts=[],
|
||||
),
|
||||
latest_checkpoint_id=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
runs.mark_pending_dispatch("run-999009")
|
||||
sched.poll(intended)
|
||||
assert runs.get_run("run-999009").status.value == "failed"
|
||||
assert not runs.is_pending_dispatch("run-999009")
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_executing_admitted_is_abandoned_by_recovery(tmp_path: Path) -> None:
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
_due(sched, store, intended)
|
||||
sched.poll(intended)
|
||||
run_id = runs.list_runs()[0].id
|
||||
assert runs.is_executing(run_id)
|
||||
# Fresh objects simulate a restart: executing work is abandoned, never
|
||||
# redispatched.
|
||||
fresh_runs = FileRunStore(tmp_path / "runs")
|
||||
fresh_sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
diags = sched_recovery.recover(
|
||||
schedule_store=fresh_sched_store,
|
||||
run_store=fresh_runs,
|
||||
now=intended + timedelta(minutes=1),
|
||||
ownership=ownership,
|
||||
)
|
||||
assert any("failed-closed" in d for d in diags)
|
||||
assert fresh_runs.get_run(run_id).status.value == "failed"
|
||||
# A second poll after recovery never re-executes the abandoned run.
|
||||
sched2, _, runs2 = _harness(tmp_path, ownership, script={"*": "hang"})
|
||||
sched2.poll(intended + timedelta(minutes=2))
|
||||
assert runs2.get_run(run_id).status.value == "failed"
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_settle_hanging_run_persists_and_clears(tmp_path: Path) -> None:
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs = _harness(tmp_path, ownership, script={"*": "hang"})
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
_due(sched, store, intended)
|
||||
sched.poll(intended)
|
||||
run_id = runs.list_runs()[0].id
|
||||
assert runs.is_executing(run_id)
|
||||
# The late stopped result settles without re-invoking the dispatcher.
|
||||
dispatcher = cast(SD, sched.dispatcher)
|
||||
admission = runs.get_admission(run_id)
|
||||
sched.record_stopped_execution(
|
||||
run_id, dispatcher.finish(admission, "complete"), intended
|
||||
)
|
||||
assert runs.get_run(run_id).status.value == "completed"
|
||||
assert not runs.is_executing(run_id)
|
||||
assert not runs.is_pending_dispatch(run_id)
|
||||
checkpoint = runs.get_latest_checkpoint(run_id)
|
||||
assert checkpoint.reason.value == "completed"
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def _child_poll_and_die(
|
||||
root: str, side_effect: str, intended_iso: str, sched_id: str = "a"
|
||||
) -> None:
|
||||
"""Child entry: poll once; the dispatcher records a side effect then dies."""
|
||||
import os as _os
|
||||
from datetime import datetime as _dt
|
||||
|
||||
from tests.scheduling.controlled import (
|
||||
DictDeployments as _Dict,
|
||||
)
|
||||
from tests.scheduling.controlled import (
|
||||
ScriptedDispatcher as _SD,
|
||||
)
|
||||
from tests.scheduling.controlled import (
|
||||
fixture_environment as _env,
|
||||
)
|
||||
from wf_artifacts.runs.store import FileRunStore as _Runs
|
||||
from wf_scheduling.calendar import OneShotSource as _OneShot
|
||||
from wf_scheduling.ownership import SchedulerOwnership as _Own
|
||||
from wf_scheduling.poll import Scheduler as _Sched
|
||||
from wf_scheduling.prepare import SchedulePreparer as _Prep
|
||||
from wf_scheduling.store import FileScheduleStore as _SchedStore
|
||||
|
||||
base = Path(root)
|
||||
intended = _dt.fromisoformat(intended_iso)
|
||||
|
||||
def killer(admission: Any, now: _dt) -> Any:
|
||||
with open(side_effect, "a", encoding="utf-8") as fh:
|
||||
fh.write(admission.id + "\n")
|
||||
_os._exit(1)
|
||||
|
||||
sched = _Sched(
|
||||
schedule_store=_SchedStore(base / "sched"),
|
||||
run_store=_Runs(base / "runs"),
|
||||
sources={sched_id: _OneShot(intended)},
|
||||
capacity=4,
|
||||
preparer=_Prep(_Dict({"dep-1": {"rev": 1, "required": []}}), _env),
|
||||
dispatcher=_SD({"*": killer}),
|
||||
ownership=_Own(base / "sched", owner="child").acquire(),
|
||||
)
|
||||
sched.poll(intended)
|
||||
|
||||
|
||||
def test_subprocess_death_after_side_effect_dispatches_once(tmp_path: Path) -> None:
|
||||
ctx = mp.get_context("spawn")
|
||||
root = tmp_path / "death"
|
||||
root.mkdir()
|
||||
sched_store = FileScheduleStore(root / "sched")
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
sched_store.save_consumed("a", intended - timedelta(hours=1))
|
||||
side_effect = root / "effects.log"
|
||||
proc = ctx.Process(
|
||||
target=_child_poll_and_die,
|
||||
args=(str(root), str(side_effect), intended.isoformat()),
|
||||
)
|
||||
proc.start()
|
||||
proc.join(120)
|
||||
assert proc.exitcode not in (None, 0), f"child must die, got {proc.exitcode}"
|
||||
assert side_effect.read_text(encoding="utf-8").strip() != ""
|
||||
first_effects = side_effect.read_text(encoding="utf-8").strip().splitlines()
|
||||
|
||||
# Restart with fresh objects: the side effect count stays one and the run
|
||||
# is abandoned, never redispatched.
|
||||
ownership = SchedulerOwnership(root / "sched", owner="parent").acquire()
|
||||
try:
|
||||
sched, store, runs = _harness(root, ownership, script={"*": "hang"})
|
||||
sched.sources["a"] = OneShotSource(intended)
|
||||
diags = sched_recovery.recover(
|
||||
schedule_store=store,
|
||||
run_store=runs,
|
||||
now=intended + timedelta(minutes=1),
|
||||
ownership=ownership,
|
||||
)
|
||||
assert any("failed-closed" in d for d in diags)
|
||||
sched.poll(intended + timedelta(minutes=1))
|
||||
effects = side_effect.read_text(encoding="utf-8").strip().splitlines()
|
||||
assert effects == first_effects
|
||||
assert len(effects) == 1
|
||||
assert runs.get_run(first_effects[0]).status.value == "failed"
|
||||
finally:
|
||||
ownership.release()
|
||||
Reference in New Issue
Block a user