fix: serialize scheduler transitions and coalesce misfires

This commit is contained in:
lda
2026-09-09 19:21:03 +07:00 Verified
parent 787a3c433f
commit 953543c5cb
6 changed files with 242 additions and 8 deletions
+76
View File
@@ -173,6 +173,82 @@ def test_latest_coalesces_to_one_candidate_and_no_double_admit() -> None:
sched.ownership.release()
def test_latest_short_downtime_does_not_burst_admit(tmp_path: Path) -> None:
"""Latest misfire coalesces several bounded missed starts to one run."""
sched, store, runs, sources = _harness(tmp_path, script={"*": "complete"})
start = ts(2026, 9, 8, 9, 0)
now = ts(2026, 9, 8, 12, 0)
_add(
sched,
store,
sources,
"m",
PeriodicSource(timedelta(hours=1), start),
start,
misfire="latest",
)
sched.poll(now)
admitted = [row for row in _history(store, "m") if row["kind"] == "admitted"]
assert len(admitted) == 1
assert datetime.fromisoformat(admitted[0]["resolved_at"]) == now
sched.ownership.release()
def test_schedule_edit_between_poll_snapshot_and_admission_cannot_overwrite_terms(
tmp_path: Path,
) -> None:
"""An edit between the poll snapshot and admission wins authoritatively."""
class EditOnPollRead(FileScheduleStore):
def __init__(self, root: Path) -> None:
super().__init__(root)
self.armed = False
def get_schedule(self, schedule_id: str) -> Schedule:
current = super().get_schedule(schedule_id)
if self.armed and schedule_id == "a":
self.armed = False
edited = current.model_copy(
update={
"revision": current.revision + 1,
"updated_at": current.updated_at + timedelta(seconds=1),
}
)
super().update_schedule(edited, expected_revision=current.revision)
return current
return current
sched_store = EditOnPollRead(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
sources: dict[str, Any] = {}
delegate = SchedulePreparer(
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
fixture_environment,
)
sched = Scheduler(
schedule_store=sched_store,
run_store=run_store,
sources=sources,
capacity=1,
preparer=delegate,
dispatcher=ScriptedDispatcher({"*": "complete"}),
ownership=SchedulerOwnership(tmp_path, owner="test").acquire(),
)
t0 = ts(2026, 9, 8, 12, 0)
_add(sched, sched_store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1))
sched_store.armed = True
try:
result = sched.poll(t0)
assert result["a"] == "admit:schedule-changed"
assert sched_store.get_schedule("a").revision == 2
assert run_store.list_admissions() == []
finally:
sched.ownership.release()
def test_parallel_limits_and_interrupted_slots() -> None:
import tempfile
+41
View File
@@ -110,6 +110,47 @@ def test_recovery_fails_abandoned_admitted_without_replay(tmp_path: Path) -> Non
assert run_store.get_run(admission.id).status.value == "failed"
def test_recovery_fails_corrupt_view_without_admission(tmp_path: Path) -> None:
"""A view with no admission is failed closed instead of left active."""
from tests.artifacts.test_run_store import artifact as _artifact
from tests.artifacts.test_run_store import deployment as _deployment
from wf_artifacts import PinnedRunEnvironment, ResumeReadiness, WorkflowRunRecord
from wf_artifacts.runs.models import StoredRunStatus
sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
sched_store.create_schedule(_sched_model("a"))
now = ts(2026, 9, 8, 12, 0)
run_id = run_store.allocate_run_id()
run_store.save_run(
WorkflowRunRecord(
id=run_id,
status=StoredRunStatus.ADMITTED,
resume_readiness=ResumeReadiness.NOT_APPLICABLE,
environment=PinnedRunEnvironment(
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
),
latest_checkpoint_id=None,
created_at=now,
updated_at=now,
)
)
ownership = SchedulerOwnership(tmp_path, owner="test").acquire()
try:
diags = sched_recovery.recover(
schedule_store=sched_store,
run_store=run_store,
now=now,
ownership=ownership,
)
finally:
ownership.release()
assert any(f"{run_id}:failed-closed" in item for item in diags)
assert run_store.get_run(run_id).status.value == "failed"
def test_recovery_never_executes_pending_until_poll(tmp_path: Path) -> None:
from tests.artifacts.test_run_store import artifact as _artifact