306 lines
11 KiB
Python
306 lines
11 KiB
Python
"""Scheduler poll loop against real file stores (T08)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, cast
|
|
|
|
from tests.scheduling.controlled import (
|
|
DictDeployments,
|
|
ScriptedDispatcher,
|
|
fixture_environment,
|
|
)
|
|
from wf_artifacts.runs.models import StoredRunStatus
|
|
from wf_artifacts.runs.store import FileRunStore
|
|
from wf_scheduling.calendar import OneShotSource
|
|
from wf_scheduling.models import Schedule
|
|
from wf_scheduling.ownership import SchedulerOwnership
|
|
from wf_scheduling.poll import SCAN_CAP, Scheduler
|
|
from wf_scheduling.prepare import SchedulePreparer
|
|
from wf_scheduling.store import FileScheduleStore
|
|
|
|
UTC_TZ = UTC
|
|
|
|
|
|
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0, s: int = 0) -> datetime:
|
|
return datetime(y, mo, d, h, mi, s, tzinfo=UTC_TZ)
|
|
|
|
|
|
class PeriodicSource:
|
|
def __init__(self, period: timedelta, start: datetime) -> None:
|
|
self.period = period
|
|
self.start = start
|
|
self.next_calls = 0
|
|
self.prev_calls = 0
|
|
|
|
def next_after(self, instant: datetime) -> datetime | None:
|
|
self.next_calls += 1
|
|
if instant < self.start:
|
|
return self.start
|
|
n = (instant - self.start) // self.period + 1
|
|
return self.start + n * self.period
|
|
|
|
def prev_before(self, instant: datetime) -> datetime | None:
|
|
self.prev_calls += 1
|
|
if instant <= self.start:
|
|
return None
|
|
n = (instant - self.start - timedelta(microseconds=1)) // self.period
|
|
return self.start + n * self.period
|
|
|
|
|
|
def _sched_model(sid: str, start_hint: str = "cron", **kw: Any) -> Schedule:
|
|
now = ts(2026, 9, 8, 12, 0)
|
|
base: dict[str, Any] = {
|
|
"id": sid,
|
|
"deployment_id": "dep-1",
|
|
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
|
|
"input_bindings": [],
|
|
"created_at": now.isoformat(),
|
|
"updated_at": now.isoformat(),
|
|
}
|
|
base.update(kw)
|
|
return Schedule.model_validate(base)
|
|
|
|
|
|
def _harness(
|
|
tmp_path: Path,
|
|
*,
|
|
capacity: int = 4,
|
|
script: dict | None = None,
|
|
deployments: dict[str, dict] | None = None,
|
|
) -> tuple[Scheduler, FileScheduleStore, FileRunStore, dict]:
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sources: dict = {}
|
|
preparer = SchedulePreparer(
|
|
DictDeployments(
|
|
deployments
|
|
if deployments is not None
|
|
else {"dep-1": {"rev": 1, "required": []}}
|
|
),
|
|
fixture_environment,
|
|
)
|
|
sched = Scheduler(
|
|
schedule_store=sched_store,
|
|
run_store=run_store,
|
|
sources=sources,
|
|
capacity=capacity,
|
|
preparer=preparer,
|
|
dispatcher=ScriptedDispatcher(script),
|
|
ownership=SchedulerOwnership(tmp_path, owner="test").acquire(),
|
|
)
|
|
return sched, sched_store, run_store, sources
|
|
|
|
|
|
def _add(
|
|
sched: Scheduler,
|
|
store: FileScheduleStore,
|
|
sources: dict,
|
|
sid: str,
|
|
src: Any,
|
|
consumed: datetime,
|
|
**kw: Any,
|
|
) -> Any:
|
|
model = _sched_model(sid, **kw)
|
|
store.create_schedule(model)
|
|
store.save_consumed(sid, consumed)
|
|
sources[sid] = src
|
|
return store.get_schedule(sid)
|
|
|
|
|
|
def _history(store: FileScheduleStore, sid: str) -> list[dict]:
|
|
page = store.list_occurrences(sid, limit=100)
|
|
return page["occurrences"] # type: ignore[return-value]
|
|
|
|
|
|
def test_overlap_skip_blocks_and_late_drops() -> None:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
sched, store, runs, sources = _harness(root, capacity=4, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"a",
|
|
PeriodicSource(timedelta(minutes=10), t0),
|
|
t0 - timedelta(minutes=10),
|
|
)
|
|
sched.poll(t0)
|
|
assert len(runs.list_runs()) == 1
|
|
sched.poll(t0 + timedelta(minutes=10))
|
|
kinds = [(r["kind"], r["resolved_at"]) for r in _history(store, "a")]
|
|
assert any(k == "skipped-overlap" and "12:10" in str(v) for k, v in kinds)
|
|
sched.poll(t0 + timedelta(minutes=30))
|
|
kinds = [(r["kind"], r["resolved_at"]) for r in _history(store, "a")]
|
|
assert any(k == "skipped-misfire" for k, _ in kinds)
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_latest_coalesces_to_one_candidate_and_no_double_admit() -> None:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
sched, store, runs, sources = _harness(root, capacity=0)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"h",
|
|
PeriodicSource(timedelta(hours=1), ts(2026, 9, 8, 9, 0)),
|
|
ts(2026, 9, 8, 9, 0),
|
|
misfire="latest",
|
|
)
|
|
sched.poll(ts(2026, 9, 8, 12, 20))
|
|
cand = store.get_candidate("h")
|
|
assert cand is not None and cand.intended_at == ts(2026, 9, 8, 12, 0)
|
|
assert [r for r in _history(store, "h") if r["kind"] == "admitted"] == []
|
|
# Capacity returns at 13:00 while 13:00 is also due: exactly one
|
|
# admission for 13:00, 12:00 superseded, never both.
|
|
sched.capacity = 4
|
|
cast(ScriptedDispatcher, sched.dispatcher).script = {"*": "complete"}
|
|
sched.poll(ts(2026, 9, 8, 13, 0))
|
|
admitted = [r for r in _history(store, "h") if r["kind"] == "admitted"]
|
|
assert len(admitted) == 1
|
|
assert datetime.fromisoformat(admitted[0]["resolved_at"]) == ts(
|
|
2026, 9, 8, 13, 0
|
|
)
|
|
assert store.get_candidate("h") is None
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_parallel_limits_and_interrupted_slots() -> None:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
sched, store, runs, sources = _harness(root, capacity=4, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"p",
|
|
PeriodicSource(timedelta(minutes=5), t0),
|
|
t0 - timedelta(minutes=5),
|
|
overlap="parallel",
|
|
max_active_runs=2,
|
|
)
|
|
sched.poll(t0)
|
|
assert sched._task_load() == 1
|
|
first = runs.list_runs()[0]
|
|
runs.save_run(first.model_copy(update={"status": StoredRunStatus.INTERRUPTED}))
|
|
assert sched._task_load() == 0
|
|
sched.poll(t0 + timedelta(minutes=5))
|
|
assert len(runs.list_runs()) == 2
|
|
sched.poll(t0 + timedelta(minutes=10))
|
|
assert any(r["kind"] == "skipped-overlap" for r in _history(store, "p"))
|
|
# Lowering the limit never cancels; blocks new admission instead.
|
|
sched_model = store.get_schedule("p")
|
|
sched_model.max_active_runs = 1
|
|
store.save_schedule(sched_model)
|
|
second = [r for r in runs.list_runs() if r.id != first.id][0]
|
|
runs.save_run(second.model_copy(update={"status": StoredRunStatus.INTERRUPTED}))
|
|
sched.poll(t0 + timedelta(minutes=15))
|
|
assert any(
|
|
r["kind"] == "skipped-overlap" and "12:15" in str(r["resolved_at"])
|
|
for r in _history(store, "p")
|
|
)
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_pause_is_not_downtime() -> None:
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
sched, store, runs, sources = _harness(
|
|
root, capacity=4, script={"*": "complete"}
|
|
)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"a",
|
|
PeriodicSource(timedelta(hours=1), ts(2026, 9, 8, 9, 0)),
|
|
ts(2026, 9, 8, 9, 0),
|
|
misfire="latest",
|
|
)
|
|
sched.poll(ts(2026, 9, 8, 9, 0))
|
|
model = store.get_schedule("a")
|
|
model.paused = True
|
|
store.save_schedule(model)
|
|
sched.poll(ts(2026, 9, 8, 10, 30))
|
|
assert store.get_candidate("a") is None
|
|
sched.resume_schedule("a", ts(2026, 9, 8, 12, 30))
|
|
sched.poll(ts(2026, 9, 8, 12, 30))
|
|
admitted = [
|
|
r["resolved_at"] for r in _history(store, "a") if r["kind"] == "admitted"
|
|
]
|
|
assert "2026-09-08T10:00:00+00:00" not in admitted
|
|
assert "2026-09-08T11:00:00+00:00" not in admitted
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_long_downtime_is_bounded(tmp_path: Path) -> None:
|
|
sched, store, runs, sources = _harness(
|
|
tmp_path, capacity=4, script={"*": "complete"}
|
|
)
|
|
src = PeriodicSource(timedelta(minutes=1), ts(2023, 9, 8, 12, 0))
|
|
_add(sched, store, sources, "m", src, ts(2023, 9, 8, 12, 0), misfire="latest")
|
|
sched.poll(ts(2026, 9, 8, 12, 0, 0))
|
|
assert src.next_calls + src.prev_calls <= SCAN_CAP + 2
|
|
admitted = [r for r in _history(store, "m") if r["kind"] == "admitted"]
|
|
assert len(admitted) == 1
|
|
# Latest-eligible <= now: a poll exactly at a due instant selects that
|
|
# instant (F3), not its exclusive predecessor.
|
|
assert datetime.fromisoformat(admitted[0]["resolved_at"]) == ts(2026, 9, 8, 12, 0)
|
|
assert (
|
|
len([r for r in _history(store, "m") if r["kind"] == "interval-summary"]) == 1
|
|
)
|
|
# A second poll at the same instant admits nothing more.
|
|
sched.poll(ts(2026, 9, 8, 12, 0, 0))
|
|
assert len([r for r in _history(store, "m") if r["kind"] == "admitted"]) == 1
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_fairness_slow_schedule_not_starved(tmp_path: Path) -> None:
|
|
sched, store, runs, sources = _harness(tmp_path, capacity=1, script={"*": "hang"})
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(
|
|
sched,
|
|
store,
|
|
sources,
|
|
"fast",
|
|
PeriodicSource(timedelta(minutes=1), t0),
|
|
t0 - timedelta(minutes=1),
|
|
)
|
|
_add(sched, store, sources, "slow", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
sched.poll(t0)
|
|
assert any(
|
|
r["kind"] == "admitted" and r["schedule_id"] == "fast"
|
|
for r in _history(store, "fast")
|
|
)
|
|
for run in runs.list_runs():
|
|
runs.save_run(run.model_copy(update={"status": StoredRunStatus.COMPLETED}))
|
|
sched.poll(t0 + timedelta(seconds=30))
|
|
assert any(r["kind"] == "admitted" for r in _history(store, "slow"))
|
|
sched.ownership.release()
|
|
|
|
|
|
def test_capacity_wait_then_expire_for_skip(tmp_path: Path) -> None:
|
|
sched, store, runs, sources = _harness(tmp_path, capacity=0)
|
|
t0 = ts(2026, 9, 8, 12, 0)
|
|
_add(sched, store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1))
|
|
assert sched.poll(t0) == {"a": "admit:held-undecided"}
|
|
assert store.get_candidate("a") is None
|
|
sched.capacity = 1
|
|
cast(ScriptedDispatcher, sched.dispatcher).script = {"*": "complete"}
|
|
sched.poll(t0 + timedelta(seconds=30))
|
|
assert len([r for r in _history(store, "a") if r["kind"] == "admitted"]) == 1
|
|
sched.ownership.release()
|