sched: add poll loop with overlap, misfire, fairness, and capacity (T08)

2026-09-08 10:32:30 +07:00
parent 7ed759053a
commit 73cb08e5ca
2 changed files with 883 additions and 0 deletions
@@ -0,0 +1,293 @@
+"""Scheduler poll loop against real file stores (T08)."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from wf_artifacts.runs.models import StoredRunStatus
+from wf_artifacts.runs.store import FileRunStore
+from wf_scheduling.models import Schedule
+from wf_scheduling.poll import SCAN_CAP, Scheduler
+from wf_scheduling.store import FileScheduleStore
+
+UTC_TZ = UTC
+
+
+def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0, s: int = 0) -> datetime:
+    return datetime(y, mo, d, h, mi, s, tzinfo=UTC_TZ)
+
+
+class PeriodicSource:
+    def __init__(self, period: timedelta, start: datetime) -> None:
+        self.period = period
+        self.start = start
+        self.next_calls = 0
+        self.prev_calls = 0
+
+    def next_after(self, instant: datetime) -> datetime | None:
+        self.next_calls += 1
+        if instant < self.start:
+            return self.start
+        n = (instant - self.start) // self.period + 1
+        return self.start + n * self.period
+
+    def prev_before(self, instant: datetime) -> datetime | None:
+        self.prev_calls += 1
+        if instant <= self.start:
+            return None
+        n = (instant - self.start - timedelta(microseconds=1)) // self.period
+        return self.start + n * self.period
+
+
+class OneShotSource:
+    def __init__(self, at: datetime) -> None:
+        self.at = at
+        self.next_calls = 0
+        self.prev_calls = 0
+
+    def next_after(self, instant: datetime) -> datetime | None:
+        self.next_calls += 1
+        return self.at if instant < self.at else None
+
+    def prev_before(self, instant: datetime) -> datetime | None:
+        self.prev_calls += 1
+        return self.at if instant > self.at else None
+
+
+def _sched_model(sid: str, start_hint: str = "cron", **kw: Any) -> Schedule:
+    now = ts(2026, 9, 8, 12, 0)
+    base: dict[str, Any] = {
+        "id": sid,
+        "deployment_id": "dep-1",
+        "trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
+        "input_bindings": [],
+        "created_at": now.isoformat(),
+        "updated_at": now.isoformat(),
+    }
+    base.update(kw)
+    return Schedule.model_validate(base)
+
+
+def _harness(
+    tmp_path: Path,
+    *,
+    capacity: int = 4,
+    outcomes: dict[str, str] | None = None,
+    deployments: dict[str, dict] | None = None,
+) -> tuple[Scheduler, FileScheduleStore, FileRunStore, dict]:
+    sched_store = FileScheduleStore(tmp_path / "sched")
+    run_store = FileRunStore(tmp_path / "runs")
+    sources: dict = {}
+    sched = Scheduler(
+        schedule_store=sched_store,
+        run_store=run_store,
+        sources=sources,
+        capacity=capacity,
+        outcomes=outcomes,
+        deployments=deployments
+        if deployments is not None
+        else {"dep-1": {"rev": 1, "required": []}},
+    )
+    return sched, sched_store, run_store, sources
+
+
+def _add(
+    sched: Scheduler,
+    store: FileScheduleStore,
+    sources: dict,
+    sid: str,
+    src: Any,
+    consumed: datetime,
+    **kw: Any,
+) -> Any:
+    model = _sched_model(sid, **kw)
+    store.create_schedule(model)
+    store.save_consumed(sid, consumed)
+    sources[sid] = src
+    return store.get_schedule(sid)
+
+
+def _history(store: FileScheduleStore, sid: str) -> list[dict]:
+    page = store.list_occurrences(sid, limit=100)
+    return page["occurrences"]  # type: ignore[return-value]
+
+
+def test_overlap_skip_blocks_and_late_drops() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        sched, store, runs, sources = _harness(root, capacity=4, outcomes={"*": "hang"})
+        t0 = ts(2026, 9, 8, 12, 0)
+        _add(
+            sched,
+            store,
+            sources,
+            "a",
+            PeriodicSource(timedelta(minutes=10), t0),
+            t0 - timedelta(minutes=10),
+        )
+        sched.poll(t0)
+        assert len(runs.list_runs()) == 1
+        sched.poll(t0 + timedelta(minutes=10))
+        kinds = [(r["kind"], r["resolved_at"]) for r in _history(store, "a")]
+        assert any(k == "skipped-overlap" and "12:10" in str(v) for k, v in kinds)
+        sched.poll(t0 + timedelta(minutes=30))
+        kinds = [(r["kind"], r["resolved_at"]) for r in _history(store, "a")]
+        assert any(k == "skipped-misfire" for k, _ in kinds)
+
+
+def test_latest_coalesces_to_one_candidate_and_no_double_admit() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        sched, store, runs, sources = _harness(root, capacity=0)
+        _add(
+            sched,
+            store,
+            sources,
+            "h",
+            PeriodicSource(timedelta(hours=1), ts(2026, 9, 8, 9, 0)),
+            ts(2026, 9, 8, 9, 0),
+            misfire="latest",
+        )
+        sched.poll(ts(2026, 9, 8, 12, 20))
+        cand = store.get_candidate("h")
+        assert cand is not None and cand.intended_at == ts(2026, 9, 8, 12, 0)
+        assert [r for r in _history(store, "h") if r["kind"] == "admitted"] == []
+        # Capacity returns at 13:00 while 13:00 is also due: exactly one
+        # admission for 13:00, 12:00 superseded, never both.
+        sched.capacity = 4
+        sched.outcomes = {"*": "complete"}
+        sched.poll(ts(2026, 9, 8, 13, 0))
+        admitted = [r for r in _history(store, "h") if r["kind"] == "admitted"]
+        assert len(admitted) == 1
+        assert datetime.fromisoformat(admitted[0]["resolved_at"]) == ts(
+            2026, 9, 8, 13, 0
+        )
+        assert store.get_candidate("h") is None
+
+
+def test_parallel_limits_and_interrupted_slots() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        sched, store, runs, sources = _harness(root, capacity=4, outcomes={"*": "hang"})
+        t0 = ts(2026, 9, 8, 12, 0)
+        _add(
+            sched,
+            store,
+            sources,
+            "p",
+            PeriodicSource(timedelta(minutes=5), t0),
+            t0 - timedelta(minutes=5),
+            overlap="parallel",
+            max_active_runs=2,
+        )
+        sched.poll(t0)
+        assert sched._task_load() == 1
+        first = runs.list_runs()[0]
+        runs.save_run(first.model_copy(update={"status": StoredRunStatus.INTERRUPTED}))
+        assert sched._task_load() == 0
+        sched.poll(t0 + timedelta(minutes=5))
+        assert len(runs.list_runs()) == 2
+        sched.poll(t0 + timedelta(minutes=10))
+        assert any(r["kind"] == "skipped-overlap" for r in _history(store, "p"))
+        # Lowering the limit never cancels; blocks new admission instead.
+        sched_model = store.get_schedule("p")
+        sched_model.max_active_runs = 1
+        store.save_schedule(sched_model)
+        second = [r for r in runs.list_runs() if r.id != first.id][0]
+        runs.save_run(second.model_copy(update={"status": StoredRunStatus.INTERRUPTED}))
+        sched.poll(t0 + timedelta(minutes=15))
+        assert any(
+            r["kind"] == "skipped-overlap" and "12:15" in str(r["resolved_at"])
+            for r in _history(store, "p")
+        )
+
+
+def test_pause_is_not_downtime() -> None:
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        sched, store, runs, sources = _harness(
+            root, capacity=4, outcomes={"*": "complete"}
+        )
+        _add(
+            sched,
+            store,
+            sources,
+            "a",
+            PeriodicSource(timedelta(hours=1), ts(2026, 9, 8, 9, 0)),
+            ts(2026, 9, 8, 9, 0),
+            misfire="latest",
+        )
+        sched.poll(ts(2026, 9, 8, 9, 0))
+        model = store.get_schedule("a")
+        model.paused = True
+        store.save_schedule(model)
+        sched.poll(ts(2026, 9, 8, 10, 30))
+        assert store.get_candidate("a") is None
+        sched.resume_schedule("a", ts(2026, 9, 8, 12, 30))
+        sched.poll(ts(2026, 9, 8, 12, 30))
+        admitted = [
+            r["resolved_at"] for r in _history(store, "a") if r["kind"] == "admitted"
+        ]
+        assert "2026-09-08T10:00:00+00:00" not in admitted
+        assert "2026-09-08T11:00:00+00:00" not in admitted
+
+
+def test_long_downtime_is_bounded(tmp_path: Path) -> None:
+    sched, store, runs, sources = _harness(
+        tmp_path, capacity=4, outcomes={"*": "complete"}
+    )
+    src = PeriodicSource(timedelta(minutes=1), ts(2023, 9, 8, 12, 0))
+    _add(sched, store, sources, "m", src, ts(2023, 9, 8, 12, 0), misfire="latest")
+    sched.poll(ts(2026, 9, 8, 12, 0, 0))
+    assert src.next_calls + src.prev_calls <= SCAN_CAP + 2
+    admitted = [r for r in _history(store, "m") if r["kind"] == "admitted"]
+    assert len(admitted) == 1
+    assert datetime.fromisoformat(admitted[0]["resolved_at"]) == ts(2026, 9, 8, 11, 59)
+    assert (
+        len([r for r in _history(store, "m") if r["kind"] == "interval-summary"]) == 1
+    )
+
+
+def test_fairness_slow_schedule_not_starved(tmp_path: Path) -> None:
+    sched, store, runs, sources = _harness(tmp_path, capacity=1, outcomes={"*": "hang"})
+    t0 = ts(2026, 9, 8, 12, 0)
+    _add(
+        sched,
+        store,
+        sources,
+        "fast",
+        PeriodicSource(timedelta(minutes=1), t0),
+        t0 - timedelta(minutes=1),
+    )
+    _add(sched, store, sources, "slow", OneShotSource(t0), t0 - timedelta(hours=1))
+    sched.poll(t0)
+    assert any(
+        r["kind"] == "admitted" and r["schedule_id"] == "fast"
+        for r in _history(store, "fast")
+    )
+    for run in runs.list_runs():
+        runs.save_run(run.model_copy(update={"status": StoredRunStatus.COMPLETED}))
+    sched.poll(t0 + timedelta(seconds=30))
+    assert any(r["kind"] == "admitted" for r in _history(store, "slow"))
+
+
+def test_capacity_wait_then_expire_for_skip(tmp_path: Path) -> None:
+    sched, store, runs, sources = _harness(tmp_path, capacity=0)
+    t0 = ts(2026, 9, 8, 12, 0)
+    _add(sched, store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1))
+    assert sched.poll(t0) == {"a": "admit:held-undecided"}
+    assert store.get_candidate("a") is None
+    sched.capacity = 1
+    sched.outcomes = {"*": "complete"}
+    sched.poll(t0 + timedelta(seconds=30))
+    assert len([r for r in _history(store, "a") if r["kind"] == "admitted"]) == 1