sched: select latest eligible occurrence <= now; watermarks never regress (F3)
This commit is contained in:
@@ -0,0 +1,256 @@
|
||||
"""Exact-boundary latest catch-up admits one occurrence, not two (R4/F3).
|
||||
|
||||
The scan-cap/latest branch used exclusive ``prev_before(now)`` for an
|
||||
inclusive latest-due query and let admission move the consumed watermark
|
||||
backwards, so a poll exactly at a due instant admitted the previous tick
|
||||
and the next poll admitted the due tick. The fix selects the latest
|
||||
eligible occurrence <= now (one bounded forward probe reusing the adapter)
|
||||
and never moves the consumed watermark backwards.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
from tests.scheduling.controlled import (
|
||||
DictDeployments,
|
||||
ScriptedDispatcher,
|
||||
fixture_environment,
|
||||
)
|
||||
from wf_artifacts.runs.store import FileRunStore
|
||||
from wf_scheduling.calendar import CronSource
|
||||
from wf_scheduling.models import Schedule
|
||||
from wf_scheduling.ownership import SchedulerOwnership
|
||||
from wf_scheduling.poll import SCAN_CAP, Scheduler
|
||||
from wf_scheduling.prepare import SchedulePreparer
|
||||
from wf_scheduling.store import FileScheduleStore
|
||||
|
||||
|
||||
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0, s: int = 0) -> datetime:
|
||||
return datetime(y, mo, d, h, mi, s, tzinfo=UTC)
|
||||
|
||||
|
||||
class CountingCron:
|
||||
"""CronSource wrapper counting bounded-lookup calls."""
|
||||
|
||||
def __init__(self, expression: str, zone: str) -> None:
|
||||
self._source = CronSource(expression, zone)
|
||||
self.next_calls = 0
|
||||
self.prev_calls = 0
|
||||
|
||||
def next_after(self, instant: datetime) -> datetime | None:
|
||||
self.next_calls += 1
|
||||
return self._source.next_after(instant)
|
||||
|
||||
def prev_before(self, instant: datetime) -> datetime | None:
|
||||
self.prev_calls += 1
|
||||
return self._source.prev_before(instant)
|
||||
|
||||
|
||||
def _sched_model(sid: str, **kw: Any) -> Schedule:
|
||||
now = ts(2026, 9, 8, 12, 0)
|
||||
base: dict[str, Any] = {
|
||||
"id": sid,
|
||||
"deployment_id": "dep-1",
|
||||
"trigger": {"kind": "cron", "expression": "* * * * *", "timezone": "UTC"},
|
||||
"input_bindings": [],
|
||||
"created_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
}
|
||||
base.update(kw)
|
||||
return Schedule.model_validate(base)
|
||||
|
||||
|
||||
def _harness(
|
||||
root: Path,
|
||||
ownership: SchedulerOwnership,
|
||||
*,
|
||||
capacity: int = 4,
|
||||
script: dict | None = None,
|
||||
) -> tuple[Scheduler, FileScheduleStore, FileRunStore, dict]:
|
||||
sched_store = FileScheduleStore(root / "sched")
|
||||
run_store = FileRunStore(root / "runs")
|
||||
sources: dict = {}
|
||||
sched = Scheduler(
|
||||
schedule_store=sched_store,
|
||||
run_store=run_store,
|
||||
sources=sources,
|
||||
capacity=capacity,
|
||||
preparer=SchedulePreparer(
|
||||
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
|
||||
fixture_environment,
|
||||
),
|
||||
dispatcher=ScriptedDispatcher(script),
|
||||
ownership=ownership,
|
||||
)
|
||||
return sched, sched_store, run_store, sources
|
||||
|
||||
|
||||
def _admitted(store: FileScheduleStore, sid: str) -> list[dict[str, Any]]:
|
||||
page = store.list_occurrences(sid, limit=100)
|
||||
return [
|
||||
r
|
||||
for r in cast(list[dict[str, Any]], page["occurrences"])
|
||||
if r["kind"] == "admitted"
|
||||
]
|
||||
|
||||
|
||||
def test_exact_boundary_admits_latest_once(tmp_path: Path) -> None:
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs, sources = _harness(
|
||||
tmp_path, ownership, script={"*": "hang"}
|
||||
)
|
||||
now = ts(2026, 9, 8, 12, 0)
|
||||
store.create_schedule(
|
||||
_sched_model("m", misfire="latest", overlap="parallel", max_active_runs=4)
|
||||
)
|
||||
store.save_consumed("m", now - timedelta(days=3))
|
||||
sources["m"] = CronSource("* * * * *", "UTC")
|
||||
sched.poll(now)
|
||||
assert len(runs.list_runs()) == 1
|
||||
admitted = _admitted(store, "m")
|
||||
assert len(admitted) == 1
|
||||
assert datetime.fromisoformat(admitted[0]["resolved_at"]) == now
|
||||
assert store.get_consumed("m") == now
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_second_poll_at_same_instant_admits_nothing(tmp_path: Path) -> None:
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs, sources = _harness(
|
||||
tmp_path, ownership, script={"*": "hang"}
|
||||
)
|
||||
now = ts(2026, 9, 8, 12, 0)
|
||||
store.create_schedule(
|
||||
_sched_model("m", misfire="latest", overlap="parallel", max_active_runs=4)
|
||||
)
|
||||
store.save_consumed("m", now - timedelta(days=3))
|
||||
sources["m"] = CronSource("* * * * *", "UTC")
|
||||
sched.poll(now)
|
||||
sched.poll(now)
|
||||
assert len(runs.list_runs()) == 1
|
||||
assert len(_admitted(store, "m")) == 1
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_between_ticks_admits_latest(tmp_path: Path) -> None:
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs, sources = _harness(
|
||||
tmp_path, ownership, script={"*": "hang"}
|
||||
)
|
||||
now = ts(2026, 9, 8, 12, 0, 30)
|
||||
store.create_schedule(
|
||||
_sched_model("m", misfire="latest", overlap="parallel", max_active_runs=4)
|
||||
)
|
||||
store.save_consumed("m", now - timedelta(days=3))
|
||||
sources["m"] = CronSource("* * * * *", "UTC")
|
||||
sched.poll(now)
|
||||
admitted = _admitted(store, "m")
|
||||
assert len(admitted) == 1
|
||||
assert datetime.fromisoformat(admitted[0]["resolved_at"]) == ts(
|
||||
2026, 9, 8, 12, 0
|
||||
)
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_restart_after_catchup_admits_nothing(tmp_path: Path) -> None:
|
||||
now = ts(2026, 9, 8, 12, 0)
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, _, sources = _harness(tmp_path, ownership, script={"*": "hang"})
|
||||
store.create_schedule(
|
||||
_sched_model("m", misfire="latest", overlap="parallel", max_active_runs=4)
|
||||
)
|
||||
store.save_consumed("m", now - timedelta(days=3))
|
||||
sources["m"] = CronSource("* * * * *", "UTC")
|
||||
sched.poll(now)
|
||||
finally:
|
||||
ownership.release()
|
||||
ownership2 = SchedulerOwnership(tmp_path / "sched", owner="test2").acquire()
|
||||
try:
|
||||
sched2, store2, runs2, sources2 = _harness(
|
||||
tmp_path, ownership2, script={"*": "hang"}
|
||||
)
|
||||
sources2["m"] = CronSource("* * * * *", "UTC")
|
||||
sched2.poll(now)
|
||||
assert len(runs2.list_runs()) == 1
|
||||
assert len(_admitted(store2, "m")) == 1
|
||||
finally:
|
||||
ownership2.release()
|
||||
|
||||
|
||||
def test_long_catchup_stays_bounded(tmp_path: Path) -> None:
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs, sources = _harness(
|
||||
tmp_path, ownership, script={"*": "complete"}
|
||||
)
|
||||
now = ts(2026, 9, 8, 12, 0)
|
||||
store.create_schedule(_sched_model("m", misfire="latest"))
|
||||
store.save_consumed("m", now - timedelta(days=3))
|
||||
src = CountingCron("* * * * *", "UTC")
|
||||
sources["m"] = src
|
||||
sched.poll(now)
|
||||
assert src.next_calls + src.prev_calls <= SCAN_CAP + 2
|
||||
assert len(_admitted(store, "m")) == 1
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_skip_policy_exact_boundary_spans_without_admission(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs, sources = _harness(
|
||||
tmp_path, ownership, script={"*": "hang"}
|
||||
)
|
||||
now = ts(2026, 9, 8, 12, 0)
|
||||
store.create_schedule(_sched_model("s", misfire="skip"))
|
||||
store.save_consumed("s", now - timedelta(days=3))
|
||||
sources["s"] = CronSource("* * * * *", "UTC")
|
||||
sched.poll(now)
|
||||
assert runs.list_runs() == []
|
||||
assert _admitted(store, "s") == []
|
||||
assert store.get_consumed("s") == now
|
||||
sched.poll(now)
|
||||
assert runs.list_runs() == []
|
||||
assert _admitted(store, "s") == []
|
||||
finally:
|
||||
ownership.release()
|
||||
|
||||
|
||||
def test_overlap_skip_exact_boundary_skips_once(tmp_path: Path) -> None:
|
||||
ownership = SchedulerOwnership(tmp_path / "sched", owner="test").acquire()
|
||||
try:
|
||||
sched, store, runs, sources = _harness(
|
||||
tmp_path, ownership, script={"*": "hang"}
|
||||
)
|
||||
now = ts(2026, 9, 8, 12, 0)
|
||||
store.create_schedule(_sched_model("o", misfire="latest", overlap="skip"))
|
||||
store.save_consumed("o", now - timedelta(minutes=1))
|
||||
sources["o"] = CronSource("* * * * *", "UTC")
|
||||
# Poll exactly at the due instant: timely admission of 12:00.
|
||||
sched.poll(now)
|
||||
assert len(runs.list_runs()) == 1
|
||||
# Exactly at the next due instant with the first run still active:
|
||||
# one terminal skip, no candidate resurrection afterwards.
|
||||
sched.poll(now + timedelta(minutes=1))
|
||||
page = store.list_occurrences("o", limit=100)
|
||||
kinds = [r["kind"] for r in cast(list[dict[str, Any]], page["occurrences"])]
|
||||
assert kinds.count("skipped-overlap") == 1
|
||||
assert store.get_candidate("o") is None
|
||||
sched.poll(now + timedelta(minutes=1))
|
||||
page = store.list_occurrences("o", limit=100)
|
||||
kinds = [r["kind"] for r in cast(list[dict[str, Any]], page["occurrences"])]
|
||||
assert kinds.count("skipped-overlap") == 1
|
||||
finally:
|
||||
ownership.release()
|
||||
Reference in New Issue
Block a user