143 lines
5.3 KiB
Python
143 lines
5.3 KiB
Python
"""Startup recovery and reconciliation with real stores (T10)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from tests.scheduling.controlled import (
|
|
DictDeployments,
|
|
ScriptedDispatcher,
|
|
fixture_environment,
|
|
)
|
|
from wf_artifacts.runs.store import FileRunStore
|
|
from wf_scheduling import recovery as sched_recovery
|
|
from wf_scheduling.calendar import OneShotSource
|
|
from wf_scheduling.models import Schedule
|
|
from wf_scheduling.poll import Scheduler
|
|
from wf_scheduling.prepare import SchedulePreparer
|
|
from wf_scheduling.store import FileScheduleStore
|
|
|
|
|
|
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime:
|
|
return datetime(y, mo, d, h, mi, tzinfo=UTC)
|
|
|
|
|
|
def _sched_model(sid: str, **kw: Any) -> Schedule:
|
|
now = ts(2026, 9, 8, 12, 0)
|
|
base: dict[str, Any] = {
|
|
"id": sid,
|
|
"deployment_id": "dep-1",
|
|
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
|
|
"input_bindings": [],
|
|
"created_at": now.isoformat(),
|
|
"updated_at": now.isoformat(),
|
|
}
|
|
base.update(kw)
|
|
return Schedule.model_validate(base)
|
|
|
|
|
|
def test_recovery_materializes_missing_view_as_pending(tmp_path: Path) -> None:
|
|
from tests.artifacts.test_run_store import artifact as _artifact
|
|
from tests.artifacts.test_run_store import deployment as _deployment
|
|
from wf_api.run_lifecycle import persist_admission
|
|
from wf_artifacts import PinnedRunEnvironment
|
|
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
env = PinnedRunEnvironment(
|
|
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
|
|
)
|
|
admission = persist_admission(
|
|
store=run_store,
|
|
run_id=run_store.allocate_run_id(),
|
|
environment=env,
|
|
resolved_input={},
|
|
max_steps=None,
|
|
)
|
|
diags = sched_recovery.recover(
|
|
schedule_store=sched_store, run_store=run_store, now=ts(2026, 9, 8, 12, 0)
|
|
)
|
|
assert any("pending-dispatch" in d for d in diags)
|
|
assert run_store.get_run(admission.id).status.value == "admitted"
|
|
assert sched_recovery._is_pending(run_store, admission.id)
|
|
|
|
|
|
def test_recovery_fails_abandoned_admitted_without_replay(tmp_path: Path) -> None:
|
|
from tests.artifacts.test_run_store import artifact as _artifact
|
|
from tests.artifacts.test_run_store import deployment as _deployment
|
|
from wf_api.run_lifecycle import materialize_admitted_view, persist_admission
|
|
from wf_artifacts import PinnedRunEnvironment
|
|
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
env = PinnedRunEnvironment(
|
|
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
|
|
)
|
|
admission = persist_admission(
|
|
store=run_store,
|
|
run_id=run_store.allocate_run_id(),
|
|
environment=env,
|
|
resolved_input={},
|
|
max_steps=None,
|
|
scheduled_at=ts(2026, 9, 8, 12, 0),
|
|
schedule_id="a",
|
|
schedule_revision=1,
|
|
)
|
|
materialize_admitted_view(store=run_store, admission=admission)
|
|
diags = sched_recovery.recover(
|
|
schedule_store=sched_store, run_store=run_store, now=ts(2026, 9, 8, 12, 0)
|
|
)
|
|
assert any("failed-closed" in d for d in diags)
|
|
assert run_store.get_run(admission.id).status.value == "failed"
|
|
|
|
|
|
def test_recovery_never_executes_pending_until_poll(tmp_path: Path) -> None:
|
|
|
|
from tests.artifacts.test_run_store import artifact as _artifact
|
|
from tests.artifacts.test_run_store import deployment as _deployment
|
|
from wf_api.run_lifecycle import persist_admission
|
|
from wf_artifacts import PinnedRunEnvironment
|
|
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
env = PinnedRunEnvironment(
|
|
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
|
|
)
|
|
admission = persist_admission(
|
|
store=run_store,
|
|
run_id=run_store.allocate_run_id(),
|
|
environment=env,
|
|
resolved_input={},
|
|
max_steps=None,
|
|
scheduled_at=ts(2026, 9, 8, 12, 0),
|
|
schedule_id="a",
|
|
schedule_revision=1,
|
|
)
|
|
diags = sched_recovery.recover(
|
|
schedule_store=sched_store, run_store=run_store, now=ts(2026, 9, 8, 12, 0)
|
|
)
|
|
assert any("pending-dispatch" in d for d in diags)
|
|
# Recovery itself produced no terminal history; the poll sweep dispatches.
|
|
assert sched_store.list_occurrences("a", limit=100)["total"] == 0
|
|
sources = {"a": OneShotSource(ts(2026, 9, 8, 12, 0))}
|
|
sched = Scheduler(
|
|
schedule_store=sched_store,
|
|
run_store=run_store,
|
|
sources=sources, # type: ignore[arg-type]
|
|
capacity=4,
|
|
preparer=SchedulePreparer(
|
|
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
|
|
fixture_environment,
|
|
),
|
|
dispatcher=ScriptedDispatcher({"*": "complete"}),
|
|
)
|
|
sched_store.save_consumed("a", ts(2026, 9, 8, 12, 0))
|
|
sched.poll(ts(2026, 9, 8, 12, 1))
|
|
assert run_store.get_run(admission.id).status.value == "completed"
|
|
assert not sched_recovery._is_pending(run_store, admission.id)
|