chore: tighten scheduling review coverage
This commit is contained in:
@@ -13,6 +13,8 @@ from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.artifacts.test_run_store import artifact as _artifact
|
||||
from tests.artifacts.test_run_store import deployment as _deployment
|
||||
from wf_api.run_lifecycle import (
|
||||
@@ -23,7 +25,7 @@ from wf_api.run_lifecycle import (
|
||||
restore_interrupted_run,
|
||||
)
|
||||
from wf_artifacts import PinnedRunEnvironment
|
||||
from wf_artifacts.runs.models import ResumeAttempt
|
||||
from wf_artifacts.runs.models import ResumeAttempt, StoredRunStatus
|
||||
from wf_artifacts.runs.store import FileRunStore
|
||||
from wf_core import RunState, RunStatus
|
||||
from wf_scheduling import recovery as sched_recovery
|
||||
@@ -128,180 +130,147 @@ def _entries(store: FileScheduleStore, sid: str, kind: str) -> list[dict[str, An
|
||||
]
|
||||
|
||||
|
||||
def test_abandonment_decision_is_stable_across_recovery() -> None:
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
sched_store = FileScheduleStore(root / "sched")
|
||||
run_store = FileRunStore(root / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
_mark_active(run_store, run_id, 9, intended)
|
||||
first = _recover(sched_store, run_store, intended)
|
||||
assert any("failed-closed" in d for d in first)
|
||||
record = run_store.get_run(run_id)
|
||||
assert record.status.value == "failed"
|
||||
assert record.resume_readiness.value == "not_applicable"
|
||||
assert len(record.diagnostics) == 1
|
||||
# Fresh store objects across the restart boundary: everything stable.
|
||||
sched_store2 = FileScheduleStore(root / "sched")
|
||||
run_store2 = FileRunStore(root / "runs")
|
||||
second = _recover(sched_store2, run_store2, intended + timedelta(minutes=1))
|
||||
assert not any(run_id in d for d in second)
|
||||
again = run_store2.get_run(run_id)
|
||||
assert again.status.value == "failed"
|
||||
assert again.resume_readiness.value == "not_applicable"
|
||||
assert len(again.diagnostics) == 1
|
||||
assert _entries(sched_store2, "a", "failed") == _entries(
|
||||
sched_store, "a", "failed"
|
||||
)
|
||||
assert len(_entries(sched_store2, "a", "failed")) == 1
|
||||
def test_abandonment_decision_is_stable_across_recovery(tmp_path: Path) -> None:
|
||||
sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
run_store = FileRunStore(tmp_path / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
_mark_active(run_store, run_id, 9, intended)
|
||||
first = _recover(sched_store, run_store, intended)
|
||||
assert any("failed-closed" in d for d in first)
|
||||
record = run_store.get_run(run_id)
|
||||
assert record.status.value == "failed"
|
||||
assert record.resume_readiness.value == "not_applicable"
|
||||
assert len(record.diagnostics) == 1
|
||||
# Fresh store objects across the restart boundary: everything stable.
|
||||
sched_store2 = FileScheduleStore(tmp_path / "sched")
|
||||
run_store2 = FileRunStore(tmp_path / "runs")
|
||||
second = _recover(sched_store2, run_store2, intended + timedelta(minutes=1))
|
||||
assert not any(run_id in d for d in second)
|
||||
again = run_store2.get_run(run_id)
|
||||
assert again.status.value == "failed"
|
||||
assert again.resume_readiness.value == "not_applicable"
|
||||
assert len(again.diagnostics) == 1
|
||||
assert _entries(sched_store2, "a", "failed") == _entries(sched_store, "a", "failed")
|
||||
assert len(_entries(sched_store2, "a", "failed")) == 1
|
||||
|
||||
|
||||
def test_failed_readiness_and_inspection_agree() -> None:
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
sched_store = FileScheduleStore(root / "sched")
|
||||
run_store = FileRunStore(root / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
_mark_active(run_store, run_id, 9, intended)
|
||||
_recover(sched_store, run_store, intended)
|
||||
record, _ = load_stored_run(run_store, run_id)
|
||||
assert record.resume_readiness.value == "not_applicable"
|
||||
try:
|
||||
restore_interrupted_run(run_store, run_id)
|
||||
raise AssertionError("failed run must not restore as interrupted")
|
||||
except ValueError:
|
||||
pass
|
||||
def test_failed_readiness_and_inspection_agree(tmp_path: Path) -> None:
|
||||
sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
run_store = FileRunStore(tmp_path / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
_mark_active(run_store, run_id, 9, intended)
|
||||
_recover(sched_store, run_store, intended)
|
||||
record, _ = load_stored_run(run_store, run_id)
|
||||
assert record.resume_readiness.value == "not_applicable"
|
||||
with pytest.raises(ValueError):
|
||||
restore_interrupted_run(run_store, run_id)
|
||||
|
||||
|
||||
def test_genuinely_newer_result_repairs_after_decision() -> None:
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
sched_store = FileScheduleStore(root / "sched")
|
||||
run_store = FileRunStore(root / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
_mark_active(run_store, run_id, 9, intended)
|
||||
_recover(sched_store, run_store, intended)
|
||||
assert run_store.get_run(run_id).status.value == "failed"
|
||||
# A genuinely newer stopped result under a new matching attempt: the
|
||||
# newer checkpoint repairs the summary and completes the attempt.
|
||||
_mark_active(run_store, run_id, 10, intended)
|
||||
_stopped(run_store, run_id, RunStatus.INTERRUPTED, attempt_id=10)
|
||||
diags = _recover(sched_store, run_store, intended + timedelta(minutes=1))
|
||||
assert any("fresh-result-resumable" in d for d in diags)
|
||||
record = run_store.get_run(run_id)
|
||||
assert record.status.value == "interrupted"
|
||||
assert record.resume_readiness.value == "ready"
|
||||
assert run_store.get_resume_attempt(run_id).state == "DONE" # type: ignore[union-attr]
|
||||
interrupted = _entries(sched_store, "a", "interrupted")
|
||||
assert {e["checkpoint_id"] for e in interrupted} == {f"{run_id}.000002"}
|
||||
def test_genuinely_newer_result_repairs_after_decision(tmp_path: Path) -> None:
|
||||
sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
run_store = FileRunStore(tmp_path / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
_mark_active(run_store, run_id, 9, intended)
|
||||
_recover(sched_store, run_store, intended)
|
||||
assert run_store.get_run(run_id).status.value == "failed"
|
||||
# A genuinely newer stopped result under a new matching attempt: the
|
||||
# newer checkpoint repairs the summary and completes the attempt.
|
||||
_mark_active(run_store, run_id, 10, intended)
|
||||
_stopped(run_store, run_id, RunStatus.INTERRUPTED, attempt_id=10)
|
||||
diags = _recover(sched_store, run_store, intended + timedelta(minutes=1))
|
||||
assert any("fresh-result-resumable" in d for d in diags)
|
||||
record = run_store.get_run(run_id)
|
||||
assert record.status.value == "interrupted"
|
||||
assert record.resume_readiness.value == "ready"
|
||||
assert run_store.get_resume_attempt(run_id).state == "DONE" # type: ignore[union-attr]
|
||||
interrupted = _entries(sched_store, "a", "interrupted")
|
||||
assert {e["checkpoint_id"] for e in interrupted} == {f"{run_id}.000002"}
|
||||
|
||||
|
||||
def test_completed_mismatch_decision_is_stable() -> None:
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
sched_store = FileScheduleStore(root / "sched")
|
||||
run_store = FileRunStore(root / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = run_store.allocate_run_id()
|
||||
admission = persist_admission(
|
||||
store=run_store,
|
||||
run_id=run_id,
|
||||
environment=_env(),
|
||||
resolved_input={},
|
||||
max_steps=None,
|
||||
scheduled_at=intended,
|
||||
schedule_id="a",
|
||||
schedule_revision=1,
|
||||
)
|
||||
materialize_admitted_view(store=run_store, admission=admission)
|
||||
_stopped(run_store, run_id, RunStatus.COMPLETED, attempt_id=3)
|
||||
_mark_active(run_store, run_id, 5, intended)
|
||||
_recover(sched_store, run_store, intended)
|
||||
assert run_store.get_run(run_id).status.value == "failed"
|
||||
second = _recover(
|
||||
FileScheduleStore(root / "sched"),
|
||||
FileRunStore(root / "runs"),
|
||||
intended + timedelta(minutes=1),
|
||||
)
|
||||
assert not any(run_id in d for d in second)
|
||||
assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1
|
||||
def test_completed_mismatch_decision_is_stable(tmp_path: Path) -> None:
|
||||
sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
run_store = FileRunStore(tmp_path / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = run_store.allocate_run_id()
|
||||
admission = persist_admission(
|
||||
store=run_store,
|
||||
run_id=run_id,
|
||||
environment=_env(),
|
||||
resolved_input={},
|
||||
max_steps=None,
|
||||
scheduled_at=intended,
|
||||
schedule_id="a",
|
||||
schedule_revision=1,
|
||||
)
|
||||
materialize_admitted_view(store=run_store, admission=admission)
|
||||
_stopped(run_store, run_id, RunStatus.COMPLETED, attempt_id=3)
|
||||
_mark_active(run_store, run_id, 5, intended)
|
||||
_recover(sched_store, run_store, intended)
|
||||
assert run_store.get_run(run_id).status.value == "failed"
|
||||
second = _recover(
|
||||
FileScheduleStore(tmp_path / "sched"),
|
||||
FileRunStore(tmp_path / "runs"),
|
||||
intended + timedelta(minutes=1),
|
||||
)
|
||||
assert not any(run_id in d for d in second)
|
||||
assert len(FileRunStore(tmp_path / "runs").get_run(run_id).diagnostics) == 1
|
||||
|
||||
|
||||
def test_legacy_failed_run_gains_history_without_refail() -> None:
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
sched_store = FileScheduleStore(root / "sched")
|
||||
run_store = FileRunStore(root / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
record = run_store.get_run(run_id)
|
||||
from wf_artifacts.runs.models import StoredRunStatus
|
||||
|
||||
run_store.save_run(record.model_copy(update={"status": StoredRunStatus.FAILED}))
|
||||
assert _entries(sched_store, "a", "failed") == []
|
||||
diags = _recover(sched_store, run_store, intended)
|
||||
assert any("terminal-reconciled" in d for d in diags)
|
||||
assert not any("failed-closed" in d for d in diags)
|
||||
assert run_store.get_run(run_id).diagnostics == []
|
||||
assert len(_entries(sched_store, "a", "failed")) == 1
|
||||
def test_legacy_failed_run_gains_history_without_refail(tmp_path: Path) -> None:
|
||||
sched_store = FileScheduleStore(tmp_path / "sched")
|
||||
run_store = FileRunStore(tmp_path / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
record = run_store.get_run(run_id)
|
||||
run_store.save_run(record.model_copy(update={"status": StoredRunStatus.FAILED}))
|
||||
assert _entries(sched_store, "a", "failed") == []
|
||||
diags = _recover(sched_store, run_store, intended)
|
||||
assert any("terminal-reconciled" in d for d in diags)
|
||||
assert not any("failed-closed" in d for d in diags)
|
||||
assert run_store.get_run(run_id).diagnostics == []
|
||||
assert len(_entries(sched_store, "a", "failed")) == 1
|
||||
|
||||
|
||||
def test_crash_between_decision_writes_recovers_history_once() -> None:
|
||||
import tempfile
|
||||
def test_crash_between_decision_writes_recovers_history_once(tmp_path: Path) -> None:
|
||||
class FailFailedHistoryOnce(FileScheduleStore):
|
||||
def __init__(self, root: Path) -> None:
|
||||
super().__init__(root)
|
||||
self.armed = False
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
|
||||
class FailFailedHistoryOnce(FileScheduleStore):
|
||||
def __init__(self, root: Path) -> None:
|
||||
super().__init__(root)
|
||||
def append_history(self, record: Any) -> None:
|
||||
if self.armed and getattr(record, "kind", None) == "failed":
|
||||
self.armed = False
|
||||
raise OSError("injected failed-history failure")
|
||||
super().append_history(record)
|
||||
|
||||
def append_history(self, record: Any) -> None:
|
||||
if self.armed and getattr(record, "kind", None) == "failed":
|
||||
self.armed = False
|
||||
raise OSError("injected failed-history failure")
|
||||
super().append_history(record)
|
||||
sched_store = FailFailedHistoryOnce(tmp_path / "sched")
|
||||
run_store = FileRunStore(tmp_path / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
_mark_active(run_store, run_id, 9, intended)
|
||||
sched_store.armed = True
|
||||
|
||||
sched_store = FailFailedHistoryOnce(root / "sched")
|
||||
run_store = FileRunStore(root / "runs")
|
||||
sched_store.create_schedule(_sched_model("a"))
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
run_id = _admit_interrupted(run_store, intended)
|
||||
_mark_active(run_store, run_id, 9, intended)
|
||||
sched_store.armed = True
|
||||
import pytest
|
||||
|
||||
with pytest.raises(OSError, match="injected failed-history failure"):
|
||||
_recover(sched_store, run_store, intended)
|
||||
record = run_store.get_run(run_id)
|
||||
assert record.status.value == "failed"
|
||||
assert len(record.diagnostics) == 1
|
||||
assert _entries(sched_store, "a", "failed") == []
|
||||
# The decision (status + reason) survived; only history is missing.
|
||||
second = _recover(
|
||||
FileScheduleStore(root / "sched"),
|
||||
FileRunStore(root / "runs"),
|
||||
intended + timedelta(minutes=1),
|
||||
)
|
||||
assert not any("failed-closed" in d for d in second)
|
||||
assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1
|
||||
assert len(_entries(FileScheduleStore(root / "sched"), "a", "failed")) == 1
|
||||
with pytest.raises(OSError, match="injected failed-history failure"):
|
||||
_recover(sched_store, run_store, intended)
|
||||
record = run_store.get_run(run_id)
|
||||
assert record.status.value == "failed"
|
||||
assert len(record.diagnostics) == 1
|
||||
assert _entries(sched_store, "a", "failed") == []
|
||||
# The decision (status + reason) survived; only history is missing.
|
||||
second = _recover(
|
||||
FileScheduleStore(tmp_path / "sched"),
|
||||
FileRunStore(tmp_path / "runs"),
|
||||
intended + timedelta(minutes=1),
|
||||
)
|
||||
assert not any("failed-closed" in d for d in second)
|
||||
assert len(FileRunStore(tmp_path / "runs").get_run(run_id).diagnostics) == 1
|
||||
assert len(_entries(FileScheduleStore(tmp_path / "sched"), "a", "failed")) == 1
|
||||
|
||||
Reference in New Issue
Block a user