552 lines
21 KiB
Python
552 lines
21 KiB
Python
"""Checkpoint authority: coherence and ordering before recovery trusts state.
|
|
|
|
A checkpoint becomes recovery authority only when it proves coherence —
|
|
same run identity, checkpoint id matching its sequence, runtime state
|
|
decodable through core validation, and the outer reason agreeing with the
|
|
decoded stopped state. The outer reason enum alone is never trusted.
|
|
|
|
A durable recovery decision is superseded only by a validated genuinely
|
|
newer result (referenced checkpoint present, strictly greater sequence,
|
|
attempt provenance, coherent content). Missing referenced state never
|
|
rolls a run backward.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, cast
|
|
|
|
import pytest
|
|
|
|
from tests.artifacts.test_run_store import artifact as _artifact
|
|
from tests.artifacts.test_run_store import deployment as _deployment
|
|
from tests.scheduling.controlled import (
|
|
DictDeployments,
|
|
ScriptedDispatcher,
|
|
fixture_environment,
|
|
)
|
|
from wf_api.run_lifecycle import (
|
|
load_stored_run,
|
|
materialize_admitted_view,
|
|
persist_admission,
|
|
persist_stopped_run,
|
|
restore_interrupted_run,
|
|
)
|
|
from wf_artifacts import CheckpointReason, PinnedRunEnvironment
|
|
from wf_artifacts.runs.models import ResumeAttempt, RunCheckpoint
|
|
from wf_artifacts.runs.store import FileRunStore
|
|
from wf_core import PersistedRunState, RunState, RunStatus, dump_run_state
|
|
from wf_scheduling import recovery as sched_recovery
|
|
from wf_scheduling.calendar import OneShotSource
|
|
from wf_scheduling.models import Schedule
|
|
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
|
|
from wf_scheduling.poll import Scheduler
|
|
from wf_scheduling.prepare import SchedulePreparer
|
|
from wf_scheduling.store import FileScheduleStore
|
|
|
|
|
|
def ts(y: int, mo: int, d: int, h: int = 0, mi: int = 0) -> datetime:
|
|
return datetime(y, mo, d, h, mi, tzinfo=UTC)
|
|
|
|
|
|
def _sched_model(sid: str) -> Schedule:
|
|
now = ts(2026, 9, 8, 12, 0)
|
|
return Schedule.model_validate(
|
|
{
|
|
"id": sid,
|
|
"deployment_id": "dep-1",
|
|
"trigger": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"},
|
|
"input_bindings": [],
|
|
"created_at": now.isoformat(),
|
|
"updated_at": now.isoformat(),
|
|
}
|
|
)
|
|
|
|
|
|
def _env() -> PinnedRunEnvironment:
|
|
return PinnedRunEnvironment(
|
|
deployment=_deployment(), root_artifact=_artifact(), child_artifacts=[]
|
|
)
|
|
|
|
|
|
def _recover(
|
|
sched_store: FileScheduleStore, run_store: FileRunStore, now: datetime
|
|
) -> list[str]:
|
|
ownership = SchedulerOwnership(sched_store.root.parent, owner="test").acquire()
|
|
try:
|
|
return sched_recovery.recover(
|
|
schedule_store=sched_store,
|
|
run_store=run_store,
|
|
now=now,
|
|
ownership=ownership,
|
|
)
|
|
finally:
|
|
ownership.release()
|
|
|
|
|
|
def _admit_interrupted(
|
|
run_store: FileRunStore, intended: datetime, *, attempt_id: int | None = None
|
|
) -> str:
|
|
run_id = run_store.allocate_run_id()
|
|
admission = persist_admission(
|
|
store=run_store,
|
|
run_id=run_id,
|
|
environment=_env(),
|
|
resolved_input={},
|
|
max_steps=None,
|
|
scheduled_at=intended,
|
|
schedule_id="a",
|
|
schedule_revision=1,
|
|
)
|
|
materialize_admitted_view(store=run_store, admission=admission)
|
|
record = run_store.get_run(run_id)
|
|
persist_stopped_run(
|
|
store=run_store,
|
|
environment=record.environment,
|
|
run=RunState(
|
|
workflow_name="sched",
|
|
status=RunStatus.INTERRUPTED,
|
|
workflow_input={},
|
|
state={},
|
|
),
|
|
run_id=run_id,
|
|
attempt_id=attempt_id,
|
|
)
|
|
return run_id
|
|
|
|
|
|
def _mark_attempt(
|
|
run_store: FileRunStore, run_id: str, attempt_id: int, state: str, now: datetime
|
|
) -> None:
|
|
run_store.save_resume_attempt(
|
|
ResumeAttempt(
|
|
run_id=run_id,
|
|
attempt_id=attempt_id,
|
|
state=cast(Any, state),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
)
|
|
|
|
|
|
def _entries(store: FileScheduleStore, sid: str, kind: str) -> list[dict[str, Any]]:
|
|
page = store.list_occurrences(sid, limit=100)
|
|
return [
|
|
r for r in cast(list[dict[str, Any]], page["occurrences"]) if r["kind"] == kind
|
|
]
|
|
|
|
|
|
def _craft_checkpoint(
|
|
run_id: str,
|
|
sequence: int,
|
|
status: RunStatus,
|
|
*,
|
|
checkpoint_run_id: str | None = None,
|
|
checkpoint_id: str | None = None,
|
|
attempt_id: int | None = None,
|
|
now: datetime | None = None,
|
|
) -> RunCheckpoint:
|
|
"""Build a storable checkpoint with caller-controlled identity fields."""
|
|
return RunCheckpoint(
|
|
id=checkpoint_id or f"{run_id}.{sequence:06d}",
|
|
run_id=checkpoint_run_id or run_id,
|
|
sequence=sequence,
|
|
reason=CheckpointReason(status.value),
|
|
state=PersistedRunState.model_validate(
|
|
dump_run_state(
|
|
RunState(
|
|
workflow_name="sched",
|
|
status=status,
|
|
workflow_input={},
|
|
state={},
|
|
)
|
|
)
|
|
),
|
|
attempt_id=attempt_id,
|
|
created_at=now or datetime.now(UTC),
|
|
)
|
|
|
|
|
|
def _checkpoint_path(root: Path, run_id: str, sequence: int) -> Path:
|
|
return root / "runs" / "runs" / run_id / "checkpoints" / f"{sequence:06d}.json"
|
|
|
|
|
|
def _tamper_reason(root: Path, run_id: str, sequence: int, reason: str) -> None:
|
|
path = _checkpoint_path(root, run_id, sequence)
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["reason"] = reason
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
|
|
|
|
def _decided_at_000002(
|
|
tmp_path: Path, intended: datetime
|
|
) -> tuple[FileScheduleStore, FileRunStore, str]:
|
|
"""Failed decision pointing at .000002 with only .000001 older state.
|
|
|
|
Genuine multi-restart sequence: ambiguous fail at .000001, a newer
|
|
stopped result under attempt 10, then stale under attempt 11. The
|
|
second durable abandonment points at .000002; attempt 11 is DONE so
|
|
no attempt matching can mask the ordering decision.
|
|
"""
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
run_id = _admit_interrupted(run_store, intended)
|
|
_mark_attempt(run_store, run_id, 9, "ACTIVE", intended)
|
|
first = _recover(sched_store, run_store, intended)
|
|
assert any("failed-closed" in d for d in first)
|
|
_mark_attempt(run_store, run_id, 10, "ACTIVE", intended)
|
|
record = run_store.get_run(run_id)
|
|
persist_stopped_run(
|
|
store=run_store,
|
|
environment=record.environment,
|
|
run=RunState(
|
|
workflow_name="sched",
|
|
status=RunStatus.INTERRUPTED,
|
|
workflow_input={},
|
|
state={},
|
|
),
|
|
run_id=run_id,
|
|
attempt_id=10,
|
|
)
|
|
_mark_attempt(run_store, run_id, 11, "ACTIVE", intended)
|
|
second = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=1),
|
|
)
|
|
assert any("failed-closed" in d for d in second)
|
|
decided = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert decided.status.value == "failed"
|
|
assert decided.latest_checkpoint_id == f"{run_id}.000002"
|
|
assert decided.resume_readiness.value == "not_applicable"
|
|
_mark_attempt(FileRunStore(tmp_path / "runs"), run_id, 11, "DONE", intended)
|
|
return (
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
run_id,
|
|
)
|
|
|
|
|
|
def test_reason_state_disagreement_fails_closed(tmp_path: Path) -> None:
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
intended = ts(2026, 9, 8, 12, 0)
|
|
run_id = _admit_interrupted(run_store, intended)
|
|
_tamper_reason(tmp_path, run_id, 1, "completed")
|
|
diags = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=1),
|
|
)
|
|
assert any("failed-closed" in d for d in diags)
|
|
record = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert record.status.value == "failed"
|
|
assert record.resume_readiness.value == "not_applicable"
|
|
assert any("contradicts its runtime state" in d.message for d in record.diagnostics)
|
|
# Nothing successful fabricated: no completed/interrupted terminal entries.
|
|
fresh_sched = FileScheduleStore(tmp_path / "sched")
|
|
assert _entries(fresh_sched, "a", "completed") == []
|
|
assert _entries(fresh_sched, "a", "interrupted") == []
|
|
assert len(_entries(fresh_sched, "a", "failed")) == 1
|
|
# Repeat recovery is stable: the decision stands, nothing new appended.
|
|
rerun = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=2),
|
|
)
|
|
assert not any(run_id in d for d in rerun)
|
|
again = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert again.status.value == "failed"
|
|
assert len(again.diagnostics) == len(record.diagnostics)
|
|
|
|
|
|
def test_run_identity_disagreement_fails_closed(tmp_path: Path) -> None:
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
intended = ts(2026, 9, 8, 12, 0)
|
|
run_id = _admit_interrupted(run_store, intended)
|
|
foreign = _craft_checkpoint(
|
|
run_id, 2, RunStatus.INTERRUPTED, checkpoint_run_id="run-999999"
|
|
)
|
|
_checkpoint_path(tmp_path, run_id, 2).write_text(
|
|
json.dumps(foreign.model_dump(mode="json")), encoding="utf-8"
|
|
)
|
|
diags = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=1),
|
|
)
|
|
assert any("failed-closed" in d for d in diags)
|
|
record = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert record.status.value == "failed"
|
|
assert any("names run" in d.message for d in record.diagnostics)
|
|
|
|
|
|
def test_sequence_pointer_incoherence_fails_closed(tmp_path: Path) -> None:
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
intended = ts(2026, 9, 8, 12, 0)
|
|
run_id = _admit_interrupted(run_store, intended)
|
|
mismatched = _craft_checkpoint(
|
|
run_id, 2, RunStatus.INTERRUPTED, checkpoint_id=f"{run_id}.000001"
|
|
)
|
|
_checkpoint_path(tmp_path, run_id, 2).write_text(
|
|
json.dumps(mismatched.model_dump(mode="json")), encoding="utf-8"
|
|
)
|
|
diags = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=1),
|
|
)
|
|
assert any("failed-closed" in d for d in diags)
|
|
record = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert record.status.value == "failed"
|
|
assert any("disagrees with its sequence" in d.message for d in record.diagnostics)
|
|
|
|
|
|
def test_undecodable_runtime_state_fails_closed(tmp_path: Path) -> None:
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
intended = ts(2026, 9, 8, 12, 0)
|
|
run_id = _admit_interrupted(run_store, intended)
|
|
bogus = _craft_checkpoint(run_id, 2, RunStatus.INTERRUPTED)
|
|
payload = bogus.model_dump(mode="json")
|
|
payload["state"] = {"version": 2, "state": {"bogus": True}}
|
|
_checkpoint_path(tmp_path, run_id, 2).write_text(
|
|
json.dumps(payload), encoding="utf-8"
|
|
)
|
|
diags = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=1),
|
|
)
|
|
assert any("failed-closed" in d for d in diags)
|
|
record = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert record.status.value == "failed"
|
|
assert record.resume_readiness.value == "not_applicable"
|
|
assert any("undecodable runtime state" in d.message for d in record.diagnostics)
|
|
|
|
|
|
def test_genuine_newer_completed_repairs_torn_summary(tmp_path: Path) -> None:
|
|
"""Coherent newer checkpoints still repair torn summaries (no over-block)."""
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
intended = ts(2026, 9, 8, 12, 0)
|
|
run_id = _admit_interrupted(run_store, intended)
|
|
# Torn boundary: the completed checkpoint persisted, the summary did not.
|
|
run_store.save_checkpoint(_craft_checkpoint(run_id, 2, RunStatus.COMPLETED))
|
|
diags = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=1),
|
|
)
|
|
assert any("summary-reconciled" in d for d in diags)
|
|
record = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert record.status.value == "completed"
|
|
assert record.latest_checkpoint_id == f"{run_id}.000002"
|
|
fresh_sched = FileScheduleStore(tmp_path / "sched")
|
|
assert {e["checkpoint_id"] for e in _entries(fresh_sched, "a", "completed")} == {
|
|
f"{run_id}.000002"
|
|
}
|
|
|
|
|
|
def test_missing_referenced_checkpoint_keeps_decision(tmp_path: Path) -> None:
|
|
sched_store, run_store, run_id = _decided_at_000002(tmp_path, ts(2026, 9, 8, 12, 0))
|
|
before = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert len(before.diagnostics) == 1
|
|
# Referenced durable state goes missing; only older .000001 remains.
|
|
_checkpoint_path(tmp_path, run_id, 2).unlink()
|
|
diags = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
ts(2026, 9, 8, 12, 2),
|
|
)
|
|
assert any("decision-kept-noted" in d for d in diags)
|
|
assert not any("summary-reconciled" in d for d in diags)
|
|
kept = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert kept.status.value == "failed"
|
|
assert kept.resume_readiness.value == "not_applicable"
|
|
assert len(kept.diagnostics) == 2
|
|
assert any("is missing" in d.message for d in kept.diagnostics)
|
|
# Both genuine decisions keep their terminal entries; the keep adds none.
|
|
assert len(_entries(FileScheduleStore(tmp_path / "sched"), "a", "failed")) == 2
|
|
# Repeat recovery is fully silent and history stays idempotent.
|
|
rerun = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
ts(2026, 9, 8, 12, 3),
|
|
)
|
|
assert not any(run_id in d for d in rerun)
|
|
again = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert again.status.value == "failed"
|
|
assert len(again.diagnostics) == 2
|
|
assert _entries(FileScheduleStore(tmp_path / "sched"), "a", "failed") == _entries(
|
|
sched_store, "a", "failed"
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("newer_attempt", "tamper", "expect_repair"),
|
|
[
|
|
(11, False, True), # newer sequence + newer attempt: genuine repair
|
|
(9, False, False), # newer sequence + older attempt: stale, keep decision
|
|
(None, False, False), # newer sequence + unattributed: keep decision
|
|
(11, True, False), # newer + provenance but incoherent: keep + note
|
|
],
|
|
)
|
|
def test_attempt_provenance_combinations(
|
|
tmp_path: Path, newer_attempt: int | None, tamper: bool, expect_repair: bool
|
|
) -> None:
|
|
intended = ts(2026, 9, 8, 12, 0)
|
|
_, _, run_id = _decided_at_000002(tmp_path, intended)
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
_mark_attempt(run_store, run_id, 11, "ACTIVE", intended)
|
|
run_store.save_checkpoint(
|
|
_craft_checkpoint(run_id, 3, RunStatus.INTERRUPTED, attempt_id=newer_attempt)
|
|
)
|
|
if tamper:
|
|
_tamper_reason(tmp_path, run_id, 3, "completed")
|
|
diags = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=5),
|
|
)
|
|
record = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
if expect_repair:
|
|
assert any("fresh-result-resumable" in d for d in diags)
|
|
assert record.status.value == "interrupted"
|
|
assert record.resume_readiness.value == "ready"
|
|
assert run_store.get_resume_attempt(run_id) is not None
|
|
assert run_store.get_resume_attempt(run_id).state == "DONE" # type: ignore[union-attr]
|
|
else:
|
|
assert record.status.value == "failed"
|
|
assert record.resume_readiness.value == "not_applicable"
|
|
assert not any("summary-reconciled" in d for d in diags)
|
|
if tamper:
|
|
assert any("decision-kept-noted" in d for d in diags)
|
|
assert any("is incoherent" in d.message for d in record.diagnostics)
|
|
else:
|
|
# Stale provenance is expected debris, not corruption: silent keep.
|
|
assert not any("decision-kept-noted" in d for d in diags)
|
|
rerun = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=6),
|
|
)
|
|
assert not any(run_id in d for d in rerun)
|
|
|
|
|
|
def test_pointerless_decision_ignores_stray_checkpoints(tmp_path: Path) -> None:
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
run_store = FileRunStore(tmp_path / "runs")
|
|
sched_store.create_schedule(_sched_model("a"))
|
|
intended = ts(2026, 9, 8, 12, 0)
|
|
run_id = run_store.allocate_run_id()
|
|
admission = persist_admission(
|
|
store=run_store,
|
|
run_id=run_id,
|
|
environment=_env(),
|
|
resolved_input={},
|
|
max_steps=None,
|
|
scheduled_at=intended,
|
|
schedule_id="a",
|
|
schedule_revision=1,
|
|
)
|
|
materialize_admitted_view(store=run_store, admission=admission)
|
|
run_store.mark_executing(run_id)
|
|
diags = _recover(sched_store, run_store, intended)
|
|
assert any("failed-closed" in d for d in diags)
|
|
decided = run_store.get_run(run_id)
|
|
assert decided.status.value == "failed"
|
|
assert decided.latest_checkpoint_id is None
|
|
# A stray checkpoint with no ordering basis cannot reopen the decision.
|
|
run_store.save_checkpoint(_craft_checkpoint(run_id, 1, RunStatus.INTERRUPTED))
|
|
rerun = _recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=1),
|
|
)
|
|
assert not any("summary-reconciled" in d for d in rerun)
|
|
kept = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert kept.status.value == "failed"
|
|
assert kept.resume_readiness.value == "not_applicable"
|
|
|
|
|
|
def test_inspection_resume_and_poll_agree_after_reconciliation(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
intended = ts(2026, 9, 8, 12, 0)
|
|
_, _, run_id = _decided_at_000002(tmp_path, intended)
|
|
_checkpoint_path(tmp_path, run_id, 2).unlink()
|
|
_recover(
|
|
FileScheduleStore(tmp_path / "sched"),
|
|
FileRunStore(tmp_path / "runs"),
|
|
intended + timedelta(minutes=2),
|
|
)
|
|
# Inspection agrees: failed, not applicable.
|
|
record, _ = load_stored_run(FileRunStore(tmp_path / "runs"), run_id)
|
|
assert record.status.value == "failed"
|
|
assert record.resume_readiness.value == "not_applicable"
|
|
# Resume refuses.
|
|
try:
|
|
restore_interrupted_run(FileRunStore(tmp_path / "runs"), run_id)
|
|
raise AssertionError("failed run must not restore as interrupted")
|
|
except ValueError:
|
|
pass
|
|
# A poll sweep with nothing due leaves the decided run untouched and
|
|
# dispatches nothing from its slot.
|
|
sched_store = FileScheduleStore(tmp_path / "sched")
|
|
sched_store.save_consumed("a", intended)
|
|
ownership = SchedulerOwnership(tmp_path, owner="poll").acquire()
|
|
try:
|
|
sched = Scheduler(
|
|
schedule_store=sched_store,
|
|
run_store=FileRunStore(tmp_path / "runs"),
|
|
sources={"a": OneShotSource(intended)},
|
|
capacity=4,
|
|
preparer=SchedulePreparer(
|
|
DictDeployments({"dep-1": {"rev": 1, "required": []}}),
|
|
fixture_environment,
|
|
),
|
|
dispatcher=ScriptedDispatcher({"*": "hang"}),
|
|
ownership=ownership,
|
|
)
|
|
sched.poll(intended)
|
|
finally:
|
|
ownership.release()
|
|
after = FileRunStore(tmp_path / "runs").get_run(run_id)
|
|
assert after.status.value == "failed"
|
|
assert [d.message for d in after.diagnostics] == [
|
|
d.message for d in record.diagnostics
|
|
]
|
|
assert [r.id for r in FileRunStore(tmp_path / "runs").list_runs()] == [run_id]
|
|
|
|
|
|
def test_wrong_root_rejected_before_writes_or_dispatch(tmp_path: Path) -> None:
|
|
comp = tmp_path / "composition"
|
|
sched_store = FileScheduleStore(comp / "sched")
|
|
run_store = FileRunStore(comp / "runs")
|
|
outsider = SchedulerOwnership(tmp_path, owner="outsider").acquire()
|
|
try:
|
|
assert outsider.held
|
|
assert outsider.covers(sched_store.root, run_store.root) is False
|
|
with pytest.raises(SecondOwnerError):
|
|
sched_recovery.recover(
|
|
schedule_store=sched_store,
|
|
run_store=run_store,
|
|
now=datetime.now(UTC),
|
|
ownership=outsider,
|
|
)
|
|
assert run_store.list_runs() == []
|
|
assert run_store.list_admissions() == []
|
|
finally:
|
|
outsider.release()
|