sched: address R2 fail-closed and fault-proof findings
This commit is contained in:
@@ -11,6 +11,7 @@ from wf_api.run_lifecycle import (
|
||||
load_stored_run,
|
||||
materialize_admitted_view,
|
||||
persist_admission,
|
||||
recover_admission_view,
|
||||
)
|
||||
from wf_artifacts import (
|
||||
PinnedRunEnvironment,
|
||||
@@ -102,8 +103,6 @@ def test_run_ids_are_store_backed_across_restart(tmp_path: Path) -> None:
|
||||
|
||||
|
||||
def test_fault_before_admission_persists_nothing(tmp_path: Path, monkeypatch) -> None:
|
||||
from wf_artifacts.runs import store as store_module
|
||||
|
||||
store = FileRunStore(tmp_path)
|
||||
run_id = store.allocate_run_id()
|
||||
|
||||
@@ -122,13 +121,13 @@ def test_fault_before_admission_persists_nothing(tmp_path: Path, monkeypatch) ->
|
||||
schedule_id=None,
|
||||
schedule_revision=None,
|
||||
)
|
||||
assert store_module.FileRunStore(tmp_path).list_admissions() == []
|
||||
assert FileRunStore(tmp_path).list_admissions() == []
|
||||
with pytest.raises(KeyError):
|
||||
store.get_admission(run_id)
|
||||
|
||||
|
||||
def test_fault_between_admission_and_view_recovers_without_dispatch(
|
||||
tmp_path: Path, monkeypatch
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
store = FileRunStore(tmp_path)
|
||||
run_id = store.allocate_run_id()
|
||||
@@ -147,8 +146,6 @@ def test_fault_between_admission_and_view_recovers_without_dispatch(
|
||||
with pytest.raises(KeyError):
|
||||
store.get_run(run_id)
|
||||
# Recovery materializes the missing view but never executes work.
|
||||
from wf_api.run_lifecycle import recover_admission_view
|
||||
|
||||
record = recover_admission_view(store=store, run_id=run_id)
|
||||
assert record.status is StoredRunStatus.ADMITTED
|
||||
assert store.get_run(run_id).id == run_id
|
||||
@@ -179,3 +176,46 @@ def test_admitted_run_has_no_fabricated_checkpoint_or_output(
|
||||
# fabricate trace/output/step counts.
|
||||
with pytest.raises(ValueError, match="admitted"):
|
||||
load_stored_run(store, run_id)
|
||||
|
||||
|
||||
def test_corrupt_run_id_sequence_fails_closed(tmp_path: Path) -> None:
|
||||
FileRunStore(tmp_path)
|
||||
seq_path = tmp_path / "runs" / "_run_id_seq.json"
|
||||
seq_path.write_text('{"seq": "not-an-int"}', encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="corrupt run-id sequence"):
|
||||
FileRunStore(tmp_path).allocate_run_id()
|
||||
|
||||
|
||||
def test_view_without_admission_fails_closed(tmp_path: Path) -> None:
|
||||
store = FileRunStore(tmp_path)
|
||||
run_id = store.allocate_run_id()
|
||||
admission = persist_admission(
|
||||
store=store,
|
||||
run_id=run_id,
|
||||
environment=_env(),
|
||||
resolved_input={},
|
||||
max_steps=None,
|
||||
scheduled_at=None,
|
||||
schedule_id=None,
|
||||
schedule_revision=None,
|
||||
)
|
||||
materialize_admitted_view(store=store, admission=admission)
|
||||
(tmp_path / "runs" / run_id / "admission.json").unlink()
|
||||
with pytest.raises(KeyError, match="unknown run admission"):
|
||||
recover_admission_view(store=store, run_id=run_id)
|
||||
|
||||
|
||||
def test_non_json_resolved_input_rejected_before_dispatch(tmp_path: Path) -> None:
|
||||
store = FileRunStore(tmp_path)
|
||||
run_id = store.allocate_run_id()
|
||||
with pytest.raises(ValueError, match="must be finite"):
|
||||
persist_admission(
|
||||
store=store,
|
||||
run_id=run_id,
|
||||
environment=_env(),
|
||||
resolved_input={"x": float("inf")}, # type: ignore[dict-item]
|
||||
max_steps=None,
|
||||
scheduled_at=None,
|
||||
schedule_id=None,
|
||||
schedule_revision=None,
|
||||
)
|
||||
|
||||
@@ -65,15 +65,26 @@ def test_manual_run_persists_admission_before_dispatch(tmp_path: Path) -> None:
|
||||
|
||||
|
||||
def test_fault_before_admission_never_dispatches(tmp_path: Path, monkeypatch) -> None:
|
||||
import wf_api.runs as runs_module
|
||||
|
||||
api, store = _api_with_echo(tmp_path / "fault-before")
|
||||
real_write = store._write_json
|
||||
dispatched: list[str] = []
|
||||
real_plan = runs_module.raw_plan_from_artifact
|
||||
|
||||
def _fail_save(admission) -> None: # type: ignore[no-untyped-def]
|
||||
raise OSError("injected admission failure")
|
||||
def _fail_on_admission(path: Path, payload: object) -> None:
|
||||
if path.name == "admission.json":
|
||||
raise OSError("injected admission failure")
|
||||
real_write(path, payload)
|
||||
|
||||
# Admission persist precedes dispatch in run_deployment ordering: a failure
|
||||
# here must propagate before any run view exists, so no dispatch could
|
||||
# have produced a stopped checkpoint.
|
||||
monkeypatch.setattr(store, "save_admission", _fail_save)
|
||||
def _spy_plan(artifact): # type: ignore[no-untyped-def]
|
||||
dispatched.append("dispatch")
|
||||
return real_plan(artifact)
|
||||
|
||||
# Real serialization fault at the admission file: nothing after it
|
||||
# (plan building, dispatch, stopped persist) may run.
|
||||
monkeypatch.setattr(store, "_write_json", _fail_on_admission)
|
||||
monkeypatch.setattr(runs_module, "raw_plan_from_artifact", _spy_plan)
|
||||
try:
|
||||
asyncio.run(
|
||||
api.run_deployment(
|
||||
@@ -84,8 +95,10 @@ def test_fault_before_admission_never_dispatches(tmp_path: Path, monkeypatch) ->
|
||||
raise AssertionError("fault must propagate")
|
||||
except OSError:
|
||||
pass
|
||||
assert store.list_admissions() == []
|
||||
assert store.list_runs() == []
|
||||
assert dispatched == []
|
||||
fresh = FileRunStore(store.root)
|
||||
assert fresh.list_admissions() == []
|
||||
assert fresh.list_runs() == []
|
||||
|
||||
|
||||
def test_captured_invocation_freezes_input(tmp_path: Path) -> None:
|
||||
|
||||
Reference in New Issue
Block a user