sched: address R2 fail-closed and fault-proof findings
This commit is contained in:
@@ -11,6 +11,7 @@ from wf_api.run_lifecycle import (
|
||||
load_stored_run,
|
||||
materialize_admitted_view,
|
||||
persist_admission,
|
||||
recover_admission_view,
|
||||
)
|
||||
from wf_artifacts import (
|
||||
PinnedRunEnvironment,
|
||||
@@ -102,8 +103,6 @@ def test_run_ids_are_store_backed_across_restart(tmp_path: Path) -> None:
|
||||
|
||||
|
||||
def test_fault_before_admission_persists_nothing(tmp_path: Path, monkeypatch) -> None:
|
||||
from wf_artifacts.runs import store as store_module
|
||||
|
||||
store = FileRunStore(tmp_path)
|
||||
run_id = store.allocate_run_id()
|
||||
|
||||
@@ -122,13 +121,13 @@ def test_fault_before_admission_persists_nothing(tmp_path: Path, monkeypatch) ->
|
||||
schedule_id=None,
|
||||
schedule_revision=None,
|
||||
)
|
||||
assert store_module.FileRunStore(tmp_path).list_admissions() == []
|
||||
assert FileRunStore(tmp_path).list_admissions() == []
|
||||
with pytest.raises(KeyError):
|
||||
store.get_admission(run_id)
|
||||
|
||||
|
||||
def test_fault_between_admission_and_view_recovers_without_dispatch(
|
||||
tmp_path: Path, monkeypatch
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
store = FileRunStore(tmp_path)
|
||||
run_id = store.allocate_run_id()
|
||||
@@ -147,8 +146,6 @@ def test_fault_between_admission_and_view_recovers_without_dispatch(
|
||||
with pytest.raises(KeyError):
|
||||
store.get_run(run_id)
|
||||
# Recovery materializes the missing view but never executes work.
|
||||
from wf_api.run_lifecycle import recover_admission_view
|
||||
|
||||
record = recover_admission_view(store=store, run_id=run_id)
|
||||
assert record.status is StoredRunStatus.ADMITTED
|
||||
assert store.get_run(run_id).id == run_id
|
||||
@@ -179,3 +176,46 @@ def test_admitted_run_has_no_fabricated_checkpoint_or_output(
|
||||
# fabricate trace/output/step counts.
|
||||
with pytest.raises(ValueError, match="admitted"):
|
||||
load_stored_run(store, run_id)
|
||||
|
||||
|
||||
def test_corrupt_run_id_sequence_fails_closed(tmp_path: Path) -> None:
|
||||
FileRunStore(tmp_path)
|
||||
seq_path = tmp_path / "runs" / "_run_id_seq.json"
|
||||
seq_path.write_text('{"seq": "not-an-int"}', encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="corrupt run-id sequence"):
|
||||
FileRunStore(tmp_path).allocate_run_id()
|
||||
|
||||
|
||||
def test_view_without_admission_fails_closed(tmp_path: Path) -> None:
|
||||
store = FileRunStore(tmp_path)
|
||||
run_id = store.allocate_run_id()
|
||||
admission = persist_admission(
|
||||
store=store,
|
||||
run_id=run_id,
|
||||
environment=_env(),
|
||||
resolved_input={},
|
||||
max_steps=None,
|
||||
scheduled_at=None,
|
||||
schedule_id=None,
|
||||
schedule_revision=None,
|
||||
)
|
||||
materialize_admitted_view(store=store, admission=admission)
|
||||
(tmp_path / "runs" / run_id / "admission.json").unlink()
|
||||
with pytest.raises(KeyError, match="unknown run admission"):
|
||||
recover_admission_view(store=store, run_id=run_id)
|
||||
|
||||
|
||||
def test_non_json_resolved_input_rejected_before_dispatch(tmp_path: Path) -> None:
|
||||
store = FileRunStore(tmp_path)
|
||||
run_id = store.allocate_run_id()
|
||||
with pytest.raises(ValueError, match="must be finite"):
|
||||
persist_admission(
|
||||
store=store,
|
||||
run_id=run_id,
|
||||
environment=_env(),
|
||||
resolved_input={"x": float("inf")}, # type: ignore[dict-item]
|
||||
max_steps=None,
|
||||
scheduled_at=None,
|
||||
schedule_id=None,
|
||||
schedule_revision=None,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user