sched: address R2 fail-closed and fault-proof findings

This commit is contained in:
lda
2026-09-08 10:27:37 +07:00 Verified
parent 3cd61be96d
commit cf8d28f1ff
5 changed files with 113 additions and 30 deletions
+46 -6
View File
@@ -11,6 +11,7 @@ from wf_api.run_lifecycle import (
load_stored_run,
materialize_admitted_view,
persist_admission,
recover_admission_view,
)
from wf_artifacts import (
PinnedRunEnvironment,
@@ -102,8 +103,6 @@ def test_run_ids_are_store_backed_across_restart(tmp_path: Path) -> None:
def test_fault_before_admission_persists_nothing(tmp_path: Path, monkeypatch) -> None:
from wf_artifacts.runs import store as store_module
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
@@ -122,13 +121,13 @@ def test_fault_before_admission_persists_nothing(tmp_path: Path, monkeypatch) ->
schedule_id=None,
schedule_revision=None,
)
assert store_module.FileRunStore(tmp_path).list_admissions() == []
assert FileRunStore(tmp_path).list_admissions() == []
with pytest.raises(KeyError):
store.get_admission(run_id)
def test_fault_between_admission_and_view_recovers_without_dispatch(
tmp_path: Path, monkeypatch
tmp_path: Path,
) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
@@ -147,8 +146,6 @@ def test_fault_between_admission_and_view_recovers_without_dispatch(
with pytest.raises(KeyError):
store.get_run(run_id)
# Recovery materializes the missing view but never executes work.
from wf_api.run_lifecycle import recover_admission_view
record = recover_admission_view(store=store, run_id=run_id)
assert record.status is StoredRunStatus.ADMITTED
assert store.get_run(run_id).id == run_id
@@ -179,3 +176,46 @@ def test_admitted_run_has_no_fabricated_checkpoint_or_output(
# fabricate trace/output/step counts.
with pytest.raises(ValueError, match="admitted"):
load_stored_run(store, run_id)
def test_corrupt_run_id_sequence_fails_closed(tmp_path: Path) -> None:
FileRunStore(tmp_path)
seq_path = tmp_path / "runs" / "_run_id_seq.json"
seq_path.write_text('{"seq": "not-an-int"}', encoding="utf-8")
with pytest.raises(ValueError, match="corrupt run-id sequence"):
FileRunStore(tmp_path).allocate_run_id()
def test_view_without_admission_fails_closed(tmp_path: Path) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
admission = persist_admission(
store=store,
run_id=run_id,
environment=_env(),
resolved_input={},
max_steps=None,
scheduled_at=None,
schedule_id=None,
schedule_revision=None,
)
materialize_admitted_view(store=store, admission=admission)
(tmp_path / "runs" / run_id / "admission.json").unlink()
with pytest.raises(KeyError, match="unknown run admission"):
recover_admission_view(store=store, run_id=run_id)
def test_non_json_resolved_input_rejected_before_dispatch(tmp_path: Path) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
with pytest.raises(ValueError, match="must be finite"):
persist_admission(
store=store,
run_id=run_id,
environment=_env(),
resolved_input={"x": float("inf")}, # type: ignore[dict-item]
max_steps=None,
scheduled_at=None,
schedule_id=None,
schedule_revision=None,
)