sched: address R2 fail-closed and fault-proof findings

This commit is contained in:
lda
2026-09-08 10:27:37 +07:00 Verified
parent 3cd61be96d
commit cf8d28f1ff
5 changed files with 113 additions and 30 deletions
+46 -6
View File
@@ -11,6 +11,7 @@ from wf_api.run_lifecycle import (
load_stored_run,
materialize_admitted_view,
persist_admission,
recover_admission_view,
)
from wf_artifacts import (
PinnedRunEnvironment,
@@ -102,8 +103,6 @@ def test_run_ids_are_store_backed_across_restart(tmp_path: Path) -> None:
def test_fault_before_admission_persists_nothing(tmp_path: Path, monkeypatch) -> None:
from wf_artifacts.runs import store as store_module
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
@@ -122,13 +121,13 @@ def test_fault_before_admission_persists_nothing(tmp_path: Path, monkeypatch) ->
schedule_id=None,
schedule_revision=None,
)
assert store_module.FileRunStore(tmp_path).list_admissions() == []
assert FileRunStore(tmp_path).list_admissions() == []
with pytest.raises(KeyError):
store.get_admission(run_id)
def test_fault_between_admission_and_view_recovers_without_dispatch(
tmp_path: Path, monkeypatch
tmp_path: Path,
) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
@@ -147,8 +146,6 @@ def test_fault_between_admission_and_view_recovers_without_dispatch(
with pytest.raises(KeyError):
store.get_run(run_id)
# Recovery materializes the missing view but never executes work.
from wf_api.run_lifecycle import recover_admission_view
record = recover_admission_view(store=store, run_id=run_id)
assert record.status is StoredRunStatus.ADMITTED
assert store.get_run(run_id).id == run_id
@@ -179,3 +176,46 @@ def test_admitted_run_has_no_fabricated_checkpoint_or_output(
# fabricate trace/output/step counts.
with pytest.raises(ValueError, match="admitted"):
load_stored_run(store, run_id)
def test_corrupt_run_id_sequence_fails_closed(tmp_path: Path) -> None:
FileRunStore(tmp_path)
seq_path = tmp_path / "runs" / "_run_id_seq.json"
seq_path.write_text('{"seq": "not-an-int"}', encoding="utf-8")
with pytest.raises(ValueError, match="corrupt run-id sequence"):
FileRunStore(tmp_path).allocate_run_id()
def test_view_without_admission_fails_closed(tmp_path: Path) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
admission = persist_admission(
store=store,
run_id=run_id,
environment=_env(),
resolved_input={},
max_steps=None,
scheduled_at=None,
schedule_id=None,
schedule_revision=None,
)
materialize_admitted_view(store=store, admission=admission)
(tmp_path / "runs" / run_id / "admission.json").unlink()
with pytest.raises(KeyError, match="unknown run admission"):
recover_admission_view(store=store, run_id=run_id)
def test_non_json_resolved_input_rejected_before_dispatch(tmp_path: Path) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
with pytest.raises(ValueError, match="must be finite"):
persist_admission(
store=store,
run_id=run_id,
environment=_env(),
resolved_input={"x": float("inf")}, # type: ignore[dict-item]
max_steps=None,
scheduled_at=None,
schedule_id=None,
schedule_revision=None,
)
+21 -8
View File
@@ -65,15 +65,26 @@ def test_manual_run_persists_admission_before_dispatch(tmp_path: Path) -> None:
def test_fault_before_admission_never_dispatches(tmp_path: Path, monkeypatch) -> None:
import wf_api.runs as runs_module
api, store = _api_with_echo(tmp_path / "fault-before")
real_write = store._write_json
dispatched: list[str] = []
real_plan = runs_module.raw_plan_from_artifact
def _fail_save(admission) -> None: # type: ignore[no-untyped-def]
raise OSError("injected admission failure")
def _fail_on_admission(path: Path, payload: object) -> None:
if path.name == "admission.json":
raise OSError("injected admission failure")
real_write(path, payload)
# Admission persist precedes dispatch in run_deployment ordering: a failure
# here must propagate before any run view exists, so no dispatch could
# have produced a stopped checkpoint.
monkeypatch.setattr(store, "save_admission", _fail_save)
def _spy_plan(artifact): # type: ignore[no-untyped-def]
dispatched.append("dispatch")
return real_plan(artifact)
# Real serialization fault at the admission file: nothing after it
# (plan building, dispatch, stopped persist) may run.
monkeypatch.setattr(store, "_write_json", _fail_on_admission)
monkeypatch.setattr(runs_module, "raw_plan_from_artifact", _spy_plan)
try:
asyncio.run(
api.run_deployment(
@@ -84,8 +95,10 @@ def test_fault_before_admission_never_dispatches(tmp_path: Path, monkeypatch) ->
raise AssertionError("fault must propagate")
except OSError:
pass
assert store.list_admissions() == []
assert store.list_runs() == []
assert dispatched == []
fresh = FileRunStore(store.root)
assert fresh.list_admissions() == []
assert fresh.list_runs() == []
def test_captured_invocation_freezes_input(tmp_path: Path) -> None: