fix: harden run step budget checkpoint validation

This commit is contained in:
lda
2026-09-05 20:25:11 +07:00 Verified
parent 57b53eb3f6
commit 344902c17e
2 changed files with 196 additions and 15 deletions
+76 -15
View File
@@ -1,7 +1,7 @@
from __future__ import annotations from __future__ import annotations
from copy import deepcopy from copy import deepcopy
from typing import Any, Literal from typing import Any, Literal, cast
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@@ -37,45 +37,91 @@ def dump_run_state(run: RunState) -> dict[str, object]:
).model_dump(mode="json") ).model_dump(mode="json")
def _is_strict_int(value: object) -> bool:
"""Return True only for actual ints, excluding bools.
Pydantic's lax mode coerces ``True``/``"10"`` to ``1``/``10`` before
``RunLimits.__post_init__`` runs, which would defeat the positive
non-boolean integer contract. Raw-envelope checks must therefore use
exact-type comparison instead of ``isinstance``.
"""
return type(value) is int
def _check_step_number(value: object, *, steps_executed: int, what: str) -> None:
"""Reject corrupt step numbers that are not None or coherent history.
``None`` survives for unadmitted pending frames and upgraded pre-budget
history. Any assigned number is one-based and can never exceed the
authoritative ``steps_executed`` counter.
"""
if value is None:
return
if not _is_strict_int(value) or not 1 <= value <= steps_executed: # type: ignore[operator]
raise ValueError(
"invalid persisted workflow run state: "
f"{what} has incoherent step number {value!r}"
)
def _inject_v1_budget_defaults(state: dict[str, Any]) -> dict[str, Any]: def _inject_v1_budget_defaults(state: dict[str, Any]) -> dict[str, Any]:
"""Copy a v1 state dict with the one-time step budget defaults applied. """Copy a v1 state dict with the one-time step budget defaults applied.
Version-1 envelopes predate step budgets, so they receive the default Version-1 envelopes predate step budgets, so they receive the default
limit, a zeroed counter, and an unassigned number per frame exactly once limit, a zeroed counter, and an unassigned number per frame exactly once
at load time. Pre-budget trace entries and any outstanding interrupt keep at load time. Assignment overwrites rather than preserves: any budget
an unassigned (``None``) number: attempts made before the upgrade are fields present in a v1 envelope are smuggled (v1 writers never emitted
outside the new budget. them) and must not survive the upgrade. Pre-budget trace entries and any
outstanding interrupt keep an unassigned (``None``) number: attempts made
before the upgrade are outside the new budget.
""" """
upgraded = deepcopy(state) upgraded = deepcopy(state)
upgraded.setdefault("limits", {"max_steps": RunLimits().max_steps}) upgraded["limits"] = {"max_steps": RunLimits().max_steps}
upgraded.setdefault("steps_executed", 0) upgraded["steps_executed"] = 0
frames = upgraded.get("frames") frames = upgraded.get("frames")
if isinstance(frames, dict): if isinstance(frames, dict):
for frame in frames.values(): for frame in frames.values():
if isinstance(frame, dict): if isinstance(frame, dict):
frame.setdefault("step_number", None) frame["step_number"] = None
trace = upgraded.get("trace") trace = upgraded.get("trace")
if isinstance(trace, list): if isinstance(trace, list):
for entry in trace: for entry in trace:
if isinstance(entry, dict): if isinstance(entry, dict):
entry.setdefault("step_number", None) entry["step_number"] = None
interrupt = upgraded.get("interrupt") interrupt = upgraded.get("interrupt")
if isinstance(interrupt, dict): if isinstance(interrupt, dict):
interrupt.setdefault("step_number", None) interrupt["step_number"] = None
return upgraded return upgraded
def _require_v2_budget_fields(state: dict[str, Any]) -> None: def _require_v2_budget_fields(state: dict[str, Any]) -> None:
"""Reject v2 payloads missing budget fields as corrupt state. """Reject v2 payloads with missing or incoherent budget fields as corrupt.
Unlike v1, a v2 envelope promises budget fields; a missing counter is Unlike v1, a v2 envelope promises budget fields; a missing counter is
corruption, not another request for defaults. Trace and interrupt entries corruption, not another request for defaults. Values are validated
always carry the key (``None`` only for upgraded pre-budget history), so a strictly on the raw envelope (exact ints, ranges, coherence) because lax
missing key is likewise corrupt even though the dataclass default would coercion would otherwise accept bools, numeric strings, negatives, or
otherwise mask it. future frame numbers and silently inflate or distort the budget. Trace
and interrupt entries always carry the key (``None`` only for unadmitted
or upgraded pre-budget history), so a missing key is likewise corrupt
even though the dataclass default would otherwise mask it.
""" """
if "limits" not in state or "steps_executed" not in state: limits = state.get("limits")
if not isinstance(limits, dict) or "max_steps" not in limits:
raise ValueError("invalid persisted workflow run state: missing step budget") raise ValueError("invalid persisted workflow run state: missing step budget")
max_steps = limits["max_steps"]
if not _is_strict_int(max_steps) or max_steps < 1: # type: ignore[operator]
raise ValueError(
"invalid persisted workflow run state: corrupt step budget limit"
)
steps_executed = state.get("steps_executed")
if not _is_strict_int(steps_executed):
raise ValueError("invalid persisted workflow run state: missing step budget")
if not 0 <= steps_executed <= max_steps: # type: ignore[operator]
raise ValueError(
"invalid persisted workflow run state: corrupt step counter"
)
exec_count = cast(int, steps_executed)
frames = state.get("frames") frames = state.get("frames")
if not isinstance(frames, dict): if not isinstance(frames, dict):
raise ValueError("invalid persisted workflow run state: missing frames") raise ValueError("invalid persisted workflow run state: missing frames")
@@ -85,6 +131,11 @@ def _require_v2_budget_fields(state: dict[str, Any]) -> None:
"invalid persisted workflow run state: " "invalid persisted workflow run state: "
f"frame {frame_id!r} is missing its step number" f"frame {frame_id!r} is missing its step number"
) )
_check_step_number(
frame["step_number"],
steps_executed=exec_count,
what=f"frame {frame_id!r}",
)
trace = state.get("trace") trace = state.get("trace")
if isinstance(trace, list): if isinstance(trace, list):
for position, entry in enumerate(trace): for position, entry in enumerate(trace):
@@ -93,6 +144,11 @@ def _require_v2_budget_fields(state: dict[str, Any]) -> None:
"invalid persisted workflow run state: " "invalid persisted workflow run state: "
f"trace entry {position!r} is missing its step number" f"trace entry {position!r} is missing its step number"
) )
_check_step_number(
entry["step_number"],
steps_executed=exec_count,
what=f"trace entry {position!r}",
)
interrupt = state.get("interrupt") interrupt = state.get("interrupt")
if interrupt is not None: if interrupt is not None:
if not isinstance(interrupt, dict) or "step_number" not in interrupt: if not isinstance(interrupt, dict) or "step_number" not in interrupt:
@@ -100,6 +156,11 @@ def _require_v2_budget_fields(state: dict[str, Any]) -> None:
"invalid persisted workflow run state: " "invalid persisted workflow run state: "
"interrupt is missing its step number" "interrupt is missing its step number"
) )
_check_step_number(
interrupt["step_number"],
steps_executed=exec_count,
what="interrupt",
)
def _restore_root_alias(run: RunState) -> RunState: def _restore_root_alias(run: RunState) -> RunState:
+120
View File
@@ -263,6 +263,126 @@ def test_v2_missing_frame_step_number_is_corrupt() -> None:
load_run_state_with_upgrade(stored) load_run_state_with_upgrade(stored)
def test_v1_smuggled_budget_fields_receive_defaults() -> None:
"""V1 envelopes predate budgets; smuggled fields must not survive."""
from typing import Any, cast
workflow = _minimal_workflow()
run = create_run_state(workflow, {}, limits=RunLimits(max_steps=7))
stored = _strip_to_v1(dump_run_state(run))
state = cast(dict[str, Any], stored["state"])
state["limits"] = {"max_steps": 999_999}
state["steps_executed"] = 999
restored, upgraded = load_run_state_with_upgrade(stored)
assert upgraded is True
assert restored.limits.max_steps == 10_000
assert restored.steps_executed == 0
def test_v2_bool_max_steps_is_corrupt() -> None:
from typing import Any, cast
workflow = _minimal_workflow()
run = create_run_state(workflow, {}, limits=RunLimits(max_steps=2))
stored = dump_run_state(run)
cast(dict[str, Any], stored["state"])["limits"] = {"max_steps": True}
with pytest.raises(ValueError):
load_run_state_with_upgrade(stored)
def test_v2_str_max_steps_is_corrupt() -> None:
from typing import Any, cast
workflow = _minimal_workflow()
run = create_run_state(workflow, {}, limits=RunLimits(max_steps=2))
stored = dump_run_state(run)
cast(dict[str, Any], stored["state"])["limits"] = {"max_steps": "10"}
with pytest.raises(ValueError):
load_run_state_with_upgrade(stored)
def test_v2_negative_steps_executed_is_corrupt() -> None:
from typing import Any, cast
workflow = _minimal_workflow()
run = create_run_state(workflow, {}, limits=RunLimits(max_steps=2))
stored = dump_run_state(run)
cast(dict[str, Any], stored["state"])["steps_executed"] = -100
with pytest.raises(ValueError):
load_run_state_with_upgrade(stored)
def test_v2_exceeding_steps_executed_is_corrupt() -> None:
from typing import Any, cast
workflow = _minimal_workflow()
run = create_run_state(workflow, {}, limits=RunLimits(max_steps=2))
stored = dump_run_state(run)
cast(dict[str, Any], stored["state"])["steps_executed"] = 5
with pytest.raises(ValueError):
load_run_state_with_upgrade(stored)
def test_v2_bool_steps_executed_is_corrupt() -> None:
from typing import Any, cast
workflow = _minimal_workflow()
run = create_run_state(workflow, {}, limits=RunLimits(max_steps=2))
stored = dump_run_state(run)
cast(dict[str, Any], stored["state"])["steps_executed"] = True
with pytest.raises(ValueError):
load_run_state_with_upgrade(stored)
def test_v2_incoherent_frame_step_number_is_corrupt() -> None:
from typing import Any, cast
workflow = _minimal_workflow()
run = create_run_state(workflow, {}, limits=RunLimits(max_steps=2))
admit_step_attempt(run, run.current_frame(), workflow.start)
stored = dump_run_state(run)
state = cast(dict[str, Any], stored["state"])
frames = cast(dict[str, Any], state["frames"])
cast(dict[str, Any], frames["root"])["step_number"] = 999
with pytest.raises(ValueError):
load_run_state_with_upgrade(stored)
def test_v2_bool_frame_step_number_is_corrupt() -> None:
from typing import Any, cast
workflow = _minimal_workflow()
run = create_run_state(workflow, {}, limits=RunLimits(max_steps=2))
admit_step_attempt(run, run.current_frame(), workflow.start)
stored = dump_run_state(run)
state = cast(dict[str, Any], stored["state"])
frames = cast(dict[str, Any], state["frames"])
cast(dict[str, Any], frames["root"])["step_number"] = True
with pytest.raises(ValueError):
load_run_state_with_upgrade(stored)
def test_v2_pending_frame_none_number_round_trips() -> None:
"""Fresh V2 runs legitimately persist unadmitted (None) frame numbers."""
workflow = _minimal_workflow()
run = create_run_state(workflow, {}, limits=RunLimits(max_steps=2))
restored, upgraded = load_run_state_with_upgrade(dump_run_state(run))
assert upgraded is False
assert restored.steps_executed == 0
assert restored.frames["root"].step_number is None
# --- Task 2: sync dispatch and trace numbering --- # --- Task 2: sync dispatch and trace numbering ---