sched: address R2 fail-closed and fault-proof findings

This commit is contained in:
lda
2026-09-08 10:27:37 +07:00 Verified
parent 3cd61be96d
commit cf8d28f1ff
5 changed files with 113 additions and 30 deletions
+20 -5
View File
@@ -28,6 +28,7 @@ from wf_core import (
load_run_state,
load_run_state_with_upgrade,
)
from wf_core.models.json_values import validate_strict_json_value
def create_pinned_environment(
@@ -62,8 +63,6 @@ def persist_admission(
single authority for the run identity and frozen invocation. A failed
durable admission must never dispatch.
"""
from wf_core.models.json_values import validate_strict_json_value
frozen = validate_strict_json_value(dict(resolved_input))
if not isinstance(frozen, dict):
raise ValueError("resolved workflow input must be a JSON object")
@@ -91,6 +90,8 @@ def materialize_admitted_view(
The view carries no checkpoint, trace, output, or step counts: the outcome
is unknown until dispatch completes and persists a stopped checkpoint.
A view without a matching admission is corrupt and fails closed instead
of being silently returned (recovery authority is the admission record).
"""
now = datetime.now(UTC)
try:
@@ -98,6 +99,11 @@ def materialize_admitted_view(
except KeyError:
existing = None
if existing is not None:
stored = store.get_admission(admission.id)
if stored.id != existing.id or stored.environment != existing.environment:
raise ValueError(
f"run view {admission.id!r} contradicts its admission record"
)
return existing
record = WorkflowRunRecord(
id=admission.id,
@@ -116,12 +122,18 @@ def recover_admission_view(*, store: RunStore, run_id: str) -> WorkflowRunRecord
"""Reconcile a missing run view from its admission record.
Recovery never executes work: it only completes the missing view so a
later poll can dispatch the captured invocation exactly once.
later poll can dispatch the captured invocation exactly once. A run view
without an admission record is corrupt and fails closed.
"""
try:
return store.get_run(run_id)
existing = store.get_run(run_id)
except KeyError:
pass
existing = None
if existing is not None:
# Fail closed on a corrupt view-without-admission (F5): do not
# silently return it to clear overlap.
store.get_admission(run_id)
return existing
admission = store.get_admission(run_id)
return materialize_admitted_view(store=store, admission=admission)
@@ -143,6 +155,9 @@ def persist_stopped_run(
f"cannot persist active workflow run with status {run.status!s}"
)
# New admissions use store-backed run-###### ids (allocate_run_id); the
# run_ hex fallback only serves pre-admission stopped runs and legacy
# tests that persist without an admission record.
key = run_id or f"run_{uuid4().hex}"
now = datetime.now(UTC)
sequence = 1
+16 -5
View File
@@ -100,10 +100,13 @@ class WorkflowRunApi:
max_steps=limits.max_steps,
)
# Durable admission ordering: recheck -> allocate/freeze -> persist
# admission -> materialize view -> dispatch captured -> persist
# stopped -> reconcile. A failed durable admission never dispatches,
# and dispatch never re-resolves the deployment.
# Durable admission ordering for manual runs: deployment recheck ->
# allocate/freeze -> persist admission -> materialize view -> dispatch
# captured -> persist stopped. A failed durable admission never
# dispatches, and dispatch never re-resolves the deployment.
# TODO(T11): hold the single-owner admission lock around this sequence
# once scheduler ownership lands; manual recheck here is only
# deployment validation (no schedule/capacity/overlap yet).
store = self._run_store()
run_id = store.allocate_run_id()
environment = create_pinned_environment(
@@ -119,6 +122,9 @@ class WorkflowRunApi:
max_steps=limits.max_steps,
)
materialize_admitted_view(store=store, admission=admission)
# TODO(T10): record a dispatch mark between materialize and execute so
# crash-after-dispatch (abandoned, failed without replay) is
# distinguishable from pending-dispatch (safe to dispatch later).
plan = raw_plan_from_artifact(admission.environment.root_artifact)
captured_tree = saved_subgraph_tree_from_snapshots(
admission.environment.child_artifacts
@@ -282,7 +288,12 @@ class WorkflowRunApi:
}
async def inspect_run(self, *, run_id: str) -> RunResult:
"""Return one durable stopped-run summary without debug trace entries."""
"""Return one durable stopped-run summary without debug trace entries.
Admitted runs with no stopped checkpoint fail closed here (no
fabricated trace/output); checkpoint-free inspection arrives with
the scheduling administration surface (T13).
"""
record, run = load_stored_run(self._run_store(), run_id)
environment = record.environment
return _run_payload(