sched: add durable admission record and admitted run view (T05)

This commit is contained in:
lda
2026-09-08 10:19:28 +07:00 Verified
parent 46c2763b66
commit d0d9fd581e
6 changed files with 364 additions and 7 deletions
+97 -3
View File
@@ -11,6 +11,7 @@ from wf_artifacts import (
DiagnosticSeverity,
PinnedRunEnvironment,
ResumeReadiness,
RunAdmission,
RunCheckpoint,
RunStore,
StoredRunStatus,
@@ -43,6 +44,88 @@ def create_pinned_environment(
)
def persist_admission(
*,
store: RunStore,
run_id: str,
environment: PinnedRunEnvironment,
resolved_input: dict[str, object],
max_steps: int | None,
scheduled_at: datetime | None = None,
schedule_id: str | None = None,
schedule_revision: int | None = None,
deployment_revision: int | None = None,
) -> RunAdmission:
"""Persist the authoritative admission record before dispatch.
The occurrence is decided here: the caller must treat this persist as the
single authority for the run identity and frozen invocation. A failed
durable admission must never dispatch.
"""
from wf_core.models.json_values import validate_strict_json_value
frozen = validate_strict_json_value(dict(resolved_input))
if not isinstance(frozen, dict):
raise ValueError("resolved workflow input must be a JSON object")
admission = RunAdmission(
id=run_id,
environment=environment,
resolved_input=frozen,
max_steps=max_steps,
scheduled_at=scheduled_at,
schedule_id=schedule_id,
schedule_revision=schedule_revision,
deployment_revision=deployment_revision,
created_at=datetime.now(UTC),
)
store.save_admission(admission)
return admission
def materialize_admitted_view(
*,
store: RunStore,
admission: RunAdmission,
) -> WorkflowRunRecord:
"""Materialize the run admission view using the admission identity.
The view carries no checkpoint, trace, output, or step counts: the outcome
is unknown until dispatch completes and persists a stopped checkpoint.
"""
now = datetime.now(UTC)
try:
existing = store.get_run(admission.id)
except KeyError:
existing = None
if existing is not None:
return existing
record = WorkflowRunRecord(
id=admission.id,
status=StoredRunStatus.ADMITTED,
resume_readiness=ResumeReadiness.NOT_APPLICABLE,
environment=admission.environment,
latest_checkpoint_id=None,
created_at=admission.created_at,
updated_at=now,
)
store.save_run(record)
return record
def recover_admission_view(*, store: RunStore, run_id: str) -> WorkflowRunRecord:
"""Reconcile a missing run view from its admission record.
Recovery never executes work: it only completes the missing view so a
later poll can dispatch the captured invocation exactly once.
"""
try:
return store.get_run(run_id)
except KeyError:
pass
admission = store.get_admission(run_id)
return materialize_admitted_view(store=store, admission=admission)
def persist_stopped_run(
*,
store: RunStore,
@@ -65,9 +148,16 @@ def persist_stopped_run(
sequence = 1
created_at = now
if run_id is not None:
existing = store.get_run(run_id)
created_at = existing.created_at
sequence = store.get_latest_checkpoint(run_id).sequence + 1
try:
existing = store.get_run(run_id)
except KeyError:
existing = None
if existing is not None:
created_at = existing.created_at
try:
sequence = store.get_latest_checkpoint(run_id).sequence + 1
except KeyError:
sequence = 1
status = StoredRunStatus(run.status.value)
readiness = (
@@ -130,6 +220,10 @@ def restore_interrupted_run(
def load_stored_run(store: RunStore, run_id: str) -> tuple[WorkflowRunRecord, RunState]:
"""Load any stopped run record together with its latest typed checkpoint."""
record = store.get_run(run_id)
if record.status is StoredRunStatus.ADMITTED:
raise ValueError(
f"workflow run {run_id!r} is admitted but has no stopped checkpoint yet"
)
checkpoint = store.get_latest_checkpoint(run_id)
return record, load_run_state(checkpoint.state.model_dump(mode="json"))
+2
View File
@@ -46,6 +46,7 @@ from .runs import (
FileRunStore,
PinnedRunEnvironment,
ResumeReadiness,
RunAdmission,
RunCheckpoint,
RunStore,
StoredRunStatus,
@@ -72,6 +73,7 @@ __all__ = [
"PinnedRunEnvironment",
"RequiredCapability",
"ResumeReadiness",
"RunAdmission",
"RunCheckpoint",
"RunStore",
"SourceBinding",
+2
View File
@@ -2,6 +2,7 @@ from .models import (
CheckpointReason,
PinnedRunEnvironment,
ResumeReadiness,
RunAdmission,
RunCheckpoint,
StoredRunStatus,
VersionedCheckpointState,
@@ -15,6 +16,7 @@ __all__ = [
"FileRunStore",
"PinnedRunEnvironment",
"ResumeReadiness",
"RunAdmission",
"RunCheckpoint",
"RunStore",
"StoredRunStatus",
+30 -2
View File
@@ -25,8 +25,13 @@ def ensure_run_id(run_id: str) -> str:
class StoredRunStatus(StrEnum):
"""Stopped runtime statuses supported by durable run persistence."""
"""Durable run statuses including pre-dispatch admission.
``ADMITTED`` marks a durably admitted run with no stopped checkpoint yet.
Never fabricate a checkpoint, trace, output, or step count for such runs.
"""
ADMITTED = "admitted"
INTERRUPTED = "interrupted"
COMPLETED = "completed"
FAILED = "failed"
@@ -67,12 +72,35 @@ class WorkflowRunRecord(BaseModel):
status: StoredRunStatus
resume_readiness: ResumeReadiness
environment: PinnedRunEnvironment
latest_checkpoint_id: str = Field(pattern=RUN_ID_PATTERN)
latest_checkpoint_id: str | None = Field(default=None, pattern=RUN_ID_PATTERN)
diagnostics: list[DependencyDiagnostic] = Field(default_factory=list)
created_at: datetime
updated_at: datetime
class RunAdmission(BaseModel):
"""Authoritative admission record persisted before dispatch.
The admission freezes the invocation (pinned environment, resolved input,
limits, deployment/schedule revisions, resolved UTC instant) under a
preassigned run identity. It is the recovery authority for partial
multi-file writes: the run view (``WorkflowRunRecord``) is materialized
from this same identity, and dispatch uses only the captured invocation.
"""
model_config = ConfigDict(extra="forbid")
id: str = Field(pattern=RUN_ID_PATTERN)
environment: PinnedRunEnvironment
resolved_input: dict[str, Any]
max_steps: int | None = None
scheduled_at: datetime | None = None
schedule_id: str | None = None
schedule_revision: int | None = None
deployment_revision: int | None = None
created_at: datetime
class VersionedCheckpointState(BaseModel):
"""Lenient read envelope for stopped-run checkpoints.
+52 -2
View File
@@ -4,11 +4,11 @@ import json
from pathlib import Path
from threading import RLock
from .models import RunCheckpoint, WorkflowRunRecord, ensure_run_id
from .models import RunAdmission, RunCheckpoint, WorkflowRunRecord, ensure_run_id
class RunStore:
"""Persistence boundary for stopped run summaries and checkpoints."""
"""Persistence boundary for admitted runs, summaries, and checkpoints."""
def save_run(self, run: WorkflowRunRecord) -> None:
raise NotImplementedError
@@ -28,6 +28,18 @@ class RunStore:
def list_checkpoints(self, run_id: str) -> list[RunCheckpoint]:
raise NotImplementedError
def save_admission(self, admission: RunAdmission) -> None:
raise NotImplementedError
def get_admission(self, run_id: str) -> RunAdmission:
raise NotImplementedError
def list_admissions(self) -> list[RunAdmission]:
raise NotImplementedError
def allocate_run_id(self) -> str:
raise NotImplementedError
class FileRunStore(RunStore):
"""JSON file-backed stopped-run store for local development and tests.
@@ -87,6 +99,41 @@ class FileRunStore(RunStore):
for path in sorted(directory.glob("*.json"))
]
def save_admission(self, admission: RunAdmission) -> None:
with self._lock:
self._write_json(
self._admission_path(admission.id),
admission.model_dump(mode="json"),
)
def get_admission(self, run_id: str) -> RunAdmission:
path = self._admission_path(run_id)
if not path.exists():
raise KeyError(f"unknown run admission {run_id!r}")
return RunAdmission.model_validate_json(path.read_text(encoding="utf-8"))
def list_admissions(self) -> list[RunAdmission]:
return [
RunAdmission.model_validate_json(path.read_text(encoding="utf-8"))
for path in sorted(self.runs_dir.glob("*/admission.json"))
]
def allocate_run_id(self) -> str:
"""Allocate a store-backed run identity that survives restart."""
with self._lock:
seq_path = self.runs_dir / "_run_id_seq.json"
seq = 0
if seq_path.exists():
try:
seq = int(
json.loads(seq_path.read_text(encoding="utf-8")).get("seq", 0)
)
except ValueError, AttributeError:
seq = 0
seq += 1
self._write_json(seq_path, {"seq": seq})
return f"run-{seq:06d}"
def _write_json(self, path: Path, payload: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temp_path = path.with_suffix(".json.tmp")
@@ -104,5 +151,8 @@ class FileRunStore(RunStore):
def _run_path(self, run_id: str) -> Path:
return self._run_directory(run_id) / "run.json"
def _admission_path(self, run_id: str) -> Path:
return self._run_directory(run_id) / "admission.json"
def _checkpoint_path(self, run_id: str, sequence: int) -> Path:
return self._run_directory(run_id) / "checkpoints" / f"{sequence:06d}.json"
+181
View File
@@ -0,0 +1,181 @@
"""Durable admission representation: persist before dispatch (T05)."""
from __future__ import annotations
from datetime import UTC, datetime
from pathlib import Path
import pytest
from wf_api.run_lifecycle import (
load_stored_run,
materialize_admitted_view,
persist_admission,
)
from wf_artifacts import (
PinnedRunEnvironment,
StoredRunStatus,
WorkflowArtifact,
WorkflowDeployment,
)
from wf_artifacts.runs.store import FileRunStore
def _env() -> PinnedRunEnvironment:
deployment = WorkflowDeployment(
id="dep-1",
artifact_id="wf-1",
artifact_version=1,
bindings=[{"logical_source": "demo", "concrete_source": "demo.personal"}],
)
artifact = WorkflowArtifact(
id="wf-1",
version=1,
title="Wf-1",
input_schema={"type": "object", "properties": {}},
output_schema={"type": "object", "properties": {}},
outcomes=("ok",),
plan={"name": "wf-1", "nodes": [], "edges": []},
)
return PinnedRunEnvironment(
deployment=deployment, root_artifact=artifact, child_artifacts=[]
)
def test_admission_persists_authority_before_view(tmp_path: Path) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
at = datetime(2026, 9, 8, 13, 0, tzinfo=UTC)
admission = persist_admission(
store=store,
run_id=run_id,
environment=_env(),
resolved_input={"team": "eng"},
max_steps=100,
scheduled_at=at,
schedule_id="sched-1",
schedule_revision=2,
)
assert admission.id == run_id
assert admission.scheduled_at == at
# Admission record exists before any run view or checkpoint.
assert store.get_admission(run_id).id == run_id
with pytest.raises(KeyError):
store.get_run(run_id)
record = materialize_admitted_view(store=store, admission=admission)
assert record.status is StoredRunStatus.ADMITTED
assert record.latest_checkpoint_id is None
# Inspection distinguishes admitted (no checkpoint) from stopped.
with pytest.raises(KeyError, match="no checkpoints"):
store.get_latest_checkpoint(run_id)
def test_admission_view_round_trips_real_serialization(tmp_path: Path) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
admission = persist_admission(
store=store,
run_id=run_id,
environment=_env(),
resolved_input={"a": [1, None, "x"]},
max_steps=None,
scheduled_at=None,
schedule_id=None,
schedule_revision=None,
)
record = materialize_admitted_view(store=store, admission=admission)
reloaded = store.get_run(run_id)
assert reloaded.id == record.id
assert reloaded.status is StoredRunStatus.ADMITTED
reloaded_admission = store.get_admission(run_id)
assert reloaded_admission.resolved_input == {"a": [1, None, "x"]}
assert reloaded_admission.environment.deployment.id == "dep-1"
def test_run_ids_are_store_backed_across_restart(tmp_path: Path) -> None:
first = FileRunStore(tmp_path).allocate_run_id()
second = FileRunStore(tmp_path).allocate_run_id()
assert first != second
# Reopening the store never reuses identities.
third = FileRunStore(tmp_path).allocate_run_id()
assert third not in {first, second}
def test_fault_before_admission_persists_nothing(tmp_path: Path, monkeypatch) -> None:
from wf_artifacts.runs import store as store_module
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
def _boom(path: Path, payload: object) -> None:
raise OSError("injected write failure")
monkeypatch.setattr(store, "_write_json", _boom)
with pytest.raises(OSError, match="injected"):
persist_admission(
store=store,
run_id=run_id,
environment=_env(),
resolved_input={},
max_steps=None,
scheduled_at=None,
schedule_id=None,
schedule_revision=None,
)
assert store_module.FileRunStore(tmp_path).list_admissions() == []
with pytest.raises(KeyError):
store.get_admission(run_id)
def test_fault_between_admission_and_view_recovers_without_dispatch(
tmp_path: Path, monkeypatch
) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
admission = persist_admission(
store=store,
run_id=run_id,
environment=_env(),
resolved_input={"team": "eng"},
max_steps=None,
scheduled_at=None,
schedule_id=None,
schedule_revision=None,
)
assert admission.id == run_id
# Crash before the view is materialized: admission exists, view missing.
with pytest.raises(KeyError):
store.get_run(run_id)
# Recovery materializes the missing view but never executes work.
from wf_api.run_lifecycle import recover_admission_view
record = recover_admission_view(store=store, run_id=run_id)
assert record.status is StoredRunStatus.ADMITTED
assert store.get_run(run_id).id == run_id
# Idempotent: second recovery returns the same view.
again = recover_admission_view(store=store, run_id=run_id)
assert again.id == run_id
def test_admitted_run_has_no_fabricated_checkpoint_or_output(
tmp_path: Path,
) -> None:
store = FileRunStore(tmp_path)
run_id = store.allocate_run_id()
admission = persist_admission(
store=store,
run_id=run_id,
environment=_env(),
resolved_input={},
max_steps=None,
scheduled_at=None,
schedule_id=None,
schedule_revision=None,
)
record = materialize_admitted_view(store=store, admission=admission)
assert record.latest_checkpoint_id is None
assert store.list_checkpoints(run_id) == []
# Loading a stopped run from an admitted view must fail closed, not
# fabricate trace/output/step counts.
with pytest.raises(ValueError, match="admitted"):
load_stored_run(store, run_id)