sched: add durable admission record and admitted run view (T05)
This commit is contained in:
@@ -11,6 +11,7 @@ from wf_artifacts import (
|
||||
DiagnosticSeverity,
|
||||
PinnedRunEnvironment,
|
||||
ResumeReadiness,
|
||||
RunAdmission,
|
||||
RunCheckpoint,
|
||||
RunStore,
|
||||
StoredRunStatus,
|
||||
@@ -43,6 +44,88 @@ def create_pinned_environment(
|
||||
)
|
||||
|
||||
|
||||
def persist_admission(
|
||||
*,
|
||||
store: RunStore,
|
||||
run_id: str,
|
||||
environment: PinnedRunEnvironment,
|
||||
resolved_input: dict[str, object],
|
||||
max_steps: int | None,
|
||||
scheduled_at: datetime | None = None,
|
||||
schedule_id: str | None = None,
|
||||
schedule_revision: int | None = None,
|
||||
deployment_revision: int | None = None,
|
||||
) -> RunAdmission:
|
||||
"""Persist the authoritative admission record before dispatch.
|
||||
|
||||
The occurrence is decided here: the caller must treat this persist as the
|
||||
single authority for the run identity and frozen invocation. A failed
|
||||
durable admission must never dispatch.
|
||||
"""
|
||||
from wf_core.models.json_values import validate_strict_json_value
|
||||
|
||||
frozen = validate_strict_json_value(dict(resolved_input))
|
||||
if not isinstance(frozen, dict):
|
||||
raise ValueError("resolved workflow input must be a JSON object")
|
||||
admission = RunAdmission(
|
||||
id=run_id,
|
||||
environment=environment,
|
||||
resolved_input=frozen,
|
||||
max_steps=max_steps,
|
||||
scheduled_at=scheduled_at,
|
||||
schedule_id=schedule_id,
|
||||
schedule_revision=schedule_revision,
|
||||
deployment_revision=deployment_revision,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
store.save_admission(admission)
|
||||
return admission
|
||||
|
||||
|
||||
def materialize_admitted_view(
|
||||
*,
|
||||
store: RunStore,
|
||||
admission: RunAdmission,
|
||||
) -> WorkflowRunRecord:
|
||||
"""Materialize the run admission view using the admission identity.
|
||||
|
||||
The view carries no checkpoint, trace, output, or step counts: the outcome
|
||||
is unknown until dispatch completes and persists a stopped checkpoint.
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
try:
|
||||
existing = store.get_run(admission.id)
|
||||
except KeyError:
|
||||
existing = None
|
||||
if existing is not None:
|
||||
return existing
|
||||
record = WorkflowRunRecord(
|
||||
id=admission.id,
|
||||
status=StoredRunStatus.ADMITTED,
|
||||
resume_readiness=ResumeReadiness.NOT_APPLICABLE,
|
||||
environment=admission.environment,
|
||||
latest_checkpoint_id=None,
|
||||
created_at=admission.created_at,
|
||||
updated_at=now,
|
||||
)
|
||||
store.save_run(record)
|
||||
return record
|
||||
|
||||
|
||||
def recover_admission_view(*, store: RunStore, run_id: str) -> WorkflowRunRecord:
|
||||
"""Reconcile a missing run view from its admission record.
|
||||
|
||||
Recovery never executes work: it only completes the missing view so a
|
||||
later poll can dispatch the captured invocation exactly once.
|
||||
"""
|
||||
try:
|
||||
return store.get_run(run_id)
|
||||
except KeyError:
|
||||
pass
|
||||
admission = store.get_admission(run_id)
|
||||
return materialize_admitted_view(store=store, admission=admission)
|
||||
|
||||
|
||||
def persist_stopped_run(
|
||||
*,
|
||||
store: RunStore,
|
||||
@@ -65,9 +148,16 @@ def persist_stopped_run(
|
||||
sequence = 1
|
||||
created_at = now
|
||||
if run_id is not None:
|
||||
existing = store.get_run(run_id)
|
||||
created_at = existing.created_at
|
||||
sequence = store.get_latest_checkpoint(run_id).sequence + 1
|
||||
try:
|
||||
existing = store.get_run(run_id)
|
||||
except KeyError:
|
||||
existing = None
|
||||
if existing is not None:
|
||||
created_at = existing.created_at
|
||||
try:
|
||||
sequence = store.get_latest_checkpoint(run_id).sequence + 1
|
||||
except KeyError:
|
||||
sequence = 1
|
||||
|
||||
status = StoredRunStatus(run.status.value)
|
||||
readiness = (
|
||||
@@ -130,6 +220,10 @@ def restore_interrupted_run(
|
||||
def load_stored_run(store: RunStore, run_id: str) -> tuple[WorkflowRunRecord, RunState]:
|
||||
"""Load any stopped run record together with its latest typed checkpoint."""
|
||||
record = store.get_run(run_id)
|
||||
if record.status is StoredRunStatus.ADMITTED:
|
||||
raise ValueError(
|
||||
f"workflow run {run_id!r} is admitted but has no stopped checkpoint yet"
|
||||
)
|
||||
checkpoint = store.get_latest_checkpoint(run_id)
|
||||
return record, load_run_state(checkpoint.state.model_dump(mode="json"))
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ from .runs import (
|
||||
FileRunStore,
|
||||
PinnedRunEnvironment,
|
||||
ResumeReadiness,
|
||||
RunAdmission,
|
||||
RunCheckpoint,
|
||||
RunStore,
|
||||
StoredRunStatus,
|
||||
@@ -72,6 +73,7 @@ __all__ = [
|
||||
"PinnedRunEnvironment",
|
||||
"RequiredCapability",
|
||||
"ResumeReadiness",
|
||||
"RunAdmission",
|
||||
"RunCheckpoint",
|
||||
"RunStore",
|
||||
"SourceBinding",
|
||||
|
||||
@@ -2,6 +2,7 @@ from .models import (
|
||||
CheckpointReason,
|
||||
PinnedRunEnvironment,
|
||||
ResumeReadiness,
|
||||
RunAdmission,
|
||||
RunCheckpoint,
|
||||
StoredRunStatus,
|
||||
VersionedCheckpointState,
|
||||
@@ -15,6 +16,7 @@ __all__ = [
|
||||
"FileRunStore",
|
||||
"PinnedRunEnvironment",
|
||||
"ResumeReadiness",
|
||||
"RunAdmission",
|
||||
"RunCheckpoint",
|
||||
"RunStore",
|
||||
"StoredRunStatus",
|
||||
|
||||
@@ -25,8 +25,13 @@ def ensure_run_id(run_id: str) -> str:
|
||||
|
||||
|
||||
class StoredRunStatus(StrEnum):
|
||||
"""Stopped runtime statuses supported by durable run persistence."""
|
||||
"""Durable run statuses including pre-dispatch admission.
|
||||
|
||||
``ADMITTED`` marks a durably admitted run with no stopped checkpoint yet.
|
||||
Never fabricate a checkpoint, trace, output, or step count for such runs.
|
||||
"""
|
||||
|
||||
ADMITTED = "admitted"
|
||||
INTERRUPTED = "interrupted"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
@@ -67,12 +72,35 @@ class WorkflowRunRecord(BaseModel):
|
||||
status: StoredRunStatus
|
||||
resume_readiness: ResumeReadiness
|
||||
environment: PinnedRunEnvironment
|
||||
latest_checkpoint_id: str = Field(pattern=RUN_ID_PATTERN)
|
||||
latest_checkpoint_id: str | None = Field(default=None, pattern=RUN_ID_PATTERN)
|
||||
diagnostics: list[DependencyDiagnostic] = Field(default_factory=list)
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class RunAdmission(BaseModel):
|
||||
"""Authoritative admission record persisted before dispatch.
|
||||
|
||||
The admission freezes the invocation (pinned environment, resolved input,
|
||||
limits, deployment/schedule revisions, resolved UTC instant) under a
|
||||
preassigned run identity. It is the recovery authority for partial
|
||||
multi-file writes: the run view (``WorkflowRunRecord``) is materialized
|
||||
from this same identity, and dispatch uses only the captured invocation.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str = Field(pattern=RUN_ID_PATTERN)
|
||||
environment: PinnedRunEnvironment
|
||||
resolved_input: dict[str, Any]
|
||||
max_steps: int | None = None
|
||||
scheduled_at: datetime | None = None
|
||||
schedule_id: str | None = None
|
||||
schedule_revision: int | None = None
|
||||
deployment_revision: int | None = None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class VersionedCheckpointState(BaseModel):
|
||||
"""Lenient read envelope for stopped-run checkpoints.
|
||||
|
||||
|
||||
@@ -4,11 +4,11 @@ import json
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
|
||||
from .models import RunCheckpoint, WorkflowRunRecord, ensure_run_id
|
||||
from .models import RunAdmission, RunCheckpoint, WorkflowRunRecord, ensure_run_id
|
||||
|
||||
|
||||
class RunStore:
|
||||
"""Persistence boundary for stopped run summaries and checkpoints."""
|
||||
"""Persistence boundary for admitted runs, summaries, and checkpoints."""
|
||||
|
||||
def save_run(self, run: WorkflowRunRecord) -> None:
|
||||
raise NotImplementedError
|
||||
@@ -28,6 +28,18 @@ class RunStore:
|
||||
def list_checkpoints(self, run_id: str) -> list[RunCheckpoint]:
|
||||
raise NotImplementedError
|
||||
|
||||
def save_admission(self, admission: RunAdmission) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_admission(self, run_id: str) -> RunAdmission:
|
||||
raise NotImplementedError
|
||||
|
||||
def list_admissions(self) -> list[RunAdmission]:
|
||||
raise NotImplementedError
|
||||
|
||||
def allocate_run_id(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FileRunStore(RunStore):
|
||||
"""JSON file-backed stopped-run store for local development and tests.
|
||||
@@ -87,6 +99,41 @@ class FileRunStore(RunStore):
|
||||
for path in sorted(directory.glob("*.json"))
|
||||
]
|
||||
|
||||
def save_admission(self, admission: RunAdmission) -> None:
|
||||
with self._lock:
|
||||
self._write_json(
|
||||
self._admission_path(admission.id),
|
||||
admission.model_dump(mode="json"),
|
||||
)
|
||||
|
||||
def get_admission(self, run_id: str) -> RunAdmission:
|
||||
path = self._admission_path(run_id)
|
||||
if not path.exists():
|
||||
raise KeyError(f"unknown run admission {run_id!r}")
|
||||
return RunAdmission.model_validate_json(path.read_text(encoding="utf-8"))
|
||||
|
||||
def list_admissions(self) -> list[RunAdmission]:
|
||||
return [
|
||||
RunAdmission.model_validate_json(path.read_text(encoding="utf-8"))
|
||||
for path in sorted(self.runs_dir.glob("*/admission.json"))
|
||||
]
|
||||
|
||||
def allocate_run_id(self) -> str:
|
||||
"""Allocate a store-backed run identity that survives restart."""
|
||||
with self._lock:
|
||||
seq_path = self.runs_dir / "_run_id_seq.json"
|
||||
seq = 0
|
||||
if seq_path.exists():
|
||||
try:
|
||||
seq = int(
|
||||
json.loads(seq_path.read_text(encoding="utf-8")).get("seq", 0)
|
||||
)
|
||||
except ValueError, AttributeError:
|
||||
seq = 0
|
||||
seq += 1
|
||||
self._write_json(seq_path, {"seq": seq})
|
||||
return f"run-{seq:06d}"
|
||||
|
||||
def _write_json(self, path: Path, payload: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp_path = path.with_suffix(".json.tmp")
|
||||
@@ -104,5 +151,8 @@ class FileRunStore(RunStore):
|
||||
def _run_path(self, run_id: str) -> Path:
|
||||
return self._run_directory(run_id) / "run.json"
|
||||
|
||||
def _admission_path(self, run_id: str) -> Path:
|
||||
return self._run_directory(run_id) / "admission.json"
|
||||
|
||||
def _checkpoint_path(self, run_id: str, sequence: int) -> Path:
|
||||
return self._run_directory(run_id) / "checkpoints" / f"{sequence:06d}.json"
|
||||
|
||||
Reference in New Issue
Block a user