lda.chat/lda-wf
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

durable stopped workflow runs and resume

Browse Source
This commit is contained in:
lda
2026-05-26 12:20:40 +07:00 Verified
parent d37077397a
commit ba8fd2b614
30 changed files with 1216 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/wf_mcp/workflow_surface/handlers.py
+120 -60
View File
@@ -1,9 +1,8 @@
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import asdict, dataclass
from dataclasses import asdict
from typing import TYPE_CHECKING, Any
from uuid import uuid4
from wf_artifacts import (
ArtifactKind,
@@ -12,6 +11,7 @@ from wf_artifacts import (
DependencyDiagnostic,
DraftWorkspaceStore,
RequiredCapability,
RunStore,
WorkflowArtifact,
WorkflowCapabilityRef,
WorkflowDeployment,
@@ -53,10 +53,21 @@ from .constants import (
from .models import TraceRange
from .refs import parse_workflow_surface_capability_id
from .saved_subgraphs import (
SavedSubgraphTree,
direct_wrapper_interrupt_diagnostic,
resolve_saved_subgraph_tree,
saved_subgraph_tree_from_snapshots,
validate_saved_subgraph_tree,
)
from .run_lifecycle import (
create_pinned_environment,
has_blocking_diagnostics,
mark_resume_blocked,
persist_stopped_run,
load_stored_run,
restore_interrupted_run,
validate_pinned_resume_environment,
)
from .wrapper_hints import wrapper_hints_for_capability
if TYPE_CHECKING:
@@ -65,27 +76,11 @@ if TYPE_CHECKING:
from ..broker.service import WfMcpService
@dataclass(slots=True)
class ActiveWorkflowRun:
"""In-memory paused deployment run.
This is intentionally not durable. It only makes interrupt/resume usable
while the current MCP server process is alive; persisted run storage remains
a separate platform concern.
"""
deployment: WorkflowDeployment
artifact: WorkflowArtifact
plan: RawWorkflowPlan
run: "RunState"
class WorkflowSurfaceHandlers:
"""Reusable implementation behind MCP workflow artifact tools."""
def __init__(self, service: WfMcpService) -> None:
self.service = service
self._active_runs: dict[str, ActiveWorkflowRun] = {}
async def list_artifacts(
self,
@@ -903,7 +898,9 @@ class WorkflowSurfaceHandlers:
}
async def validate_deployment(self, *, deployment_id: str) -> dict[str, Any]:
deployment, artifact, diagnostics = self._deployment_validation(deployment_id)
deployment, artifact, diagnostics, _tree = self._deployment_validation(
deployment_id
)
return {
"deployment_id": deployment.id,
"artifact_id": artifact.id,
@@ -921,7 +918,9 @@ class WorkflowSurfaceHandlers:
workflow_input: dict[str, Any],
trace_range: TraceRange | None = None,
) -> dict[str, Any]:
deployment, artifact, diagnostics = self._deployment_validation(deployment_id)
deployment, artifact, diagnostics, tree = self._deployment_validation(
deployment_id
)
if diagnostics:
return _run_payload(
deployment=deployment,
@@ -936,18 +935,23 @@ class WorkflowSurfaceHandlers:
workflow_input,
deployment=deployment,
artifact=artifact,
saved_subgraph_tree=tree,
)
run_id = self._save_active_run(
deployment=deployment,
artifact=artifact,
plan=plan,
record = persist_stopped_run(
store=self._run_store(),
environment=create_pinned_environment(
deployment=deployment,
artifact=artifact,
tree=tree,
),
run=run,
)
return _run_payload(
deployment=deployment,
artifact=artifact,
status=run.status.value,
run_id=run_id,
run_id=record.id,
resume_readiness=record.resume_readiness.value,
interrupt=_interrupt_payload(run),
outcome=run.outcome,
output=run.output,
@@ -978,29 +982,54 @@ class WorkflowSurfaceHandlers:
resume_outcome: str = "submitted",
trace_range: TraceRange | None = None,
) -> dict[str, Any]:
"""Resume one interrupted in-memory deployment run."""
active = self._active_runs[run_id]
"""Resume one durable interrupted deployment run."""
record, stopped_run = restore_interrupted_run(self._run_store(), run_id)
environment = record.environment
diagnostics = validate_pinned_resume_environment(
record=record,
sources=_available_sources(self.service),
)
if has_blocking_diagnostics(diagnostics):
blocked = mark_resume_blocked(
store=self._run_store(),
record=record,
diagnostics=diagnostics,
)
return _run_payload(
deployment=environment.deployment,
artifact=environment.root_artifact,
status=stopped_run.status.value,
run_id=blocked.id,
resume_readiness=blocked.resume_readiness.value,
interrupt=_interrupt_payload(stopped_run),
outcome=stopped_run.outcome,
output=stopped_run.output,
diagnostics=diagnostics,
trace_count=len(stopped_run.trace),
)
plan = _raw_plan_from_artifact(environment.root_artifact)
tree = saved_subgraph_tree_from_snapshots(environment.child_artifacts)
run = await self.service.resume_workflow_from_plan(
active.plan,
active.run,
plan,
stopped_run,
resume_payload=resume_payload,
resume_outcome=resume_outcome,
deployment=active.deployment,
artifact=active.artifact,
deployment=environment.deployment,
artifact=environment.root_artifact,
saved_subgraph_tree=tree,
)
active.run = run
next_run_id = self._save_active_run(
deployment=active.deployment,
artifact=active.artifact,
plan=active.plan,
next_record = persist_stopped_run(
store=self._run_store(),
environment=environment,
run=run,
run_id=run_id,
)
return _run_payload(
deployment=active.deployment,
artifact=active.artifact,
deployment=environment.deployment,
artifact=environment.root_artifact,
status=run.status.value,
run_id=next_run_id,
run_id=next_record.id,
resume_readiness=next_record.resume_readiness.value,
interrupt=_interrupt_payload(run),
outcome=run.outcome,
output=run.output,
@@ -1023,33 +1052,62 @@ class WorkflowSurfaceHandlers:
),
)
def _save_active_run(
async def inspect_run(self, *, run_id: str) -> dict[str, Any]:
"""Return one durable stopped-run summary without debug trace entries."""
record, run = load_stored_run(self._run_store(), run_id)
environment = record.environment
return _run_payload(
deployment=environment.deployment,
artifact=environment.root_artifact,
status=record.status.value,
run_id=record.id,
resume_readiness=record.resume_readiness.value,
interrupt=_interrupt_payload(run),
outcome=run.outcome,
output=run.output,
diagnostics=record.diagnostics,
trace_count=len(run.trace),
)
async def read_run_trace(
self,
*,
deployment: WorkflowDeployment,
artifact: WorkflowArtifact,
plan: RawWorkflowPlan,
run: RunState,
run_id: str | None = None,
) -> str | None:
"""Store only interrupted runs; terminal runs leave no resume handle."""
if run.status.value != "interrupted":
if run_id is not None:
self._active_runs.pop(run_id, None)
return None
key = run_id or f"run_{uuid4().hex}"
self._active_runs[key] = ActiveWorkflowRun(
deployment=deployment,
artifact=artifact,
plan=plan,
run=run,
run_id: str,
trace_range: TraceRange,
) -> dict[str, Any]:
"""Return only a caller-bounded debug trace slice from a stopped run."""
record, run = load_stored_run(self._run_store(), run_id)
environment = record.environment
end = trace_range.start + trace_range.limit
return _run_payload(
deployment=environment.deployment,
artifact=environment.root_artifact,
status=record.status.value,
run_id=record.id,
resume_readiness=record.resume_readiness.value,
diagnostics=record.diagnostics,
trace_count=len(run.trace),
trace=[asdict(entry) for entry in run.trace[trace_range.start : end]],
trace_start=trace_range.start,
trace_limit=trace_range.limit,
trace_truncated=len(run.trace) > end,
)
return key
def _run_store(self) -> RunStore:
"""Return the configured durable run store required by workflow runs."""
if self.service.run_store is None:
raise KeyError("workflow run store is not configured")
return self.service.run_store
def _deployment_validation(
self,
deployment_id: str,
) -> tuple[WorkflowDeployment, WorkflowArtifact, list[DependencyDiagnostic]]:
) -> tuple[
WorkflowDeployment,
WorkflowArtifact,
list[DependencyDiagnostic],
SavedSubgraphTree,
]:
if self.service.artifact_store is None:
raise KeyError("workflow artifact store is not configured")
deployment = self.service.artifact_store.get_deployment(deployment_id)
@@ -1074,7 +1132,7 @@ class WorkflowSurfaceHandlers:
sources=available_sources,
)
)
return deployment, artifact, diagnostics
return deployment, artifact, diagnostics, tree
def _available_sources(service: WfMcpService) -> list[AvailableSource]:
@@ -1346,6 +1404,7 @@ def _run_payload(
artifact: WorkflowArtifact,
status: str,
run_id: str | None = None,
resume_readiness: str | None = None,
interrupt: dict[str, Any] | None = None,
outcome: str | None = None,
diagnostics: list[DependencyDiagnostic] | None = None,
@@ -1362,6 +1421,7 @@ def _run_payload(
"artifact_version": artifact.version,
"status": status,
"run_id": run_id,
"resume_readiness": resume_readiness,
"interrupt": interrupt,
"outcome": outcome,
"output": output,
src/wf_mcp/workflow_surface/run_lifecycle.py
+161
View File
@@ -0,0 +1,161 @@
from __future__ import annotations
from datetime import UTC, datetime
from uuid import uuid4
from wf_artifacts import (
AvailableSource,
CheckpointReason,
DependencyDiagnostic,
DiagnosticSeverity,
PinnedRunEnvironment,
ResumeReadiness,
RunCheckpoint,
RunStore,
StoredRunStatus,
WorkflowArtifact,
WorkflowDeployment,
WorkflowRunRecord,
validate_deployment_dependencies,
)
from wf_core import (
PersistedRunState,
RunState,
RunStatus,
dump_run_state,
load_run_state,
)
from .saved_subgraphs import SavedSubgraphTree
def create_pinned_environment(
*,
deployment: WorkflowDeployment,
artifact: WorkflowArtifact,
tree: SavedSubgraphTree,
) -> PinnedRunEnvironment:
"""Capture exact root, deployment, and child definitions for one run."""
return PinnedRunEnvironment(
deployment=deployment,
root_artifact=artifact,
child_artifacts=list(tree.artifacts_by_ref.values()),
)
def persist_stopped_run(
*,
store: RunStore,
environment: PinnedRunEnvironment,
run: RunState,
run_id: str | None = None,
) -> WorkflowRunRecord:
"""Persist one externally visible stopped state and its typed checkpoint."""
if run.status not in {
RunStatus.INTERRUPTED,
RunStatus.COMPLETED,
RunStatus.FAILED,
}:
raise ValueError(
f"cannot persist active workflow run with status {run.status!s}"
)
key = run_id or f"run_{uuid4().hex}"
now = datetime.now(UTC)
sequence = 1
created_at = now
if run_id is not None:
existing = store.get_run(run_id)
created_at = existing.created_at
sequence = store.get_latest_checkpoint(run_id).sequence + 1
status = StoredRunStatus(run.status.value)
readiness = (
ResumeReadiness.READY
if status is StoredRunStatus.INTERRUPTED
else ResumeReadiness.NOT_APPLICABLE
)
checkpoint_id = f"{key}.{sequence:06d}"
checkpoint = RunCheckpoint(
id=checkpoint_id,
run_id=key,
sequence=sequence,
reason=CheckpointReason(status.value),
state=PersistedRunState.model_validate(dump_run_state(run)),
created_at=now,
)
record = WorkflowRunRecord(
id=key,
status=status,
resume_readiness=readiness,
environment=environment,
latest_checkpoint_id=checkpoint_id,
created_at=created_at,
updated_at=now,
)
store.save_checkpoint(checkpoint)
store.save_run(record)
return record
def restore_interrupted_run(
store: RunStore, run_id: str
) -> tuple[WorkflowRunRecord, RunState]:
"""Load a persisted interrupted run and its latest typed runtime state."""
record, run = load_stored_run(store, run_id)
if record.status is not StoredRunStatus.INTERRUPTED:
raise ValueError(f"workflow run {run_id!r} is not interrupted")
return record, run
def load_stored_run(store: RunStore, run_id: str) -> tuple[WorkflowRunRecord, RunState]:
"""Load any stopped run record together with its latest typed checkpoint."""
record = store.get_run(run_id)
checkpoint = store.get_latest_checkpoint(run_id)
return record, load_run_state(checkpoint.state.model_dump(mode="json"))
def validate_pinned_resume_environment(
*,
record: WorkflowRunRecord,
sources: list[AvailableSource],
) -> list[DependencyDiagnostic]:
"""Revalidate exact stored graph definitions before a resume mutates state."""
environment = record.environment
diagnostics = validate_deployment_dependencies(
artifact=environment.root_artifact,
deployment=environment.deployment,
sources=sources,
)
for child in environment.child_artifacts:
diagnostics.extend(
validate_deployment_dependencies(
artifact=child,
deployment=environment.deployment,
sources=sources,
)
)
return diagnostics
def has_blocking_diagnostics(diagnostics: list[DependencyDiagnostic]) -> bool:
"""Return whether dependency diagnostics prohibit executing a resume."""
return any(item.severity is DiagnosticSeverity.ERROR for item in diagnostics)
def mark_resume_blocked(
*,
store: RunStore,
record: WorkflowRunRecord,
diagnostics: list[DependencyDiagnostic],
) -> WorkflowRunRecord:
"""Record blocked readiness without writing a new execution checkpoint."""
blocked = record.model_copy(
update={
"resume_readiness": ResumeReadiness.BLOCKED,
"diagnostics": diagnostics,
"updated_at": datetime.now(UTC),
}
)
store.save_run(blocked)
return blocked
src/wf_mcp/workflow_surface/saved_subgraphs.py
+15 -2
View File
@@ -40,6 +40,19 @@ class SavedSubgraphTree:
diagnostics: list[DependencyDiagnostic]
def saved_subgraph_tree_from_snapshots(
child_artifacts: list[WorkflowArtifact],
) -> SavedSubgraphTree:
"""Restore the exact saved-child definitions pinned by a durable run."""
return SavedSubgraphTree(
artifacts_by_ref={
f"workflow.{artifact.id}.v{artifact.version}": artifact
for artifact in child_artifacts
},
diagnostics=[],
)
def resolve_saved_subgraph_tree(
*,
root_artifact: WorkflowArtifact,
@@ -127,8 +140,8 @@ def direct_wrapper_interrupt_diagnostic(
) -> DependencyDiagnostic | None:
"""Reject direct wrapper calls that cannot return a resumable run handle.
Deployment execution supports interrupt/resume through an in-memory
`run_id`; `call_capability` remains a single-call authoring probe.
Deployment execution supports interrupt/resume through a durable `run_id`;
`call_capability` remains a single-call authoring probe.
"""
if not any(isinstance(node, InterruptNode) for node in _artifact_steps(artifact)):
return None
src/wf_mcp/workflow_surface/tools.py
+36 -2
View File
@@ -633,8 +633,9 @@ def register_workflow_tools(server: FastMCP[Any], service: WfMcpService) -> None
name="wf.workflow.resume_run",
title="Resume Workflow Run",
description=(
"Resume an interrupted in-memory deployment run returned by "
"run_deployment. Run IDs are process-local and are not durable."
"Resume an interrupted durable deployment run returned by "
"run_deployment. Resume can remain blocked when a pinned source "
"dependency is unavailable."
),
)
async def resume_run(
@@ -657,3 +658,36 @@ def register_workflow_tools(server: FastMCP[Any], service: WfMcpService) -> None
resume_outcome=resume_outcome,
trace_range=trace_range,
)
@server.tool(
name="wf.workflow.inspect_run",
title="Inspect Workflow Run",
description=(
"Return a durable stopped-run summary and result without debug trace "
"entries. Use read_run_trace only when trace detail is required."
),
)
async def inspect_run(run_id: str) -> dict[str, Any]:
return await handlers.inspect_run(run_id=run_id)
@server.tool(
name="wf.workflow.read_run_trace",
title="Read Workflow Run Trace",
description="Read an explicit bounded debug trace slice for a durable run.",
)
async def read_run_trace(
run_id: str,
trace_range: Annotated[
TraceRange,
Field(
description=(
"Debug traces range to return. Keep the range small because "
"entries can include resolved inputs, outputs, and state changes."
)
),
],
) -> dict[str, Any]:
return await handlers.read_run_trace(
run_id=run_id,
trace_range=trace_range,
)