concurrent foreach preparation, types, validation, refactors
This commit is contained in:
@@ -8,6 +8,21 @@
|
||||
|
||||
**Tech Stack:** Python 3.14, Pydantic v2, dataclasses, pytest, basedpyright, ruff, existing `wf_core` scheduler/runtime modules.
|
||||
|
||||
## Implementation Status
|
||||
|
||||
- Phase 1 is implemented: foreach policy models, legacy `parallel` parse-only
|
||||
compatibility, derived `completed_with_errors` outcomes, and collect
|
||||
destination validation exist.
|
||||
- Phase 2 is implemented: node output writes can be split into
|
||||
`build_output_patch(...)` and `commit_state_patch(...)`; the old
|
||||
`apply_output_bindings(...)` helper remains as the compatibility wrapper.
|
||||
- Phase 3 is implemented: `wf_core.runtime.foreach_state` owns typed barrier
|
||||
metadata and serial foreach progress now uses that metadata instead of ad hoc
|
||||
`foreach_progress`.
|
||||
- Phase 4 is not implemented: `foreach(mode="concurrent")` still validates as a
|
||||
model shape but runtime execution rejects it until concurrent scheduling,
|
||||
barrier commits, and item failure handling are implemented.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Foreach Policy Models
|
||||
|
||||
@@ -53,6 +53,7 @@ def _add_step(builder: WorkflowBuilder, step_id: str, step: DraftStep):
|
||||
as_=step.foreach.as_,
|
||||
mode=step.foreach.mode,
|
||||
on_item_error=step.foreach.on_item_error,
|
||||
concurrent=step.foreach.concurrent,
|
||||
)
|
||||
if isinstance(step, DraftInterruptStep):
|
||||
return builder.interrupt(
|
||||
|
||||
@@ -97,8 +97,24 @@ class DraftForeachPayload(BaseModel):
|
||||
|
||||
over: GraphSourcePath
|
||||
as_: str = Field(alias="as")
|
||||
mode: Literal["serial", "parallel"] = "serial"
|
||||
mode: Literal["serial", "concurrent"] = "serial"
|
||||
on_item_error: Literal["fail", "collect", "skip"] = "fail"
|
||||
concurrent: JsonObject | None = None
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _coerce_legacy_parallel_policy(cls, data: object) -> object:
|
||||
"""Accept old draft foreach parallel names as parse-only compatibility."""
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
normalized = dict(data)
|
||||
if normalized.get("mode") == "parallel":
|
||||
normalized["mode"] = "concurrent"
|
||||
if "parallel" in normalized:
|
||||
if "concurrent" in normalized:
|
||||
raise ValueError("cannot mix deprecated parallel with concurrent")
|
||||
normalized["concurrent"] = normalized.pop("parallel")
|
||||
return normalized
|
||||
|
||||
|
||||
class DraftForeachStep(BaseModel):
|
||||
|
||||
@@ -394,9 +394,16 @@ class WorkflowBuilder:
|
||||
id: str | None = None,
|
||||
over: PathArg,
|
||||
as_: str,
|
||||
mode: Literal["serial", "parallel"] = "serial",
|
||||
mode: Literal["serial", "concurrent"] = "serial",
|
||||
on_item_error: Literal["fail", "collect", "skip"] = "fail",
|
||||
concurrent: Mapping[str, object] | None = None,
|
||||
) -> ForeachNode:
|
||||
"""Add a foreach step.
|
||||
|
||||
Concurrent mode is intentionally model-only for now: it validates saved
|
||||
shape, but runtime execution still rejects it until barrier commits are
|
||||
implemented.
|
||||
"""
|
||||
node = ForeachNode.model_validate(
|
||||
{
|
||||
"id": id or self._next_step_id(f"foreach_{slug_id(as_)}"),
|
||||
@@ -405,6 +412,7 @@ class WorkflowBuilder:
|
||||
"as": as_,
|
||||
"mode": mode,
|
||||
"on_item_error": on_item_error,
|
||||
"concurrent": concurrent,
|
||||
}
|
||||
)
|
||||
self.nodes.append(node)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from .models import (
|
||||
ConditionNode,
|
||||
Edge,
|
||||
ForeachConcurrentPolicy,
|
||||
ForeachItemErrorPolicy,
|
||||
ForeachNode,
|
||||
InterruptNode,
|
||||
JoinNode,
|
||||
@@ -47,6 +49,8 @@ from .validation import (
|
||||
__all__ = [
|
||||
"ConditionNode",
|
||||
"Edge",
|
||||
"ForeachConcurrentPolicy",
|
||||
"ForeachItemErrorPolicy",
|
||||
"ForeachNode",
|
||||
"InterruptNode",
|
||||
"JoinNode",
|
||||
|
||||
@@ -13,6 +13,8 @@ from wf_core.models.reducers import ReducerRef, ReducerSpec
|
||||
from wf_core.models.schemas import NodeDef, SchemaRef, StateField, StateSchema
|
||||
from wf_core.models.steps import (
|
||||
ConditionNode,
|
||||
ForeachConcurrentPolicy,
|
||||
ForeachItemErrorPolicy,
|
||||
ForeachNode,
|
||||
InterruptNode,
|
||||
JoinNode,
|
||||
@@ -27,6 +29,8 @@ __all__ = [
|
||||
"ConditionNode",
|
||||
"Edge",
|
||||
"ExistsCondition",
|
||||
"ForeachConcurrentPolicy",
|
||||
"ForeachItemErrorPolicy",
|
||||
"ForeachNode",
|
||||
"InterruptNode",
|
||||
"JoinNode",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Annotated, Literal
|
||||
from typing import Annotated, Literal, Self
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
@@ -163,6 +163,44 @@ class ConditionNode(BaseModel):
|
||||
check: Condition
|
||||
|
||||
|
||||
class ForeachItemErrorPolicy(BaseModel):
|
||||
"""Policy for runtime failures inside one foreach item lineage."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
action: Literal["fail", "skip", "collect"] = "fail"
|
||||
collect_to: StatePath | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_collect_to(self) -> Self:
|
||||
if self.action == "collect" and self.collect_to is None:
|
||||
raise ValueError("collect item error policy requires collect_to")
|
||||
if self.action != "collect" and self.collect_to is not None:
|
||||
raise ValueError("collect_to is only valid when action='collect'")
|
||||
return self
|
||||
|
||||
|
||||
class ForeachConcurrentPolicy(BaseModel):
|
||||
"""Concurrency policy for foreach frame admission.
|
||||
|
||||
This controls workflow-level child frame admission. It does not imply
|
||||
thread/process execution for sync node handlers; async runtime can use the
|
||||
same policy to admit simultaneous async handler calls later.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
max_active: int = Field(default=4, ge=1)
|
||||
max_outstanding: int = Field(default=20, ge=1)
|
||||
interrupt: Literal["quiesce"] = "quiesce"
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_capacity(self) -> Self:
|
||||
if self.max_outstanding < self.max_active:
|
||||
raise ValueError("max_outstanding must be >= max_active")
|
||||
return self
|
||||
|
||||
|
||||
class ForeachNode(BaseModel):
|
||||
"""Control-flow step that iterates over an input or state list."""
|
||||
|
||||
@@ -174,8 +212,50 @@ class ForeachNode(BaseModel):
|
||||
description="Workflow input/state/context path that must resolve to a list."
|
||||
)
|
||||
as_: str = Field(alias="as", description="Context key for the current item.")
|
||||
mode: Literal["serial", "parallel"] = "serial"
|
||||
on_item_error: Literal["fail", "collect", "skip"] = "fail"
|
||||
mode: Literal["serial", "concurrent"] = "serial"
|
||||
item_error: ForeachItemErrorPolicy = Field(
|
||||
default_factory=ForeachItemErrorPolicy
|
||||
)
|
||||
concurrent: ForeachConcurrentPolicy | None = None
|
||||
on_item_error: Literal["fail", "collect", "skip"] | None = Field(
|
||||
default=None,
|
||||
exclude=True,
|
||||
description="Deprecated parse-only shorthand; use item_error.action.",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _coerce_legacy_policy_shape(cls, data: object) -> object:
|
||||
"""Accept old foreach policy names while saving the new canonical shape."""
|
||||
if not isinstance(data, Mapping):
|
||||
return data
|
||||
|
||||
normalized = dict(data)
|
||||
if normalized.get("mode") == "parallel":
|
||||
normalized["mode"] = "concurrent"
|
||||
|
||||
if "parallel" in normalized:
|
||||
if "concurrent" in normalized:
|
||||
raise ValueError("cannot mix deprecated parallel with concurrent")
|
||||
normalized["concurrent"] = normalized.pop("parallel")
|
||||
|
||||
old_item_error = normalized.pop("on_item_error", None)
|
||||
if old_item_error is not None:
|
||||
if "item_error" in normalized:
|
||||
raise ValueError("cannot mix deprecated on_item_error with item_error")
|
||||
normalized["item_error"] = {"action": old_item_error}
|
||||
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_concurrent_policy(self) -> Self:
|
||||
if self.mode == "concurrent" and self.concurrent is None:
|
||||
raise ValueError("concurrent foreach requires concurrent policy")
|
||||
if self.mode == "serial" and self.concurrent is not None:
|
||||
raise ValueError(
|
||||
"concurrent policy is only valid when mode='concurrent'"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class JoinNode(BaseModel):
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal
|
||||
|
||||
from wf_core.errors import WorkflowExecutionError
|
||||
from wf_core.run_state import ExecutionFrame
|
||||
from wf_core.runtime.ops.state import StatePatch
|
||||
|
||||
_BARRIER_METADATA_KEY = "foreach_barriers"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ItemErrorRecord:
|
||||
"""Structured runtime failure record for one foreach item."""
|
||||
|
||||
index: int
|
||||
frame_id: str
|
||||
node_id: str
|
||||
error_type: str
|
||||
message: str
|
||||
item: Any = None
|
||||
|
||||
@classmethod
|
||||
def from_metadata(cls, raw: object) -> ItemErrorRecord:
|
||||
if not isinstance(raw, dict):
|
||||
raise WorkflowExecutionError("malformed foreach item error record")
|
||||
try:
|
||||
index = raw["index"]
|
||||
frame_id = raw["frame_id"]
|
||||
node_id = raw["node_id"]
|
||||
error_type = raw["error_type"]
|
||||
message = raw["message"]
|
||||
except KeyError as exc:
|
||||
raise WorkflowExecutionError(
|
||||
f"malformed foreach item error record missing {exc.args[0]!r}"
|
||||
) from exc
|
||||
if not isinstance(index, int):
|
||||
raise WorkflowExecutionError("malformed foreach item error index")
|
||||
if not all(
|
||||
isinstance(value, str)
|
||||
for value in (frame_id, node_id, error_type, message)
|
||||
):
|
||||
raise WorkflowExecutionError("malformed foreach item error text fields")
|
||||
return cls(
|
||||
index=index,
|
||||
frame_id=frame_id,
|
||||
node_id=node_id,
|
||||
error_type=error_type,
|
||||
message=message,
|
||||
item=raw.get("item"),
|
||||
)
|
||||
|
||||
def to_metadata(self) -> dict[str, Any]:
|
||||
return {
|
||||
"index": self.index,
|
||||
"frame_id": self.frame_id,
|
||||
"node_id": self.node_id,
|
||||
"error_type": self.error_type,
|
||||
"message": self.message,
|
||||
"item": self.item,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PendingItemResult:
|
||||
"""Buffered item result waiting for a future foreach barrier commit."""
|
||||
|
||||
index: int
|
||||
frame_id: str
|
||||
status: Literal["succeeded", "failed"]
|
||||
patch: StatePatch = field(default_factory=StatePatch)
|
||||
error: ItemErrorRecord | None = None
|
||||
|
||||
@classmethod
|
||||
def from_metadata(cls, raw: object) -> PendingItemResult:
|
||||
if not isinstance(raw, dict):
|
||||
raise WorkflowExecutionError("malformed pending foreach result")
|
||||
index = raw.get("index")
|
||||
frame_id = raw.get("frame_id")
|
||||
status = raw.get("status")
|
||||
patch_changes = raw.get("patch_changes", {})
|
||||
if not isinstance(index, int):
|
||||
raise WorkflowExecutionError("malformed pending foreach result index")
|
||||
if not isinstance(frame_id, str):
|
||||
raise WorkflowExecutionError("malformed pending foreach result frame id")
|
||||
if status not in {"succeeded", "failed"}:
|
||||
raise WorkflowExecutionError("malformed pending foreach result status")
|
||||
if not isinstance(patch_changes, dict):
|
||||
raise WorkflowExecutionError("malformed pending foreach result patch")
|
||||
raw_error = raw.get("error")
|
||||
return cls(
|
||||
index=index,
|
||||
frame_id=frame_id,
|
||||
status=status,
|
||||
patch=StatePatch(changes=dict(patch_changes)),
|
||||
error=ItemErrorRecord.from_metadata(raw_error)
|
||||
if raw_error is not None
|
||||
else None,
|
||||
)
|
||||
|
||||
def to_metadata(self) -> dict[str, Any]:
|
||||
return {
|
||||
"index": self.index,
|
||||
"frame_id": self.frame_id,
|
||||
"status": self.status,
|
||||
"patch_changes": dict(self.patch.changes),
|
||||
"error": self.error.to_metadata() if self.error is not None else None,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ForeachBarrierState:
|
||||
"""Resumable state owned by one foreach parent frame."""
|
||||
|
||||
next_index: int = 0
|
||||
active_frame_ids: tuple[str, ...] = ()
|
||||
outstanding_frame_ids: tuple[str, ...] = ()
|
||||
pending_results: dict[int, PendingItemResult] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def from_frame(
|
||||
cls,
|
||||
frame: ExecutionFrame,
|
||||
foreach_node_id: str,
|
||||
) -> ForeachBarrierState | None:
|
||||
"""Load one foreach barrier state from frame metadata.
|
||||
|
||||
Missing metadata means the foreach has not started on this frame yet.
|
||||
Malformed metadata means runtime state is corrupt and should fail fast.
|
||||
"""
|
||||
all_barriers = frame.metadata.get(_BARRIER_METADATA_KEY)
|
||||
if all_barriers is None:
|
||||
return None
|
||||
if not isinstance(all_barriers, dict):
|
||||
raise WorkflowExecutionError(
|
||||
f"malformed foreach barrier table for frame {frame.id!r}"
|
||||
)
|
||||
raw = all_barriers.get(foreach_node_id)
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, dict):
|
||||
raise WorkflowExecutionError(
|
||||
f"malformed foreach barrier state for frame {frame.id!r}"
|
||||
)
|
||||
return cls.from_metadata(raw)
|
||||
|
||||
@classmethod
|
||||
def from_metadata(cls, raw: object) -> ForeachBarrierState:
|
||||
if not isinstance(raw, dict):
|
||||
raise WorkflowExecutionError("malformed foreach barrier state")
|
||||
next_index = raw.get("next_index")
|
||||
active_frame_ids = _string_tuple(raw.get("active_frame_ids", ()))
|
||||
outstanding_frame_ids = _string_tuple(raw.get("outstanding_frame_ids", ()))
|
||||
pending_results = raw.get("pending_results", {})
|
||||
if not isinstance(next_index, int):
|
||||
raise WorkflowExecutionError("malformed foreach barrier next_index")
|
||||
if not isinstance(pending_results, dict):
|
||||
raise WorkflowExecutionError("malformed foreach barrier pending results")
|
||||
parsed_results: dict[int, PendingItemResult] = {}
|
||||
for raw_index, raw_result in pending_results.items():
|
||||
try:
|
||||
index = int(raw_index)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise WorkflowExecutionError(
|
||||
"malformed foreach barrier pending result index"
|
||||
) from exc
|
||||
parsed_results[index] = PendingItemResult.from_metadata(raw_result)
|
||||
return cls(
|
||||
next_index=next_index,
|
||||
active_frame_ids=active_frame_ids,
|
||||
outstanding_frame_ids=outstanding_frame_ids,
|
||||
pending_results=parsed_results,
|
||||
)
|
||||
|
||||
def save_to_frame(self, frame: ExecutionFrame, foreach_node_id: str) -> None:
|
||||
"""Store this barrier state in frame metadata under its foreach node id."""
|
||||
raw = frame.metadata.setdefault(_BARRIER_METADATA_KEY, {})
|
||||
if not isinstance(raw, dict):
|
||||
raise WorkflowExecutionError(
|
||||
f"malformed foreach barrier table for frame {frame.id!r}"
|
||||
)
|
||||
raw[foreach_node_id] = self.to_metadata()
|
||||
|
||||
def to_metadata(self) -> dict[str, Any]:
|
||||
return {
|
||||
"next_index": self.next_index,
|
||||
"active_frame_ids": list(self.active_frame_ids),
|
||||
"outstanding_frame_ids": list(self.outstanding_frame_ids),
|
||||
"pending_results": {
|
||||
str(index): result.to_metadata()
|
||||
for index, result in self.pending_results.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _string_tuple(raw: object) -> tuple[str, ...]:
|
||||
if isinstance(raw, tuple) and all(isinstance(item, str) for item in raw):
|
||||
return raw
|
||||
if isinstance(raw, list) and all(isinstance(item, str) for item in raw):
|
||||
return tuple(raw)
|
||||
raise WorkflowExecutionError("malformed foreach barrier frame id list")
|
||||
@@ -5,6 +5,7 @@ from wf_core.errors import WorkflowExecutionError
|
||||
from wf_core.models.steps import ForeachNode
|
||||
from wf_core.models.workflow import Workflow
|
||||
from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult
|
||||
from wf_core.runtime.foreach_state import ForeachBarrierState
|
||||
from wf_core.runtime.ops.flow import advance_frame, append_step_result_trace
|
||||
from wf_core.runtime.ops.frames import frame_context_values
|
||||
from wf_core.runtime.ops.index import WorkflowIndex
|
||||
@@ -23,12 +24,11 @@ def step_foreach(
|
||||
) -> RunState:
|
||||
if step.mode != "serial":
|
||||
raise WorkflowExecutionError(
|
||||
"parallel foreach execution is not implemented yet"
|
||||
"concurrent foreach execution is not implemented yet"
|
||||
)
|
||||
|
||||
frame = run.current_frame()
|
||||
progress_map = frame.metadata.setdefault("foreach_progress", {})
|
||||
progress = progress_map.setdefault(step.id, {"index": 0})
|
||||
barrier = ForeachBarrierState.from_frame(frame, step.id) or ForeachBarrierState()
|
||||
|
||||
iterable = safe_resolve_path(
|
||||
str(step.over),
|
||||
@@ -41,7 +41,7 @@ def step_foreach(
|
||||
f"foreach source {str(step.over)!r} must resolve to a list"
|
||||
)
|
||||
|
||||
loop_index = progress["index"]
|
||||
loop_index = barrier.next_index
|
||||
if loop_index >= len(iterable):
|
||||
outcome = "done"
|
||||
next_node_id = index.next_node_id(frame.node_id, outcome)
|
||||
@@ -64,7 +64,8 @@ def step_foreach(
|
||||
loop_start = index.next_node_id(frame.node_id, "loop")
|
||||
|
||||
item = iterable[loop_index]
|
||||
progress["index"] = loop_index + 1
|
||||
barrier.next_index = loop_index + 1
|
||||
barrier.save_to_frame(frame, step.id)
|
||||
child_id = f"{frame.id}:{step.id}:{loop_index}"
|
||||
child_metadata = ForeachIterationMetadata(
|
||||
foreach_node_id=step.id,
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field as dataclass_field
|
||||
from typing import Any
|
||||
|
||||
from wf_core.errors import WorkflowExecutionError
|
||||
@@ -23,6 +24,24 @@ from wf_core.runtime.ops.schemas import validate_payload_against_schema
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class StatePatch:
|
||||
"""Validated state writes produced by one step before commit.
|
||||
|
||||
`changes` is the public trace-facing view: the incoming values keyed by
|
||||
state path. `_prepared_writes` and `_staged_state` are the executor internals
|
||||
needed to commit reducer-aware values atomically without recomputing the
|
||||
patch.
|
||||
"""
|
||||
|
||||
changes: dict[str, Any] = dataclass_field(default_factory=dict)
|
||||
_prepared_writes: dict[StatePath, tuple[list[str], Any]] = dataclass_field(
|
||||
default_factory=dict,
|
||||
repr=False,
|
||||
)
|
||||
_staged_state: dict[str, Any] = dataclass_field(default_factory=dict, repr=False)
|
||||
|
||||
|
||||
def apply_output_map(
|
||||
workflow: Workflow,
|
||||
node: NodeUse,
|
||||
@@ -58,6 +77,27 @@ def apply_output_bindings(
|
||||
missing_field_message: str = "node output did not include required field {field}",
|
||||
) -> dict[str, Any]:
|
||||
"""Prepare and commit one atomic state patch from canonical output bindings."""
|
||||
patch = build_output_patch(
|
||||
workflow,
|
||||
bindings,
|
||||
node_output,
|
||||
state,
|
||||
reducers=reducers,
|
||||
missing_field_message=missing_field_message,
|
||||
)
|
||||
return commit_state_patch(state, patch)
|
||||
|
||||
|
||||
def build_output_patch(
|
||||
workflow: Workflow,
|
||||
bindings: Sequence[OutputBinding],
|
||||
node_output: Mapping[str, Any],
|
||||
state: dict[str, Any],
|
||||
*,
|
||||
reducers: Mapping[str, ReducerDefinition] | None = None,
|
||||
missing_field_message: str = "node output did not include required field {field}",
|
||||
) -> StatePatch:
|
||||
"""Build and validate one reducer-aware state patch without mutating state."""
|
||||
if has_overlapping_paths(str(binding.target) for binding in bindings):
|
||||
raise WorkflowExecutionError(
|
||||
"mapped state patch has overlapping destination paths"
|
||||
@@ -91,9 +131,18 @@ def apply_output_bindings(
|
||||
for _destination_path, (key_path, merged_value) in prepared_patch.items():
|
||||
safe_set_nested_value(staged_state, key_path, merged_value)
|
||||
validate_staged_state_patch(staged_state, prepared_patch, state_fields)
|
||||
return StatePatch(
|
||||
changes={str(path): value for path, value in resolved_patch.items()},
|
||||
_prepared_writes=prepared_patch,
|
||||
_staged_state=staged_state,
|
||||
)
|
||||
|
||||
|
||||
def commit_state_patch(state: dict[str, Any], patch: StatePatch) -> dict[str, Any]:
|
||||
"""Commit a prevalidated patch to state and return trace-facing changes."""
|
||||
state.clear()
|
||||
state.update(staged_state)
|
||||
return {str(path): value for path, value in resolved_patch.items()}
|
||||
state.update(patch._staged_state)
|
||||
return dict(patch.changes)
|
||||
|
||||
|
||||
def apply_mapped_state(
|
||||
|
||||
@@ -75,7 +75,12 @@ def _validate_nodes(
|
||||
)
|
||||
elif isinstance(node, ForeachNode):
|
||||
validate_foreach_node(
|
||||
node, index, report, state_root_fields, input_root_fields
|
||||
node,
|
||||
index,
|
||||
report,
|
||||
state_root_fields,
|
||||
input_root_fields,
|
||||
workflow,
|
||||
)
|
||||
elif isinstance(node, InterruptNode):
|
||||
validate_interrupt_node(
|
||||
|
||||
@@ -21,6 +21,7 @@ class ValidationIssueCode(StrEnum):
|
||||
EMPTY_CONDITION_ARGS = "empty_condition_args"
|
||||
INVALID_CONDITION_PATH = "invalid_condition_path"
|
||||
INVALID_FOREACH_SOURCE = "invalid_foreach_source"
|
||||
INVALID_FOREACH_COLLECT_DESTINATION = "invalid_foreach_collect_destination"
|
||||
INVALID_INTERRUPT_SOURCE = "invalid_interrupt_source"
|
||||
INVALID_INTERRUPT_DESTINATION = "invalid_interrupt_destination"
|
||||
|
||||
|
||||
@@ -13,7 +13,10 @@ def declared_outcomes_for_step(step: Step, node_defs: dict[str, NodeDef]) -> set
|
||||
if step.type == "condition":
|
||||
return {"true", "false"}
|
||||
if step.type == "foreach":
|
||||
return {"loop", "done"}
|
||||
outcomes = {"loop", "done"}
|
||||
if step.item_error.action in {"skip", "collect"}:
|
||||
outcomes.add("completed_with_errors")
|
||||
return outcomes
|
||||
if step.type == "join":
|
||||
return {"done"}
|
||||
if isinstance(step, InterruptNode):
|
||||
|
||||
@@ -149,6 +149,7 @@ def validate_foreach_node(
|
||||
report: ValidationReport,
|
||||
state_root_fields: set[str],
|
||||
input_root_fields: set[str],
|
||||
workflow: Workflow,
|
||||
) -> None:
|
||||
if not is_valid_source_path(node.over, state_root_fields, input_root_fields):
|
||||
report.add(
|
||||
@@ -156,6 +157,27 @@ def validate_foreach_node(
|
||||
f"nodes[{index}].over",
|
||||
"foreach source path must start with input. or state. and reference a declared root field",
|
||||
)
|
||||
if node.item_error.action != "collect":
|
||||
return
|
||||
collect_to = node.item_error.collect_to
|
||||
if collect_to is None:
|
||||
return
|
||||
destination_root = _state_destination_root(collect_to)
|
||||
state_fields = workflow.state_schema.field_index()
|
||||
field = state_fields.get(collect_to)
|
||||
if destination_root is None or destination_root not in state_root_fields:
|
||||
report.add(
|
||||
ValidationIssueCode.INVALID_FOREACH_COLLECT_DESTINATION,
|
||||
f"nodes[{index}].item_error.collect_to",
|
||||
"collect_to must start with state. and reference a declared state field",
|
||||
)
|
||||
return
|
||||
if field is None or field.type != "array":
|
||||
report.add(
|
||||
ValidationIssueCode.INVALID_FOREACH_COLLECT_DESTINATION,
|
||||
f"nodes[{index}].item_error.collect_to",
|
||||
"collect_to must reference a declared array state field",
|
||||
)
|
||||
|
||||
|
||||
def validate_interrupt_node(
|
||||
|
||||
@@ -17,7 +17,11 @@ from wf_core import (
|
||||
from wf_core.models.steps import OutputBinding
|
||||
from wf_core.runtime.engine import resume_workflow
|
||||
from wf_core.runtime.ops.runs import create_run_state
|
||||
from wf_core.runtime.ops.state import apply_output_bindings
|
||||
from wf_core.runtime.ops.state import (
|
||||
apply_output_bindings,
|
||||
build_output_patch,
|
||||
commit_state_patch,
|
||||
)
|
||||
|
||||
|
||||
def test_output_bindings_commit_patch_atomically_when_source_is_missing() -> None:
|
||||
@@ -171,6 +175,41 @@ def test_full_workflow_execution_writes_canonical_output_bindings() -> None:
|
||||
assert run.trace[0].state_changes["state.person.name"] == "Ada"
|
||||
|
||||
|
||||
def test_build_output_patch_does_not_mutate_until_commit() -> None:
|
||||
workflow = _workflow(fields={"person.name": StateField(type="string")})
|
||||
state = {"person": {"name": "old"}}
|
||||
|
||||
patch = build_output_patch(
|
||||
workflow,
|
||||
[_binding("person.name", "state.person.name")],
|
||||
{"person": {"name": "Ada"}},
|
||||
state,
|
||||
)
|
||||
|
||||
assert state["person"]["name"] == "old"
|
||||
assert patch.changes["state.person.name"] == "Ada"
|
||||
|
||||
committed = commit_state_patch(state, patch)
|
||||
|
||||
assert committed["state.person.name"] == "Ada"
|
||||
assert state["person"]["name"] == "Ada"
|
||||
|
||||
|
||||
def test_build_and_commit_patch_matches_apply_output_bindings() -> None:
|
||||
workflow = _workflow(fields={"person.name": StateField(type="string")})
|
||||
state_from_apply = {"person": {"name": "old"}}
|
||||
state_from_patch = {"person": {"name": "old"}}
|
||||
bindings = [_binding("person.name", "state.person.name")]
|
||||
output = {"person": {"name": "Ada"}}
|
||||
|
||||
applied = apply_output_bindings(workflow, bindings, output, state_from_apply)
|
||||
patch = build_output_patch(workflow, bindings, output, state_from_patch)
|
||||
committed = commit_state_patch(state_from_patch, patch)
|
||||
|
||||
assert applied["state.person.name"] == committed["state.person.name"]
|
||||
assert state_from_apply["person"]["name"] == state_from_patch["person"]["name"]
|
||||
|
||||
|
||||
def _binding(source: str, target: str) -> OutputBinding:
|
||||
return OutputBinding.model_validate({"source": source, "target": target})
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from wf_core.errors import WorkflowExecutionError
|
||||
from wf_core.run_state import ExecutionFrame
|
||||
from wf_core.runtime.foreach_state import (
|
||||
ForeachBarrierState,
|
||||
ItemErrorRecord,
|
||||
PendingItemResult,
|
||||
)
|
||||
from wf_core.runtime.ops.state import StatePatch
|
||||
|
||||
|
||||
def test_foreach_barrier_state_round_trips_through_frame_metadata() -> None:
|
||||
frame = ExecutionFrame(id="root", kind="root", node_id="each")
|
||||
barrier = ForeachBarrierState(
|
||||
next_index=2,
|
||||
active_frame_ids=("child-1",),
|
||||
outstanding_frame_ids=("child-1", "child-2"),
|
||||
pending_results={
|
||||
1: PendingItemResult(
|
||||
index=1,
|
||||
frame_id="child-1",
|
||||
status="failed",
|
||||
patch=StatePatch(changes={"state.count": 1}),
|
||||
error=ItemErrorRecord(
|
||||
index=1,
|
||||
frame_id="child-1",
|
||||
node_id="work",
|
||||
error_type="ValueError",
|
||||
message="bad item",
|
||||
item={"id": "a"},
|
||||
),
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
barrier.save_to_frame(frame, "each")
|
||||
loaded = ForeachBarrierState.from_frame(frame, "each")
|
||||
|
||||
assert loaded is not None
|
||||
assert loaded.next_index == 2
|
||||
assert loaded.active_frame_ids == ("child-1",)
|
||||
assert loaded.outstanding_frame_ids == ("child-1", "child-2")
|
||||
assert loaded.pending_results[1].patch.changes["state.count"] == 1
|
||||
assert loaded.pending_results[1].error is not None
|
||||
assert loaded.pending_results[1].error.message == "bad item"
|
||||
|
||||
|
||||
def test_foreach_barrier_state_returns_none_when_missing() -> None:
|
||||
frame = ExecutionFrame(id="root", kind="root", node_id="each")
|
||||
|
||||
assert ForeachBarrierState.from_frame(frame, "each") is None
|
||||
|
||||
|
||||
def test_foreach_barrier_state_rejects_malformed_metadata() -> None:
|
||||
frame = ExecutionFrame(
|
||||
id="root",
|
||||
kind="root",
|
||||
node_id="each",
|
||||
metadata={"foreach_barriers": {"each": {"next_index": "bad"}}},
|
||||
)
|
||||
|
||||
with pytest.raises(WorkflowExecutionError, match="next_index"):
|
||||
ForeachBarrierState.from_frame(frame, "each")
|
||||
@@ -0,0 +1,181 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from wf_core import END, Workflow, validate_workflow
|
||||
from wf_core.models.steps import ForeachNode
|
||||
|
||||
|
||||
def test_serial_foreach_defaults_to_fail_item_policy() -> None:
|
||||
node = ForeachNode.model_validate(
|
||||
{
|
||||
"id": "each",
|
||||
"type": "foreach",
|
||||
"over": {"root": "state", "parts": ["items"]},
|
||||
"as": "item",
|
||||
}
|
||||
)
|
||||
|
||||
assert node.mode == "serial"
|
||||
assert node.item_error.action == "fail"
|
||||
assert node.item_error.collect_to is None
|
||||
assert node.concurrent is None
|
||||
|
||||
|
||||
def test_deprecated_on_item_error_parses_to_nested_policy() -> None:
|
||||
node = ForeachNode.model_validate(
|
||||
{
|
||||
"id": "each",
|
||||
"type": "foreach",
|
||||
"over": "state.items",
|
||||
"as": "item",
|
||||
"on_item_error": "skip",
|
||||
}
|
||||
)
|
||||
|
||||
dumped = node.model_dump(mode="json", by_alias=True)
|
||||
|
||||
assert node.item_error.action == "skip"
|
||||
assert "on_item_error" not in dumped
|
||||
assert dumped["item_error"]["action"] == "skip"
|
||||
|
||||
|
||||
def test_collect_item_policy_requires_collect_to() -> None:
|
||||
with pytest.raises(ValidationError, match="collect_to"):
|
||||
ForeachNode.model_validate(
|
||||
{
|
||||
"id": "each",
|
||||
"type": "foreach",
|
||||
"over": {"root": "state", "parts": ["items"]},
|
||||
"as": "item",
|
||||
"item_error": {"action": "collect"},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_policy_requires_concurrent_mode() -> None:
|
||||
with pytest.raises(ValidationError, match="concurrent policy"):
|
||||
ForeachNode.model_validate(
|
||||
{
|
||||
"id": "each",
|
||||
"type": "foreach",
|
||||
"over": {"root": "state", "parts": ["items"]},
|
||||
"as": "item",
|
||||
"concurrent": {"max_active": 4, "max_outstanding": 20},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_policy_validates_capacity_order() -> None:
|
||||
with pytest.raises(ValidationError, match="max_outstanding"):
|
||||
ForeachNode.model_validate(
|
||||
{
|
||||
"id": "each",
|
||||
"type": "foreach",
|
||||
"over": {"root": "state", "parts": ["items"]},
|
||||
"as": "item",
|
||||
"mode": "concurrent",
|
||||
"concurrent": {"max_active": 10, "max_outstanding": 4},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_deprecated_parallel_policy_parses_to_concurrent_policy() -> None:
|
||||
node = ForeachNode.model_validate(
|
||||
{
|
||||
"id": "each",
|
||||
"type": "foreach",
|
||||
"over": "state.items",
|
||||
"as": "item",
|
||||
"mode": "parallel",
|
||||
"parallel": {"max_active": 2, "max_outstanding": 5},
|
||||
}
|
||||
)
|
||||
|
||||
dumped = node.model_dump(mode="json", by_alias=True)
|
||||
|
||||
assert node.mode == "concurrent"
|
||||
assert node.concurrent is not None
|
||||
assert node.concurrent.max_active == 2
|
||||
assert "parallel" not in dumped
|
||||
assert dumped["mode"] == "concurrent"
|
||||
assert dumped["concurrent"]["max_outstanding"] == 5
|
||||
|
||||
|
||||
def test_collect_policy_requires_completed_with_errors_edge() -> None:
|
||||
workflow = _workflow(
|
||||
item_error={"action": "collect", "collect_to": "state.item_errors"},
|
||||
edges=[
|
||||
{"from": "each", "outcome": "loop", "to": END},
|
||||
{"from": "each", "outcome": "done", "to": END},
|
||||
],
|
||||
)
|
||||
|
||||
report = validate_workflow(workflow)
|
||||
|
||||
assert report.errors
|
||||
assert report.errors[0].code == "missing_outcome_edge"
|
||||
assert "completed_with_errors" in report.errors[0].message
|
||||
|
||||
|
||||
def test_collect_policy_destination_must_be_declared_array_field() -> None:
|
||||
workflow = _workflow(
|
||||
item_error={"action": "collect", "collect_to": "state.not_array"},
|
||||
state_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {"type": "array"},
|
||||
"not_array": {"type": "string"},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
report = validate_workflow(workflow)
|
||||
|
||||
matching = [
|
||||
issue
|
||||
for issue in report.errors
|
||||
if issue.code == "invalid_foreach_collect_destination"
|
||||
]
|
||||
assert matching
|
||||
assert "array state field" in matching[0].message
|
||||
|
||||
|
||||
def _workflow(
|
||||
*,
|
||||
item_error: dict[str, object] | None = None,
|
||||
state_schema: dict[str, object] | None = None,
|
||||
edges: list[dict[str, str]] | None = None,
|
||||
) -> Workflow:
|
||||
return Workflow.model_validate(
|
||||
{
|
||||
"name": "foreach_policy",
|
||||
"input_schema": {"type": "object", "properties": {}},
|
||||
"state_schema": state_schema
|
||||
or {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {"type": "array"},
|
||||
"item_errors": {"type": "array"},
|
||||
},
|
||||
},
|
||||
"output_schema": {"type": "object", "properties": {}},
|
||||
"start": "each",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "each",
|
||||
"type": "foreach",
|
||||
"over": "state.items",
|
||||
"as": "item",
|
||||
"item_error": item_error or {"action": "fail"},
|
||||
}
|
||||
],
|
||||
"edges": edges
|
||||
or [
|
||||
{"from": "each", "outcome": "loop", "to": END},
|
||||
{"from": "each", "outcome": "done", "to": END},
|
||||
],
|
||||
"node_defs": [],
|
||||
}
|
||||
)
|
||||
Reference in New Issue
Block a user