concurrent foreach preparation, types, validation, refactors

src/wf_core/runtime/foreach_state.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from wf_core.errors import WorkflowExecutionError
+from wf_core.run_state import ExecutionFrame
+from wf_core.runtime.ops.state import StatePatch
+
+_BARRIER_METADATA_KEY = "foreach_barriers"
+
+
+@dataclass(slots=True)
+class ItemErrorRecord:
+    """Structured runtime failure record for one foreach item."""
+
+    index: int
+    frame_id: str
+    node_id: str
+    error_type: str
+    message: str
+    item: Any = None
+
+    @classmethod
+    def from_metadata(cls, raw: object) -> ItemErrorRecord:
+        if not isinstance(raw, dict):
+            raise WorkflowExecutionError("malformed foreach item error record")
+        try:
+            index = raw["index"]
+            frame_id = raw["frame_id"]
+            node_id = raw["node_id"]
+            error_type = raw["error_type"]
+            message = raw["message"]
+        except KeyError as exc:
+            raise WorkflowExecutionError(
+                f"malformed foreach item error record missing {exc.args[0]!r}"
+            ) from exc
+        if not isinstance(index, int):
+            raise WorkflowExecutionError("malformed foreach item error index")
+        if not all(
+            isinstance(value, str)
+            for value in (frame_id, node_id, error_type, message)
+        ):
+            raise WorkflowExecutionError("malformed foreach item error text fields")
+        return cls(
+            index=index,
+            frame_id=frame_id,
+            node_id=node_id,
+            error_type=error_type,
+            message=message,
+            item=raw.get("item"),
+        )
+
+    def to_metadata(self) -> dict[str, Any]:
+        return {
+            "index": self.index,
+            "frame_id": self.frame_id,
+            "node_id": self.node_id,
+            "error_type": self.error_type,
+            "message": self.message,
+            "item": self.item,
+        }
+
+
+@dataclass(slots=True)
+class PendingItemResult:
+    """Buffered item result waiting for a future foreach barrier commit."""
+
+    index: int
+    frame_id: str
+    status: Literal["succeeded", "failed"]
+    patch: StatePatch = field(default_factory=StatePatch)
+    error: ItemErrorRecord | None = None
+
+    @classmethod
+    def from_metadata(cls, raw: object) -> PendingItemResult:
+        if not isinstance(raw, dict):
+            raise WorkflowExecutionError("malformed pending foreach result")
+        index = raw.get("index")
+        frame_id = raw.get("frame_id")
+        status = raw.get("status")
+        patch_changes = raw.get("patch_changes", {})
+        if not isinstance(index, int):
+            raise WorkflowExecutionError("malformed pending foreach result index")
+        if not isinstance(frame_id, str):
+            raise WorkflowExecutionError("malformed pending foreach result frame id")
+        if status not in {"succeeded", "failed"}:
+            raise WorkflowExecutionError("malformed pending foreach result status")
+        if not isinstance(patch_changes, dict):
+            raise WorkflowExecutionError("malformed pending foreach result patch")
+        raw_error = raw.get("error")
+        return cls(
+            index=index,
+            frame_id=frame_id,
+            status=status,
+            patch=StatePatch(changes=dict(patch_changes)),
+            error=ItemErrorRecord.from_metadata(raw_error)
+            if raw_error is not None
+            else None,
+        )
+
+    def to_metadata(self) -> dict[str, Any]:
+        return {
+            "index": self.index,
+            "frame_id": self.frame_id,
+            "status": self.status,
+            "patch_changes": dict(self.patch.changes),
+            "error": self.error.to_metadata() if self.error is not None else None,
+        }
+
+
+@dataclass(slots=True)
+class ForeachBarrierState:
+    """Resumable state owned by one foreach parent frame."""
+
+    next_index: int = 0
+    active_frame_ids: tuple[str, ...] = ()
+    outstanding_frame_ids: tuple[str, ...] = ()
+    pending_results: dict[int, PendingItemResult] = field(default_factory=dict)
+
+    @classmethod
+    def from_frame(
+        cls,
+        frame: ExecutionFrame,
+        foreach_node_id: str,
+    ) -> ForeachBarrierState | None:
+        """Load one foreach barrier state from frame metadata.
+
+        Missing metadata means the foreach has not started on this frame yet.
+        Malformed metadata means runtime state is corrupt and should fail fast.
+        """
+        all_barriers = frame.metadata.get(_BARRIER_METADATA_KEY)
+        if all_barriers is None:
+            return None
+        if not isinstance(all_barriers, dict):
+            raise WorkflowExecutionError(
+                f"malformed foreach barrier table for frame {frame.id!r}"
+            )
+        raw = all_barriers.get(foreach_node_id)
+        if raw is None:
+            return None
+        if not isinstance(raw, dict):
+            raise WorkflowExecutionError(
+                f"malformed foreach barrier state for frame {frame.id!r}"
+            )
+        return cls.from_metadata(raw)
+
+    @classmethod
+    def from_metadata(cls, raw: object) -> ForeachBarrierState:
+        if not isinstance(raw, dict):
+            raise WorkflowExecutionError("malformed foreach barrier state")
+        next_index = raw.get("next_index")
+        active_frame_ids = _string_tuple(raw.get("active_frame_ids", ()))
+        outstanding_frame_ids = _string_tuple(raw.get("outstanding_frame_ids", ()))
+        pending_results = raw.get("pending_results", {})
+        if not isinstance(next_index, int):
+            raise WorkflowExecutionError("malformed foreach barrier next_index")
+        if not isinstance(pending_results, dict):
+            raise WorkflowExecutionError("malformed foreach barrier pending results")
+        parsed_results: dict[int, PendingItemResult] = {}
+        for raw_index, raw_result in pending_results.items():
+            try:
+                index = int(raw_index)
+            except (TypeError, ValueError) as exc:
+                raise WorkflowExecutionError(
+                    "malformed foreach barrier pending result index"
+                ) from exc
+            parsed_results[index] = PendingItemResult.from_metadata(raw_result)
+        return cls(
+            next_index=next_index,
+            active_frame_ids=active_frame_ids,
+            outstanding_frame_ids=outstanding_frame_ids,
+            pending_results=parsed_results,
+        )
+
+    def save_to_frame(self, frame: ExecutionFrame, foreach_node_id: str) -> None:
+        """Store this barrier state in frame metadata under its foreach node id."""
+        raw = frame.metadata.setdefault(_BARRIER_METADATA_KEY, {})
+        if not isinstance(raw, dict):
+            raise WorkflowExecutionError(
+                f"malformed foreach barrier table for frame {frame.id!r}"
+            )
+        raw[foreach_node_id] = self.to_metadata()
+
+    def to_metadata(self) -> dict[str, Any]:
+        return {
+            "next_index": self.next_index,
+            "active_frame_ids": list(self.active_frame_ids),
+            "outstanding_frame_ids": list(self.outstanding_frame_ids),
+            "pending_results": {
+                str(index): result.to_metadata()
+                for index, result in self.pending_results.items()
+            },
+        }
+
+
+def _string_tuple(raw: object) -> tuple[str, ...]:
+    if isinstance(raw, tuple) and all(isinstance(item, str) for item in raw):
+        return raw
+    if isinstance(raw, list) and all(isinstance(item, str) for item in raw):
+        return tuple(raw)
+    raise WorkflowExecutionError("malformed foreach barrier frame id list")

src/wf_core/runtime/ops/foreach.py

@@ -5,6 +5,7 @@ from wf_core.errors import WorkflowExecutionError
 from wf_core.models.steps import ForeachNode
 from wf_core.models.workflow import Workflow
 from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult
+from wf_core.runtime.foreach_state import ForeachBarrierState
 from wf_core.runtime.ops.flow import advance_frame, append_step_result_trace
 from wf_core.runtime.ops.frames import frame_context_values
 from wf_core.runtime.ops.index import WorkflowIndex
@@ -23,12 +24,11 @@ def step_foreach(
 ) -> RunState:
     if step.mode != "serial":
         raise WorkflowExecutionError(
-            "parallel foreach execution is not implemented yet"
+            "concurrent foreach execution is not implemented yet"
         )
 
     frame = run.current_frame()
-    progress_map = frame.metadata.setdefault("foreach_progress", {})
-    progress = progress_map.setdefault(step.id, {"index": 0})
+    barrier = ForeachBarrierState.from_frame(frame, step.id) or ForeachBarrierState()
 
     iterable = safe_resolve_path(
         str(step.over),
@@ -41,7 +41,7 @@ def step_foreach(
             f"foreach source {str(step.over)!r} must resolve to a list"
         )
 
-    loop_index = progress["index"]
+    loop_index = barrier.next_index
     if loop_index >= len(iterable):
         outcome = "done"
         next_node_id = index.next_node_id(frame.node_id, outcome)
@@ -64,7 +64,8 @@ def step_foreach(
     loop_start = index.next_node_id(frame.node_id, "loop")
 
     item = iterable[loop_index]
-    progress["index"] = loop_index + 1
+    barrier.next_index = loop_index + 1
+    barrier.save_to_frame(frame, step.id)
     child_id = f"{frame.id}:{step.id}:{loop_index}"
     child_metadata = ForeachIterationMetadata(
         foreach_node_id=step.id,

src/wf_core/runtime/ops/state.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from collections.abc import Mapping, Sequence
 from copy import deepcopy
+from dataclasses import dataclass, field as dataclass_field
 from typing import Any
 
 from wf_core.errors import WorkflowExecutionError
@@ -23,6 +24,24 @@ from wf_core.runtime.ops.schemas import validate_payload_against_schema
 _MISSING = object()
 
 
+@dataclass(slots=True)
+class StatePatch:
+    """Validated state writes produced by one step before commit.
+
+    `changes` is the public trace-facing view: the incoming values keyed by
+    state path. `_prepared_writes` and `_staged_state` are the executor internals
+    needed to commit reducer-aware values atomically without recomputing the
+    patch.
+    """
+
+    changes: dict[str, Any] = dataclass_field(default_factory=dict)
+    _prepared_writes: dict[StatePath, tuple[list[str], Any]] = dataclass_field(
+        default_factory=dict,
+        repr=False,
+    )
+    _staged_state: dict[str, Any] = dataclass_field(default_factory=dict, repr=False)
+
+
 def apply_output_map(
     workflow: Workflow,
     node: NodeUse,
@@ -58,6 +77,27 @@ def apply_output_bindings(
     missing_field_message: str = "node output did not include required field {field}",
 ) -> dict[str, Any]:
     """Prepare and commit one atomic state patch from canonical output bindings."""
+    patch = build_output_patch(
+        workflow,
+        bindings,
+        node_output,
+        state,
+        reducers=reducers,
+        missing_field_message=missing_field_message,
+    )
+    return commit_state_patch(state, patch)
+
+
+def build_output_patch(
+    workflow: Workflow,
+    bindings: Sequence[OutputBinding],
+    node_output: Mapping[str, Any],
+    state: dict[str, Any],
+    *,
+    reducers: Mapping[str, ReducerDefinition] | None = None,
+    missing_field_message: str = "node output did not include required field {field}",
+) -> StatePatch:
+    """Build and validate one reducer-aware state patch without mutating state."""
     if has_overlapping_paths(str(binding.target) for binding in bindings):
         raise WorkflowExecutionError(
             "mapped state patch has overlapping destination paths"
@@ -91,9 +131,18 @@ def apply_output_bindings(
     for _destination_path, (key_path, merged_value) in prepared_patch.items():
         safe_set_nested_value(staged_state, key_path, merged_value)
     validate_staged_state_patch(staged_state, prepared_patch, state_fields)
+    return StatePatch(
+        changes={str(path): value for path, value in resolved_patch.items()},
+        _prepared_writes=prepared_patch,
+        _staged_state=staged_state,
+    )
+
+
+def commit_state_patch(state: dict[str, Any], patch: StatePatch) -> dict[str, Any]:
+    """Commit a prevalidated patch to state and return trace-facing changes."""
     state.clear()
-    state.update(staged_state)
-    return {str(path): value for path, value in resolved_patch.items()}
+    state.update(patch._staged_state)
+    return dict(patch.changes)
 
 
 def apply_mapped_state(