feat: identify dynamic foreach activations

This commit is contained in:
lda
2026-09-04 07:25:55 +07:00 Verified
parent 4bbd9f9650
commit f69c4502cf
12 changed files with 499 additions and 76 deletions
+202 -3
View File
@@ -11,6 +11,31 @@ from wf_core.runtime.ops.state import StatePatch
from wf_core.runtime.scheduler import ForeachIterationMetadata from wf_core.runtime.scheduler import ForeachIterationMetadata
_BARRIER_METADATA_KEY = "foreach_barriers" _BARRIER_METADATA_KEY = "foreach_barriers"
_ACTIVATION_METADATA_KEY = "foreach_activations"
@dataclass(slots=True)
class ForeachActivationState:
"""Persisted state for one dynamic visit to a foreach controller.
A parent frame creates a fresh activation on first entry, reuses it while
admitting items, and closes it before emitting ``done``. The id is opaque:
callers compare it by name and never parse it.
"""
id: str
foreach_node_id: str
barrier: ForeachBarrierState
@dataclass(frozen=True, slots=True)
class ForeachItemOwner:
"""Named ownership record for one foreach item frame."""
parent_frame_id: str
foreach_node_id: str
activation_id: str
item_index: int
@dataclass(slots=True) @dataclass(slots=True)
@@ -320,14 +345,188 @@ class ForeachBarrierState:
) )
def item_frame_owner(frame: ExecutionFrame) -> tuple[str, str, int] | None: def load_or_begin_foreach_activation(
"""Return parent frame id, foreach node id, and item index for item frames.""" frame: ExecutionFrame,
foreach_node_id: str,
*,
mode: Literal["serial", "concurrent"],
) -> ForeachActivationState:
"""Load the active activation or begin a fresh visit.
The first entry for one visit allocates an opaque id from the parent frame
id, foreach node id, and a persisted per-frame sequence. Later calls reuse
the active activation; closing it makes the next visit allocate a new id
with fresh barrier state. Mode mismatches and malformed tables fail fast.
"""
table = _activation_table(frame)
entry = table.get(foreach_node_id)
if entry is None:
entry = {"next_sequence": 0, "active": None}
table[foreach_node_id] = entry
if not isinstance(entry, dict):
raise WorkflowExecutionError(
f"malformed foreach activation entry for frame {frame.id!r}"
)
next_sequence = entry.get("next_sequence", 0)
if not isinstance(next_sequence, int) or next_sequence < 0:
raise WorkflowExecutionError(
f"malformed foreach activation sequence for frame {frame.id!r}"
)
active = entry.get("active")
if active is not None:
activation = _activation_from_metadata(
active, frame_id=frame.id, foreach_node_id=foreach_node_id
)
if activation.barrier.mode != mode:
raise WorkflowExecutionError(
f"foreach {foreach_node_id!r} activation {activation.id!r} "
f"has mode {activation.barrier.mode!r}, got {mode!r}"
)
return activation
activation_id = f"{frame.id}:{foreach_node_id}#{next_sequence}"
activation = ForeachActivationState(
id=activation_id,
foreach_node_id=foreach_node_id,
barrier=ForeachBarrierState(mode=mode),
)
entry["next_sequence"] = next_sequence + 1
entry["active"] = {
"id": activation.id,
"barrier": activation.barrier.to_metadata(),
}
return activation
def save_foreach_activation(
frame: ExecutionFrame, activation: ForeachActivationState
) -> None:
"""Persist barrier progress for the named active activation."""
table = _activation_table(frame)
entry = table.get(activation.foreach_node_id)
if not isinstance(entry, dict):
raise WorkflowExecutionError(
f"malformed foreach activation entry for frame {frame.id!r}"
)
active = entry.get("active")
if not isinstance(active, dict) or active.get("id") != activation.id:
raise WorkflowExecutionError(
f"cannot save stale foreach activation {activation.id!r} "
f"for frame {frame.id!r}"
)
active["barrier"] = activation.barrier.to_metadata()
def close_foreach_activation(
frame: ExecutionFrame, activation: ForeachActivationState
) -> None:
"""Close the named active activation, preserving the visit sequence.
The barrier is removed so a later visit starts fresh; the sequence keeps
increasing so child and lineage ids cannot collide across visits.
"""
table = _activation_table(frame)
entry = table.get(activation.foreach_node_id)
if not isinstance(entry, dict):
raise WorkflowExecutionError(
f"malformed foreach activation entry for frame {frame.id!r}"
)
active = entry.get("active")
if not isinstance(active, dict) or active.get("id") != activation.id:
raise WorkflowExecutionError(
f"cannot close stale foreach activation {activation.id!r} "
f"for frame {frame.id!r}"
)
entry["active"] = None
def load_foreach_activation(
frame: ExecutionFrame, foreach_node_id: str, activation_id: str
) -> ForeachActivationState | None:
"""Return the active activation only when its id matches the child.
A child result naming a closed or different activation must fail closed in
the caller rather than buffering into the wrong barrier.
"""
table = _activation_table(frame)
entry = table.get(foreach_node_id)
if not isinstance(entry, dict):
raise WorkflowExecutionError(
f"malformed foreach activation entry for frame {frame.id!r}"
)
active = entry.get("active")
if active is None:
return None
activation = _activation_from_metadata(
active, frame_id=frame.id, foreach_node_id=foreach_node_id
)
if activation.id != activation_id:
return None
return activation
def require_foreach_activation(
frame: ExecutionFrame, foreach_node_id: str, activation_id: str
) -> ForeachActivationState:
"""Load the named activation or raise when it is closed or superseded."""
activation = load_foreach_activation(frame, foreach_node_id, activation_id)
if activation is None:
raise WorkflowExecutionError(
f"foreach item activation {activation_id!r} for node "
f"{foreach_node_id!r} is closed or superseded"
)
return activation
def item_frame_owner(frame: ExecutionFrame) -> ForeachItemOwner | None:
"""Return the named foreach ownership record for item frames.
Malformed item metadata fails closed via ``ForeachIterationMetadata``;
only non-item frames return ``None``.
"""
if frame.kind != "foreach_iteration" or frame.parent_frame_id is None: if frame.kind != "foreach_iteration" or frame.parent_frame_id is None:
return None return None
metadata = ForeachIterationMetadata.from_frame(frame) metadata = ForeachIterationMetadata.from_frame(frame)
if metadata is None: if metadata is None:
return None return None
return frame.parent_frame_id, metadata.foreach_node_id, metadata.loop_index return ForeachItemOwner(
parent_frame_id=frame.parent_frame_id,
foreach_node_id=metadata.foreach_node_id,
activation_id=metadata.activation_id,
item_index=metadata.loop_index,
)
def _activation_table(frame: ExecutionFrame) -> dict[str, Any]:
raw = frame.metadata.get(_ACTIVATION_METADATA_KEY)
if raw is None:
table: dict[str, Any] = {}
frame.metadata[_ACTIVATION_METADATA_KEY] = table
return table
if not isinstance(raw, dict):
raise WorkflowExecutionError(
f"malformed foreach activation table for frame {frame.id!r}"
)
return raw
def _activation_from_metadata(
raw: object, *, frame_id: str, foreach_node_id: str
) -> ForeachActivationState:
if not isinstance(raw, dict):
raise WorkflowExecutionError(
f"malformed foreach activation for frame {frame_id!r}"
)
activation_id = raw.get("id")
barrier_raw = raw.get("barrier")
if not isinstance(activation_id, str) or not activation_id:
raise WorkflowExecutionError(
f"malformed foreach activation id for frame {frame_id!r}"
)
return ForeachActivationState(
id=activation_id,
foreach_node_id=foreach_node_id,
barrier=ForeachBarrierState.from_metadata(barrier_raw),
)
def _string_tuple(raw: object) -> tuple[str, ...]: def _string_tuple(raw: object) -> tuple[str, ...]:
+10 -7
View File
@@ -7,7 +7,7 @@ from typing import Any
from wf_core.errors import WorkflowExecutionError from wf_core.errors import WorkflowExecutionError
from wf_core.run_state import ExecutionFrame, LineageState, RunState, StateWrite from wf_core.run_state import ExecutionFrame, LineageState, RunState, StateWrite
from wf_core.runtime.foreach_state import ForeachBarrierState, item_frame_owner from wf_core.runtime.foreach_state import item_frame_owner, load_foreach_activation
from wf_core.runtime.ops.state import ( from wf_core.runtime.ops.state import (
StatePatch, StatePatch,
commit_state_patch, commit_state_patch,
@@ -61,21 +61,24 @@ def lineage_writes_for_frame(
# Compatibility fallback: concurrent foreach used barrier-local patches # Compatibility fallback: concurrent foreach used barrier-local patches
# before `RunState.lineages` became the primary write store. Keep reading # before `RunState.lineages` became the primary write store. Keep reading
# those patches so old serialized runs and direct barrier tests still work. # those patches so old serialized runs and direct barrier tests still work.
# Barrier lookup includes the activation so a stale visit cannot read a
# later activation's buffered writes.
owner = item_frame_owner(frame) owner = item_frame_owner(frame)
if owner is None: if owner is None:
return () return ()
parent_frame_id, foreach_node_id, item_index = owner parent_frame = run.frames.get(owner.parent_frame_id)
parent_frame = run.frames.get(parent_frame_id)
if parent_frame is None: if parent_frame is None:
raise WorkflowExecutionError( raise WorkflowExecutionError(
"foreach lineage compatibility state references missing parent frame " "foreach lineage compatibility state references missing parent frame "
f"{parent_frame_id!r} for child frame {frame.id!r}" f"{owner.parent_frame_id!r} for child frame {frame.id!r}"
) )
barrier = ForeachBarrierState.from_frame(parent_frame, foreach_node_id) activation = load_foreach_activation(
if barrier is None or barrier.mode != "concurrent": parent_frame, owner.foreach_node_id, owner.activation_id
)
if activation is None or activation.barrier.mode != "concurrent":
return () return ()
pending = barrier.pending_results.get(item_index) pending = activation.barrier.pending_results.get(owner.item_index)
if pending is None: if pending is None:
return () return ()
return pending.patch.writes return pending.patch.writes
+29 -15
View File
@@ -8,9 +8,12 @@ from wf_core.models.steps import ForeachNode
from wf_core.models.workflow import Workflow from wf_core.models.workflow import Workflow
from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult
from wf_core.runtime.foreach_state import ( from wf_core.runtime.foreach_state import (
ForeachActivationState,
ForeachBarrierState, ForeachBarrierState,
ItemErrorRecord, ItemErrorRecord,
PendingItemResult, PendingItemResult,
load_or_begin_foreach_activation,
save_foreach_activation,
) )
from wf_core.runtime.lineage import ( from wf_core.runtime.lineage import (
add_lineage, add_lineage,
@@ -63,7 +66,8 @@ def _step_foreach_serial(
raise WorkflowExecutionError("serial foreach helper received non-serial mode") raise WorkflowExecutionError("serial foreach helper received non-serial mode")
frame = run.current_frame() frame = run.current_frame()
barrier = ForeachBarrierState.from_frame(frame, step.id) or ForeachBarrierState() activation = load_or_begin_foreach_activation(frame, step.id, mode="serial")
barrier = activation.barrier
iterable = _resolve_foreach_iterable(run, frame, step) iterable = _resolve_foreach_iterable(run, frame, step)
loop_index = barrier.next_index loop_index = barrier.next_index
@@ -89,9 +93,9 @@ def _step_foreach_serial(
loop_start = index.next_node_id(frame.node_id, "loop") loop_start = index.next_node_id(frame.node_id, "loop")
item = iterable[loop_index] item = iterable[loop_index]
barrier.next_index = loop_index + 1 barrier.next_index = loop_index + 1
barrier.save_to_frame(frame, step.id) save_foreach_activation(frame, activation)
child_id = f"{frame.id}:{step.id}:{loop_index}" child_id = _child_frame_id(activation, loop_index)
child_lineage_id = _child_lineage_id(frame, step, loop_index) child_lineage_id = _child_lineage_id(activation, loop_index)
add_frame( add_frame(
run, run,
ExecutionFrame( ExecutionFrame(
@@ -105,6 +109,7 @@ def _step_foreach_serial(
parent_lineage_id=frame.lineage_id, parent_lineage_id=frame.lineage_id,
metadata=ForeachIterationMetadata( metadata=ForeachIterationMetadata(
foreach_node_id=step.id, foreach_node_id=step.id,
activation_id=activation.id,
loop_index=loop_index, loop_index=loop_index,
loop_item=item, loop_item=item,
loop_alias=step.as_, loop_alias=step.as_,
@@ -141,11 +146,8 @@ def _step_foreach_concurrent(
if step.concurrent is None: if step.concurrent is None:
raise WorkflowExecutionError("concurrent foreach requires concurrent policy") raise WorkflowExecutionError("concurrent foreach requires concurrent policy")
frame = run.current_frame() frame = run.current_frame()
barrier = ForeachBarrierState.from_frame(frame, step.id) activation = load_or_begin_foreach_activation(frame, step.id, mode="concurrent")
if barrier is None: barrier = activation.barrier
barrier = ForeachBarrierState(mode="concurrent")
elif barrier.mode != "concurrent":
raise WorkflowExecutionError("malformed concurrent foreach barrier mode")
_finish_completed_children(run, step, barrier) _finish_completed_children(run, step, barrier)
iterable = _resolve_foreach_iterable(run, frame, step) iterable = _resolve_foreach_iterable(run, frame, step)
@@ -154,6 +156,7 @@ def _step_foreach_concurrent(
frame=frame, frame=frame,
step=step, step=step,
index=index, index=index,
activation=activation,
barrier=barrier, barrier=barrier,
iterable=iterable, iterable=iterable,
) )
@@ -169,7 +172,7 @@ def _step_foreach_concurrent(
reducers=reducers, reducers=reducers,
) )
barrier.save_to_frame(frame, step.id) save_foreach_activation(frame, activation)
block_frame_on_children(run, frame.id, barrier.outstanding_frame_ids) block_frame_on_children(run, frame.id, barrier.outstanding_frame_ids)
run.sync_from_current_frame() run.sync_from_current_frame()
return run return run
@@ -242,6 +245,7 @@ def _admit_concurrent_children(
frame: ExecutionFrame, frame: ExecutionFrame,
step: ForeachNode, step: ForeachNode,
index: WorkflowIndex, index: WorkflowIndex,
activation: ForeachActivationState,
barrier: ForeachBarrierState, barrier: ForeachBarrierState,
iterable: list[object], iterable: list[object],
) -> None: ) -> None:
@@ -256,8 +260,8 @@ def _admit_concurrent_children(
): ):
loop_index = barrier.next_index loop_index = barrier.next_index
item = iterable[loop_index] item = iterable[loop_index]
child_id = f"{frame.id}:{step.id}:{loop_index}" child_id = _child_frame_id(activation, loop_index)
child_lineage_id = _child_lineage_id(frame, step, loop_index) child_lineage_id = _child_lineage_id(activation, loop_index)
add_lineage( add_lineage(
run, run,
scope_id=frame.scope_id, scope_id=frame.scope_id,
@@ -280,6 +284,7 @@ def _admit_concurrent_children(
parent_lineage_id=frame.lineage_id, parent_lineage_id=frame.lineage_id,
metadata=ForeachIterationMetadata( metadata=ForeachIterationMetadata(
foreach_node_id=step.id, foreach_node_id=step.id,
activation_id=activation.id,
loop_index=loop_index, loop_index=loop_index,
loop_item=item, loop_item=item,
loop_alias=step.as_, loop_alias=step.as_,
@@ -372,13 +377,22 @@ def _finish_concurrent_foreach(
return run return run
def _child_lineage_id(frame: ExecutionFrame, step: ForeachNode, loop_index: int) -> str: def _child_frame_id(activation: ForeachActivationState, loop_index: int) -> str:
"""Return a deterministic opaque lineage id for one foreach child frame. """Return a deterministic opaque child frame id for one activation item.
The id embeds the activation so a later visit at item zero cannot collide
with the first visit. Compare full ids; never parse them.
"""
return f"{activation.id}:{loop_index}"
def _child_lineage_id(activation: ForeachActivationState, loop_index: int) -> str:
"""Return a deterministic opaque lineage id for one activation item.
The readable shape is only for diagnostics. Runtime code should compare the The readable shape is only for diagnostics. Runtime code should compare the
full id, not parse it; future structured lineage refs can replace this. full id, not parse it; future structured lineage refs can replace this.
""" """
return f"{frame.lineage_id}/{step.id}[{loop_index}]" return f"{activation.id}[{loop_index}]"
def _patch_for_successful_item( def _patch_for_successful_item(
+15 -7
View File
@@ -15,7 +15,11 @@ from wf_core.run_state import (
RuntimeContext, RuntimeContext,
StepExecutionResult, StepExecutionResult,
) )
from wf_core.runtime.foreach_state import ForeachBarrierState, item_frame_owner from wf_core.runtime.foreach_state import (
item_frame_owner,
require_foreach_activation,
save_foreach_activation,
)
from wf_core.runtime.input_bindings import resolve_step_input_bindings from wf_core.runtime.input_bindings import resolve_step_input_bindings
from wf_core.runtime.lineage import ( from wf_core.runtime.lineage import (
append_lineage_writes, append_lineage_writes,
@@ -116,10 +120,14 @@ def _finalize_node_execution(
if owner is None: if owner is None:
state_changes = commit_patch_for_frame(run, frame, patch) state_changes = commit_patch_for_frame(run, frame, patch)
else: else:
parent_frame_id, foreach_node_id, item_index = owner parent_frame = run.frames[owner.parent_frame_id]
parent_frame = run.frames[parent_frame_id] # Fail closed when the child names a closed or superseded activation:
barrier = ForeachBarrierState.from_frame(parent_frame, foreach_node_id) # its writes must not land in a later visit's barrier.
if barrier is not None and barrier.mode == "concurrent": activation = require_foreach_activation(
parent_frame, owner.foreach_node_id, owner.activation_id
)
barrier = activation.barrier
if barrier.mode == "concurrent":
# New concurrent foreach stores writes in the child lineage; the # New concurrent foreach stores writes in the child lineage; the
# barrier keeps only result metadata plus old patch fallback. # barrier keeps only result metadata plus old patch fallback.
append_lineage_writes( append_lineage_writes(
@@ -129,12 +137,12 @@ def _finalize_node_execution(
writes=patch.writes, writes=patch.writes,
) )
barrier.add_success_patch( barrier.add_success_patch(
index=item_index, index=owner.item_index,
frame_id=frame.id, frame_id=frame.id,
patch=StatePatch(), patch=StatePatch(),
lineage_id=frame.lineage_id, lineage_id=frame.lineage_id,
) )
barrier.save_to_frame(parent_frame, foreach_node_id) save_foreach_activation(parent_frame, activation)
state_changes = {} state_changes = {}
else: else:
state_changes = commit_patch_for_frame(run, parent_frame, patch) state_changes = commit_patch_for_frame(run, parent_frame, patch)
+39 -2
View File
@@ -38,9 +38,15 @@ class BlockedOnChildren:
@dataclass(slots=True, frozen=True) @dataclass(slots=True, frozen=True)
class ForeachIterationMetadata: class ForeachIterationMetadata:
"""Typed metadata for a foreach iteration frame.""" """Typed metadata for a foreach iteration frame.
``activation_id`` names the dynamic foreach visit that owns this item.
It separates fresh barrier state from earlier visits to the same node use
and must survive checkpoint serialization.
"""
foreach_node_id: str foreach_node_id: str
activation_id: str
loop_index: int loop_index: int
loop_item: Any loop_item: Any
loop_alias: str loop_alias: str
@@ -51,12 +57,17 @@ class ForeachIterationMetadata:
return None return None
metadata = frame.metadata metadata = frame.metadata
foreach_node_id = metadata.get("foreach_node_id") foreach_node_id = metadata.get("foreach_node_id")
activation_id = metadata.get("activation_id")
loop_index = metadata.get("loop_index") loop_index = metadata.get("loop_index")
loop_alias = metadata.get("loop_alias") loop_alias = metadata.get("loop_alias")
if not isinstance(foreach_node_id, str) or not foreach_node_id: if not isinstance(foreach_node_id, str) or not foreach_node_id:
raise WorkflowExecutionError( raise WorkflowExecutionError(
f"malformed foreach node id for frame {frame.id!r}" f"malformed foreach node id for frame {frame.id!r}"
) )
if not isinstance(activation_id, str) or not activation_id:
raise WorkflowExecutionError(
f"malformed foreach activation id for frame {frame.id!r}"
)
if not isinstance(loop_index, int): if not isinstance(loop_index, int):
raise WorkflowExecutionError( raise WorkflowExecutionError(
f"malformed foreach loop index for frame {frame.id!r}" f"malformed foreach loop index for frame {frame.id!r}"
@@ -71,6 +82,7 @@ class ForeachIterationMetadata:
) )
return cls( return cls(
foreach_node_id=foreach_node_id, foreach_node_id=foreach_node_id,
activation_id=activation_id,
loop_index=loop_index, loop_index=loop_index,
loop_item=metadata["loop_item"], loop_item=metadata["loop_item"],
loop_alias=loop_alias, loop_alias=loop_alias,
@@ -79,6 +91,7 @@ class ForeachIterationMetadata:
def to_metadata(self) -> dict[str, object]: def to_metadata(self) -> dict[str, object]:
return { return {
"foreach_node_id": self.foreach_node_id, "foreach_node_id": self.foreach_node_id,
"activation_id": self.activation_id,
"loop_index": self.loop_index, "loop_index": self.loop_index,
"loop_item": self.loop_item, "loop_item": self.loop_item,
"loop_alias": self.loop_alias, "loop_alias": self.loop_alias,
@@ -178,7 +191,11 @@ def wake_parent_if_children_complete(run: RunState, child_frame_id: str) -> None
def wake_parent_for_child_progress(run: RunState, child_frame_id: str) -> None: def wake_parent_for_child_progress(run: RunState, child_frame_id: str) -> None:
"""Wake a blocked parent after one child finishes so it can refill slots.""" """Wake a blocked parent after one child finishes so it can refill slots.
The wake-up includes the foreach activation: a child naming a closed or
superseded activation cannot wake a parent waiting on a later visit.
"""
child = _frame(run, child_frame_id) child = _frame(run, child_frame_id)
parent_id = child.parent_frame_id parent_id = child.parent_frame_id
if parent_id is None: if parent_id is None:
@@ -189,6 +206,26 @@ def wake_parent_for_child_progress(run: RunState, child_frame_id: str) -> None:
block = BlockedOnChildren.from_frame(parent) block = BlockedOnChildren.from_frame(parent)
if block is None or child_frame_id not in block.child_frame_ids: if block is None or child_frame_id not in block.child_frame_ids:
return return
# Lazy import avoids a cycle: foreach_state owns activation persistence on
# top of this scheduler's frame metadata types.
from wf_core.runtime.foreach_state import (
item_frame_owner,
load_foreach_activation,
)
try:
owner = item_frame_owner(child)
except WorkflowExecutionError:
raise
if owner is not None:
activation = load_foreach_activation(
parent, owner.foreach_node_id, owner.activation_id
)
if activation is None:
raise WorkflowExecutionError(
f"foreach item frame {child_frame_id!r} names closed activation "
f"{owner.activation_id!r} and cannot wake parent {parent_id!r}"
)
wake_frame(run, parent_id) wake_frame(run, parent_id)
+16 -8
View File
@@ -16,7 +16,7 @@ from wf_core.models.steps import (
) )
from wf_core.models.workflow import Workflow from wf_core.models.workflow import Workflow
from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult
from wf_core.runtime.foreach_state import ForeachBarrierState, item_frame_owner from wf_core.runtime.foreach_state import item_frame_owner, load_foreach_activation
from wf_core.runtime.ops.flow import advance_frame, append_step_result_trace from wf_core.runtime.ops.flow import advance_frame, append_step_result_trace
from wf_core.runtime.ops.foreach import step_foreach from wf_core.runtime.ops.foreach import step_foreach
from wf_core.runtime.ops.handlers import ( from wf_core.runtime.ops.handlers import (
@@ -361,10 +361,15 @@ def _claim_matching_async_item_frames(
index: WorkflowIndex, index: WorkflowIndex,
first_frame: ExecutionFrame, first_frame: ExecutionFrame,
) -> list[ExecutionFrame]: ) -> list[ExecutionFrame]:
"""Claim sibling item frames from the same activation for async batching.
Batching never mixes activations: only frames naming the same parent,
foreach, and activation id run together, preserving deterministic barrier
commits across revisits.
"""
owner = item_frame_owner(first_frame) owner = item_frame_owner(first_frame)
if owner is None: if owner is None:
return [] return []
parent_frame_id, foreach_node_id, _item_index = owner
claimed: list[ExecutionFrame] = [] claimed: list[ExecutionFrame] = []
remaining_ready: list[str] = [] remaining_ready: list[str] = []
for frame_id in run.ready_frame_ids: for frame_id in run.ready_frame_ids:
@@ -373,7 +378,9 @@ def _claim_matching_async_item_frames(
if ( if (
frame.status == FrameStatus.PENDING frame.status == FrameStatus.PENDING
and frame_owner is not None and frame_owner is not None
and frame_owner[:2] == (parent_frame_id, foreach_node_id) and frame_owner.parent_frame_id == owner.parent_frame_id
and frame_owner.foreach_node_id == owner.foreach_node_id
and frame_owner.activation_id == owner.activation_id
and isinstance(index.nodes_by_id.get(frame.node_id), NodeUse) and isinstance(index.nodes_by_id.get(frame.node_id), NodeUse)
): ):
frame.status = FrameStatus.RUNNING frame.status = FrameStatus.RUNNING
@@ -392,14 +399,15 @@ def _can_batch_async_foreach_item(
owner = item_frame_owner(frame) owner = item_frame_owner(frame)
if owner is None: if owner is None:
return False return False
parent_frame_id, foreach_node_id, _item_index = owner parent_frame = run.frames.get(owner.parent_frame_id)
parent_frame = run.frames.get(parent_frame_id)
if parent_frame is None: if parent_frame is None:
return False return False
barrier = ForeachBarrierState.from_frame(parent_frame, foreach_node_id) activation = load_foreach_activation(
parent_frame, owner.foreach_node_id, owner.activation_id
)
return ( return (
barrier is not None activation is not None
and barrier.mode == "concurrent" and activation.barrier.mode == "concurrent"
and isinstance(index.nodes_by_id.get(frame.node_id), NodeUse) and isinstance(index.nodes_by_id.get(frame.node_id), NodeUse)
) )
+36 -14
View File
@@ -19,7 +19,11 @@ from wf_core import (
execute_workflow, execute_workflow,
) )
from wf_core.run_state import ExecutionFrame, RunState, RuntimeContext from wf_core.run_state import ExecutionFrame, RunState, RuntimeContext
from wf_core.runtime.foreach_state import ForeachBarrierState from wf_core.runtime.foreach_state import (
ForeachItemOwner,
item_frame_owner,
load_or_begin_foreach_activation,
)
from wf_core.runtime.scheduler import ForeachIterationMetadata from wf_core.runtime.scheduler import ForeachIterationMetadata
@@ -143,7 +147,11 @@ def test_concurrent_foreach_item_frames_use_distinct_lineages() -> None:
assert run.frames["root"].lineage_id == "root" assert run.frames["root"].lineage_id == "root"
assert run.frames["root"].parent_lineage_id is None assert run.frames["root"].parent_lineage_id is None
assert len(item_frames) == 2 assert len(item_frames) == 2
assert item_lineage_ids == {"root/each[0]", "root/each[1]"} assert item_lineage_ids == {"root:each#0[0]", "root:each#0[1]"}
for frame in item_frames:
owner = item_frame_owner(frame)
assert isinstance(owner, ForeachItemOwner)
assert owner.activation_id == "root:each#0"
assert set(context_lineage_ids) == item_lineage_ids assert set(context_lineage_ids) == item_lineage_ids
assert all(frame.scope_id == "root" for frame in item_frames) assert all(frame.scope_id == "root" for frame in item_frames)
assert all(frame.parent_lineage_id == "root" for frame in item_frames) assert all(frame.parent_lineage_id == "root" for frame in item_frames)
@@ -162,15 +170,27 @@ def test_nested_concurrent_foreach_records_parent_child_lineages() -> None:
inner_frames = _foreach_frames(run, "inner_each") inner_frames = _foreach_frames(run, "inner_each")
assert {frame.lineage_id for frame in outer_frames} == { assert {frame.lineage_id for frame in outer_frames} == {
"root/outer_each[0]", "root:outer_each#0[0]",
"root/outer_each[1]", "root:outer_each#0[1]",
} }
assert all(frame.parent_lineage_id == "root" for frame in outer_frames) assert all(frame.parent_lineage_id == "root" for frame in outer_frames)
assert {(frame.parent_lineage_id, frame.lineage_id) for frame in inner_frames} == { assert {(frame.parent_lineage_id, frame.lineage_id) for frame in inner_frames} == {
("root/outer_each[0]", "root/outer_each[0]/inner_each[0]"), (
("root/outer_each[0]", "root/outer_each[0]/inner_each[1]"), "root:outer_each#0[0]",
("root/outer_each[1]", "root/outer_each[1]/inner_each[0]"), "root:outer_each#0:0:inner_each#0[0]",
("root/outer_each[1]", "root/outer_each[1]/inner_each[1]"), ),
(
"root:outer_each#0[0]",
"root:outer_each#0:0:inner_each#0[1]",
),
(
"root:outer_each#0[1]",
"root:outer_each#0:1:inner_each#0[0]",
),
(
"root:outer_each#0[1]",
"root:outer_each#0:1:inner_each#0[1]",
),
} }
@@ -264,12 +284,14 @@ def test_sync_concurrent_foreach_barrier_replays_add_reducer_inputs() -> None:
assert run.state["number"] == 6 assert run.state["number"] == 6
assert run.output["number"] == 6 assert run.output["number"] == 6
assert run.lineages["root/each[0]"].writes[0].incoming_value == 3 assert run.lineages["root:each#0[0]"].writes[0].incoming_value == 3
assert run.lineages["root/each[1]"].writes[0].incoming_value == 1 assert run.lineages["root:each#0[1]"].writes[0].incoming_value == 1
barrier = ForeachBarrierState.from_frame(run.frames["root"], "each") active = load_or_begin_foreach_activation(
assert barrier is not None run.frames["root"], "each", mode="concurrent"
assert barrier.pending_results[0].lineage_id == "root/each[0]" )
assert barrier.pending_results[0].patch.writes == [] assert active.id == "root:each#0"
assert active.barrier.pending_results[0].lineage_id == "root:each#0[0]"
assert active.barrier.pending_results[0].patch.writes == []
foreach_entries = [entry for entry in run.trace if entry.step_type == "foreach"] foreach_entries = [entry for entry in run.trace if entry.step_type == "foreach"]
assert foreach_entries[-1].state_changes["state.number"] == 6 assert foreach_entries[-1].state_changes["state.number"] == 6
+2 -2
View File
@@ -27,7 +27,7 @@ def test_concurrent_foreach_skip_emits_completed_with_errors() -> None:
) )
assert run.state["seen"] == ["a", "c"] assert run.state["seen"] == ["a", "c"]
assert run.frames["root:each:1"].status == "failed" assert run.frames["root:each#0:1"].status == "failed"
foreach_entries = [entry for entry in run.trace if entry.step_type == "foreach"] foreach_entries = [entry for entry in run.trace if entry.step_type == "foreach"]
assert foreach_entries[-1].outcome == "completed_with_errors" assert foreach_entries[-1].outcome == "completed_with_errors"
assert foreach_entries[-1].resolved_input["failed_items"] == 1 assert foreach_entries[-1].resolved_input["failed_items"] == 1
@@ -47,7 +47,7 @@ def test_concurrent_foreach_collect_writes_ordered_error_records() -> None:
assert len(run.state["errors"]) == 1 assert len(run.state["errors"]) == 1
error = run.state["errors"][0] error = run.state["errors"][0]
assert error["index"] == 1 assert error["index"] == 1
assert error["frame_id"] == "root:each:1" assert error["frame_id"] == "root:each#0:1"
assert error["node_id"] == "record" assert error["node_id"] == "record"
assert error["error_type"] == "ValueError" assert error["error_type"] == "ValueError"
assert error["message"] == "bad item" assert error["message"] == "bad item"
@@ -31,8 +31,8 @@ async def test_concurrent_foreach_interrupt_returns_before_refill() -> None:
assert run.status is RunStatus.INTERRUPTED assert run.status is RunStatus.INTERRUPTED
assert run.interrupt is not None assert run.interrupt is not None
assert run.interrupt.payload["item"] == "b" assert run.interrupt.payload["item"] == "b"
assert run.frames["root:each:1"].status == "interrupted" assert run.frames["root:each#0:1"].status == "interrupted"
assert "root:each:2" not in run.frames assert "root:each#0:2" not in run.frames
assert "seen" not in run.state assert "seen" not in run.state
@@ -54,7 +54,7 @@ async def test_resume_prioritizes_interrupted_item_before_siblings() -> None:
assert resumed.status is RunStatus.COMPLETED assert resumed.status is RunStatus.COMPLETED
assert resumed.state["seen"] == ["a", "b", "c"] assert resumed.state["seen"] == ["a", "b", "c"]
assert resumed.trace[interrupted_trace_len].frame_id == "root:each:1" assert resumed.trace[interrupted_trace_len].frame_id == "root:each#0:1"
assert resumed.trace[interrupted_trace_len].step_type == "interrupt" assert resumed.trace[interrupted_trace_len].step_type == "interrupt"
assert resumed.trace[interrupted_trace_len].outcome == "submitted" assert resumed.trace[interrupted_trace_len].outcome == "submitted"
foreach_entries = [entry for entry in resumed.trace if entry.step_type == "foreach"] foreach_entries = [entry for entry in resumed.trace if entry.step_type == "foreach"]
+102
View File
@@ -0,0 +1,102 @@
from __future__ import annotations
import pytest
from wf_core.errors import WorkflowExecutionError
from wf_core.run_state import ExecutionFrame
from wf_core.runtime.foreach_state import (
close_foreach_activation,
item_frame_owner,
load_or_begin_foreach_activation,
save_foreach_activation,
)
from wf_core.runtime.scheduler import ForeachIterationMetadata
def _frame() -> ExecutionFrame:
return ExecutionFrame(id="root", kind="workflow", node_id="each")
def test_activation_lifecycle_reuses_active_then_fresh_after_close() -> None:
frame = _frame()
first = load_or_begin_foreach_activation(frame, "each", mode="serial")
save_foreach_activation(frame, first)
restored = load_or_begin_foreach_activation(frame, "each", mode="serial")
assert restored.id == first.id
close_foreach_activation(frame, restored)
second = load_or_begin_foreach_activation(frame, "each", mode="serial")
assert second.id != first.id
assert second.barrier.next_index == 0
def test_activation_rejects_malformed_metadata() -> None:
frame = ExecutionFrame(
id="root",
kind="workflow",
node_id="each",
metadata={"foreach_activations": "corrupt"},
)
with pytest.raises(WorkflowExecutionError, match="activation"):
load_or_begin_foreach_activation(frame, "each", mode="serial")
def test_activation_rejects_mode_mismatch() -> None:
frame = _frame()
activation = load_or_begin_foreach_activation(frame, "each", mode="serial")
save_foreach_activation(frame, activation)
with pytest.raises(WorkflowExecutionError, match="mode"):
load_or_begin_foreach_activation(frame, "each", mode="concurrent")
def test_closing_stale_activation_fails_closed() -> None:
frame = _frame()
first = load_or_begin_foreach_activation(frame, "each", mode="serial")
save_foreach_activation(frame, first)
close_foreach_activation(frame, first)
second = load_or_begin_foreach_activation(frame, "each", mode="serial")
save_foreach_activation(frame, second)
with pytest.raises(WorkflowExecutionError, match="stale|closed|active"):
close_foreach_activation(frame, first)
def test_activation_json_round_trip_through_frame_metadata() -> None:
frame = _frame()
activation = load_or_begin_foreach_activation(frame, "each", mode="serial")
activation.barrier.next_index = 2
save_foreach_activation(frame, activation)
dumped = dict(frame.metadata)
restored_frame = ExecutionFrame(
id="root", kind="workflow", node_id="each", metadata=dumped
)
restored = load_or_begin_foreach_activation(restored_frame, "each", mode="serial")
assert restored.id == activation.id
assert restored.barrier.next_index == 2
def test_item_metadata_requires_activation_identity() -> None:
frame = ExecutionFrame(
id="root:each#0:0",
kind="foreach_iteration",
node_id="work",
parent_frame_id="root",
metadata={
"foreach_node_id": "each",
"loop_index": 0,
"loop_item": "a",
"loop_alias": "item",
},
)
with pytest.raises(WorkflowExecutionError, match="activation"):
ForeachIterationMetadata.from_frame(frame)
with pytest.raises(WorkflowExecutionError, match="activation"):
item_frame_owner(frame)
+23 -15
View File
@@ -8,9 +8,13 @@ from wf_core.paths import StatePath
from wf_core.run_state import ExecutionFrame, RunState, RunStatus, StateWrite from wf_core.run_state import ExecutionFrame, RunState, RunStatus, StateWrite
from wf_core.runtime.foreach_state import ( from wf_core.runtime.foreach_state import (
ForeachBarrierState, ForeachBarrierState,
ForeachItemOwner,
ItemErrorRecord, ItemErrorRecord,
PendingItemResult, PendingItemResult,
_state_write_from_metadata, _state_write_from_metadata,
item_frame_owner,
load_or_begin_foreach_activation,
save_foreach_activation,
) )
from wf_core.runtime.lineage import LineageStateView, lineage_writes_for_frame from wf_core.runtime.lineage import LineageStateView, lineage_writes_for_frame
from wf_core.runtime.ops.state import StatePatch from wf_core.runtime.ops.state import StatePatch
@@ -134,20 +138,28 @@ def test_lineage_state_view_materializes_visible_values_without_mutating_base()
def test_lineage_writes_for_frame_reads_current_foreach_pending_result() -> None: def test_lineage_writes_for_frame_reads_current_foreach_pending_result() -> None:
parent = ExecutionFrame(id="root", kind="workflow", node_id="each") parent = ExecutionFrame(id="root", kind="workflow", node_id="each")
activation = load_or_begin_foreach_activation(parent, "each", mode="concurrent")
child_lineage_id = f"{activation.id}[0]"
child = ExecutionFrame( child = ExecutionFrame(
id="root:each:0", id=f"{activation.id}:0",
kind="foreach_iteration", kind="foreach_iteration",
node_id="work", node_id="work",
parent_frame_id="root", parent_frame_id="root",
lineage_id="root/each[0]", lineage_id=child_lineage_id,
parent_lineage_id="root", parent_lineage_id="root",
metadata={ metadata={
"foreach_node_id": "each", "foreach_node_id": "each",
"activation_id": activation.id,
"loop_index": 0, "loop_index": 0,
"loop_item": "a", "loop_item": "a",
"loop_alias": "item", "loop_alias": "item",
}, },
) )
# Ownership is named, not positional.
owner = item_frame_owner(child)
assert isinstance(owner, ForeachItemOwner)
assert owner.activation_id == activation.id
assert owner.item_index == 0
patch = StatePatch( patch = StatePatch(
writes=[ writes=[
StateWrite( StateWrite(
@@ -158,19 +170,14 @@ def test_lineage_writes_for_frame_reads_current_foreach_pending_result() -> None
) )
] ]
) )
barrier = ForeachBarrierState( activation.barrier.pending_results[0] = PendingItemResult(
mode="concurrent", index=0,
pending_results={ frame_id=child.id,
0: PendingItemResult( status="succeeded",
index=0, lineage_id=child.lineage_id,
frame_id=child.id, patch=patch,
status="succeeded",
lineage_id=child.lineage_id,
patch=patch,
)
},
) )
barrier.save_to_frame(parent, "each") save_foreach_activation(parent, activation)
run = RunState( run = RunState(
workflow_name="lineage", workflow_name="lineage",
status=RunStatus.PENDING, status=RunStatus.PENDING,
@@ -188,12 +195,13 @@ def test_lineage_writes_for_frame_reads_current_foreach_pending_result() -> None
def test_lineage_writes_for_frame_rejects_missing_compatibility_parent_frame() -> None: def test_lineage_writes_for_frame_rejects_missing_compatibility_parent_frame() -> None:
child = ExecutionFrame( child = ExecutionFrame(
id="missing:each:0", id="missing:each#0:0",
kind="foreach_iteration", kind="foreach_iteration",
node_id="work", node_id="work",
parent_frame_id="missing", parent_frame_id="missing",
metadata={ metadata={
"foreach_node_id": "each", "foreach_node_id": "each",
"activation_id": "missing:each#0",
"loop_index": 0, "loop_index": 0,
"loop_item": "a", "loop_item": "a",
"loop_alias": "item", "loop_alias": "item",
+22
View File
@@ -163,8 +163,19 @@ def test_child_completion_wakes_blocked_parent() -> None:
def test_wake_parent_when_child_finishes_for_refill() -> None: def test_wake_parent_when_child_finishes_for_refill() -> None:
from wf_core.runtime.foreach_state import (
ForeachItemOwner,
item_frame_owner,
load_or_begin_foreach_activation,
save_foreach_activation,
)
run = _run() run = _run()
add_frame(run, ExecutionFrame(id="parent", kind="root", node_id="foreach")) add_frame(run, ExecutionFrame(id="parent", kind="root", node_id="foreach"))
activation = load_or_begin_foreach_activation(
run.frames["parent"], "foreach", mode="serial"
)
save_foreach_activation(run.frames["parent"], activation)
add_frame( add_frame(
run, run,
ExecutionFrame( ExecutionFrame(
@@ -172,11 +183,22 @@ def test_wake_parent_when_child_finishes_for_refill() -> None:
kind="foreach_iteration", kind="foreach_iteration",
node_id="__end__", node_id="__end__",
parent_frame_id="parent", parent_frame_id="parent",
metadata={
"foreach_node_id": "foreach",
"activation_id": activation.id,
"loop_index": 0,
"loop_item": "a",
"loop_alias": "item",
},
), ),
) )
block_frame_on_children(run, "parent", ("child", "other")) block_frame_on_children(run, "parent", ("child", "other"))
run.frames["child"].status = FrameStatus.COMPLETED run.frames["child"].status = FrameStatus.COMPLETED
owner = item_frame_owner(run.frames["child"])
assert isinstance(owner, ForeachItemOwner)
assert owner.activation_id == activation.id
wake_parent_for_child_progress(run, "child") wake_parent_for_child_progress(run, "child")
assert run.frames["parent"].status == FrameStatus.PENDING assert run.frames["parent"].status == FrameStatus.PENDING