feat: identify dynamic foreach activations
This commit is contained in:
@@ -11,6 +11,31 @@ from wf_core.runtime.ops.state import StatePatch
|
|||||||
from wf_core.runtime.scheduler import ForeachIterationMetadata
|
from wf_core.runtime.scheduler import ForeachIterationMetadata
|
||||||
|
|
||||||
_BARRIER_METADATA_KEY = "foreach_barriers"
|
_BARRIER_METADATA_KEY = "foreach_barriers"
|
||||||
|
_ACTIVATION_METADATA_KEY = "foreach_activations"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class ForeachActivationState:
|
||||||
|
"""Persisted state for one dynamic visit to a foreach controller.
|
||||||
|
|
||||||
|
A parent frame creates a fresh activation on first entry, reuses it while
|
||||||
|
admitting items, and closes it before emitting ``done``. The id is opaque:
|
||||||
|
callers compare it by name and never parse it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
id: str
|
||||||
|
foreach_node_id: str
|
||||||
|
barrier: ForeachBarrierState
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ForeachItemOwner:
|
||||||
|
"""Named ownership record for one foreach item frame."""
|
||||||
|
|
||||||
|
parent_frame_id: str
|
||||||
|
foreach_node_id: str
|
||||||
|
activation_id: str
|
||||||
|
item_index: int
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
@@ -320,14 +345,188 @@ class ForeachBarrierState:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def item_frame_owner(frame: ExecutionFrame) -> tuple[str, str, int] | None:
|
def load_or_begin_foreach_activation(
|
||||||
"""Return parent frame id, foreach node id, and item index for item frames."""
|
frame: ExecutionFrame,
|
||||||
|
foreach_node_id: str,
|
||||||
|
*,
|
||||||
|
mode: Literal["serial", "concurrent"],
|
||||||
|
) -> ForeachActivationState:
|
||||||
|
"""Load the active activation or begin a fresh visit.
|
||||||
|
|
||||||
|
The first entry for one visit allocates an opaque id from the parent frame
|
||||||
|
id, foreach node id, and a persisted per-frame sequence. Later calls reuse
|
||||||
|
the active activation; closing it makes the next visit allocate a new id
|
||||||
|
with fresh barrier state. Mode mismatches and malformed tables fail fast.
|
||||||
|
"""
|
||||||
|
table = _activation_table(frame)
|
||||||
|
entry = table.get(foreach_node_id)
|
||||||
|
if entry is None:
|
||||||
|
entry = {"next_sequence": 0, "active": None}
|
||||||
|
table[foreach_node_id] = entry
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"malformed foreach activation entry for frame {frame.id!r}"
|
||||||
|
)
|
||||||
|
next_sequence = entry.get("next_sequence", 0)
|
||||||
|
if not isinstance(next_sequence, int) or next_sequence < 0:
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"malformed foreach activation sequence for frame {frame.id!r}"
|
||||||
|
)
|
||||||
|
active = entry.get("active")
|
||||||
|
if active is not None:
|
||||||
|
activation = _activation_from_metadata(
|
||||||
|
active, frame_id=frame.id, foreach_node_id=foreach_node_id
|
||||||
|
)
|
||||||
|
if activation.barrier.mode != mode:
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"foreach {foreach_node_id!r} activation {activation.id!r} "
|
||||||
|
f"has mode {activation.barrier.mode!r}, got {mode!r}"
|
||||||
|
)
|
||||||
|
return activation
|
||||||
|
activation_id = f"{frame.id}:{foreach_node_id}#{next_sequence}"
|
||||||
|
activation = ForeachActivationState(
|
||||||
|
id=activation_id,
|
||||||
|
foreach_node_id=foreach_node_id,
|
||||||
|
barrier=ForeachBarrierState(mode=mode),
|
||||||
|
)
|
||||||
|
entry["next_sequence"] = next_sequence + 1
|
||||||
|
entry["active"] = {
|
||||||
|
"id": activation.id,
|
||||||
|
"barrier": activation.barrier.to_metadata(),
|
||||||
|
}
|
||||||
|
return activation
|
||||||
|
|
||||||
|
|
||||||
|
def save_foreach_activation(
|
||||||
|
frame: ExecutionFrame, activation: ForeachActivationState
|
||||||
|
) -> None:
|
||||||
|
"""Persist barrier progress for the named active activation."""
|
||||||
|
table = _activation_table(frame)
|
||||||
|
entry = table.get(activation.foreach_node_id)
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"malformed foreach activation entry for frame {frame.id!r}"
|
||||||
|
)
|
||||||
|
active = entry.get("active")
|
||||||
|
if not isinstance(active, dict) or active.get("id") != activation.id:
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"cannot save stale foreach activation {activation.id!r} "
|
||||||
|
f"for frame {frame.id!r}"
|
||||||
|
)
|
||||||
|
active["barrier"] = activation.barrier.to_metadata()
|
||||||
|
|
||||||
|
|
||||||
|
def close_foreach_activation(
|
||||||
|
frame: ExecutionFrame, activation: ForeachActivationState
|
||||||
|
) -> None:
|
||||||
|
"""Close the named active activation, preserving the visit sequence.
|
||||||
|
|
||||||
|
The barrier is removed so a later visit starts fresh; the sequence keeps
|
||||||
|
increasing so child and lineage ids cannot collide across visits.
|
||||||
|
"""
|
||||||
|
table = _activation_table(frame)
|
||||||
|
entry = table.get(activation.foreach_node_id)
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"malformed foreach activation entry for frame {frame.id!r}"
|
||||||
|
)
|
||||||
|
active = entry.get("active")
|
||||||
|
if not isinstance(active, dict) or active.get("id") != activation.id:
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"cannot close stale foreach activation {activation.id!r} "
|
||||||
|
f"for frame {frame.id!r}"
|
||||||
|
)
|
||||||
|
entry["active"] = None
|
||||||
|
|
||||||
|
|
||||||
|
def load_foreach_activation(
|
||||||
|
frame: ExecutionFrame, foreach_node_id: str, activation_id: str
|
||||||
|
) -> ForeachActivationState | None:
|
||||||
|
"""Return the active activation only when its id matches the child.
|
||||||
|
|
||||||
|
A child result naming a closed or different activation must fail closed in
|
||||||
|
the caller rather than buffering into the wrong barrier.
|
||||||
|
"""
|
||||||
|
table = _activation_table(frame)
|
||||||
|
entry = table.get(foreach_node_id)
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"malformed foreach activation entry for frame {frame.id!r}"
|
||||||
|
)
|
||||||
|
active = entry.get("active")
|
||||||
|
if active is None:
|
||||||
|
return None
|
||||||
|
activation = _activation_from_metadata(
|
||||||
|
active, frame_id=frame.id, foreach_node_id=foreach_node_id
|
||||||
|
)
|
||||||
|
if activation.id != activation_id:
|
||||||
|
return None
|
||||||
|
return activation
|
||||||
|
|
||||||
|
|
||||||
|
def require_foreach_activation(
|
||||||
|
frame: ExecutionFrame, foreach_node_id: str, activation_id: str
|
||||||
|
) -> ForeachActivationState:
|
||||||
|
"""Load the named activation or raise when it is closed or superseded."""
|
||||||
|
activation = load_foreach_activation(frame, foreach_node_id, activation_id)
|
||||||
|
if activation is None:
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"foreach item activation {activation_id!r} for node "
|
||||||
|
f"{foreach_node_id!r} is closed or superseded"
|
||||||
|
)
|
||||||
|
return activation
|
||||||
|
|
||||||
|
|
||||||
|
def item_frame_owner(frame: ExecutionFrame) -> ForeachItemOwner | None:
|
||||||
|
"""Return the named foreach ownership record for item frames.
|
||||||
|
|
||||||
|
Malformed item metadata fails closed via ``ForeachIterationMetadata``;
|
||||||
|
only non-item frames return ``None``.
|
||||||
|
"""
|
||||||
if frame.kind != "foreach_iteration" or frame.parent_frame_id is None:
|
if frame.kind != "foreach_iteration" or frame.parent_frame_id is None:
|
||||||
return None
|
return None
|
||||||
metadata = ForeachIterationMetadata.from_frame(frame)
|
metadata = ForeachIterationMetadata.from_frame(frame)
|
||||||
if metadata is None:
|
if metadata is None:
|
||||||
return None
|
return None
|
||||||
return frame.parent_frame_id, metadata.foreach_node_id, metadata.loop_index
|
return ForeachItemOwner(
|
||||||
|
parent_frame_id=frame.parent_frame_id,
|
||||||
|
foreach_node_id=metadata.foreach_node_id,
|
||||||
|
activation_id=metadata.activation_id,
|
||||||
|
item_index=metadata.loop_index,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _activation_table(frame: ExecutionFrame) -> dict[str, Any]:
|
||||||
|
raw = frame.metadata.get(_ACTIVATION_METADATA_KEY)
|
||||||
|
if raw is None:
|
||||||
|
table: dict[str, Any] = {}
|
||||||
|
frame.metadata[_ACTIVATION_METADATA_KEY] = table
|
||||||
|
return table
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"malformed foreach activation table for frame {frame.id!r}"
|
||||||
|
)
|
||||||
|
return raw
|
||||||
|
|
||||||
|
|
||||||
|
def _activation_from_metadata(
|
||||||
|
raw: object, *, frame_id: str, foreach_node_id: str
|
||||||
|
) -> ForeachActivationState:
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"malformed foreach activation for frame {frame_id!r}"
|
||||||
|
)
|
||||||
|
activation_id = raw.get("id")
|
||||||
|
barrier_raw = raw.get("barrier")
|
||||||
|
if not isinstance(activation_id, str) or not activation_id:
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"malformed foreach activation id for frame {frame_id!r}"
|
||||||
|
)
|
||||||
|
return ForeachActivationState(
|
||||||
|
id=activation_id,
|
||||||
|
foreach_node_id=foreach_node_id,
|
||||||
|
barrier=ForeachBarrierState.from_metadata(barrier_raw),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _string_tuple(raw: object) -> tuple[str, ...]:
|
def _string_tuple(raw: object) -> tuple[str, ...]:
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from typing import Any
|
|||||||
|
|
||||||
from wf_core.errors import WorkflowExecutionError
|
from wf_core.errors import WorkflowExecutionError
|
||||||
from wf_core.run_state import ExecutionFrame, LineageState, RunState, StateWrite
|
from wf_core.run_state import ExecutionFrame, LineageState, RunState, StateWrite
|
||||||
from wf_core.runtime.foreach_state import ForeachBarrierState, item_frame_owner
|
from wf_core.runtime.foreach_state import item_frame_owner, load_foreach_activation
|
||||||
from wf_core.runtime.ops.state import (
|
from wf_core.runtime.ops.state import (
|
||||||
StatePatch,
|
StatePatch,
|
||||||
commit_state_patch,
|
commit_state_patch,
|
||||||
@@ -61,21 +61,24 @@ def lineage_writes_for_frame(
|
|||||||
# Compatibility fallback: concurrent foreach used barrier-local patches
|
# Compatibility fallback: concurrent foreach used barrier-local patches
|
||||||
# before `RunState.lineages` became the primary write store. Keep reading
|
# before `RunState.lineages` became the primary write store. Keep reading
|
||||||
# those patches so old serialized runs and direct barrier tests still work.
|
# those patches so old serialized runs and direct barrier tests still work.
|
||||||
|
# Barrier lookup includes the activation so a stale visit cannot read a
|
||||||
|
# later activation's buffered writes.
|
||||||
owner = item_frame_owner(frame)
|
owner = item_frame_owner(frame)
|
||||||
if owner is None:
|
if owner is None:
|
||||||
return ()
|
return ()
|
||||||
parent_frame_id, foreach_node_id, item_index = owner
|
parent_frame = run.frames.get(owner.parent_frame_id)
|
||||||
parent_frame = run.frames.get(parent_frame_id)
|
|
||||||
if parent_frame is None:
|
if parent_frame is None:
|
||||||
raise WorkflowExecutionError(
|
raise WorkflowExecutionError(
|
||||||
"foreach lineage compatibility state references missing parent frame "
|
"foreach lineage compatibility state references missing parent frame "
|
||||||
f"{parent_frame_id!r} for child frame {frame.id!r}"
|
f"{owner.parent_frame_id!r} for child frame {frame.id!r}"
|
||||||
)
|
)
|
||||||
barrier = ForeachBarrierState.from_frame(parent_frame, foreach_node_id)
|
activation = load_foreach_activation(
|
||||||
if barrier is None or barrier.mode != "concurrent":
|
parent_frame, owner.foreach_node_id, owner.activation_id
|
||||||
|
)
|
||||||
|
if activation is None or activation.barrier.mode != "concurrent":
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
pending = barrier.pending_results.get(item_index)
|
pending = activation.barrier.pending_results.get(owner.item_index)
|
||||||
if pending is None:
|
if pending is None:
|
||||||
return ()
|
return ()
|
||||||
return pending.patch.writes
|
return pending.patch.writes
|
||||||
|
|||||||
@@ -8,9 +8,12 @@ from wf_core.models.steps import ForeachNode
|
|||||||
from wf_core.models.workflow import Workflow
|
from wf_core.models.workflow import Workflow
|
||||||
from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult
|
from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult
|
||||||
from wf_core.runtime.foreach_state import (
|
from wf_core.runtime.foreach_state import (
|
||||||
|
ForeachActivationState,
|
||||||
ForeachBarrierState,
|
ForeachBarrierState,
|
||||||
ItemErrorRecord,
|
ItemErrorRecord,
|
||||||
PendingItemResult,
|
PendingItemResult,
|
||||||
|
load_or_begin_foreach_activation,
|
||||||
|
save_foreach_activation,
|
||||||
)
|
)
|
||||||
from wf_core.runtime.lineage import (
|
from wf_core.runtime.lineage import (
|
||||||
add_lineage,
|
add_lineage,
|
||||||
@@ -63,7 +66,8 @@ def _step_foreach_serial(
|
|||||||
raise WorkflowExecutionError("serial foreach helper received non-serial mode")
|
raise WorkflowExecutionError("serial foreach helper received non-serial mode")
|
||||||
|
|
||||||
frame = run.current_frame()
|
frame = run.current_frame()
|
||||||
barrier = ForeachBarrierState.from_frame(frame, step.id) or ForeachBarrierState()
|
activation = load_or_begin_foreach_activation(frame, step.id, mode="serial")
|
||||||
|
barrier = activation.barrier
|
||||||
iterable = _resolve_foreach_iterable(run, frame, step)
|
iterable = _resolve_foreach_iterable(run, frame, step)
|
||||||
|
|
||||||
loop_index = barrier.next_index
|
loop_index = barrier.next_index
|
||||||
@@ -89,9 +93,9 @@ def _step_foreach_serial(
|
|||||||
loop_start = index.next_node_id(frame.node_id, "loop")
|
loop_start = index.next_node_id(frame.node_id, "loop")
|
||||||
item = iterable[loop_index]
|
item = iterable[loop_index]
|
||||||
barrier.next_index = loop_index + 1
|
barrier.next_index = loop_index + 1
|
||||||
barrier.save_to_frame(frame, step.id)
|
save_foreach_activation(frame, activation)
|
||||||
child_id = f"{frame.id}:{step.id}:{loop_index}"
|
child_id = _child_frame_id(activation, loop_index)
|
||||||
child_lineage_id = _child_lineage_id(frame, step, loop_index)
|
child_lineage_id = _child_lineage_id(activation, loop_index)
|
||||||
add_frame(
|
add_frame(
|
||||||
run,
|
run,
|
||||||
ExecutionFrame(
|
ExecutionFrame(
|
||||||
@@ -105,6 +109,7 @@ def _step_foreach_serial(
|
|||||||
parent_lineage_id=frame.lineage_id,
|
parent_lineage_id=frame.lineage_id,
|
||||||
metadata=ForeachIterationMetadata(
|
metadata=ForeachIterationMetadata(
|
||||||
foreach_node_id=step.id,
|
foreach_node_id=step.id,
|
||||||
|
activation_id=activation.id,
|
||||||
loop_index=loop_index,
|
loop_index=loop_index,
|
||||||
loop_item=item,
|
loop_item=item,
|
||||||
loop_alias=step.as_,
|
loop_alias=step.as_,
|
||||||
@@ -141,11 +146,8 @@ def _step_foreach_concurrent(
|
|||||||
if step.concurrent is None:
|
if step.concurrent is None:
|
||||||
raise WorkflowExecutionError("concurrent foreach requires concurrent policy")
|
raise WorkflowExecutionError("concurrent foreach requires concurrent policy")
|
||||||
frame = run.current_frame()
|
frame = run.current_frame()
|
||||||
barrier = ForeachBarrierState.from_frame(frame, step.id)
|
activation = load_or_begin_foreach_activation(frame, step.id, mode="concurrent")
|
||||||
if barrier is None:
|
barrier = activation.barrier
|
||||||
barrier = ForeachBarrierState(mode="concurrent")
|
|
||||||
elif barrier.mode != "concurrent":
|
|
||||||
raise WorkflowExecutionError("malformed concurrent foreach barrier mode")
|
|
||||||
|
|
||||||
_finish_completed_children(run, step, barrier)
|
_finish_completed_children(run, step, barrier)
|
||||||
iterable = _resolve_foreach_iterable(run, frame, step)
|
iterable = _resolve_foreach_iterable(run, frame, step)
|
||||||
@@ -154,6 +156,7 @@ def _step_foreach_concurrent(
|
|||||||
frame=frame,
|
frame=frame,
|
||||||
step=step,
|
step=step,
|
||||||
index=index,
|
index=index,
|
||||||
|
activation=activation,
|
||||||
barrier=barrier,
|
barrier=barrier,
|
||||||
iterable=iterable,
|
iterable=iterable,
|
||||||
)
|
)
|
||||||
@@ -169,7 +172,7 @@ def _step_foreach_concurrent(
|
|||||||
reducers=reducers,
|
reducers=reducers,
|
||||||
)
|
)
|
||||||
|
|
||||||
barrier.save_to_frame(frame, step.id)
|
save_foreach_activation(frame, activation)
|
||||||
block_frame_on_children(run, frame.id, barrier.outstanding_frame_ids)
|
block_frame_on_children(run, frame.id, barrier.outstanding_frame_ids)
|
||||||
run.sync_from_current_frame()
|
run.sync_from_current_frame()
|
||||||
return run
|
return run
|
||||||
@@ -242,6 +245,7 @@ def _admit_concurrent_children(
|
|||||||
frame: ExecutionFrame,
|
frame: ExecutionFrame,
|
||||||
step: ForeachNode,
|
step: ForeachNode,
|
||||||
index: WorkflowIndex,
|
index: WorkflowIndex,
|
||||||
|
activation: ForeachActivationState,
|
||||||
barrier: ForeachBarrierState,
|
barrier: ForeachBarrierState,
|
||||||
iterable: list[object],
|
iterable: list[object],
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -256,8 +260,8 @@ def _admit_concurrent_children(
|
|||||||
):
|
):
|
||||||
loop_index = barrier.next_index
|
loop_index = barrier.next_index
|
||||||
item = iterable[loop_index]
|
item = iterable[loop_index]
|
||||||
child_id = f"{frame.id}:{step.id}:{loop_index}"
|
child_id = _child_frame_id(activation, loop_index)
|
||||||
child_lineage_id = _child_lineage_id(frame, step, loop_index)
|
child_lineage_id = _child_lineage_id(activation, loop_index)
|
||||||
add_lineage(
|
add_lineage(
|
||||||
run,
|
run,
|
||||||
scope_id=frame.scope_id,
|
scope_id=frame.scope_id,
|
||||||
@@ -280,6 +284,7 @@ def _admit_concurrent_children(
|
|||||||
parent_lineage_id=frame.lineage_id,
|
parent_lineage_id=frame.lineage_id,
|
||||||
metadata=ForeachIterationMetadata(
|
metadata=ForeachIterationMetadata(
|
||||||
foreach_node_id=step.id,
|
foreach_node_id=step.id,
|
||||||
|
activation_id=activation.id,
|
||||||
loop_index=loop_index,
|
loop_index=loop_index,
|
||||||
loop_item=item,
|
loop_item=item,
|
||||||
loop_alias=step.as_,
|
loop_alias=step.as_,
|
||||||
@@ -372,13 +377,22 @@ def _finish_concurrent_foreach(
|
|||||||
return run
|
return run
|
||||||
|
|
||||||
|
|
||||||
def _child_lineage_id(frame: ExecutionFrame, step: ForeachNode, loop_index: int) -> str:
|
def _child_frame_id(activation: ForeachActivationState, loop_index: int) -> str:
|
||||||
"""Return a deterministic opaque lineage id for one foreach child frame.
|
"""Return a deterministic opaque child frame id for one activation item.
|
||||||
|
|
||||||
|
The id embeds the activation so a later visit at item zero cannot collide
|
||||||
|
with the first visit. Compare full ids; never parse them.
|
||||||
|
"""
|
||||||
|
return f"{activation.id}:{loop_index}"
|
||||||
|
|
||||||
|
|
||||||
|
def _child_lineage_id(activation: ForeachActivationState, loop_index: int) -> str:
|
||||||
|
"""Return a deterministic opaque lineage id for one activation item.
|
||||||
|
|
||||||
The readable shape is only for diagnostics. Runtime code should compare the
|
The readable shape is only for diagnostics. Runtime code should compare the
|
||||||
full id, not parse it; future structured lineage refs can replace this.
|
full id, not parse it; future structured lineage refs can replace this.
|
||||||
"""
|
"""
|
||||||
return f"{frame.lineage_id}/{step.id}[{loop_index}]"
|
return f"{activation.id}[{loop_index}]"
|
||||||
|
|
||||||
|
|
||||||
def _patch_for_successful_item(
|
def _patch_for_successful_item(
|
||||||
|
|||||||
@@ -15,7 +15,11 @@ from wf_core.run_state import (
|
|||||||
RuntimeContext,
|
RuntimeContext,
|
||||||
StepExecutionResult,
|
StepExecutionResult,
|
||||||
)
|
)
|
||||||
from wf_core.runtime.foreach_state import ForeachBarrierState, item_frame_owner
|
from wf_core.runtime.foreach_state import (
|
||||||
|
item_frame_owner,
|
||||||
|
require_foreach_activation,
|
||||||
|
save_foreach_activation,
|
||||||
|
)
|
||||||
from wf_core.runtime.input_bindings import resolve_step_input_bindings
|
from wf_core.runtime.input_bindings import resolve_step_input_bindings
|
||||||
from wf_core.runtime.lineage import (
|
from wf_core.runtime.lineage import (
|
||||||
append_lineage_writes,
|
append_lineage_writes,
|
||||||
@@ -116,10 +120,14 @@ def _finalize_node_execution(
|
|||||||
if owner is None:
|
if owner is None:
|
||||||
state_changes = commit_patch_for_frame(run, frame, patch)
|
state_changes = commit_patch_for_frame(run, frame, patch)
|
||||||
else:
|
else:
|
||||||
parent_frame_id, foreach_node_id, item_index = owner
|
parent_frame = run.frames[owner.parent_frame_id]
|
||||||
parent_frame = run.frames[parent_frame_id]
|
# Fail closed when the child names a closed or superseded activation:
|
||||||
barrier = ForeachBarrierState.from_frame(parent_frame, foreach_node_id)
|
# its writes must not land in a later visit's barrier.
|
||||||
if barrier is not None and barrier.mode == "concurrent":
|
activation = require_foreach_activation(
|
||||||
|
parent_frame, owner.foreach_node_id, owner.activation_id
|
||||||
|
)
|
||||||
|
barrier = activation.barrier
|
||||||
|
if barrier.mode == "concurrent":
|
||||||
# New concurrent foreach stores writes in the child lineage; the
|
# New concurrent foreach stores writes in the child lineage; the
|
||||||
# barrier keeps only result metadata plus old patch fallback.
|
# barrier keeps only result metadata plus old patch fallback.
|
||||||
append_lineage_writes(
|
append_lineage_writes(
|
||||||
@@ -129,12 +137,12 @@ def _finalize_node_execution(
|
|||||||
writes=patch.writes,
|
writes=patch.writes,
|
||||||
)
|
)
|
||||||
barrier.add_success_patch(
|
barrier.add_success_patch(
|
||||||
index=item_index,
|
index=owner.item_index,
|
||||||
frame_id=frame.id,
|
frame_id=frame.id,
|
||||||
patch=StatePatch(),
|
patch=StatePatch(),
|
||||||
lineage_id=frame.lineage_id,
|
lineage_id=frame.lineage_id,
|
||||||
)
|
)
|
||||||
barrier.save_to_frame(parent_frame, foreach_node_id)
|
save_foreach_activation(parent_frame, activation)
|
||||||
state_changes = {}
|
state_changes = {}
|
||||||
else:
|
else:
|
||||||
state_changes = commit_patch_for_frame(run, parent_frame, patch)
|
state_changes = commit_patch_for_frame(run, parent_frame, patch)
|
||||||
|
|||||||
@@ -38,9 +38,15 @@ class BlockedOnChildren:
|
|||||||
|
|
||||||
@dataclass(slots=True, frozen=True)
|
@dataclass(slots=True, frozen=True)
|
||||||
class ForeachIterationMetadata:
|
class ForeachIterationMetadata:
|
||||||
"""Typed metadata for a foreach iteration frame."""
|
"""Typed metadata for a foreach iteration frame.
|
||||||
|
|
||||||
|
``activation_id`` names the dynamic foreach visit that owns this item.
|
||||||
|
It separates fresh barrier state from earlier visits to the same node use
|
||||||
|
and must survive checkpoint serialization.
|
||||||
|
"""
|
||||||
|
|
||||||
foreach_node_id: str
|
foreach_node_id: str
|
||||||
|
activation_id: str
|
||||||
loop_index: int
|
loop_index: int
|
||||||
loop_item: Any
|
loop_item: Any
|
||||||
loop_alias: str
|
loop_alias: str
|
||||||
@@ -51,12 +57,17 @@ class ForeachIterationMetadata:
|
|||||||
return None
|
return None
|
||||||
metadata = frame.metadata
|
metadata = frame.metadata
|
||||||
foreach_node_id = metadata.get("foreach_node_id")
|
foreach_node_id = metadata.get("foreach_node_id")
|
||||||
|
activation_id = metadata.get("activation_id")
|
||||||
loop_index = metadata.get("loop_index")
|
loop_index = metadata.get("loop_index")
|
||||||
loop_alias = metadata.get("loop_alias")
|
loop_alias = metadata.get("loop_alias")
|
||||||
if not isinstance(foreach_node_id, str) or not foreach_node_id:
|
if not isinstance(foreach_node_id, str) or not foreach_node_id:
|
||||||
raise WorkflowExecutionError(
|
raise WorkflowExecutionError(
|
||||||
f"malformed foreach node id for frame {frame.id!r}"
|
f"malformed foreach node id for frame {frame.id!r}"
|
||||||
)
|
)
|
||||||
|
if not isinstance(activation_id, str) or not activation_id:
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"malformed foreach activation id for frame {frame.id!r}"
|
||||||
|
)
|
||||||
if not isinstance(loop_index, int):
|
if not isinstance(loop_index, int):
|
||||||
raise WorkflowExecutionError(
|
raise WorkflowExecutionError(
|
||||||
f"malformed foreach loop index for frame {frame.id!r}"
|
f"malformed foreach loop index for frame {frame.id!r}"
|
||||||
@@ -71,6 +82,7 @@ class ForeachIterationMetadata:
|
|||||||
)
|
)
|
||||||
return cls(
|
return cls(
|
||||||
foreach_node_id=foreach_node_id,
|
foreach_node_id=foreach_node_id,
|
||||||
|
activation_id=activation_id,
|
||||||
loop_index=loop_index,
|
loop_index=loop_index,
|
||||||
loop_item=metadata["loop_item"],
|
loop_item=metadata["loop_item"],
|
||||||
loop_alias=loop_alias,
|
loop_alias=loop_alias,
|
||||||
@@ -79,6 +91,7 @@ class ForeachIterationMetadata:
|
|||||||
def to_metadata(self) -> dict[str, object]:
|
def to_metadata(self) -> dict[str, object]:
|
||||||
return {
|
return {
|
||||||
"foreach_node_id": self.foreach_node_id,
|
"foreach_node_id": self.foreach_node_id,
|
||||||
|
"activation_id": self.activation_id,
|
||||||
"loop_index": self.loop_index,
|
"loop_index": self.loop_index,
|
||||||
"loop_item": self.loop_item,
|
"loop_item": self.loop_item,
|
||||||
"loop_alias": self.loop_alias,
|
"loop_alias": self.loop_alias,
|
||||||
@@ -178,7 +191,11 @@ def wake_parent_if_children_complete(run: RunState, child_frame_id: str) -> None
|
|||||||
|
|
||||||
|
|
||||||
def wake_parent_for_child_progress(run: RunState, child_frame_id: str) -> None:
|
def wake_parent_for_child_progress(run: RunState, child_frame_id: str) -> None:
|
||||||
"""Wake a blocked parent after one child finishes so it can refill slots."""
|
"""Wake a blocked parent after one child finishes so it can refill slots.
|
||||||
|
|
||||||
|
The wake-up includes the foreach activation: a child naming a closed or
|
||||||
|
superseded activation cannot wake a parent waiting on a later visit.
|
||||||
|
"""
|
||||||
child = _frame(run, child_frame_id)
|
child = _frame(run, child_frame_id)
|
||||||
parent_id = child.parent_frame_id
|
parent_id = child.parent_frame_id
|
||||||
if parent_id is None:
|
if parent_id is None:
|
||||||
@@ -189,6 +206,26 @@ def wake_parent_for_child_progress(run: RunState, child_frame_id: str) -> None:
|
|||||||
block = BlockedOnChildren.from_frame(parent)
|
block = BlockedOnChildren.from_frame(parent)
|
||||||
if block is None or child_frame_id not in block.child_frame_ids:
|
if block is None or child_frame_id not in block.child_frame_ids:
|
||||||
return
|
return
|
||||||
|
# Lazy import avoids a cycle: foreach_state owns activation persistence on
|
||||||
|
# top of this scheduler's frame metadata types.
|
||||||
|
from wf_core.runtime.foreach_state import (
|
||||||
|
item_frame_owner,
|
||||||
|
load_foreach_activation,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
owner = item_frame_owner(child)
|
||||||
|
except WorkflowExecutionError:
|
||||||
|
raise
|
||||||
|
if owner is not None:
|
||||||
|
activation = load_foreach_activation(
|
||||||
|
parent, owner.foreach_node_id, owner.activation_id
|
||||||
|
)
|
||||||
|
if activation is None:
|
||||||
|
raise WorkflowExecutionError(
|
||||||
|
f"foreach item frame {child_frame_id!r} names closed activation "
|
||||||
|
f"{owner.activation_id!r} and cannot wake parent {parent_id!r}"
|
||||||
|
)
|
||||||
wake_frame(run, parent_id)
|
wake_frame(run, parent_id)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from wf_core.models.steps import (
|
|||||||
)
|
)
|
||||||
from wf_core.models.workflow import Workflow
|
from wf_core.models.workflow import Workflow
|
||||||
from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult
|
from wf_core.run_state import ExecutionFrame, FrameStatus, RunState, StepExecutionResult
|
||||||
from wf_core.runtime.foreach_state import ForeachBarrierState, item_frame_owner
|
from wf_core.runtime.foreach_state import item_frame_owner, load_foreach_activation
|
||||||
from wf_core.runtime.ops.flow import advance_frame, append_step_result_trace
|
from wf_core.runtime.ops.flow import advance_frame, append_step_result_trace
|
||||||
from wf_core.runtime.ops.foreach import step_foreach
|
from wf_core.runtime.ops.foreach import step_foreach
|
||||||
from wf_core.runtime.ops.handlers import (
|
from wf_core.runtime.ops.handlers import (
|
||||||
@@ -361,10 +361,15 @@ def _claim_matching_async_item_frames(
|
|||||||
index: WorkflowIndex,
|
index: WorkflowIndex,
|
||||||
first_frame: ExecutionFrame,
|
first_frame: ExecutionFrame,
|
||||||
) -> list[ExecutionFrame]:
|
) -> list[ExecutionFrame]:
|
||||||
|
"""Claim sibling item frames from the same activation for async batching.
|
||||||
|
|
||||||
|
Batching never mixes activations: only frames naming the same parent,
|
||||||
|
foreach, and activation id run together, preserving deterministic barrier
|
||||||
|
commits across revisits.
|
||||||
|
"""
|
||||||
owner = item_frame_owner(first_frame)
|
owner = item_frame_owner(first_frame)
|
||||||
if owner is None:
|
if owner is None:
|
||||||
return []
|
return []
|
||||||
parent_frame_id, foreach_node_id, _item_index = owner
|
|
||||||
claimed: list[ExecutionFrame] = []
|
claimed: list[ExecutionFrame] = []
|
||||||
remaining_ready: list[str] = []
|
remaining_ready: list[str] = []
|
||||||
for frame_id in run.ready_frame_ids:
|
for frame_id in run.ready_frame_ids:
|
||||||
@@ -373,7 +378,9 @@ def _claim_matching_async_item_frames(
|
|||||||
if (
|
if (
|
||||||
frame.status == FrameStatus.PENDING
|
frame.status == FrameStatus.PENDING
|
||||||
and frame_owner is not None
|
and frame_owner is not None
|
||||||
and frame_owner[:2] == (parent_frame_id, foreach_node_id)
|
and frame_owner.parent_frame_id == owner.parent_frame_id
|
||||||
|
and frame_owner.foreach_node_id == owner.foreach_node_id
|
||||||
|
and frame_owner.activation_id == owner.activation_id
|
||||||
and isinstance(index.nodes_by_id.get(frame.node_id), NodeUse)
|
and isinstance(index.nodes_by_id.get(frame.node_id), NodeUse)
|
||||||
):
|
):
|
||||||
frame.status = FrameStatus.RUNNING
|
frame.status = FrameStatus.RUNNING
|
||||||
@@ -392,14 +399,15 @@ def _can_batch_async_foreach_item(
|
|||||||
owner = item_frame_owner(frame)
|
owner = item_frame_owner(frame)
|
||||||
if owner is None:
|
if owner is None:
|
||||||
return False
|
return False
|
||||||
parent_frame_id, foreach_node_id, _item_index = owner
|
parent_frame = run.frames.get(owner.parent_frame_id)
|
||||||
parent_frame = run.frames.get(parent_frame_id)
|
|
||||||
if parent_frame is None:
|
if parent_frame is None:
|
||||||
return False
|
return False
|
||||||
barrier = ForeachBarrierState.from_frame(parent_frame, foreach_node_id)
|
activation = load_foreach_activation(
|
||||||
|
parent_frame, owner.foreach_node_id, owner.activation_id
|
||||||
|
)
|
||||||
return (
|
return (
|
||||||
barrier is not None
|
activation is not None
|
||||||
and barrier.mode == "concurrent"
|
and activation.barrier.mode == "concurrent"
|
||||||
and isinstance(index.nodes_by_id.get(frame.node_id), NodeUse)
|
and isinstance(index.nodes_by_id.get(frame.node_id), NodeUse)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,11 @@ from wf_core import (
|
|||||||
execute_workflow,
|
execute_workflow,
|
||||||
)
|
)
|
||||||
from wf_core.run_state import ExecutionFrame, RunState, RuntimeContext
|
from wf_core.run_state import ExecutionFrame, RunState, RuntimeContext
|
||||||
from wf_core.runtime.foreach_state import ForeachBarrierState
|
from wf_core.runtime.foreach_state import (
|
||||||
|
ForeachItemOwner,
|
||||||
|
item_frame_owner,
|
||||||
|
load_or_begin_foreach_activation,
|
||||||
|
)
|
||||||
from wf_core.runtime.scheduler import ForeachIterationMetadata
|
from wf_core.runtime.scheduler import ForeachIterationMetadata
|
||||||
|
|
||||||
|
|
||||||
@@ -143,7 +147,11 @@ def test_concurrent_foreach_item_frames_use_distinct_lineages() -> None:
|
|||||||
assert run.frames["root"].lineage_id == "root"
|
assert run.frames["root"].lineage_id == "root"
|
||||||
assert run.frames["root"].parent_lineage_id is None
|
assert run.frames["root"].parent_lineage_id is None
|
||||||
assert len(item_frames) == 2
|
assert len(item_frames) == 2
|
||||||
assert item_lineage_ids == {"root/each[0]", "root/each[1]"}
|
assert item_lineage_ids == {"root:each#0[0]", "root:each#0[1]"}
|
||||||
|
for frame in item_frames:
|
||||||
|
owner = item_frame_owner(frame)
|
||||||
|
assert isinstance(owner, ForeachItemOwner)
|
||||||
|
assert owner.activation_id == "root:each#0"
|
||||||
assert set(context_lineage_ids) == item_lineage_ids
|
assert set(context_lineage_ids) == item_lineage_ids
|
||||||
assert all(frame.scope_id == "root" for frame in item_frames)
|
assert all(frame.scope_id == "root" for frame in item_frames)
|
||||||
assert all(frame.parent_lineage_id == "root" for frame in item_frames)
|
assert all(frame.parent_lineage_id == "root" for frame in item_frames)
|
||||||
@@ -162,15 +170,27 @@ def test_nested_concurrent_foreach_records_parent_child_lineages() -> None:
|
|||||||
inner_frames = _foreach_frames(run, "inner_each")
|
inner_frames = _foreach_frames(run, "inner_each")
|
||||||
|
|
||||||
assert {frame.lineage_id for frame in outer_frames} == {
|
assert {frame.lineage_id for frame in outer_frames} == {
|
||||||
"root/outer_each[0]",
|
"root:outer_each#0[0]",
|
||||||
"root/outer_each[1]",
|
"root:outer_each#0[1]",
|
||||||
}
|
}
|
||||||
assert all(frame.parent_lineage_id == "root" for frame in outer_frames)
|
assert all(frame.parent_lineage_id == "root" for frame in outer_frames)
|
||||||
assert {(frame.parent_lineage_id, frame.lineage_id) for frame in inner_frames} == {
|
assert {(frame.parent_lineage_id, frame.lineage_id) for frame in inner_frames} == {
|
||||||
("root/outer_each[0]", "root/outer_each[0]/inner_each[0]"),
|
(
|
||||||
("root/outer_each[0]", "root/outer_each[0]/inner_each[1]"),
|
"root:outer_each#0[0]",
|
||||||
("root/outer_each[1]", "root/outer_each[1]/inner_each[0]"),
|
"root:outer_each#0:0:inner_each#0[0]",
|
||||||
("root/outer_each[1]", "root/outer_each[1]/inner_each[1]"),
|
),
|
||||||
|
(
|
||||||
|
"root:outer_each#0[0]",
|
||||||
|
"root:outer_each#0:0:inner_each#0[1]",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"root:outer_each#0[1]",
|
||||||
|
"root:outer_each#0:1:inner_each#0[0]",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"root:outer_each#0[1]",
|
||||||
|
"root:outer_each#0:1:inner_each#0[1]",
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -264,12 +284,14 @@ def test_sync_concurrent_foreach_barrier_replays_add_reducer_inputs() -> None:
|
|||||||
|
|
||||||
assert run.state["number"] == 6
|
assert run.state["number"] == 6
|
||||||
assert run.output["number"] == 6
|
assert run.output["number"] == 6
|
||||||
assert run.lineages["root/each[0]"].writes[0].incoming_value == 3
|
assert run.lineages["root:each#0[0]"].writes[0].incoming_value == 3
|
||||||
assert run.lineages["root/each[1]"].writes[0].incoming_value == 1
|
assert run.lineages["root:each#0[1]"].writes[0].incoming_value == 1
|
||||||
barrier = ForeachBarrierState.from_frame(run.frames["root"], "each")
|
active = load_or_begin_foreach_activation(
|
||||||
assert barrier is not None
|
run.frames["root"], "each", mode="concurrent"
|
||||||
assert barrier.pending_results[0].lineage_id == "root/each[0]"
|
)
|
||||||
assert barrier.pending_results[0].patch.writes == []
|
assert active.id == "root:each#0"
|
||||||
|
assert active.barrier.pending_results[0].lineage_id == "root:each#0[0]"
|
||||||
|
assert active.barrier.pending_results[0].patch.writes == []
|
||||||
foreach_entries = [entry for entry in run.trace if entry.step_type == "foreach"]
|
foreach_entries = [entry for entry in run.trace if entry.step_type == "foreach"]
|
||||||
assert foreach_entries[-1].state_changes["state.number"] == 6
|
assert foreach_entries[-1].state_changes["state.number"] == 6
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ def test_concurrent_foreach_skip_emits_completed_with_errors() -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert run.state["seen"] == ["a", "c"]
|
assert run.state["seen"] == ["a", "c"]
|
||||||
assert run.frames["root:each:1"].status == "failed"
|
assert run.frames["root:each#0:1"].status == "failed"
|
||||||
foreach_entries = [entry for entry in run.trace if entry.step_type == "foreach"]
|
foreach_entries = [entry for entry in run.trace if entry.step_type == "foreach"]
|
||||||
assert foreach_entries[-1].outcome == "completed_with_errors"
|
assert foreach_entries[-1].outcome == "completed_with_errors"
|
||||||
assert foreach_entries[-1].resolved_input["failed_items"] == 1
|
assert foreach_entries[-1].resolved_input["failed_items"] == 1
|
||||||
@@ -47,7 +47,7 @@ def test_concurrent_foreach_collect_writes_ordered_error_records() -> None:
|
|||||||
assert len(run.state["errors"]) == 1
|
assert len(run.state["errors"]) == 1
|
||||||
error = run.state["errors"][0]
|
error = run.state["errors"][0]
|
||||||
assert error["index"] == 1
|
assert error["index"] == 1
|
||||||
assert error["frame_id"] == "root:each:1"
|
assert error["frame_id"] == "root:each#0:1"
|
||||||
assert error["node_id"] == "record"
|
assert error["node_id"] == "record"
|
||||||
assert error["error_type"] == "ValueError"
|
assert error["error_type"] == "ValueError"
|
||||||
assert error["message"] == "bad item"
|
assert error["message"] == "bad item"
|
||||||
|
|||||||
@@ -31,8 +31,8 @@ async def test_concurrent_foreach_interrupt_returns_before_refill() -> None:
|
|||||||
assert run.status is RunStatus.INTERRUPTED
|
assert run.status is RunStatus.INTERRUPTED
|
||||||
assert run.interrupt is not None
|
assert run.interrupt is not None
|
||||||
assert run.interrupt.payload["item"] == "b"
|
assert run.interrupt.payload["item"] == "b"
|
||||||
assert run.frames["root:each:1"].status == "interrupted"
|
assert run.frames["root:each#0:1"].status == "interrupted"
|
||||||
assert "root:each:2" not in run.frames
|
assert "root:each#0:2" not in run.frames
|
||||||
assert "seen" not in run.state
|
assert "seen" not in run.state
|
||||||
|
|
||||||
|
|
||||||
@@ -54,7 +54,7 @@ async def test_resume_prioritizes_interrupted_item_before_siblings() -> None:
|
|||||||
|
|
||||||
assert resumed.status is RunStatus.COMPLETED
|
assert resumed.status is RunStatus.COMPLETED
|
||||||
assert resumed.state["seen"] == ["a", "b", "c"]
|
assert resumed.state["seen"] == ["a", "b", "c"]
|
||||||
assert resumed.trace[interrupted_trace_len].frame_id == "root:each:1"
|
assert resumed.trace[interrupted_trace_len].frame_id == "root:each#0:1"
|
||||||
assert resumed.trace[interrupted_trace_len].step_type == "interrupt"
|
assert resumed.trace[interrupted_trace_len].step_type == "interrupt"
|
||||||
assert resumed.trace[interrupted_trace_len].outcome == "submitted"
|
assert resumed.trace[interrupted_trace_len].outcome == "submitted"
|
||||||
foreach_entries = [entry for entry in resumed.trace if entry.step_type == "foreach"]
|
foreach_entries = [entry for entry in resumed.trace if entry.step_type == "foreach"]
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from wf_core.errors import WorkflowExecutionError
|
||||||
|
from wf_core.run_state import ExecutionFrame
|
||||||
|
from wf_core.runtime.foreach_state import (
|
||||||
|
close_foreach_activation,
|
||||||
|
item_frame_owner,
|
||||||
|
load_or_begin_foreach_activation,
|
||||||
|
save_foreach_activation,
|
||||||
|
)
|
||||||
|
from wf_core.runtime.scheduler import ForeachIterationMetadata
|
||||||
|
|
||||||
|
|
||||||
|
def _frame() -> ExecutionFrame:
|
||||||
|
return ExecutionFrame(id="root", kind="workflow", node_id="each")
|
||||||
|
|
||||||
|
|
||||||
|
def test_activation_lifecycle_reuses_active_then_fresh_after_close() -> None:
|
||||||
|
frame = _frame()
|
||||||
|
|
||||||
|
first = load_or_begin_foreach_activation(frame, "each", mode="serial")
|
||||||
|
save_foreach_activation(frame, first)
|
||||||
|
restored = load_or_begin_foreach_activation(frame, "each", mode="serial")
|
||||||
|
|
||||||
|
assert restored.id == first.id
|
||||||
|
|
||||||
|
close_foreach_activation(frame, restored)
|
||||||
|
second = load_or_begin_foreach_activation(frame, "each", mode="serial")
|
||||||
|
|
||||||
|
assert second.id != first.id
|
||||||
|
assert second.barrier.next_index == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_activation_rejects_malformed_metadata() -> None:
|
||||||
|
frame = ExecutionFrame(
|
||||||
|
id="root",
|
||||||
|
kind="workflow",
|
||||||
|
node_id="each",
|
||||||
|
metadata={"foreach_activations": "corrupt"},
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(WorkflowExecutionError, match="activation"):
|
||||||
|
load_or_begin_foreach_activation(frame, "each", mode="serial")
|
||||||
|
|
||||||
|
|
||||||
|
def test_activation_rejects_mode_mismatch() -> None:
|
||||||
|
frame = _frame()
|
||||||
|
activation = load_or_begin_foreach_activation(frame, "each", mode="serial")
|
||||||
|
save_foreach_activation(frame, activation)
|
||||||
|
|
||||||
|
with pytest.raises(WorkflowExecutionError, match="mode"):
|
||||||
|
load_or_begin_foreach_activation(frame, "each", mode="concurrent")
|
||||||
|
|
||||||
|
|
||||||
|
def test_closing_stale_activation_fails_closed() -> None:
|
||||||
|
frame = _frame()
|
||||||
|
first = load_or_begin_foreach_activation(frame, "each", mode="serial")
|
||||||
|
save_foreach_activation(frame, first)
|
||||||
|
close_foreach_activation(frame, first)
|
||||||
|
second = load_or_begin_foreach_activation(frame, "each", mode="serial")
|
||||||
|
save_foreach_activation(frame, second)
|
||||||
|
|
||||||
|
with pytest.raises(WorkflowExecutionError, match="stale|closed|active"):
|
||||||
|
close_foreach_activation(frame, first)
|
||||||
|
|
||||||
|
|
||||||
|
def test_activation_json_round_trip_through_frame_metadata() -> None:
|
||||||
|
frame = _frame()
|
||||||
|
activation = load_or_begin_foreach_activation(frame, "each", mode="serial")
|
||||||
|
activation.barrier.next_index = 2
|
||||||
|
save_foreach_activation(frame, activation)
|
||||||
|
|
||||||
|
dumped = dict(frame.metadata)
|
||||||
|
restored_frame = ExecutionFrame(
|
||||||
|
id="root", kind="workflow", node_id="each", metadata=dumped
|
||||||
|
)
|
||||||
|
restored = load_or_begin_foreach_activation(restored_frame, "each", mode="serial")
|
||||||
|
|
||||||
|
assert restored.id == activation.id
|
||||||
|
assert restored.barrier.next_index == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_item_metadata_requires_activation_identity() -> None:
|
||||||
|
frame = ExecutionFrame(
|
||||||
|
id="root:each#0:0",
|
||||||
|
kind="foreach_iteration",
|
||||||
|
node_id="work",
|
||||||
|
parent_frame_id="root",
|
||||||
|
metadata={
|
||||||
|
"foreach_node_id": "each",
|
||||||
|
"loop_index": 0,
|
||||||
|
"loop_item": "a",
|
||||||
|
"loop_alias": "item",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(WorkflowExecutionError, match="activation"):
|
||||||
|
ForeachIterationMetadata.from_frame(frame)
|
||||||
|
with pytest.raises(WorkflowExecutionError, match="activation"):
|
||||||
|
item_frame_owner(frame)
|
||||||
@@ -8,9 +8,13 @@ from wf_core.paths import StatePath
|
|||||||
from wf_core.run_state import ExecutionFrame, RunState, RunStatus, StateWrite
|
from wf_core.run_state import ExecutionFrame, RunState, RunStatus, StateWrite
|
||||||
from wf_core.runtime.foreach_state import (
|
from wf_core.runtime.foreach_state import (
|
||||||
ForeachBarrierState,
|
ForeachBarrierState,
|
||||||
|
ForeachItemOwner,
|
||||||
ItemErrorRecord,
|
ItemErrorRecord,
|
||||||
PendingItemResult,
|
PendingItemResult,
|
||||||
_state_write_from_metadata,
|
_state_write_from_metadata,
|
||||||
|
item_frame_owner,
|
||||||
|
load_or_begin_foreach_activation,
|
||||||
|
save_foreach_activation,
|
||||||
)
|
)
|
||||||
from wf_core.runtime.lineage import LineageStateView, lineage_writes_for_frame
|
from wf_core.runtime.lineage import LineageStateView, lineage_writes_for_frame
|
||||||
from wf_core.runtime.ops.state import StatePatch
|
from wf_core.runtime.ops.state import StatePatch
|
||||||
@@ -134,20 +138,28 @@ def test_lineage_state_view_materializes_visible_values_without_mutating_base()
|
|||||||
|
|
||||||
def test_lineage_writes_for_frame_reads_current_foreach_pending_result() -> None:
|
def test_lineage_writes_for_frame_reads_current_foreach_pending_result() -> None:
|
||||||
parent = ExecutionFrame(id="root", kind="workflow", node_id="each")
|
parent = ExecutionFrame(id="root", kind="workflow", node_id="each")
|
||||||
|
activation = load_or_begin_foreach_activation(parent, "each", mode="concurrent")
|
||||||
|
child_lineage_id = f"{activation.id}[0]"
|
||||||
child = ExecutionFrame(
|
child = ExecutionFrame(
|
||||||
id="root:each:0",
|
id=f"{activation.id}:0",
|
||||||
kind="foreach_iteration",
|
kind="foreach_iteration",
|
||||||
node_id="work",
|
node_id="work",
|
||||||
parent_frame_id="root",
|
parent_frame_id="root",
|
||||||
lineage_id="root/each[0]",
|
lineage_id=child_lineage_id,
|
||||||
parent_lineage_id="root",
|
parent_lineage_id="root",
|
||||||
metadata={
|
metadata={
|
||||||
"foreach_node_id": "each",
|
"foreach_node_id": "each",
|
||||||
|
"activation_id": activation.id,
|
||||||
"loop_index": 0,
|
"loop_index": 0,
|
||||||
"loop_item": "a",
|
"loop_item": "a",
|
||||||
"loop_alias": "item",
|
"loop_alias": "item",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
# Ownership is named, not positional.
|
||||||
|
owner = item_frame_owner(child)
|
||||||
|
assert isinstance(owner, ForeachItemOwner)
|
||||||
|
assert owner.activation_id == activation.id
|
||||||
|
assert owner.item_index == 0
|
||||||
patch = StatePatch(
|
patch = StatePatch(
|
||||||
writes=[
|
writes=[
|
||||||
StateWrite(
|
StateWrite(
|
||||||
@@ -158,19 +170,14 @@ def test_lineage_writes_for_frame_reads_current_foreach_pending_result() -> None
|
|||||||
)
|
)
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
barrier = ForeachBarrierState(
|
activation.barrier.pending_results[0] = PendingItemResult(
|
||||||
mode="concurrent",
|
index=0,
|
||||||
pending_results={
|
frame_id=child.id,
|
||||||
0: PendingItemResult(
|
status="succeeded",
|
||||||
index=0,
|
lineage_id=child.lineage_id,
|
||||||
frame_id=child.id,
|
patch=patch,
|
||||||
status="succeeded",
|
|
||||||
lineage_id=child.lineage_id,
|
|
||||||
patch=patch,
|
|
||||||
)
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
barrier.save_to_frame(parent, "each")
|
save_foreach_activation(parent, activation)
|
||||||
run = RunState(
|
run = RunState(
|
||||||
workflow_name="lineage",
|
workflow_name="lineage",
|
||||||
status=RunStatus.PENDING,
|
status=RunStatus.PENDING,
|
||||||
@@ -188,12 +195,13 @@ def test_lineage_writes_for_frame_reads_current_foreach_pending_result() -> None
|
|||||||
|
|
||||||
def test_lineage_writes_for_frame_rejects_missing_compatibility_parent_frame() -> None:
|
def test_lineage_writes_for_frame_rejects_missing_compatibility_parent_frame() -> None:
|
||||||
child = ExecutionFrame(
|
child = ExecutionFrame(
|
||||||
id="missing:each:0",
|
id="missing:each#0:0",
|
||||||
kind="foreach_iteration",
|
kind="foreach_iteration",
|
||||||
node_id="work",
|
node_id="work",
|
||||||
parent_frame_id="missing",
|
parent_frame_id="missing",
|
||||||
metadata={
|
metadata={
|
||||||
"foreach_node_id": "each",
|
"foreach_node_id": "each",
|
||||||
|
"activation_id": "missing:each#0",
|
||||||
"loop_index": 0,
|
"loop_index": 0,
|
||||||
"loop_item": "a",
|
"loop_item": "a",
|
||||||
"loop_alias": "item",
|
"loop_alias": "item",
|
||||||
|
|||||||
@@ -163,8 +163,19 @@ def test_child_completion_wakes_blocked_parent() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_wake_parent_when_child_finishes_for_refill() -> None:
|
def test_wake_parent_when_child_finishes_for_refill() -> None:
|
||||||
|
from wf_core.runtime.foreach_state import (
|
||||||
|
ForeachItemOwner,
|
||||||
|
item_frame_owner,
|
||||||
|
load_or_begin_foreach_activation,
|
||||||
|
save_foreach_activation,
|
||||||
|
)
|
||||||
|
|
||||||
run = _run()
|
run = _run()
|
||||||
add_frame(run, ExecutionFrame(id="parent", kind="root", node_id="foreach"))
|
add_frame(run, ExecutionFrame(id="parent", kind="root", node_id="foreach"))
|
||||||
|
activation = load_or_begin_foreach_activation(
|
||||||
|
run.frames["parent"], "foreach", mode="serial"
|
||||||
|
)
|
||||||
|
save_foreach_activation(run.frames["parent"], activation)
|
||||||
add_frame(
|
add_frame(
|
||||||
run,
|
run,
|
||||||
ExecutionFrame(
|
ExecutionFrame(
|
||||||
@@ -172,11 +183,22 @@ def test_wake_parent_when_child_finishes_for_refill() -> None:
|
|||||||
kind="foreach_iteration",
|
kind="foreach_iteration",
|
||||||
node_id="__end__",
|
node_id="__end__",
|
||||||
parent_frame_id="parent",
|
parent_frame_id="parent",
|
||||||
|
metadata={
|
||||||
|
"foreach_node_id": "foreach",
|
||||||
|
"activation_id": activation.id,
|
||||||
|
"loop_index": 0,
|
||||||
|
"loop_item": "a",
|
||||||
|
"loop_alias": "item",
|
||||||
|
},
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
block_frame_on_children(run, "parent", ("child", "other"))
|
block_frame_on_children(run, "parent", ("child", "other"))
|
||||||
run.frames["child"].status = FrameStatus.COMPLETED
|
run.frames["child"].status = FrameStatus.COMPLETED
|
||||||
|
|
||||||
|
owner = item_frame_owner(run.frames["child"])
|
||||||
|
assert isinstance(owner, ForeachItemOwner)
|
||||||
|
assert owner.activation_id == activation.id
|
||||||
|
|
||||||
wake_parent_for_child_progress(run, "child")
|
wake_parent_for_child_progress(run, "child")
|
||||||
|
|
||||||
assert run.frames["parent"].status == FrameStatus.PENDING
|
assert run.frames["parent"].status == FrameStatus.PENDING
|
||||||
|
|||||||
Reference in New Issue
Block a user