interrupt goes first
This commit is contained in:
@@ -131,7 +131,7 @@ future explicit deep merge policy exists.
|
||||
|
||||
## Interrupt and Failure Quiescence
|
||||
|
||||
Future concurrent execution should not assume in-flight node calls can be safely
|
||||
Concurrent execution should not assume in-flight node calls can be safely
|
||||
cancelled.
|
||||
|
||||
If an interrupt or fail policy trips while sibling jobs are already started, the
|
||||
@@ -146,6 +146,14 @@ runtime should:
|
||||
For `fail`, drained sibling results are for observability/cleanup only and
|
||||
should not commit normal state progress after the failure boundary.
|
||||
|
||||
Current async concurrent foreach implements the interrupt part of this by
|
||||
prioritizing item frames that route into an `InterruptNode`. If a batched async
|
||||
node result sends one item to an interrupt while a sibling completes, the
|
||||
interrupt-bound frame is placed at the front of the ready queue before the
|
||||
parent foreach can refill capacity. Already-started async handler calls from
|
||||
the batch are awaited first, then their results are finalized sequentially.
|
||||
The foreach barrier does not commit while an item frame remains interrupted.
|
||||
|
||||
## Capacity and Runtime Limits
|
||||
|
||||
Foreach capacity is local correctness policy, not total process protection.
|
||||
|
||||
@@ -168,7 +168,7 @@ Key tests:
|
||||
|
||||
## Slice 6: Interrupt Quiescence
|
||||
|
||||
Implement after async execution exists.
|
||||
Implemented after async execution exists.
|
||||
|
||||
Scope:
|
||||
|
||||
@@ -177,6 +177,10 @@ Scope:
|
||||
- Already-started async node calls drain to pending results.
|
||||
- The caller gets control only at a quiescent point.
|
||||
- Pending results do not commit until resume/commit policy allows it.
|
||||
- Item frames that route into an `InterruptNode` are prioritized before the
|
||||
parent foreach can refill capacity.
|
||||
- Already-started async handler calls drain at the batch boundary; state
|
||||
finalization remains sequential.
|
||||
|
||||
Files likely touched:
|
||||
|
||||
@@ -188,7 +192,7 @@ Files likely touched:
|
||||
|
||||
Key tests:
|
||||
|
||||
- `test_concurrent_foreach_interrupt_returns_after_quiescence`
|
||||
- `test_concurrent_foreach_interrupt_returns_before_refill`
|
||||
- `test_resume_prioritizes_interrupted_item_frame_before_siblings`
|
||||
|
||||
## Execution Order
|
||||
|
||||
@@ -74,6 +74,7 @@ def advance_frame(
|
||||
*,
|
||||
outcome: str,
|
||||
next_node_id: str,
|
||||
front: bool = False,
|
||||
) -> None:
|
||||
frame.prior_outcome = outcome
|
||||
frame.activated_incoming_edge = frame.node_id
|
||||
@@ -84,7 +85,7 @@ def advance_frame(
|
||||
wake_parent_for_child_progress(run, frame.id)
|
||||
else:
|
||||
frame.finished_at_node_id = None
|
||||
mark_frame_pending(run, frame.id)
|
||||
mark_frame_pending(run, frame.id, front=front)
|
||||
run.sync_from_current_frame()
|
||||
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ def complete_step(
|
||||
) -> RunState:
|
||||
"""Record a completed step and advance the active frame."""
|
||||
next_node_id = index.next_node_id(node_id, outcome)
|
||||
next_step = index.nodes_by_id.get(next_node_id)
|
||||
|
||||
append_step_result_trace(
|
||||
run,
|
||||
@@ -67,6 +68,7 @@ def complete_step(
|
||||
run.frames[frame_id],
|
||||
outcome=outcome,
|
||||
next_node_id=next_node_id,
|
||||
front=isinstance(next_step, InterruptNode),
|
||||
)
|
||||
return run
|
||||
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from wf_core import (
|
||||
END,
|
||||
Edge,
|
||||
ForeachNode,
|
||||
InterruptNode,
|
||||
NodeDef,
|
||||
NodeUse,
|
||||
ReducerRef,
|
||||
RunStatus,
|
||||
SchemaRef,
|
||||
StateField,
|
||||
StateSchema,
|
||||
Workflow,
|
||||
execute_workflow_async,
|
||||
resume_workflow_async,
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_foreach_interrupt_returns_before_refill() -> None:
|
||||
asyncio.run(_assert_concurrent_foreach_interrupt_returns_before_refill())
|
||||
|
||||
|
||||
async def _assert_concurrent_foreach_interrupt_returns_before_refill() -> None:
|
||||
run = await execute_workflow_async(
|
||||
_workflow(),
|
||||
{"items": ["a", "b", "c"]},
|
||||
{"route": _interrupt_on_b},
|
||||
)
|
||||
|
||||
assert run.status is RunStatus.INTERRUPTED
|
||||
assert run.interrupt is not None
|
||||
assert run.interrupt.payload["item"] == "b"
|
||||
assert run.frames["root:each:1"].status == "interrupted"
|
||||
assert "root:each:2" not in run.frames
|
||||
assert "seen" not in run.state
|
||||
|
||||
|
||||
def test_resume_prioritizes_interrupted_item_before_siblings() -> None:
|
||||
asyncio.run(_assert_resume_prioritizes_interrupted_item_before_siblings())
|
||||
|
||||
|
||||
async def _assert_resume_prioritizes_interrupted_item_before_siblings() -> None:
|
||||
workflow = _workflow()
|
||||
run = await execute_workflow_async(
|
||||
workflow,
|
||||
{"items": ["a", "b", "c"]},
|
||||
{"route": _interrupt_on_b},
|
||||
)
|
||||
interrupted_trace_len = len(run.trace)
|
||||
|
||||
resumed = await resume_workflow_async(
|
||||
workflow,
|
||||
run,
|
||||
{"route": _interrupt_on_b},
|
||||
resume_payload={},
|
||||
)
|
||||
|
||||
assert resumed.status is RunStatus.COMPLETED
|
||||
assert resumed.state["seen"] == ["a", "b", "c"]
|
||||
assert resumed.trace[interrupted_trace_len].frame_id == "root:each:1"
|
||||
assert resumed.trace[interrupted_trace_len].step_type == "interrupt"
|
||||
assert resumed.trace[interrupted_trace_len].outcome == "submitted"
|
||||
foreach_entries = [entry for entry in resumed.trace if entry.step_type == "foreach"]
|
||||
assert foreach_entries[-1].state_changes["state.seen"] == ["a", "b", "c"]
|
||||
|
||||
|
||||
async def _interrupt_on_b(payload: dict[str, Any], _ctx: object) -> dict[str, Any]:
|
||||
await asyncio.sleep(0.01 if payload["value"] == "a" else 0.02)
|
||||
outcome = "needs_input" if payload["value"] == "b" else "ok"
|
||||
return {"outcome": outcome, "output": {"seen": payload["value"]}}
|
||||
|
||||
|
||||
def _workflow() -> Workflow:
|
||||
return Workflow(
|
||||
name="concurrent_foreach_interrupts",
|
||||
input_schema=SchemaRef(
|
||||
type="object",
|
||||
properties={"items": {"type": "array"}},
|
||||
),
|
||||
state_schema=StateSchema.from_field_map(
|
||||
{
|
||||
"items": StateField(type="array"),
|
||||
"seen": StateField(
|
||||
type="array",
|
||||
reducer=ReducerRef(name="wf.std.append"),
|
||||
),
|
||||
}
|
||||
),
|
||||
output_schema=SchemaRef(type="object", properties={"seen": {"type": "array"}}),
|
||||
node_defs=[
|
||||
NodeDef(
|
||||
name="route",
|
||||
input_schema=SchemaRef(
|
||||
type="object",
|
||||
properties={"value": {}},
|
||||
required=["value"],
|
||||
),
|
||||
output_schema=SchemaRef(
|
||||
type="object",
|
||||
properties={"seen": {}},
|
||||
required=["seen"],
|
||||
),
|
||||
outcomes=["ok", "needs_input"],
|
||||
)
|
||||
],
|
||||
start="each",
|
||||
nodes=[
|
||||
ForeachNode.model_validate(
|
||||
{
|
||||
"id": "each",
|
||||
"type": "foreach",
|
||||
"over": "state.items",
|
||||
"as": "item",
|
||||
"mode": "concurrent",
|
||||
"concurrent": {"max_active": 2, "max_outstanding": 2},
|
||||
}
|
||||
),
|
||||
NodeUse.model_validate(
|
||||
{
|
||||
"id": "route",
|
||||
"type": "node",
|
||||
"node": "route",
|
||||
"input": [{"target": "value", "path": "context.item"}],
|
||||
"output": [{"source": "seen", "target": "state.seen"}],
|
||||
}
|
||||
),
|
||||
InterruptNode.model_validate(
|
||||
{
|
||||
"id": "ask",
|
||||
"type": "interrupt",
|
||||
"kind": "approval",
|
||||
"request": [{"target": "item", "path": "context.item"}],
|
||||
}
|
||||
),
|
||||
],
|
||||
edges=[
|
||||
Edge.model_validate({"from": "each", "outcome": "loop", "to": "route"}),
|
||||
Edge.model_validate({"from": "route", "outcome": "ok", "to": END}),
|
||||
Edge.model_validate(
|
||||
{"from": "route", "outcome": "needs_input", "to": "ask"}
|
||||
),
|
||||
Edge.model_validate({"from": "ask", "outcome": "submitted", "to": END}),
|
||||
Edge.model_validate({"from": "each", "outcome": "done", "to": END}),
|
||||
],
|
||||
)
|
||||
Reference in New Issue
Block a user