interrupt goes first
This commit is contained in:
@@ -131,7 +131,7 @@ future explicit deep merge policy exists.
|
|||||||
|
|
||||||
## Interrupt and Failure Quiescence
|
## Interrupt and Failure Quiescence
|
||||||
|
|
||||||
Future concurrent execution should not assume in-flight node calls can be safely
|
Concurrent execution should not assume in-flight node calls can be safely
|
||||||
cancelled.
|
cancelled.
|
||||||
|
|
||||||
If an interrupt or fail policy trips while sibling jobs are already started, the
|
If an interrupt or fail policy trips while sibling jobs are already started, the
|
||||||
@@ -146,6 +146,14 @@ runtime should:
|
|||||||
For `fail`, drained sibling results are for observability/cleanup only and
|
For `fail`, drained sibling results are for observability/cleanup only and
|
||||||
should not commit normal state progress after the failure boundary.
|
should not commit normal state progress after the failure boundary.
|
||||||
|
|
||||||
|
Current async concurrent foreach implements the interrupt part of this by
|
||||||
|
prioritizing item frames that route into an `InterruptNode`. If a batched async
|
||||||
|
node result sends one item to an interrupt while a sibling completes, the
|
||||||
|
interrupt-bound frame is placed at the front of the ready queue before the
|
||||||
|
parent foreach can refill capacity. Already-started async handler calls from
|
||||||
|
the batch are awaited first, then their results are finalized sequentially.
|
||||||
|
The foreach barrier does not commit while an item frame remains interrupted.
|
||||||
|
|
||||||
## Capacity and Runtime Limits
|
## Capacity and Runtime Limits
|
||||||
|
|
||||||
Foreach capacity is local correctness policy, not total process protection.
|
Foreach capacity is local correctness policy, not total process protection.
|
||||||
|
|||||||
@@ -168,7 +168,7 @@ Key tests:
|
|||||||
|
|
||||||
## Slice 6: Interrupt Quiescence
|
## Slice 6: Interrupt Quiescence
|
||||||
|
|
||||||
Implement after async execution exists.
|
Implemented after async execution exists.
|
||||||
|
|
||||||
Scope:
|
Scope:
|
||||||
|
|
||||||
@@ -177,6 +177,10 @@ Scope:
|
|||||||
- Already-started async node calls drain to pending results.
|
- Already-started async node calls drain to pending results.
|
||||||
- The caller gets control only at a quiescent point.
|
- The caller gets control only at a quiescent point.
|
||||||
- Pending results do not commit until resume/commit policy allows it.
|
- Pending results do not commit until resume/commit policy allows it.
|
||||||
|
- Item frames that route into an `InterruptNode` are prioritized before the
|
||||||
|
parent foreach can refill capacity.
|
||||||
|
- Already-started async handler calls drain at the batch boundary; state
|
||||||
|
finalization remains sequential.
|
||||||
|
|
||||||
Files likely touched:
|
Files likely touched:
|
||||||
|
|
||||||
@@ -188,7 +192,7 @@ Files likely touched:
|
|||||||
|
|
||||||
Key tests:
|
Key tests:
|
||||||
|
|
||||||
- `test_concurrent_foreach_interrupt_returns_after_quiescence`
|
- `test_concurrent_foreach_interrupt_returns_before_refill`
|
||||||
- `test_resume_prioritizes_interrupted_item_frame_before_siblings`
|
- `test_resume_prioritizes_interrupted_item_frame_before_siblings`
|
||||||
|
|
||||||
## Execution Order
|
## Execution Order
|
||||||
|
|||||||
@@ -74,6 +74,7 @@ def advance_frame(
|
|||||||
*,
|
*,
|
||||||
outcome: str,
|
outcome: str,
|
||||||
next_node_id: str,
|
next_node_id: str,
|
||||||
|
front: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
frame.prior_outcome = outcome
|
frame.prior_outcome = outcome
|
||||||
frame.activated_incoming_edge = frame.node_id
|
frame.activated_incoming_edge = frame.node_id
|
||||||
@@ -84,7 +85,7 @@ def advance_frame(
|
|||||||
wake_parent_for_child_progress(run, frame.id)
|
wake_parent_for_child_progress(run, frame.id)
|
||||||
else:
|
else:
|
||||||
frame.finished_at_node_id = None
|
frame.finished_at_node_id = None
|
||||||
mark_frame_pending(run, frame.id)
|
mark_frame_pending(run, frame.id, front=front)
|
||||||
run.sync_from_current_frame()
|
run.sync_from_current_frame()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ def complete_step(
|
|||||||
) -> RunState:
|
) -> RunState:
|
||||||
"""Record a completed step and advance the active frame."""
|
"""Record a completed step and advance the active frame."""
|
||||||
next_node_id = index.next_node_id(node_id, outcome)
|
next_node_id = index.next_node_id(node_id, outcome)
|
||||||
|
next_step = index.nodes_by_id.get(next_node_id)
|
||||||
|
|
||||||
append_step_result_trace(
|
append_step_result_trace(
|
||||||
run,
|
run,
|
||||||
@@ -67,6 +68,7 @@ def complete_step(
|
|||||||
run.frames[frame_id],
|
run.frames[frame_id],
|
||||||
outcome=outcome,
|
outcome=outcome,
|
||||||
next_node_id=next_node_id,
|
next_node_id=next_node_id,
|
||||||
|
front=isinstance(next_step, InterruptNode),
|
||||||
)
|
)
|
||||||
return run
|
return run
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from wf_core import (
|
||||||
|
END,
|
||||||
|
Edge,
|
||||||
|
ForeachNode,
|
||||||
|
InterruptNode,
|
||||||
|
NodeDef,
|
||||||
|
NodeUse,
|
||||||
|
ReducerRef,
|
||||||
|
RunStatus,
|
||||||
|
SchemaRef,
|
||||||
|
StateField,
|
||||||
|
StateSchema,
|
||||||
|
Workflow,
|
||||||
|
execute_workflow_async,
|
||||||
|
resume_workflow_async,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concurrent_foreach_interrupt_returns_before_refill() -> None:
|
||||||
|
asyncio.run(_assert_concurrent_foreach_interrupt_returns_before_refill())
|
||||||
|
|
||||||
|
|
||||||
|
async def _assert_concurrent_foreach_interrupt_returns_before_refill() -> None:
|
||||||
|
run = await execute_workflow_async(
|
||||||
|
_workflow(),
|
||||||
|
{"items": ["a", "b", "c"]},
|
||||||
|
{"route": _interrupt_on_b},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert run.status is RunStatus.INTERRUPTED
|
||||||
|
assert run.interrupt is not None
|
||||||
|
assert run.interrupt.payload["item"] == "b"
|
||||||
|
assert run.frames["root:each:1"].status == "interrupted"
|
||||||
|
assert "root:each:2" not in run.frames
|
||||||
|
assert "seen" not in run.state
|
||||||
|
|
||||||
|
|
||||||
|
def test_resume_prioritizes_interrupted_item_before_siblings() -> None:
|
||||||
|
asyncio.run(_assert_resume_prioritizes_interrupted_item_before_siblings())
|
||||||
|
|
||||||
|
|
||||||
|
async def _assert_resume_prioritizes_interrupted_item_before_siblings() -> None:
|
||||||
|
workflow = _workflow()
|
||||||
|
run = await execute_workflow_async(
|
||||||
|
workflow,
|
||||||
|
{"items": ["a", "b", "c"]},
|
||||||
|
{"route": _interrupt_on_b},
|
||||||
|
)
|
||||||
|
interrupted_trace_len = len(run.trace)
|
||||||
|
|
||||||
|
resumed = await resume_workflow_async(
|
||||||
|
workflow,
|
||||||
|
run,
|
||||||
|
{"route": _interrupt_on_b},
|
||||||
|
resume_payload={},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert resumed.status is RunStatus.COMPLETED
|
||||||
|
assert resumed.state["seen"] == ["a", "b", "c"]
|
||||||
|
assert resumed.trace[interrupted_trace_len].frame_id == "root:each:1"
|
||||||
|
assert resumed.trace[interrupted_trace_len].step_type == "interrupt"
|
||||||
|
assert resumed.trace[interrupted_trace_len].outcome == "submitted"
|
||||||
|
foreach_entries = [entry for entry in resumed.trace if entry.step_type == "foreach"]
|
||||||
|
assert foreach_entries[-1].state_changes["state.seen"] == ["a", "b", "c"]
|
||||||
|
|
||||||
|
|
||||||
|
async def _interrupt_on_b(payload: dict[str, Any], _ctx: object) -> dict[str, Any]:
|
||||||
|
await asyncio.sleep(0.01 if payload["value"] == "a" else 0.02)
|
||||||
|
outcome = "needs_input" if payload["value"] == "b" else "ok"
|
||||||
|
return {"outcome": outcome, "output": {"seen": payload["value"]}}
|
||||||
|
|
||||||
|
|
||||||
|
def _workflow() -> Workflow:
|
||||||
|
return Workflow(
|
||||||
|
name="concurrent_foreach_interrupts",
|
||||||
|
input_schema=SchemaRef(
|
||||||
|
type="object",
|
||||||
|
properties={"items": {"type": "array"}},
|
||||||
|
),
|
||||||
|
state_schema=StateSchema.from_field_map(
|
||||||
|
{
|
||||||
|
"items": StateField(type="array"),
|
||||||
|
"seen": StateField(
|
||||||
|
type="array",
|
||||||
|
reducer=ReducerRef(name="wf.std.append"),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
),
|
||||||
|
output_schema=SchemaRef(type="object", properties={"seen": {"type": "array"}}),
|
||||||
|
node_defs=[
|
||||||
|
NodeDef(
|
||||||
|
name="route",
|
||||||
|
input_schema=SchemaRef(
|
||||||
|
type="object",
|
||||||
|
properties={"value": {}},
|
||||||
|
required=["value"],
|
||||||
|
),
|
||||||
|
output_schema=SchemaRef(
|
||||||
|
type="object",
|
||||||
|
properties={"seen": {}},
|
||||||
|
required=["seen"],
|
||||||
|
),
|
||||||
|
outcomes=["ok", "needs_input"],
|
||||||
|
)
|
||||||
|
],
|
||||||
|
start="each",
|
||||||
|
nodes=[
|
||||||
|
ForeachNode.model_validate(
|
||||||
|
{
|
||||||
|
"id": "each",
|
||||||
|
"type": "foreach",
|
||||||
|
"over": "state.items",
|
||||||
|
"as": "item",
|
||||||
|
"mode": "concurrent",
|
||||||
|
"concurrent": {"max_active": 2, "max_outstanding": 2},
|
||||||
|
}
|
||||||
|
),
|
||||||
|
NodeUse.model_validate(
|
||||||
|
{
|
||||||
|
"id": "route",
|
||||||
|
"type": "node",
|
||||||
|
"node": "route",
|
||||||
|
"input": [{"target": "value", "path": "context.item"}],
|
||||||
|
"output": [{"source": "seen", "target": "state.seen"}],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
InterruptNode.model_validate(
|
||||||
|
{
|
||||||
|
"id": "ask",
|
||||||
|
"type": "interrupt",
|
||||||
|
"kind": "approval",
|
||||||
|
"request": [{"target": "item", "path": "context.item"}],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
],
|
||||||
|
edges=[
|
||||||
|
Edge.model_validate({"from": "each", "outcome": "loop", "to": "route"}),
|
||||||
|
Edge.model_validate({"from": "route", "outcome": "ok", "to": END}),
|
||||||
|
Edge.model_validate(
|
||||||
|
{"from": "route", "outcome": "needs_input", "to": "ask"}
|
||||||
|
),
|
||||||
|
Edge.model_validate({"from": "ask", "outcome": "submitted", "to": END}),
|
||||||
|
Edge.model_validate({"from": "each", "outcome": "done", "to": END}),
|
||||||
|
],
|
||||||
|
)
|
||||||
Reference in New Issue
Block a user