more examples of concurrent foreach
This commit is contained in:
@@ -20,6 +20,7 @@ from wf_core import (
|
||||
END,
|
||||
ForeachConcurrentPolicy,
|
||||
ForeachItemErrorPolicy,
|
||||
WorkflowExecutionError,
|
||||
execute_workflow_async,
|
||||
)
|
||||
from wf_core.paths import StatePath
|
||||
@@ -132,6 +133,47 @@ def run_collected_errors_example() -> RunState:
|
||||
return builder.execute({"items": ["a", "bad", "c"]})
|
||||
|
||||
|
||||
def run_replace_conflict_example() -> None:
|
||||
"""Demonstrate the exact-path reducer rule for sibling item writes.
|
||||
|
||||
Concurrent sibling writes to the same state path need a mergeable reducer.
|
||||
`errors` has the default replace semantics, so writing every successful item
|
||||
to that path is rejected at the barrier.
|
||||
"""
|
||||
builder = WorkflowBuilder(
|
||||
name="authoring_concurrent_foreach_replace_conflict",
|
||||
input_schema=ItemsInput,
|
||||
state_schema=ConcurrentForeachState,
|
||||
output_schema=ConcurrentForeachOutput,
|
||||
)
|
||||
each = builder.foreach(
|
||||
id="each",
|
||||
over=state_path("items"),
|
||||
as_="item",
|
||||
mode="concurrent",
|
||||
item_error="fail",
|
||||
concurrent={"max_active": 2, "max_outstanding": 2},
|
||||
)
|
||||
record = builder.use(
|
||||
record_item,
|
||||
id="record",
|
||||
input=[
|
||||
input_from(context_path("item"), "value"),
|
||||
input_from(context_path("item"), "seen"),
|
||||
],
|
||||
output=[output_to("seen", state_path("errors"))],
|
||||
)
|
||||
builder.set_entry_point(each)
|
||||
builder.connect(each, "loop", record)
|
||||
builder.connect(record, "ok", END)
|
||||
builder.connect(each, "done", END)
|
||||
try:
|
||||
builder.execute({"items": ["a", "b"]})
|
||||
except WorkflowExecutionError:
|
||||
return
|
||||
raise AssertionError("expected same-path sibling replace writes to fail")
|
||||
|
||||
|
||||
async def run_async_ordered_example() -> RunState:
|
||||
"""Run the async example; barrier commits still preserve item order."""
|
||||
builder = build_concurrent_foreach_workflow(record_item_async)
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from wf_core import END, RuntimeContext, Workflow, execute_workflow
|
||||
from wf_core.run_state import RunState
|
||||
|
||||
|
||||
def build_raw_concurrent_foreach_workflow() -> Workflow:
|
||||
"""Build the canonical JSON/Pydantic shape for concurrent foreach.
|
||||
|
||||
This example is intentionally raw `wf_core`: it is useful for MCP/LLM-facing
|
||||
authoring surfaces that need to emit workflow JSON without Python builder
|
||||
helpers.
|
||||
"""
|
||||
return Workflow.model_validate(
|
||||
{
|
||||
"name": "raw_concurrent_foreach",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {"items": {"type": "array"}},
|
||||
"required": ["items"],
|
||||
},
|
||||
"state_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {"type": "array"},
|
||||
"seen": {
|
||||
"type": "array",
|
||||
"reducer": "wf.std.append",
|
||||
},
|
||||
"errors": {"type": "array"},
|
||||
},
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"seen": {"type": "array"},
|
||||
"errors": {"type": "array"},
|
||||
},
|
||||
},
|
||||
"node_defs": [
|
||||
{
|
||||
"name": "record_item",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"value": {},
|
||||
"seen": {},
|
||||
},
|
||||
"required": ["value", "seen"],
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {"seen": {}},
|
||||
"required": ["seen"],
|
||||
},
|
||||
"outcomes": ["ok"],
|
||||
}
|
||||
],
|
||||
"start": "each",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "each",
|
||||
"type": "foreach",
|
||||
"over": {"root": "state", "parts": ["items"]},
|
||||
"as": "item",
|
||||
"mode": "concurrent",
|
||||
"concurrent": {
|
||||
"max_active": 2,
|
||||
"max_outstanding": 2,
|
||||
},
|
||||
"item_error": {
|
||||
"action": "collect",
|
||||
"collect_to": {"root": "state", "parts": ["errors"]},
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "record",
|
||||
"type": "node",
|
||||
"node": "record_item",
|
||||
"input": [
|
||||
{
|
||||
"target": {"root": "local", "parts": ["value"]},
|
||||
"path": {"root": "context", "parts": ["item"]},
|
||||
},
|
||||
{
|
||||
"target": {"root": "local", "parts": ["seen"]},
|
||||
"path": {"root": "context", "parts": ["item"]},
|
||||
},
|
||||
],
|
||||
"output": [
|
||||
{
|
||||
"source": {"root": "local", "parts": ["seen"]},
|
||||
"target": {"root": "state", "parts": ["seen"]},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"edges": [
|
||||
{"from": "each", "outcome": "loop", "to": "record"},
|
||||
{"from": "record", "outcome": "ok", "to": END},
|
||||
{"from": "each", "outcome": "done", "to": END},
|
||||
{"from": "each", "outcome": "completed_with_errors", "to": END},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def build_raw_concurrent_foreach_registry():
|
||||
"""Return handlers for the raw concurrent foreach example."""
|
||||
|
||||
def record_item(
|
||||
payload: dict[str, object],
|
||||
_context: RuntimeContext,
|
||||
) -> dict[str, object]:
|
||||
if payload["value"] == "bad":
|
||||
raise ValueError("bad item")
|
||||
return {"outcome": "ok", "output": {"seen": payload["seen"]}}
|
||||
|
||||
return {"record_item": record_item}
|
||||
|
||||
|
||||
def run_raw_concurrent_foreach_example() -> RunState:
|
||||
"""Run raw concurrent foreach with collected item errors."""
|
||||
return execute_workflow(
|
||||
build_raw_concurrent_foreach_workflow(),
|
||||
{"items": ["a", "bad", "c"]},
|
||||
build_raw_concurrent_foreach_registry(),
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run the example directly from the command line."""
|
||||
run = run_raw_concurrent_foreach_example()
|
||||
print(run.status.value, run.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user