add a config to reducer, to enable custom ish reducers

This commit is contained in:
lda
2026-05-17 17:11:19 +07:00 Verified
parent 32bcba9b35
commit 9d45c59fdc
14 changed files with 238 additions and 38 deletions
+11 -2
View File
@@ -197,8 +197,17 @@ Reducers are a capability family, similar to reusable node specs:
- inspectable - inspectable
- dependency-trackable - dependency-trackable
State fields reference reducers declaratively. Workflow artifacts do not embed State fields reference reducers declaratively. String reducer names are accepted
arbitrary Python callables. as shorthand for unconfigured reducers; configured reducers use a `name` plus
JSON-compatible `config`. Workflow artifacts do not embed arbitrary Python
callables.
```python
StateField(
type="integer",
reducer={"name": "wf.std.modulo_add", "config": {"modulus": 10}},
)
```
Reducers should be pure: Reducers should be pure:
+1 -1
View File
@@ -416,7 +416,7 @@ At minimum each declared field may carry:
```text ```text
StateField StateField
.type .type
.reducer // wf.std.replace | wf.std.append | wf.std.merge_object | ... .reducer // "wf.std.replace" or { name, config }
.trace? // whether to include in trace by default .trace? // whether to include in trace by default
``` ```
@@ -0,0 +1,63 @@
# ReducerRef Config Validation Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Replace string-only reducer references with `ReducerRef(name, config)` and validate reducer config before state mutation.
**Architecture:** Keep string reducer input as shorthand, normalize it to a `ReducerRef`, and add `config_schema` to reducer specs. Runtime reducer application resolves the reducer spec, validates config through the existing JSON Schema backend, then calls reducer functions with `(current, incoming, config)`.
**Tech Stack:** Python, Pydantic, jsonschema, pytest, existing reducer registry.
---
## File Structure
- Modify `src/wf_core/models/reducers.py`
- add `ReducerRef`
- add `config_schema` to `ReducerSpec`
- Modify `src/wf_core/models/schemas.py`
- make `StateField.reducer` a `ReducerRef` with string shorthand parsing
- Modify `src/wf_core/runtime/ops/merges.py`
- reducer callable accepts config
- validate config against reducer spec before merge
- Modify `src/wf_core/runtime/ops/state.py`
- pass `ReducerRef` to reducer application
- Modify `src/wf_artifacts/factory.py`
- infer reducer dependencies from `ReducerRef` objects and dict payloads
- Modify tests for core reducers and artifact dependency inference
## Tasks
### Task 1: Pin ReducerRef Behavior
- [ ] Add tests proving string shorthand normalizes to `ReducerRef(name=..., config={})`.
- [ ] Add tests proving object reducer payloads preserve config.
- [ ] Add tests proving invalid config fails before mutation.
- [ ] Run focused tests and confirm failure before implementation.
### Task 2: Implement ReducerRef and Config Validation
- [ ] Add `ReducerRef`.
- [ ] Add `ReducerSpec.config_schema`.
- [ ] Update reducer callables to accept config.
- [ ] Validate config before calling a reducer.
- [ ] Keep existing no-config reducers accepting `{}` only through their empty config schemas.
### Task 3: Update Artifact Dependency Inference
- [ ] Infer reducer dependency names from string reducers and object reducers.
- [ ] Keep dependency key by reducer name, not by reducer config.
- [ ] Run artifact factory tests.
### Task 4: Verify
- [ ] Run focused core/artifact tests.
- [ ] Run full suite.
- [ ] Run basedpyright.
## Non-Goals
- implementing `modulo_add`
- reducer decorator UX
- configurable reducer factories
- caching configured reducers
+2 -1
View File
@@ -499,7 +499,8 @@ Saved workflow execution eventually needs first-class runtime support for:
- child final outcome mapping to parent node outcome - child final outcome mapping to parent node outcome
- dependency checks before execution - dependency checks before execution
- reducer capabilities referenced by declared workflow state fields are saved as - reducer capabilities referenced by declared workflow state fields are saved as
direct artifact dependencies just like node specs or tools direct artifact dependencies just like node specs or tools; reducer config is
part of the state field contract, while the dependency key is the reducer name
The first implementation should prefer artifact validation and dependency The first implementation should prefer artifact validation and dependency
diagnostics before attempting persistent nested resume. diagnostics before attempting persistent nested resume.
+14 -5
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from collections.abc import Mapping from collections.abc import Mapping
from wf_core import Workflow from wf_core import ReducerRef, Workflow
from .models import ArtifactKind, JsonObject, RequiredCapability, WorkflowArtifact from .models import ArtifactKind, JsonObject, RequiredCapability, WorkflowArtifact
@@ -86,11 +86,20 @@ def _required_reducers_from_plan(plan: JsonObject) -> dict[str, RequiredCapabili
for field in fields.values(): for field in fields.values():
if not isinstance(field, dict): if not isinstance(field, dict):
continue continue
reducer = field.get("reducer", "wf.std.replace") reducer_payload = field.get("reducer", "wf.std.replace")
if not isinstance(reducer, str) or "." not in reducer: if isinstance(reducer_payload, str):
reducer = ReducerRef(name=reducer_payload)
else:
try:
reducer = ReducerRef.model_validate(reducer_payload)
except ValueError:
continue
if "." not in reducer.name:
continue continue
logical_source, _, capability_name = reducer.rpartition(".") if "." not in reducer.name:
requirements[reducer] = RequiredCapability( continue
logical_source, _, capability_name = reducer.name.rpartition(".")
requirements[reducer.name] = RequiredCapability(
logical_source=logical_source, logical_source=logical_source,
capability_name=capability_name, capability_name=capability_name,
kind="reducer", kind="reducer",
+4 -2
View File
@@ -5,7 +5,7 @@ from typing import Any, Iterator
from pydantic import BaseModel, TypeAdapter from pydantic import BaseModel, TypeAdapter
from wf_core import SchemaRef, StateField, StateSchema from wf_core import ReducerRef, SchemaRef, StateField, StateSchema
SchemaLike = SchemaRef | type[BaseModel] | type[Any] | dict[str, Any] SchemaLike = SchemaRef | type[BaseModel] | type[Any] | dict[str, Any]
StateSchemaLike = StateSchema | type[BaseModel] | type[Any] | dict[str, Any] StateSchemaLike = StateSchema | type[BaseModel] | type[Any] | dict[str, Any]
@@ -51,7 +51,9 @@ def state_schema_from(value: StateSchemaLike) -> StateSchema:
fields = { fields = {
path: StateField( path: StateField(
type=_state_field_type(property_schema), type=_state_field_type(property_schema),
reducer=metadata_by_name.get(path, StateFieldMetadata()).reducer, reducer=ReducerRef(
name=metadata_by_name.get(path, StateFieldMetadata()).reducer
),
trace=metadata_by_name.get(path, StateFieldMetadata()).trace, trace=metadata_by_name.get(path, StateFieldMetadata()).trace,
default=_state_field_default(value, path, property_schema), default=_state_field_default(value, path, property_schema),
) )
+2
View File
@@ -7,6 +7,7 @@ from .models import (
NodeDef, NodeDef,
NodeResult, NodeResult,
NodeUse, NodeUse,
ReducerRef,
ReducerSpec, ReducerSpec,
SchemaRef, SchemaRef,
StateField, StateField,
@@ -52,6 +53,7 @@ __all__ = [
"NodeDef", "NodeDef",
"NodeResult", "NodeResult",
"NodeUse", "NodeUse",
"ReducerRef",
"ReducerSpec", "ReducerSpec",
"SchemaRef", "SchemaRef",
"StateField", "StateField",
+2 -1
View File
@@ -9,7 +9,7 @@ from wf_core.models.conditions import (
VariadicCondition, VariadicCondition,
) )
from wf_core.models.results import NodeResult from wf_core.models.results import NodeResult
from wf_core.models.reducers import ReducerSpec from wf_core.models.reducers import ReducerRef, ReducerSpec
from wf_core.models.schemas import NodeDef, SchemaRef, StateField, StateSchema from wf_core.models.schemas import NodeDef, SchemaRef, StateField, StateSchema
from wf_core.models.steps import ( from wf_core.models.steps import (
ConditionNode, ConditionNode,
@@ -33,6 +33,7 @@ __all__ = [
"LiteralOperand", "LiteralOperand",
"NodeDef", "NodeDef",
"NodeResult", "NodeResult",
"ReducerRef",
"ReducerSpec", "ReducerSpec",
"NodeUse", "NodeUse",
"NotCondition", "NotCondition",
+19 -1
View File
@@ -1,10 +1,28 @@
from __future__ import annotations from __future__ import annotations
from pydantic import BaseModel from typing import Any
from pydantic import BaseModel, ConfigDict, Field
class ReducerRef(BaseModel):
"""Reference to one reducer plus JSON-compatible configuration."""
name: str
config: dict[str, Any] = Field(default_factory=dict)
class ReducerSpec(BaseModel): class ReducerSpec(BaseModel):
"""Inspectable metadata for one named pure state reducer.""" """Inspectable metadata for one named pure state reducer."""
model_config = ConfigDict(extra="allow")
name: str name: str
description: str | None = None description: str | None = None
config_schema: dict[str, Any] = Field(
default_factory=lambda: {
"type": "object",
"properties": {},
"additionalProperties": False,
}
)
+13 -2
View File
@@ -2,7 +2,9 @@ from __future__ import annotations
from typing import Any from typing import Any
from pydantic import BaseModel, ConfigDict, Field from pydantic import BaseModel, ConfigDict, Field, field_validator
from wf_core.models.reducers import ReducerRef
class SchemaRef(BaseModel): class SchemaRef(BaseModel):
@@ -20,10 +22,19 @@ class StateField(BaseModel):
"""Declared state path plus its runtime merge behavior.""" """Declared state path plus its runtime merge behavior."""
type: str type: str
reducer: str = "wf.std.replace" reducer: ReducerRef = Field(
default_factory=lambda: ReducerRef(name="wf.std.replace")
)
trace: bool = True trace: bool = True
default: Any = None default: Any = None
@field_validator("reducer", mode="before")
@classmethod
def _coerce_reducer(cls, value: object) -> object:
if isinstance(value, str):
return {"name": value}
return value
class StateSchema(BaseModel): class StateSchema(BaseModel):
"""Workflow state schema keyed by declared exact state path.""" """Workflow state schema keyed by declared exact state path."""
+36 -11
View File
@@ -4,16 +4,22 @@ from collections.abc import Callable, Mapping
from typing import Any from typing import Any
from wf_core.errors import WorkflowExecutionError from wf_core.errors import WorkflowExecutionError
from wf_core.models.reducers import ReducerRef, ReducerSpec
from wf_core.runtime.ops.schemas import validate_payload_against_schema
Reducer = Callable[[Any, Any], Any] Reducer = Callable[[Any, Any, Mapping[str, Any]], Any]
def replace_reducer(_current_value: Any, incoming_value: Any) -> Any: def replace_reducer(
_current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
) -> Any:
"""Replace the current state value with the incoming value.""" """Replace the current state value with the incoming value."""
return incoming_value return incoming_value
def append_reducer(current_value: Any, incoming_value: Any) -> Any: def append_reducer(
current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
) -> Any:
"""Append one value or many values into a list-valued state path.""" """Append one value or many values into a list-valued state path."""
if current_value is None: if current_value is None:
return ( return (
@@ -28,7 +34,9 @@ def append_reducer(current_value: Any, incoming_value: Any) -> Any:
) )
def merge_object_reducer(current_value: Any, incoming_value: Any) -> Any: def merge_object_reducer(
current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
) -> Any:
"""Shallow-merge object values at one exact state path.""" """Shallow-merge object values at one exact state path."""
if current_value is None: if current_value is None:
if not isinstance(incoming_value, dict): if not isinstance(incoming_value, dict):
@@ -39,7 +47,9 @@ def merge_object_reducer(current_value: Any, incoming_value: Any) -> Any:
return current_value | incoming_value return current_value | incoming_value
def set_union_reducer(current_value: Any, incoming_value: Any) -> Any: def set_union_reducer(
current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
) -> Any:
"""Merge list values while preserving stable first-seen order.""" """Merge list values while preserving stable first-seen order."""
if current_value is None: if current_value is None:
current_items: list[Any] = [] current_items: list[Any] = []
@@ -58,7 +68,9 @@ def set_union_reducer(current_value: Any, incoming_value: Any) -> Any:
return merged return merged
def max_reducer(current_value: Any, incoming_value: Any) -> Any: def max_reducer(
current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
) -> Any:
"""Keep the larger of the current and incoming values.""" """Keep the larger of the current and incoming values."""
return ( return (
incoming_value if current_value is None else max(current_value, incoming_value) incoming_value if current_value is None else max(current_value, incoming_value)
@@ -73,20 +85,33 @@ DEFAULT_REDUCERS: Mapping[str, Reducer] = {
"wf.std.max": max_reducer, "wf.std.max": max_reducer,
} }
DEFAULT_REDUCER_SPECS: Mapping[str, ReducerSpec] = {
name: ReducerSpec(name=name) for name in DEFAULT_REDUCERS
}
def apply_reducer( def apply_reducer(
*, *,
reducer_name: str, reducer: ReducerRef,
current_value: Any, current_value: Any,
incoming_value: Any, incoming_value: Any,
destination_path: str, destination_path: str,
reducers: Mapping[str, Reducer] = DEFAULT_REDUCERS, reducers: Mapping[str, Reducer] = DEFAULT_REDUCERS,
reducer_specs: Mapping[str, ReducerSpec] = DEFAULT_REDUCER_SPECS,
) -> Any: ) -> Any:
"""Apply one named pure reducer to a state write.""" """Apply one named pure reducer to a state write."""
reducer = reducers.get(reducer_name) reducer_fn = reducers.get(reducer.name)
if reducer is None: if reducer_fn is None:
raise WorkflowExecutionError(f"unknown reducer {reducer_name!r}") raise WorkflowExecutionError(f"unknown reducer {reducer.name!r}")
spec = reducer_specs.get(reducer.name)
if spec is None:
raise WorkflowExecutionError(f"unknown reducer spec {reducer.name!r}")
validate_payload_against_schema(
spec.config_schema,
reducer.config,
f"reducer config for {reducer.name!r}",
)
try: try:
return reducer(current_value, incoming_value) return reducer_fn(current_value, incoming_value, reducer.config)
except TypeError as exc: except TypeError as exc:
raise WorkflowExecutionError(f"{exc} at {destination_path!r}") from exc raise WorkflowExecutionError(f"{exc} at {destination_path!r}") from exc
+3 -2
View File
@@ -4,6 +4,7 @@ from typing import Any
from wf_core.errors import WorkflowExecutionError from wf_core.errors import WorkflowExecutionError
from wf_core.local_paths import LocalPathError, get_local_value, has_overlapping_paths from wf_core.local_paths import LocalPathError, get_local_value, has_overlapping_paths
from wf_core.models.reducers import ReducerRef
from wf_core.models.steps import NodeUse from wf_core.models.steps import NodeUse
from wf_core.models.workflow import Workflow from wf_core.models.workflow import Workflow
from wf_core.paths import ( from wf_core.paths import (
@@ -73,11 +74,11 @@ def write_state_value(
declared_path = ".".join(parts) declared_path = ".".join(parts)
declared_field = workflow.state_schema.fields.get(declared_path) declared_field = workflow.state_schema.fields.get(declared_path)
reducer_name = declared_field.reducer if declared_field else "wf.std.replace" reducer = declared_field.reducer if declared_field else ReducerRef(name="wf.std.replace")
key_path = parts key_path = parts
current_value = get_nested_value(state, key_path) current_value = get_nested_value(state, key_path)
merged_value = apply_reducer( merged_value = apply_reducer(
reducer_name=reducer_name, reducer=reducer,
current_value=current_value, current_value=current_value,
incoming_value=value, incoming_value=value,
destination_path=destination_path, destination_path=destination_path,
+3 -3
View File
@@ -58,7 +58,7 @@ def test_state_basemodel_can_declare_reducer_with_annotated_metadata() -> None:
workflow = builder.compile() workflow = builder.compile()
assert workflow.state_schema.fields["items"].type == "array" assert workflow.state_schema.fields["items"].type == "array"
assert workflow.state_schema.fields["items"].reducer == "wf.std.append" assert workflow.state_schema.fields["items"].reducer.name == "wf.std.append"
def test_state_basemodel_seeds_safe_initial_defaults() -> None: def test_state_basemodel_seeds_safe_initial_defaults() -> None:
@@ -96,5 +96,5 @@ def test_nested_state_basemodel_projects_parent_and_child_paths() -> None:
assert workflow.state_schema.fields["person"].type == "object" assert workflow.state_schema.fields["person"].type == "object"
assert workflow.state_schema.fields["person.name"].type == "string" assert workflow.state_schema.fields["person.name"].type == "string"
assert workflow.state_schema.fields["person.tags"].type == "array" assert workflow.state_schema.fields["person.tags"].type == "array"
assert workflow.state_schema.fields["person"].reducer == "wf.std.replace" assert workflow.state_schema.fields["person"].reducer.name == "wf.std.replace"
assert workflow.state_schema.fields["person.tags"].reducer == "wf.std.append" assert workflow.state_schema.fields["person.tags"].reducer.name == "wf.std.append"
+65 -7
View File
@@ -1,12 +1,16 @@
from __future__ import annotations from __future__ import annotations
from wf_core import SchemaRef, StateField, StateSchema, Workflow from wf_core import ReducerRef, SchemaRef, StateField, StateSchema, Workflow
from wf_core.runtime.ops.state import write_state_value from wf_core.runtime.ops.state import write_state_value
def test_exact_nested_state_path_uses_declared_reducer() -> None: def test_exact_nested_state_path_uses_declared_reducer() -> None:
workflow = _workflow( workflow = _workflow(
fields={"person.tags": StateField(type="array", reducer="wf.std.append")} fields={
"person.tags": StateField(
reducer=ReducerRef(name="wf.std.append"), type="array"
)
}
) )
state = {"person": {"tags": ["seed"]}} state = {"person": {"tags": ["seed"]}}
@@ -17,7 +21,11 @@ def test_exact_nested_state_path_uses_declared_reducer() -> None:
def test_parent_state_declaration_does_not_apply_to_nested_write() -> None: def test_parent_state_declaration_does_not_apply_to_nested_write() -> None:
workflow = _workflow( workflow = _workflow(
fields={"person": StateField(type="object", reducer="wf.std.merge_object")} fields={
"person": StateField(
reducer=ReducerRef(name="wf.std.merge_object"), type="object"
)
}
) )
state = {"person": {"tags": ["seed"]}} state = {"person": {"tags": ["seed"]}}
@@ -36,12 +44,34 @@ def test_undeclared_nested_state_path_defaults_to_replace() -> None:
def test_state_field_defaults_to_replace_reducer() -> None: def test_state_field_defaults_to_replace_reducer() -> None:
assert StateField(type="string").reducer == "wf.std.replace" assert StateField(type="string").reducer == ReducerRef(name="wf.std.replace")
def test_state_field_accepts_string_reducer_shorthand() -> None:
field = StateField.model_validate(
{"type": "array", "reducer": "wf.std.set_union"}
)
assert field.reducer == ReducerRef(name="wf.std.set_union")
def test_state_field_accepts_configured_reducer_reference() -> None:
field = StateField(
type="integer",
reducer=ReducerRef(name="wf.std.max", config={"sample": True}),
)
assert field.reducer.name == "wf.std.max"
assert field.reducer.config == {"sample": True}
def test_unknown_state_reducer_fails_clearly() -> None: def test_unknown_state_reducer_fails_clearly() -> None:
workflow = _workflow( workflow = _workflow(
fields={"person.tags": StateField(type="array", reducer="x.nope")} fields={
"person.tags": StateField(
reducer=ReducerRef(name="x.nope"), type="array"
)
}
) )
state = {"person": {"tags": ["seed"]}} state = {"person": {"tags": ["seed"]}}
@@ -55,7 +85,11 @@ def test_unknown_state_reducer_fails_clearly() -> None:
def test_set_union_reducer_preserves_first_seen_order() -> None: def test_set_union_reducer_preserves_first_seen_order() -> None:
workflow = _workflow( workflow = _workflow(
fields={"person.tags": StateField(type="array", reducer="wf.std.set_union")} fields={
"person.tags": StateField(
reducer=ReducerRef(name="wf.std.set_union"), type="array"
)
}
) )
state = {"person": {"tags": ["alpha", "beta"]}} state = {"person": {"tags": ["alpha", "beta"]}}
@@ -64,9 +98,33 @@ def test_set_union_reducer_preserves_first_seen_order() -> None:
assert state["person"]["tags"] == ["alpha", "beta", "gamma"] assert state["person"]["tags"] == ["alpha", "beta", "gamma"]
def test_unexpected_reducer_config_fails_before_state_mutation() -> None:
workflow = _workflow(
fields={
"person.tags": StateField(
type="array",
reducer=ReducerRef(name="wf.std.set_union", config={"bad": True}),
)
}
)
state = {"person": {"tags": ["alpha"]}}
try:
write_state_value(workflow, state, "state.person.tags", ["beta"])
except Exception as exc:
assert "reducer config for 'wf.std.set_union'" in str(exc)
else:
raise AssertionError("expected invalid reducer config to fail")
assert state["person"]["tags"] == ["alpha"]
def test_max_reducer_keeps_larger_value() -> None: def test_max_reducer_keeps_larger_value() -> None:
workflow = _workflow( workflow = _workflow(
fields={"best_score": StateField(type="integer", reducer="wf.std.max")} fields={
"best_score": StateField(
reducer=ReducerRef(name="wf.std.max"), type="integer"
)
}
) )
state = {"best_score": 7} state = {"best_score": 7}