add a config to reducer, to enable custom ish reducers
This commit is contained in:
@@ -197,8 +197,17 @@ Reducers are a capability family, similar to reusable node specs:
|
||||
- inspectable
|
||||
- dependency-trackable
|
||||
|
||||
State fields reference reducers declaratively. Workflow artifacts do not embed
|
||||
arbitrary Python callables.
|
||||
State fields reference reducers declaratively. String reducer names are accepted
|
||||
as shorthand for unconfigured reducers; configured reducers use a `name` plus
|
||||
JSON-compatible `config`. Workflow artifacts do not embed arbitrary Python
|
||||
callables.
|
||||
|
||||
```python
|
||||
StateField(
|
||||
type="integer",
|
||||
reducer={"name": "wf.std.modulo_add", "config": {"modulus": 10}},
|
||||
)
|
||||
```
|
||||
|
||||
Reducers should be pure:
|
||||
|
||||
|
||||
+1
-1
@@ -416,7 +416,7 @@ At minimum each declared field may carry:
|
||||
```text
|
||||
StateField
|
||||
.type
|
||||
.reducer // wf.std.replace | wf.std.append | wf.std.merge_object | ...
|
||||
.reducer // "wf.std.replace" or { name, config }
|
||||
.trace? // whether to include in trace by default
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# ReducerRef Config Validation Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Replace string-only reducer references with `ReducerRef(name, config)` and validate reducer config before state mutation.
|
||||
|
||||
**Architecture:** Keep string reducer input as shorthand, normalize it to a `ReducerRef`, and add `config_schema` to reducer specs. Runtime reducer application resolves the reducer spec, validates config through the existing JSON Schema backend, then calls reducer functions with `(current, incoming, config)`.
|
||||
|
||||
**Tech Stack:** Python, Pydantic, jsonschema, pytest, existing reducer registry.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- Modify `src/wf_core/models/reducers.py`
|
||||
- add `ReducerRef`
|
||||
- add `config_schema` to `ReducerSpec`
|
||||
- Modify `src/wf_core/models/schemas.py`
|
||||
- make `StateField.reducer` a `ReducerRef` with string shorthand parsing
|
||||
- Modify `src/wf_core/runtime/ops/merges.py`
|
||||
- reducer callable accepts config
|
||||
- validate config against reducer spec before merge
|
||||
- Modify `src/wf_core/runtime/ops/state.py`
|
||||
- pass `ReducerRef` to reducer application
|
||||
- Modify `src/wf_artifacts/factory.py`
|
||||
- infer reducer dependencies from `ReducerRef` objects and dict payloads
|
||||
- Modify tests for core reducers and artifact dependency inference
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1: Pin ReducerRef Behavior
|
||||
|
||||
- [ ] Add tests proving string shorthand normalizes to `ReducerRef(name=..., config={})`.
|
||||
- [ ] Add tests proving object reducer payloads preserve config.
|
||||
- [ ] Add tests proving invalid config fails before mutation.
|
||||
- [ ] Run focused tests and confirm failure before implementation.
|
||||
|
||||
### Task 2: Implement ReducerRef and Config Validation
|
||||
|
||||
- [ ] Add `ReducerRef`.
|
||||
- [ ] Add `ReducerSpec.config_schema`.
|
||||
- [ ] Update reducer callables to accept config.
|
||||
- [ ] Validate config before calling a reducer.
|
||||
- [ ] Keep existing no-config reducers accepting `{}` only through their empty config schemas.
|
||||
|
||||
### Task 3: Update Artifact Dependency Inference
|
||||
|
||||
- [ ] Infer reducer dependency names from string reducers and object reducers.
|
||||
- [ ] Keep dependency key by reducer name, not by reducer config.
|
||||
- [ ] Run artifact factory tests.
|
||||
|
||||
### Task 4: Verify
|
||||
|
||||
- [ ] Run focused core/artifact tests.
|
||||
- [ ] Run full suite.
|
||||
- [ ] Run basedpyright.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- implementing `modulo_add`
|
||||
- reducer decorator UX
|
||||
- configurable reducer factories
|
||||
- caching configured reducers
|
||||
@@ -499,7 +499,8 @@ Saved workflow execution eventually needs first-class runtime support for:
|
||||
- child final outcome mapping to parent node outcome
|
||||
- dependency checks before execution
|
||||
- reducer capabilities referenced by declared workflow state fields are saved as
|
||||
direct artifact dependencies just like node specs or tools
|
||||
direct artifact dependencies just like node specs or tools; reducer config is
|
||||
part of the state field contract, while the dependency key is the reducer name
|
||||
|
||||
The first implementation should prefer artifact validation and dependency
|
||||
diagnostics before attempting persistent nested resume.
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
from wf_core import Workflow
|
||||
from wf_core import ReducerRef, Workflow
|
||||
|
||||
from .models import ArtifactKind, JsonObject, RequiredCapability, WorkflowArtifact
|
||||
|
||||
@@ -86,11 +86,20 @@ def _required_reducers_from_plan(plan: JsonObject) -> dict[str, RequiredCapabili
|
||||
for field in fields.values():
|
||||
if not isinstance(field, dict):
|
||||
continue
|
||||
reducer = field.get("reducer", "wf.std.replace")
|
||||
if not isinstance(reducer, str) or "." not in reducer:
|
||||
reducer_payload = field.get("reducer", "wf.std.replace")
|
||||
if isinstance(reducer_payload, str):
|
||||
reducer = ReducerRef(name=reducer_payload)
|
||||
else:
|
||||
try:
|
||||
reducer = ReducerRef.model_validate(reducer_payload)
|
||||
except ValueError:
|
||||
continue
|
||||
if "." not in reducer.name:
|
||||
continue
|
||||
logical_source, _, capability_name = reducer.rpartition(".")
|
||||
requirements[reducer] = RequiredCapability(
|
||||
if "." not in reducer.name:
|
||||
continue
|
||||
logical_source, _, capability_name = reducer.name.rpartition(".")
|
||||
requirements[reducer.name] = RequiredCapability(
|
||||
logical_source=logical_source,
|
||||
capability_name=capability_name,
|
||||
kind="reducer",
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import Any, Iterator
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from wf_core import SchemaRef, StateField, StateSchema
|
||||
from wf_core import ReducerRef, SchemaRef, StateField, StateSchema
|
||||
|
||||
SchemaLike = SchemaRef | type[BaseModel] | type[Any] | dict[str, Any]
|
||||
StateSchemaLike = StateSchema | type[BaseModel] | type[Any] | dict[str, Any]
|
||||
@@ -51,7 +51,9 @@ def state_schema_from(value: StateSchemaLike) -> StateSchema:
|
||||
fields = {
|
||||
path: StateField(
|
||||
type=_state_field_type(property_schema),
|
||||
reducer=metadata_by_name.get(path, StateFieldMetadata()).reducer,
|
||||
reducer=ReducerRef(
|
||||
name=metadata_by_name.get(path, StateFieldMetadata()).reducer
|
||||
),
|
||||
trace=metadata_by_name.get(path, StateFieldMetadata()).trace,
|
||||
default=_state_field_default(value, path, property_schema),
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ from .models import (
|
||||
NodeDef,
|
||||
NodeResult,
|
||||
NodeUse,
|
||||
ReducerRef,
|
||||
ReducerSpec,
|
||||
SchemaRef,
|
||||
StateField,
|
||||
@@ -52,6 +53,7 @@ __all__ = [
|
||||
"NodeDef",
|
||||
"NodeResult",
|
||||
"NodeUse",
|
||||
"ReducerRef",
|
||||
"ReducerSpec",
|
||||
"SchemaRef",
|
||||
"StateField",
|
||||
|
||||
@@ -9,7 +9,7 @@ from wf_core.models.conditions import (
|
||||
VariadicCondition,
|
||||
)
|
||||
from wf_core.models.results import NodeResult
|
||||
from wf_core.models.reducers import ReducerSpec
|
||||
from wf_core.models.reducers import ReducerRef, ReducerSpec
|
||||
from wf_core.models.schemas import NodeDef, SchemaRef, StateField, StateSchema
|
||||
from wf_core.models.steps import (
|
||||
ConditionNode,
|
||||
@@ -33,6 +33,7 @@ __all__ = [
|
||||
"LiteralOperand",
|
||||
"NodeDef",
|
||||
"NodeResult",
|
||||
"ReducerRef",
|
||||
"ReducerSpec",
|
||||
"NodeUse",
|
||||
"NotCondition",
|
||||
|
||||
@@ -1,10 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class ReducerRef(BaseModel):
|
||||
"""Reference to one reducer plus JSON-compatible configuration."""
|
||||
|
||||
name: str
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ReducerSpec(BaseModel):
|
||||
"""Inspectable metadata for one named pure state reducer."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
name: str
|
||||
description: str | None = None
|
||||
config_schema: dict[str, Any] = Field(
|
||||
default_factory=lambda: {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"additionalProperties": False,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -2,7 +2,9 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
from wf_core.models.reducers import ReducerRef
|
||||
|
||||
|
||||
class SchemaRef(BaseModel):
|
||||
@@ -20,10 +22,19 @@ class StateField(BaseModel):
|
||||
"""Declared state path plus its runtime merge behavior."""
|
||||
|
||||
type: str
|
||||
reducer: str = "wf.std.replace"
|
||||
reducer: ReducerRef = Field(
|
||||
default_factory=lambda: ReducerRef(name="wf.std.replace")
|
||||
)
|
||||
trace: bool = True
|
||||
default: Any = None
|
||||
|
||||
@field_validator("reducer", mode="before")
|
||||
@classmethod
|
||||
def _coerce_reducer(cls, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return {"name": value}
|
||||
return value
|
||||
|
||||
|
||||
class StateSchema(BaseModel):
|
||||
"""Workflow state schema keyed by declared exact state path."""
|
||||
|
||||
@@ -4,16 +4,22 @@ from collections.abc import Callable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from wf_core.errors import WorkflowExecutionError
|
||||
from wf_core.models.reducers import ReducerRef, ReducerSpec
|
||||
from wf_core.runtime.ops.schemas import validate_payload_against_schema
|
||||
|
||||
Reducer = Callable[[Any, Any], Any]
|
||||
Reducer = Callable[[Any, Any, Mapping[str, Any]], Any]
|
||||
|
||||
|
||||
def replace_reducer(_current_value: Any, incoming_value: Any) -> Any:
|
||||
def replace_reducer(
|
||||
_current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
|
||||
) -> Any:
|
||||
"""Replace the current state value with the incoming value."""
|
||||
return incoming_value
|
||||
|
||||
|
||||
def append_reducer(current_value: Any, incoming_value: Any) -> Any:
|
||||
def append_reducer(
|
||||
current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
|
||||
) -> Any:
|
||||
"""Append one value or many values into a list-valued state path."""
|
||||
if current_value is None:
|
||||
return (
|
||||
@@ -28,7 +34,9 @@ def append_reducer(current_value: Any, incoming_value: Any) -> Any:
|
||||
)
|
||||
|
||||
|
||||
def merge_object_reducer(current_value: Any, incoming_value: Any) -> Any:
|
||||
def merge_object_reducer(
|
||||
current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
|
||||
) -> Any:
|
||||
"""Shallow-merge object values at one exact state path."""
|
||||
if current_value is None:
|
||||
if not isinstance(incoming_value, dict):
|
||||
@@ -39,7 +47,9 @@ def merge_object_reducer(current_value: Any, incoming_value: Any) -> Any:
|
||||
return current_value | incoming_value
|
||||
|
||||
|
||||
def set_union_reducer(current_value: Any, incoming_value: Any) -> Any:
|
||||
def set_union_reducer(
|
||||
current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
|
||||
) -> Any:
|
||||
"""Merge list values while preserving stable first-seen order."""
|
||||
if current_value is None:
|
||||
current_items: list[Any] = []
|
||||
@@ -58,7 +68,9 @@ def set_union_reducer(current_value: Any, incoming_value: Any) -> Any:
|
||||
return merged
|
||||
|
||||
|
||||
def max_reducer(current_value: Any, incoming_value: Any) -> Any:
|
||||
def max_reducer(
|
||||
current_value: Any, incoming_value: Any, _config: Mapping[str, Any]
|
||||
) -> Any:
|
||||
"""Keep the larger of the current and incoming values."""
|
||||
return (
|
||||
incoming_value if current_value is None else max(current_value, incoming_value)
|
||||
@@ -73,20 +85,33 @@ DEFAULT_REDUCERS: Mapping[str, Reducer] = {
|
||||
"wf.std.max": max_reducer,
|
||||
}
|
||||
|
||||
DEFAULT_REDUCER_SPECS: Mapping[str, ReducerSpec] = {
|
||||
name: ReducerSpec(name=name) for name in DEFAULT_REDUCERS
|
||||
}
|
||||
|
||||
|
||||
def apply_reducer(
|
||||
*,
|
||||
reducer_name: str,
|
||||
reducer: ReducerRef,
|
||||
current_value: Any,
|
||||
incoming_value: Any,
|
||||
destination_path: str,
|
||||
reducers: Mapping[str, Reducer] = DEFAULT_REDUCERS,
|
||||
reducer_specs: Mapping[str, ReducerSpec] = DEFAULT_REDUCER_SPECS,
|
||||
) -> Any:
|
||||
"""Apply one named pure reducer to a state write."""
|
||||
reducer = reducers.get(reducer_name)
|
||||
if reducer is None:
|
||||
raise WorkflowExecutionError(f"unknown reducer {reducer_name!r}")
|
||||
reducer_fn = reducers.get(reducer.name)
|
||||
if reducer_fn is None:
|
||||
raise WorkflowExecutionError(f"unknown reducer {reducer.name!r}")
|
||||
spec = reducer_specs.get(reducer.name)
|
||||
if spec is None:
|
||||
raise WorkflowExecutionError(f"unknown reducer spec {reducer.name!r}")
|
||||
validate_payload_against_schema(
|
||||
spec.config_schema,
|
||||
reducer.config,
|
||||
f"reducer config for {reducer.name!r}",
|
||||
)
|
||||
try:
|
||||
return reducer(current_value, incoming_value)
|
||||
return reducer_fn(current_value, incoming_value, reducer.config)
|
||||
except TypeError as exc:
|
||||
raise WorkflowExecutionError(f"{exc} at {destination_path!r}") from exc
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import Any
|
||||
|
||||
from wf_core.errors import WorkflowExecutionError
|
||||
from wf_core.local_paths import LocalPathError, get_local_value, has_overlapping_paths
|
||||
from wf_core.models.reducers import ReducerRef
|
||||
from wf_core.models.steps import NodeUse
|
||||
from wf_core.models.workflow import Workflow
|
||||
from wf_core.paths import (
|
||||
@@ -73,11 +74,11 @@ def write_state_value(
|
||||
|
||||
declared_path = ".".join(parts)
|
||||
declared_field = workflow.state_schema.fields.get(declared_path)
|
||||
reducer_name = declared_field.reducer if declared_field else "wf.std.replace"
|
||||
reducer = declared_field.reducer if declared_field else ReducerRef(name="wf.std.replace")
|
||||
key_path = parts
|
||||
current_value = get_nested_value(state, key_path)
|
||||
merged_value = apply_reducer(
|
||||
reducer_name=reducer_name,
|
||||
reducer=reducer,
|
||||
current_value=current_value,
|
||||
incoming_value=value,
|
||||
destination_path=destination_path,
|
||||
|
||||
@@ -58,7 +58,7 @@ def test_state_basemodel_can_declare_reducer_with_annotated_metadata() -> None:
|
||||
workflow = builder.compile()
|
||||
|
||||
assert workflow.state_schema.fields["items"].type == "array"
|
||||
assert workflow.state_schema.fields["items"].reducer == "wf.std.append"
|
||||
assert workflow.state_schema.fields["items"].reducer.name == "wf.std.append"
|
||||
|
||||
|
||||
def test_state_basemodel_seeds_safe_initial_defaults() -> None:
|
||||
@@ -96,5 +96,5 @@ def test_nested_state_basemodel_projects_parent_and_child_paths() -> None:
|
||||
assert workflow.state_schema.fields["person"].type == "object"
|
||||
assert workflow.state_schema.fields["person.name"].type == "string"
|
||||
assert workflow.state_schema.fields["person.tags"].type == "array"
|
||||
assert workflow.state_schema.fields["person"].reducer == "wf.std.replace"
|
||||
assert workflow.state_schema.fields["person.tags"].reducer == "wf.std.append"
|
||||
assert workflow.state_schema.fields["person"].reducer.name == "wf.std.replace"
|
||||
assert workflow.state_schema.fields["person.tags"].reducer.name == "wf.std.append"
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from wf_core import SchemaRef, StateField, StateSchema, Workflow
|
||||
from wf_core import ReducerRef, SchemaRef, StateField, StateSchema, Workflow
|
||||
from wf_core.runtime.ops.state import write_state_value
|
||||
|
||||
|
||||
def test_exact_nested_state_path_uses_declared_reducer() -> None:
|
||||
workflow = _workflow(
|
||||
fields={"person.tags": StateField(type="array", reducer="wf.std.append")}
|
||||
fields={
|
||||
"person.tags": StateField(
|
||||
reducer=ReducerRef(name="wf.std.append"), type="array"
|
||||
)
|
||||
}
|
||||
)
|
||||
state = {"person": {"tags": ["seed"]}}
|
||||
|
||||
@@ -17,7 +21,11 @@ def test_exact_nested_state_path_uses_declared_reducer() -> None:
|
||||
|
||||
def test_parent_state_declaration_does_not_apply_to_nested_write() -> None:
|
||||
workflow = _workflow(
|
||||
fields={"person": StateField(type="object", reducer="wf.std.merge_object")}
|
||||
fields={
|
||||
"person": StateField(
|
||||
reducer=ReducerRef(name="wf.std.merge_object"), type="object"
|
||||
)
|
||||
}
|
||||
)
|
||||
state = {"person": {"tags": ["seed"]}}
|
||||
|
||||
@@ -36,12 +44,34 @@ def test_undeclared_nested_state_path_defaults_to_replace() -> None:
|
||||
|
||||
|
||||
def test_state_field_defaults_to_replace_reducer() -> None:
|
||||
assert StateField(type="string").reducer == "wf.std.replace"
|
||||
assert StateField(type="string").reducer == ReducerRef(name="wf.std.replace")
|
||||
|
||||
|
||||
def test_state_field_accepts_string_reducer_shorthand() -> None:
|
||||
field = StateField.model_validate(
|
||||
{"type": "array", "reducer": "wf.std.set_union"}
|
||||
)
|
||||
|
||||
assert field.reducer == ReducerRef(name="wf.std.set_union")
|
||||
|
||||
|
||||
def test_state_field_accepts_configured_reducer_reference() -> None:
|
||||
field = StateField(
|
||||
type="integer",
|
||||
reducer=ReducerRef(name="wf.std.max", config={"sample": True}),
|
||||
)
|
||||
|
||||
assert field.reducer.name == "wf.std.max"
|
||||
assert field.reducer.config == {"sample": True}
|
||||
|
||||
|
||||
def test_unknown_state_reducer_fails_clearly() -> None:
|
||||
workflow = _workflow(
|
||||
fields={"person.tags": StateField(type="array", reducer="x.nope")}
|
||||
fields={
|
||||
"person.tags": StateField(
|
||||
reducer=ReducerRef(name="x.nope"), type="array"
|
||||
)
|
||||
}
|
||||
)
|
||||
state = {"person": {"tags": ["seed"]}}
|
||||
|
||||
@@ -55,7 +85,11 @@ def test_unknown_state_reducer_fails_clearly() -> None:
|
||||
|
||||
def test_set_union_reducer_preserves_first_seen_order() -> None:
|
||||
workflow = _workflow(
|
||||
fields={"person.tags": StateField(type="array", reducer="wf.std.set_union")}
|
||||
fields={
|
||||
"person.tags": StateField(
|
||||
reducer=ReducerRef(name="wf.std.set_union"), type="array"
|
||||
)
|
||||
}
|
||||
)
|
||||
state = {"person": {"tags": ["alpha", "beta"]}}
|
||||
|
||||
@@ -64,9 +98,33 @@ def test_set_union_reducer_preserves_first_seen_order() -> None:
|
||||
assert state["person"]["tags"] == ["alpha", "beta", "gamma"]
|
||||
|
||||
|
||||
def test_unexpected_reducer_config_fails_before_state_mutation() -> None:
|
||||
workflow = _workflow(
|
||||
fields={
|
||||
"person.tags": StateField(
|
||||
type="array",
|
||||
reducer=ReducerRef(name="wf.std.set_union", config={"bad": True}),
|
||||
)
|
||||
}
|
||||
)
|
||||
state = {"person": {"tags": ["alpha"]}}
|
||||
|
||||
try:
|
||||
write_state_value(workflow, state, "state.person.tags", ["beta"])
|
||||
except Exception as exc:
|
||||
assert "reducer config for 'wf.std.set_union'" in str(exc)
|
||||
else:
|
||||
raise AssertionError("expected invalid reducer config to fail")
|
||||
assert state["person"]["tags"] == ["alpha"]
|
||||
|
||||
|
||||
def test_max_reducer_keeps_larger_value() -> None:
|
||||
workflow = _workflow(
|
||||
fields={"best_score": StateField(type="integer", reducer="wf.std.max")}
|
||||
fields={
|
||||
"best_score": StateField(
|
||||
reducer=ReducerRef(name="wf.std.max"), type="integer"
|
||||
)
|
||||
}
|
||||
)
|
||||
state = {"best_score": 7}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user