fix: address workflow authoring review findings
This commit is contained in:
@@ -280,7 +280,8 @@ stable.
|
|||||||
deployment binding suggestions, reject bare `--bind-output` state targets
|
deployment binding suggestions, reject bare `--bind-output` state targets
|
||||||
before RPC with compact guidance, and accept `wf schema --full` as an alias
|
before RPC with compact guidance, and accept `wf schema --full` as an alias
|
||||||
for `--verbose`.
|
for `--verbose`.
|
||||||
- Completed: `wf draft bind local.x -> output.y` now lowers through state
|
- Completed: `wf draft bind` with `--from local.x --to output.y` now lowers
|
||||||
|
through state
|
||||||
atomically (projecting into both state_schema and output_schema), and
|
atomically (projecting into both state_schema and output_schema), and
|
||||||
validation repair hints cover undeclared workflow input source paths.
|
validation repair hints cover undeclared workflow input source paths.
|
||||||
Implementation plan:
|
Implementation plan:
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
Planned. This design generalizes the two remaining capability-specific draft
|
Implemented. This design generalized the two capability-specific draft
|
||||||
CLI verbs:
|
CLI verbs:
|
||||||
|
|
||||||
- `wf draft create-from-capability`
|
- `wf draft create-from-capability`
|
||||||
@@ -13,10 +13,9 @@ They are replaced at the CLI layer by:
|
|||||||
- `wf draft create --capability <qualified_name>`
|
- `wf draft create --capability <qualified_name>`
|
||||||
- `wf draft add-step --capability <qualified_name>`
|
- `wf draft add-step --capability <qualified_name>`
|
||||||
|
|
||||||
The long commands should be removed from the CLI, docs, and skills rather than
|
The long commands are removed from the CLI, docs, and skills rather than kept
|
||||||
kept as aliases. The programmatic API/RPC/MCP method names are not renamed in
|
as aliases. The focused bind/remove surface is threaded through API, RPC, and
|
||||||
this slice because they are older, tested surfaces with broader callers than
|
MCP layers; CLI/docs/skills expose only the concise command vocabulary.
|
||||||
the CLI benchmark loop.
|
|
||||||
|
|
||||||
## Problem
|
## Problem
|
||||||
|
|
||||||
@@ -27,16 +26,16 @@ wf draft create --capability local.report.extract_report ...
|
|||||||
wf draft add-step --capability local.report.render_markdown_report ...
|
wf draft add-step --capability local.report.render_markdown_report ...
|
||||||
```
|
```
|
||||||
|
|
||||||
The product currently exposes:
|
Before this change, the product exposed:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
wf draft create-from-capability ...
|
wf draft create-from-capability ...
|
||||||
wf draft add-step-from-capability ...
|
wf draft add-step-from-capability ...
|
||||||
```
|
```
|
||||||
|
|
||||||
The long names are precise but hostile to discovery. Skills currently have to
|
The long names were precise but hostile to discovery. Skills had to warn agents
|
||||||
warn agents that `wf draft create --capability` does not exist, which is a sign
|
that `wf draft create --capability` did not exist, which showed that the CLI
|
||||||
that the CLI shape is wrong.
|
shape was wrong.
|
||||||
|
|
||||||
## New CLI Shape
|
## New CLI Shape
|
||||||
|
|
||||||
|
|||||||
@@ -71,7 +71,10 @@ use wf artifact create-from-plan instead.
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
For capability output to state, prefer:
|
For capability output to state, prefer:
|
||||||
wf draft bind --from local.FIELD --to state.FIELD
|
wf draft bind WORKSPACE --revision N --step STEP --from local.FIELD --to state.FIELD
|
||||||
|
|
||||||
|
For capability output to public workflow output, prefer:
|
||||||
|
wf draft bind WORKSPACE --revision N --step STEP --from local.FIELD --to output.FIELD
|
||||||
```
|
```
|
||||||
|
|
||||||
`draft_invalid` must distinguish draft shape from raw plan shape:
|
`draft_invalid` must distinguish draft shape from raw plan shape:
|
||||||
|
|||||||
+1
-1
@@ -368,7 +368,7 @@ Repair-hint examples:
|
|||||||
```bash
|
```bash
|
||||||
# Declare an undeclared workflow input field and bind it to a step input
|
# Declare an undeclared workflow input field and bind it to a step input
|
||||||
wf draft bind report_ws --revision 4 --step read --from input.path --to local.path
|
wf draft bind report_ws --revision 4 --step read --from input.path --to local.path
|
||||||
# Lower a capability output through state into workflow output
|
# Request a public workflow output; bind lowers it through state internally
|
||||||
wf draft bind report_ws --revision 5 --step render --from local.markdown --to output.markdown
|
wf draft bind report_ws --revision 5 --step render --from local.markdown --to output.markdown
|
||||||
# Set workflow output independently (no schema projection)
|
# Set workflow output independently (no schema projection)
|
||||||
wf draft set-workflow-output report_ws --revision 6 --map state.markdown=markdown
|
wf draft set-workflow-output report_ws --revision 6 --map state.markdown=markdown
|
||||||
|
|||||||
@@ -620,9 +620,10 @@ def run_v2_trial(
|
|||||||
assertion_failures.append(
|
assertion_failures.append(
|
||||||
"could not extract challenge report for success_assertions evaluation"
|
"could not extract challenge report for success_assertions evaluation"
|
||||||
)
|
)
|
||||||
if profile == InstructionProfile.DEBUG and challenge_report is not None:
|
if profile == InstructionProfile.DEBUG:
|
||||||
ux_issues = challenge_report.get("ux_issues_found")
|
if challenge_report is None:
|
||||||
if not isinstance(ux_issues, list):
|
assertion_failures.append("debug profile requires a challenge_report")
|
||||||
|
elif not isinstance(challenge_report.get("ux_issues_found"), list):
|
||||||
assertion_failures.append(
|
assertion_failures.append(
|
||||||
"debug profile requires ux_issues_found list in challenge_report"
|
"debug profile requires ux_issues_found list in challenge_report"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -114,9 +114,28 @@ def _raw_result_path_for_report(report_path: Path) -> Path:
|
|||||||
return report_path
|
return report_path
|
||||||
|
|
||||||
|
|
||||||
|
def _recorded_raw_result_path(report_path: Path) -> Path | None:
|
||||||
|
"""Read the raw result identity recorded in a machine report."""
|
||||||
|
try:
|
||||||
|
payload = json.loads(report_path.read_text(encoding="utf-8"))
|
||||||
|
except OSError, json.JSONDecodeError:
|
||||||
|
return None
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return None
|
||||||
|
identity = payload.get("identity")
|
||||||
|
if not isinstance(identity, dict):
|
||||||
|
return None
|
||||||
|
raw_result_path = identity.get("raw_result_path")
|
||||||
|
if not isinstance(raw_result_path, str) or not raw_result_path:
|
||||||
|
return None
|
||||||
|
return Path(raw_result_path)
|
||||||
|
|
||||||
|
|
||||||
def _sort_mtime(path: Path, *, sort_by: str) -> float:
|
def _sort_mtime(path: Path, *, sort_by: str) -> float:
|
||||||
if sort_by == "result":
|
if sort_by == "result":
|
||||||
raw_result = _raw_result_path_for_report(path)
|
raw_result = _recorded_raw_result_path(path) or _raw_result_path_for_report(
|
||||||
|
path
|
||||||
|
)
|
||||||
if raw_result.exists():
|
if raw_result.exists():
|
||||||
return raw_result.stat().st_mtime
|
return raw_result.stat().st_mtime
|
||||||
return path.stat().st_mtime
|
return path.stat().st_mtime
|
||||||
|
|||||||
@@ -59,10 +59,11 @@ reported in wrapper-hint notes; bind them explicitly with `wf draft bind` or
|
|||||||
`wf draft set-input --merge` only when the workflow should expose them.
|
`wf draft set-input --merge` only when the workflow should expose them.
|
||||||
|
|
||||||
When `wf draft validate` returns a `repair_hint`, run that exact focused command
|
When `wf draft validate` returns a `repair_hint`, run that exact focused command
|
||||||
before writing JSON Patch manually. Use `wf draft bind local.x -> output.y` when
|
before writing JSON Patch manually. To make one capability output public, use
|
||||||
one capability output should become public workflow output; it creates the
|
`wf draft bind <workspace_id> --revision <n> --step <step_id> --from local.x
|
||||||
required state intermediary and projects the field schema into both state and
|
--to output.y`; it creates the required state intermediary and projects the
|
||||||
output schemas atomically. Re-run `wf draft validate` after the repair.
|
field schema into both state and output schemas atomically. Re-run
|
||||||
|
`wf draft validate` after the repair.
|
||||||
|
|
||||||
wf artifact create-from-plan workflow.plan.json --artifact <artifact_id> --version <n> --title <title>
|
wf artifact create-from-plan workflow.plan.json --artifact <artifact_id> --version <n> --title <title>
|
||||||
wf deploy save <deployment_id> --artifact <artifact_id> --version <n> --binding <logical>=<concrete>
|
wf deploy save <deployment_id> --artifact <artifact_id> --version <n> --binding <logical>=<concrete>
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ low-level escape hatch or you already have a complete compiler/generated plan.
|
|||||||
2. Inspect one candidate with `wf cap inspect`.
|
2. Inspect one candidate with `wf cap inspect`.
|
||||||
3. Call one candidate with `wf cap call` when payload shape or upstream source
|
3. Call one candidate with `wf cap call` when payload shape or upstream source
|
||||||
reachability is uncertain.
|
reachability is uncertain.
|
||||||
4. Create a patchable draft workspace with `wf draft create --capability`.
|
4. Create a patchable draft workspace with
|
||||||
|
`wf draft create <workspace_id> --capability <qualified_name>`.
|
||||||
5. Patch targeted fields with focused draft commands or JSON Patch.
|
5. Patch targeted fields with focused draft commands or JSON Patch.
|
||||||
6. Validate with `wf draft validate`.
|
6. Validate with `wf draft validate`.
|
||||||
7. Save an artifact with `wf draft save`, or import a complete raw plan with
|
7. Save an artifact with `wf draft save`, or import a complete raw plan with
|
||||||
|
|||||||
@@ -53,11 +53,15 @@ def _graph_parts(path: str) -> tuple[str, tuple[str, ...]]:
|
|||||||
|
|
||||||
|
|
||||||
def _local_field(path: str) -> str:
|
def _local_field(path: str) -> str:
|
||||||
raw = path.removeprefix("local.")
|
parts = _local_parts(path)
|
||||||
parsed = LocalPath.parse(raw)
|
if len(parts) != 1:
|
||||||
if len(parsed.parts) != 1:
|
|
||||||
raise ValueError("local path must name one capability field")
|
raise ValueError("local path must name one capability field")
|
||||||
return parsed.parts[0]
|
return parts[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _local_parts(path: str) -> tuple[str, ...]:
|
||||||
|
"""Parse a CLI local-root path as the rootless core LocalPath value."""
|
||||||
|
return LocalPath.parse(path.removeprefix("local.")).parts
|
||||||
|
|
||||||
|
|
||||||
class WorkflowDraftAuthoringApi:
|
class WorkflowDraftAuthoringApi:
|
||||||
@@ -193,7 +197,7 @@ class WorkflowDraftAuthoringApi:
|
|||||||
source_root, source_parts = (
|
source_root, source_parts = (
|
||||||
_graph_parts(source_path)
|
_graph_parts(source_path)
|
||||||
if not source_path.startswith("local.")
|
if not source_path.startswith("local.")
|
||||||
else ("local", LocalPath.parse(source_path).parts)
|
else ("local", _local_parts(source_path))
|
||||||
)
|
)
|
||||||
if target_path.startswith("output."):
|
if target_path.startswith("output."):
|
||||||
# GraphSourcePath excludes output targets, but output fields still
|
# GraphSourcePath excludes output targets, but output fields still
|
||||||
@@ -205,7 +209,7 @@ class WorkflowDraftAuthoringApi:
|
|||||||
raise ValueError("output path must name a field, such as output.result")
|
raise ValueError("output path must name a field, such as output.result")
|
||||||
elif target_path.startswith("local."):
|
elif target_path.startswith("local."):
|
||||||
target_root = "local"
|
target_root = "local"
|
||||||
target_parts = LocalPath.parse(target_path).parts
|
target_parts = _local_parts(target_path)
|
||||||
else:
|
else:
|
||||||
target_root, target_parts = _graph_parts(target_path)
|
target_root, target_parts = _graph_parts(target_path)
|
||||||
|
|
||||||
@@ -271,10 +275,12 @@ class WorkflowDraftAuthoringApi:
|
|||||||
allow_existing_equivalent=True,
|
allow_existing_equivalent=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
current_output_map = self.drafts._step_output_map(
|
||||||
|
workspace_id=workspace_id, step_id=step_id
|
||||||
|
)
|
||||||
|
previous_state_path = current_output_map.get(local_field)
|
||||||
output_map = {
|
output_map = {
|
||||||
**self.drafts._step_output_map(
|
**current_output_map,
|
||||||
workspace_id=workspace_id, step_id=step_id
|
|
||||||
),
|
|
||||||
local_field: state_path_str,
|
local_field: state_path_str,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -284,7 +290,15 @@ class WorkflowDraftAuthoringApi:
|
|||||||
b
|
b
|
||||||
for b in existing_output
|
for b in existing_output
|
||||||
if not (
|
if not (
|
||||||
isinstance(b, dict) and b.get("target") == output_target_str
|
isinstance(b, dict)
|
||||||
|
and (
|
||||||
|
b.get("target") == output_target_str
|
||||||
|
or b.get("path") == state_path_str
|
||||||
|
or (
|
||||||
|
previous_state_path is not None
|
||||||
|
and b.get("path") == previous_state_path
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
@@ -667,6 +681,14 @@ class WorkflowDraftAuthoringApi:
|
|||||||
raise ValueError(f"input bindings for step {step_id!r} must be a list")
|
raise ValueError(f"input bindings for step {step_id!r} must be a list")
|
||||||
if not isinstance(current_outputs, list):
|
if not isinstance(current_outputs, list):
|
||||||
raise ValueError(f"output bindings for step {step_id!r} must be a list")
|
raise ValueError(f"output bindings for step {step_id!r} must be a list")
|
||||||
|
if not all(isinstance(item, dict) for item in current_inputs):
|
||||||
|
raise ValueError(
|
||||||
|
f"input binding entries for step {step_id!r} must be objects"
|
||||||
|
)
|
||||||
|
if not all(isinstance(item, dict) for item in current_outputs):
|
||||||
|
raise ValueError(
|
||||||
|
f"output binding entries for step {step_id!r} must be objects"
|
||||||
|
)
|
||||||
input_targets = set(inputs)
|
input_targets = set(inputs)
|
||||||
output_sources = set(outputs)
|
output_sources = set(outputs)
|
||||||
next_inputs = [
|
next_inputs = [
|
||||||
|
|||||||
@@ -535,7 +535,7 @@ def _draft_repair_hint(
|
|||||||
target_field = details.get("target_field")
|
target_field = details.get("target_field")
|
||||||
if not isinstance(source_path, str) or not isinstance(target_field, str):
|
if not isinstance(source_path, str) or not isinstance(target_field, str):
|
||||||
return None
|
return None
|
||||||
if source_path.startswith("input."):
|
if source_path.startswith(("input.", "state.")):
|
||||||
return (
|
return (
|
||||||
f"wf draft bind {workspace_id} --revision {revision} "
|
f"wf draft bind {workspace_id} --revision {revision} "
|
||||||
f"--step {step_id} --from {source_path} --to local.{target_field}"
|
f"--step {step_id} --from {source_path} --to local.{target_field}"
|
||||||
|
|||||||
@@ -154,8 +154,10 @@ EXPLAIN_CARDS: tuple[ExplainCard, ...] = (
|
|||||||
"A draft patch changed output bindings without changing the matching schema.",
|
"A draft patch changed output bindings without changing the matching schema.",
|
||||||
],
|
],
|
||||||
how_to_fix=[
|
how_to_fix=[
|
||||||
"For capability output to state, prefer `wf draft bind --from local.FIELD --to state.FIELD`.",
|
"For capability output to state, prefer `wf draft bind <workspace_id> --revision N --step STEP --from local.FIELD --to state.FIELD`.",
|
||||||
|
"To publish one capability output, use the same bind command with `--to output.FIELD`.",
|
||||||
"For multiple output bindings, use `wf draft set-output --merge` when preserving existing mappings.",
|
"For multiple output bindings, use `wf draft set-output --merge` when preserving existing mappings.",
|
||||||
|
"For an existing state-to-public-output projection, use `wf draft set-workflow-output <workspace_id> --revision N --merge --map state.FIELD=FIELD`.",
|
||||||
"Read any `repair_hint` returned by `wf draft validate` before writing JSON Patch.",
|
"Read any `repair_hint` returned by `wf draft validate` before writing JSON Patch.",
|
||||||
"Run `wf draft validate <workspace_id>` after the edit.",
|
"Run `wf draft validate <workspace_id>` after the edit.",
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -51,6 +51,10 @@ _SEARCH_ALWAYS_VISIBLE_TOOL_NAMES = [
|
|||||||
"wf.workflow.set_step_input_map",
|
"wf.workflow.set_step_input_map",
|
||||||
"wf.workflow.set_step_output_map",
|
"wf.workflow.set_step_output_map",
|
||||||
"wf.workflow.set_workflow_output_map",
|
"wf.workflow.set_workflow_output_map",
|
||||||
|
"wf.workflow.bind",
|
||||||
|
"wf.workflow.remove_draft_route",
|
||||||
|
"wf.workflow.remove_draft_step",
|
||||||
|
"wf.workflow.remove_draft_binding",
|
||||||
"wf.workflow.create_minimal_draft_workspace",
|
"wf.workflow.create_minimal_draft_workspace",
|
||||||
"wf.workflow.create_artifact_from_workspace",
|
"wf.workflow.create_artifact_from_workspace",
|
||||||
"wf.workflow.create_wrapper_from_workspace",
|
"wf.workflow.create_wrapper_from_workspace",
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ DraftPathMap = Annotated[
|
|||||||
)
|
)
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
NonEmptyString = Annotated[str, Field(min_length=1)]
|
||||||
DraftInputBindings = Annotated[
|
DraftInputBindings = Annotated[
|
||||||
list[InputBinding],
|
list[InputBinding],
|
||||||
Field(
|
Field(
|
||||||
@@ -216,7 +217,9 @@ class SetStepInputMapRequest(BaseModel):
|
|||||||
|
|
||||||
workspace_id: WorkspaceId
|
workspace_id: WorkspaceId
|
||||||
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
||||||
step_id: str = Field(description="Draft step id whose input map should change.")
|
step_id: NonEmptyString = Field(
|
||||||
|
description="Draft step id whose input map should change."
|
||||||
|
)
|
||||||
input_map: DraftPathMap
|
input_map: DraftPathMap
|
||||||
merge: bool = Field(
|
merge: bool = Field(
|
||||||
default=False,
|
default=False,
|
||||||
@@ -232,7 +235,9 @@ class SetStepOutputMapRequest(BaseModel):
|
|||||||
|
|
||||||
workspace_id: WorkspaceId
|
workspace_id: WorkspaceId
|
||||||
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
||||||
step_id: str = Field(description="Draft step id whose output map should change.")
|
step_id: NonEmptyString = Field(
|
||||||
|
description="Draft step id whose output map should change."
|
||||||
|
)
|
||||||
output_map: DraftPathMap
|
output_map: DraftPathMap
|
||||||
merge: bool = Field(
|
merge: bool = Field(
|
||||||
default=False,
|
default=False,
|
||||||
@@ -263,9 +268,13 @@ class BindDraftRequest(BaseModel):
|
|||||||
|
|
||||||
workspace_id: WorkspaceId
|
workspace_id: WorkspaceId
|
||||||
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
||||||
step_id: str = Field(description="Capability-backed draft step id.")
|
step_id: NonEmptyString = Field(description="Capability-backed draft step id.")
|
||||||
source_path: str = Field(description="Source path, for example input.x or local.y.")
|
source_path: NonEmptyString = Field(
|
||||||
target_path: str = Field(description="Target path, for example local.x or state.y.")
|
description="Source path, for example input.x or local.y."
|
||||||
|
)
|
||||||
|
target_path: NonEmptyString = Field(
|
||||||
|
description="Target path, for example local.x or state.y."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class AddStepFromCapabilityRequest(BaseModel):
|
class AddStepFromCapabilityRequest(BaseModel):
|
||||||
@@ -273,13 +282,14 @@ class AddStepFromCapabilityRequest(BaseModel):
|
|||||||
|
|
||||||
workspace_id: WorkspaceId
|
workspace_id: WorkspaceId
|
||||||
revision: int = Field(ge=1, description="Expected workspace revision.")
|
revision: int = Field(ge=1, description="Expected workspace revision.")
|
||||||
step_id: str = Field(description="New draft step id.")
|
step_id: NonEmptyString = Field(description="New draft step id.")
|
||||||
capability_name: str = Field(description="Qualified capability name.")
|
capability_name: NonEmptyString = Field(description="Qualified capability name.")
|
||||||
route_from_step: str | None = Field(
|
route_from_step: str | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
|
min_length=1,
|
||||||
description="Optional existing step whose outcome should route to the new step.",
|
description="Optional existing step whose outcome should route to the new step.",
|
||||||
)
|
)
|
||||||
route_from_outcome: str = Field(
|
route_from_outcome: NonEmptyString = Field(
|
||||||
default="ok",
|
default="ok",
|
||||||
description="Outcome on route_from_step that should route to the new step.",
|
description="Outcome on route_from_step that should route to the new step.",
|
||||||
)
|
)
|
||||||
@@ -306,8 +316,12 @@ class RemoveDraftRouteRequest(BaseModel):
|
|||||||
|
|
||||||
workspace_id: WorkspaceId
|
workspace_id: WorkspaceId
|
||||||
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
||||||
step_id: str = Field(description="Draft step id whose route should be removed.")
|
step_id: NonEmptyString = Field(
|
||||||
outcome: str = Field(description="Outcome label to remove from the step route map.")
|
description="Draft step id whose route should be removed."
|
||||||
|
)
|
||||||
|
outcome: NonEmptyString = Field(
|
||||||
|
description="Outcome label to remove from the step route map."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RemoveDraftStepRequest(BaseModel):
|
class RemoveDraftStepRequest(BaseModel):
|
||||||
@@ -315,7 +329,7 @@ class RemoveDraftStepRequest(BaseModel):
|
|||||||
|
|
||||||
workspace_id: WorkspaceId
|
workspace_id: WorkspaceId
|
||||||
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
||||||
step_id: str = Field(description="Draft step id to remove.")
|
step_id: NonEmptyString = Field(description="Draft step id to remove.")
|
||||||
|
|
||||||
|
|
||||||
class RemoveDraftBindingRequest(BaseModel):
|
class RemoveDraftBindingRequest(BaseModel):
|
||||||
@@ -323,7 +337,9 @@ class RemoveDraftBindingRequest(BaseModel):
|
|||||||
|
|
||||||
workspace_id: WorkspaceId
|
workspace_id: WorkspaceId
|
||||||
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
revision: int = Field(ge=1, description="Expected current workspace revision.")
|
||||||
step_id: str = Field(description="Draft step id whose bindings should be removed.")
|
step_id: NonEmptyString = Field(
|
||||||
|
description="Draft step id whose bindings should be removed."
|
||||||
|
)
|
||||||
inputs: list[str] = Field(
|
inputs: list[str] = Field(
|
||||||
default_factory=list,
|
default_factory=list,
|
||||||
description="Local input target names to remove.",
|
description="Local input target names to remove.",
|
||||||
|
|||||||
@@ -1400,6 +1400,60 @@ def test_v2_runner_debug_profile_requires_ux_issues_found(
|
|||||||
assert any("ux_issues_found" in f for f in result["assertion_failures"])
|
assert any("ux_issues_found" in f for f in result["assertion_failures"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_v2_runner_debug_profile_requires_challenge_report(tmp_path: Path) -> None:
|
||||||
|
from examples.agent_challenges.runner import run_v2_trial
|
||||||
|
|
||||||
|
manifest_path = _write_manifest(tmp_path / "challenge")
|
||||||
|
manifest_path.write_text(
|
||||||
|
manifest_path.read_text(encoding="utf-8").replace(
|
||||||
|
" required_fields: [value, run_failed]\n"
|
||||||
|
" success_assertions:\n"
|
||||||
|
" value: expected\n"
|
||||||
|
" run_failed: false\n",
|
||||||
|
" required_fields: []\n success_assertions: {}\n",
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
challenge = load_challenge_manifest(manifest_path)
|
||||||
|
bundle = ROOT / "examples/agent_challenges/instruction_bundles/workflow_cli.yaml"
|
||||||
|
results_dir = tmp_path / "results"
|
||||||
|
results_dir.mkdir()
|
||||||
|
|
||||||
|
def fake_run(
|
||||||
|
command: list[str],
|
||||||
|
*,
|
||||||
|
cwd: str,
|
||||||
|
text: bool,
|
||||||
|
capture_output: bool,
|
||||||
|
timeout: float | None,
|
||||||
|
check: bool,
|
||||||
|
) -> object:
|
||||||
|
return type(
|
||||||
|
"Result",
|
||||||
|
(),
|
||||||
|
{
|
||||||
|
"returncode": 0,
|
||||||
|
"stdout": json.dumps({"text": "Completed without a report."}),
|
||||||
|
"stderr": "",
|
||||||
|
},
|
||||||
|
)()
|
||||||
|
|
||||||
|
result = run_v2_trial(
|
||||||
|
challenge,
|
||||||
|
profile=InstructionProfile.DEBUG,
|
||||||
|
model="test-model",
|
||||||
|
variant="high",
|
||||||
|
index=1,
|
||||||
|
workspaces_dir=tmp_path / "workspaces",
|
||||||
|
results_dir=results_dir,
|
||||||
|
instruction_bundle=bundle,
|
||||||
|
run_fn=fake_run,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["task_outcome"] == "failed"
|
||||||
|
assert any("challenge_report" in f for f in result["assertion_failures"])
|
||||||
|
|
||||||
|
|
||||||
def test_v2_runner_assertions_fail_on_mismatched_report(tmp_path: Path) -> None:
|
def test_v2_runner_assertions_fail_on_mismatched_report(tmp_path: Path) -> None:
|
||||||
from examples.agent_challenges.runner import run_v2_trial
|
from examples.agent_challenges.runner import run_v2_trial
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ def _write_report(
|
|||||||
manual: str | None = "pass",
|
manual: str | None = "pass",
|
||||||
) -> Path:
|
) -> Path:
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
raw_result_path = path.with_name(path.name.removesuffix(".report.json") + ".json")
|
||||||
path.write_text(
|
path.write_text(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
@@ -24,7 +25,7 @@ def _write_report(
|
|||||||
"variant": "high",
|
"variant": "high",
|
||||||
"instruction_profile": profile,
|
"instruction_profile": profile,
|
||||||
"trial_index": trial,
|
"trial_index": trial,
|
||||||
"raw_result_path": str(path.with_suffix(".json")),
|
"raw_result_path": str(raw_result_path),
|
||||||
"workspace_path": str(
|
"workspace_path": str(
|
||||||
path.parent.parent / "workspaces" / path.stem
|
path.parent.parent / "workspaces" / path.stem
|
||||||
),
|
),
|
||||||
@@ -166,6 +167,35 @@ def test_find_report_files_prefers_raw_result_mtime_for_last(tmp_path: Path) ->
|
|||||||
assert find_report_files([challenge], last=1, sort_by="report") == [old_report]
|
assert find_report_files([challenge], last=1, sort_by="report") == [old_report]
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_report_files_uses_recorded_raw_result_path_for_last(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
from examples.agent_challenges.summarize_trials import find_report_files
|
||||||
|
|
||||||
|
challenge = tmp_path / "browser_click_challenge"
|
||||||
|
old_report = _write_report(challenge / "results" / "old.report.json")
|
||||||
|
new_report = _write_report(challenge / "results" / "new.report.json")
|
||||||
|
actual_old_raw = challenge / "raw" / "actual-old.json"
|
||||||
|
actual_new_raw = challenge / "raw" / "actual-new.json"
|
||||||
|
actual_old_raw.parent.mkdir()
|
||||||
|
actual_old_raw.write_text("{}", encoding="utf-8")
|
||||||
|
actual_new_raw.write_text("{}", encoding="utf-8")
|
||||||
|
for report, raw_path in (
|
||||||
|
(old_report, actual_old_raw),
|
||||||
|
(new_report, actual_new_raw),
|
||||||
|
):
|
||||||
|
payload = json.loads(report.read_text(encoding="utf-8"))
|
||||||
|
payload["identity"]["raw_result_path"] = str(raw_path)
|
||||||
|
report.write_text(json.dumps(payload), encoding="utf-8")
|
||||||
|
|
||||||
|
os.utime(actual_old_raw, (1000, 1000))
|
||||||
|
os.utime(actual_new_raw, (2000, 2000))
|
||||||
|
os.utime(old_report, (3000, 3000))
|
||||||
|
os.utime(new_report, (1500, 1500))
|
||||||
|
|
||||||
|
assert find_report_files([challenge], last=1) == [new_report]
|
||||||
|
|
||||||
|
|
||||||
def test_summarize_trials_direct_script_execution_smoke(tmp_path: Path) -> None:
|
def test_summarize_trials_direct_script_execution_smoke(tmp_path: Path) -> None:
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
|||||||
@@ -1581,6 +1581,61 @@ async def test_bind_draft_local_output_to_quoted_workflow_output_field(
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_bind_draft_replaces_previous_public_output_for_local_field(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
artifact_store = FileWorkflowArtifactStore(tmp_path / "drafts_rebind_output")
|
||||||
|
api, _service, authoring = _draft_api(artifact_store, register_echo=True)
|
||||||
|
draft = _echo_draft()
|
||||||
|
draft["state_schema"] = {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"old": {"type": "string"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
draft["output_schema"] = {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"old": {"type": "string"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
draft["steps"]["echo"]["output"] = [{"source": "echoed", "target": "state.old"}]
|
||||||
|
draft["output"] = [{"path": "state.old", "target": "old"}]
|
||||||
|
await api.create_draft_workspace(workspace_id="report", draft=draft)
|
||||||
|
|
||||||
|
result = await authoring.bind_draft(
|
||||||
|
workspace_id="report",
|
||||||
|
revision=1,
|
||||||
|
step_id="echo",
|
||||||
|
source_path="local.echoed",
|
||||||
|
target_path="output.echoed",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "valid"
|
||||||
|
workspace = await api.get_draft_workspace(workspace_id="report", include_draft=True)
|
||||||
|
assert workspace["draft"]["output"] == [
|
||||||
|
{"path": "state.echoed", "target": "echoed"}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_remove_draft_binding_rejects_non_object_entries(tmp_path: Path) -> None:
|
||||||
|
artifact_store = FileWorkflowArtifactStore(tmp_path / "drafts_bad_bindings")
|
||||||
|
api, _service, authoring = _draft_api(artifact_store, register_echo=True)
|
||||||
|
draft = _echo_draft()
|
||||||
|
draft["steps"]["echo"]["input"] = ["not-an-object"]
|
||||||
|
await api.create_draft_workspace(workspace_id="bad", draft=draft)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="input binding entries.*objects"):
|
||||||
|
await authoring.remove_draft_binding(
|
||||||
|
workspace_id="bad",
|
||||||
|
revision=1,
|
||||||
|
step_id="echo",
|
||||||
|
inputs=["text"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_validate_draft_workspace_omits_unusable_nested_input_repair_hint(
|
async def test_validate_draft_workspace_omits_unusable_nested_input_repair_hint(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
@@ -1698,3 +1753,39 @@ async def test_validate_draft_workspace_hints_input_schema_projection(
|
|||||||
"wf draft bind wait_ws --revision 1 "
|
"wf draft bind wait_ws --revision 1 "
|
||||||
"--step wait --from input.undeclared --to local.text"
|
"--step wait --from input.undeclared --to local.text"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_validate_draft_workspace_hints_state_schema_projection(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
artifact_store = FileWorkflowArtifactStore(tmp_path / "drafts_state_schema_hint")
|
||||||
|
api, _service, _authoring = _draft_api(artifact_store, register_echo=True)
|
||||||
|
await api.create_draft_workspace(
|
||||||
|
workspace_id="wait_ws",
|
||||||
|
draft={
|
||||||
|
"name": "wait",
|
||||||
|
"input_schema": {"type": "object", "properties": {}},
|
||||||
|
"state_schema": {"type": "object", "properties": {}},
|
||||||
|
"output_schema": {"type": "object", "properties": {}},
|
||||||
|
"start": "wait",
|
||||||
|
"steps": {
|
||||||
|
"wait": {
|
||||||
|
"use": "demo.personal.echo_tool",
|
||||||
|
"input": [{"target": "text", "path": "state.undeclared"}],
|
||||||
|
"output": [],
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"routes": {"wait": {"ok": "__end__"}},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
payload = await api.validate_draft_workspace(workspace_id="wait_ws")
|
||||||
|
|
||||||
|
diagnostic = next(
|
||||||
|
item for item in payload["diagnostics"] if item["code"] == "invalid_source_path"
|
||||||
|
)
|
||||||
|
assert diagnostic["repair_hint"] == (
|
||||||
|
"wf draft bind wait_ws --revision 1 "
|
||||||
|
"--step wait --from state.undeclared --to local.text"
|
||||||
|
)
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ from typer.testing import CliRunner
|
|||||||
import wf_cli.context as cli_context
|
import wf_cli.context as cli_context
|
||||||
from wf_api.models import RawWorkflowPlan
|
from wf_api.models import RawWorkflowPlan
|
||||||
from wf_cli.app import app
|
from wf_cli.app import app
|
||||||
from wf_cli.commands import drafts as drafts_module
|
|
||||||
from wf_cli.context import CliContext, load_cli_context, load_local_cli_context
|
from wf_cli.context import CliContext, load_cli_context, load_local_cli_context
|
||||||
from wf_core import END
|
from wf_core import END
|
||||||
from wf_server import build_local_static_workflow_server
|
from wf_server import build_local_static_workflow_server
|
||||||
@@ -1177,29 +1176,27 @@ def test_wf_draft_focused_edit_commands_use_rpc_target(monkeypatch, tmp_path) ->
|
|||||||
|
|
||||||
|
|
||||||
def test_wf_draft_set_workflow_output_uses_rpc_target(monkeypatch, tmp_path) -> None:
|
def test_wf_draft_set_workflow_output_uses_rpc_target(monkeypatch, tmp_path) -> None:
|
||||||
calls: list[dict[str, object]] = []
|
server = build_local_static_workflow_server(tmp_path / "store")
|
||||||
|
_patch_rpc_client_to_server(monkeypatch, server)
|
||||||
class FakeDrafts:
|
config_path = tmp_path / "wf.json"
|
||||||
async def set_workflow_output_map(self, **kwargs: object) -> dict[str, object]:
|
config_path.write_text('{"version": 1}', encoding="utf-8")
|
||||||
calls.append(kwargs)
|
|
||||||
return {"workspace_id": "report", "revision": 2}
|
|
||||||
|
|
||||||
def _fake_context(ctx: object) -> CliContext:
|
|
||||||
return CliContext(
|
|
||||||
config_path=Path("dummy"),
|
|
||||||
service=None,
|
|
||||||
handlers=FakeDrafts(), # type: ignore[arg-type]
|
|
||||||
source_admin=cast(Any, object()),
|
|
||||||
admin=cast(Any, object()),
|
|
||||||
)
|
|
||||||
|
|
||||||
monkeypatch.setattr(drafts_module, "load_cli_context", _fake_context)
|
|
||||||
runner = CliRunner()
|
runner = CliRunner()
|
||||||
|
base_args = ["--config", str(config_path), "--url", "http://test/rpc"]
|
||||||
|
created = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
*base_args,
|
||||||
|
"draft",
|
||||||
|
"create",
|
||||||
|
"report",
|
||||||
|
"--capability",
|
||||||
|
"wf.std.constant",
|
||||||
|
],
|
||||||
|
)
|
||||||
result = runner.invoke(
|
result = runner.invoke(
|
||||||
app,
|
app,
|
||||||
[
|
[
|
||||||
"--url",
|
*base_args,
|
||||||
"http://example.test/rpc",
|
|
||||||
"draft",
|
"draft",
|
||||||
"set-workflow-output",
|
"set-workflow-output",
|
||||||
"report",
|
"report",
|
||||||
@@ -1209,16 +1206,16 @@ def test_wf_draft_set_workflow_output_uses_rpc_target(monkeypatch, tmp_path) ->
|
|||||||
"state.markdown=markdown",
|
"state.markdown=markdown",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
inspected = runner.invoke(
|
||||||
|
app,
|
||||||
|
[*base_args, "draft", "inspect", "report", "--include-draft"],
|
||||||
|
)
|
||||||
|
|
||||||
assert result.exit_code == 0
|
assert created.exit_code == 0, created.output
|
||||||
assert calls == [
|
assert result.exit_code == 0, result.output
|
||||||
{
|
assert inspected.exit_code == 0, inspected.output
|
||||||
"workspace_id": "report",
|
draft = json.loads(inspected.output)["draft"]
|
||||||
"revision": 1,
|
assert draft["output"] == [{"path": "state.markdown", "target": "markdown"}]
|
||||||
"output_map": {"state.markdown": "markdown"},
|
|
||||||
"merge": False,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_wf_draft_remove_route_uses_rpc_target(monkeypatch, tmp_path) -> None:
|
def test_wf_draft_remove_route_uses_rpc_target(monkeypatch, tmp_path) -> None:
|
||||||
@@ -1263,6 +1260,13 @@ def test_wf_draft_remove_route_uses_rpc_target(monkeypatch, tmp_path) -> None:
|
|||||||
assert result.exit_code == 0, result.output
|
assert result.exit_code == 0, result.output
|
||||||
payload = json.loads(result.output)
|
payload = json.loads(result.output)
|
||||||
assert payload["revision"] == 2
|
assert payload["revision"] == 2
|
||||||
|
inspected = runner.invoke(
|
||||||
|
app,
|
||||||
|
[*base_args, "draft", "inspect", "remove_route_ws", "--include-draft"],
|
||||||
|
)
|
||||||
|
assert inspected.exit_code == 0, inspected.output
|
||||||
|
draft = json.loads(inspected.output)["draft"]
|
||||||
|
assert "ok" not in draft["routes"]["call"]
|
||||||
|
|
||||||
|
|
||||||
def test_wf_draft_bind_uses_rpc_target(monkeypatch, tmp_path) -> None:
|
def test_wf_draft_bind_uses_rpc_target(monkeypatch, tmp_path) -> None:
|
||||||
@@ -1309,6 +1313,15 @@ def test_wf_draft_bind_uses_rpc_target(monkeypatch, tmp_path) -> None:
|
|||||||
assert result.exit_code == 0, result.output
|
assert result.exit_code == 0, result.output
|
||||||
payload = json.loads(result.output)
|
payload = json.loads(result.output)
|
||||||
assert payload["revision"] == 2
|
assert payload["revision"] == 2
|
||||||
|
inspected = runner.invoke(
|
||||||
|
app,
|
||||||
|
[*base_args, "draft", "inspect", "snapshot_ws", "--include-draft"],
|
||||||
|
)
|
||||||
|
assert inspected.exit_code == 0, inspected.output
|
||||||
|
draft = json.loads(inspected.output)["draft"]
|
||||||
|
assert draft["steps"]["call"]["output"] == [
|
||||||
|
{"source": "value", "target": "state.result"}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_wf_draft_add_step_from_capability_uses_rpc_target(
|
def test_wf_draft_add_step_from_capability_uses_rpc_target(
|
||||||
|
|||||||
@@ -126,6 +126,18 @@ def test_server_exposes_upstream_admin_and_workflow_tools() -> None:
|
|||||||
]
|
]
|
||||||
assert "output_map" in set_workflow_output_request["properties"]
|
assert "output_map" in set_workflow_output_request["properties"]
|
||||||
assert "merge" in set_workflow_output_request["properties"]
|
assert "merge" in set_workflow_output_request["properties"]
|
||||||
|
bind_schema = tools_by_name["wf.workflow.bind"].inputSchema
|
||||||
|
bind_request = bind_schema["properties"]["request"]
|
||||||
|
assert set(bind_request["required"]) == {
|
||||||
|
"workspace_id",
|
||||||
|
"revision",
|
||||||
|
"step_id",
|
||||||
|
"source_path",
|
||||||
|
"target_path",
|
||||||
|
}
|
||||||
|
assert bind_request["properties"]["step_id"]["minLength"] == 1
|
||||||
|
assert bind_request["properties"]["source_path"]["minLength"] == 1
|
||||||
|
assert bind_request["properties"]["target_path"]["minLength"] == 1
|
||||||
add_step_schema = tools_by_name[
|
add_step_schema = tools_by_name[
|
||||||
"wf.workflow.add_step_from_capability"
|
"wf.workflow.add_step_from_capability"
|
||||||
].inputSchema
|
].inputSchema
|
||||||
|
|||||||
@@ -41,6 +41,10 @@ def test_server_search_mode_pins_stable_control_and_workflow_tools() -> None:
|
|||||||
assert "wf.workflow.set_step_input_map" in names
|
assert "wf.workflow.set_step_input_map" in names
|
||||||
assert "wf.workflow.set_step_output_map" in names
|
assert "wf.workflow.set_step_output_map" in names
|
||||||
assert "wf.workflow.set_workflow_output_map" in names
|
assert "wf.workflow.set_workflow_output_map" in names
|
||||||
|
assert "wf.workflow.bind" in names
|
||||||
|
assert "wf.workflow.remove_draft_route" in names
|
||||||
|
assert "wf.workflow.remove_draft_step" in names
|
||||||
|
assert "wf.workflow.remove_draft_binding" in names
|
||||||
assert "wf.workflow.create_minimal_draft_workspace" in names
|
assert "wf.workflow.create_minimal_draft_workspace" in names
|
||||||
assert "wf.workflow.create_artifact_from_workspace" in names
|
assert "wf.workflow.create_artifact_from_workspace" in names
|
||||||
assert "wf.workflow.create_wrapper_from_workspace" in names
|
assert "wf.workflow.create_wrapper_from_workspace" in names
|
||||||
|
|||||||
@@ -589,6 +589,7 @@ async def test_rpc_client_draft_remove_methods(tmp_path) -> None:
|
|||||||
assert calls[1]["method"] == "workflow.draft_workspaces.remove_step"
|
assert calls[1]["method"] == "workflow.draft_workspaces.remove_step"
|
||||||
assert calls[2]["method"] == "workflow.draft_workspaces.remove_binding"
|
assert calls[2]["method"] == "workflow.draft_workspaces.remove_binding"
|
||||||
assert calls[2]["params"]["inputs"] == ["message"]
|
assert calls[2]["params"]["inputs"] == ["message"]
|
||||||
|
assert calls[2]["params"]["outputs"] == ["debug"]
|
||||||
|
|
||||||
|
|
||||||
async def test_rpc_client_draft_workspace_add_step_from_capability(tmp_path) -> None:
|
async def test_rpc_client_draft_workspace_add_step_from_capability(tmp_path) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user