docs: prepare agent challenge reruns
This commit is contained in:
@@ -171,8 +171,8 @@ uv run wf --config wf.python.config.json cap call local.ops.echo --input '{"text
|
||||
Turn it into a saved workflow:
|
||||
|
||||
```powershell
|
||||
uv run wf --config wf.python.config.json draft create-from-capability `
|
||||
python_echo_ws local.ops.echo --name python_echo
|
||||
uv run wf --config wf.python.config.json draft create `
|
||||
python_echo_ws --capability local.ops.echo --name python_echo
|
||||
|
||||
uv run wf --config wf.python.config.json draft save python_echo_ws `
|
||||
--artifact python_echo `
|
||||
|
||||
@@ -240,8 +240,8 @@ uv run wf --config wf.python.config.json cap call local.ops.echo --input '{"text
|
||||
Turn it into a saved workflow:
|
||||
|
||||
```powershell
|
||||
uv run wf --config wf.python.config.json draft create-from-capability `
|
||||
python_echo_ws local.ops.echo --name python_echo
|
||||
uv run wf --config wf.python.config.json draft create `
|
||||
python_echo_ws --capability local.ops.echo --name python_echo
|
||||
|
||||
uv run wf --config wf.python.config.json draft save python_echo_ws `
|
||||
--artifact python_echo `
|
||||
|
||||
@@ -1018,7 +1018,7 @@ The draft path demonstrates agent-oriented authoring. A draft workspace can be
|
||||
seeded from one capability's input and output schemas:
|
||||
|
||||
```powershell
|
||||
wf draft create-from-capability report_ws local.report.extract_report
|
||||
wf draft create report_ws --capability local.report.extract_report
|
||||
```
|
||||
|
||||
That command is intentionally a best-effort bootstrap, not a complete workflow
|
||||
@@ -1550,13 +1550,13 @@ cap call local.report.extract_report `
|
||||
|
||||
## Draft Bootstrap And Focused Edits
|
||||
|
||||
`create-from-capability` is a best-effort bootstrap. It creates a one-step
|
||||
`wf draft create --capability` is a best-effort bootstrap. It creates a one-step
|
||||
draft from the selected capability's wrapper hints. Focused commands then cover
|
||||
common edits without requiring the agent to write RFC 6902 patches by hand.
|
||||
|
||||
```powershell
|
||||
uv run wf --config examples/report_workflow/wf.config.json `
|
||||
draft create-from-capability report_ws local.report.extract_report `
|
||||
draft create report_ws --capability local.report.extract_report `
|
||||
--name report_case_study --title "Report Case Study"
|
||||
|
||||
uv run wf --config examples/report_workflow/wf.config.json `
|
||||
|
||||
@@ -411,7 +411,7 @@ wf config validate
|
||||
-> wf source list
|
||||
-> wf source resources / prompts
|
||||
-> wf cap list / inspect / call
|
||||
-> wf draft create-from-capability
|
||||
-> wf draft create --capability
|
||||
-> wf draft save
|
||||
-> wf deploy save / validate
|
||||
-> wf run start
|
||||
|
||||
@@ -64,9 +64,9 @@ clear operator feedback before adding more architecture.
|
||||
compact JSON outlines for agent discovery, and emits valid self-contained
|
||||
JSON Schema with `--verbose`.
|
||||
- Completed: `wf draft bind --from ... --to ...` composes input/state/output
|
||||
schema projection with step binding merge, replacing the narrower
|
||||
`bind-output-to-state` helper and reducing manual draft patch repairs in
|
||||
agent challenge runs.
|
||||
schema projection with step binding merge, replacing the prior narrower
|
||||
output-to-state helper and reducing manual draft patch repairs in agent
|
||||
challenge runs.
|
||||
- Completed: draft CLI vocabulary now uses `wf draft create --capability` and
|
||||
`wf draft add-step --capability`, replacing the longer
|
||||
`*-from-capability` commands that agents repeatedly guessed around.
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# Agent Challenge Review Notes
|
||||
|
||||
Before reviewing, grading, or summarizing challenge runs, read:
|
||||
|
||||
- [`docs/runbooks/agent-challenge-evaluation.md`](../../docs/runbooks/agent-challenge-evaluation.md)
|
||||
|
||||
Before changing or running a specific challenge, also read that challenge's
|
||||
local docs:
|
||||
|
||||
- `browser_click_challenge/README.md`
|
||||
- `report_workflow_challenge/README.md`
|
||||
|
||||
Keep challenge-specific instructions in each challenge's `challenge-prompt.md`.
|
||||
Keep shared harness behavior in the root `examples/agent_challenges` modules.
|
||||
@@ -44,7 +44,15 @@ def _dict(value: object) -> dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _attempts(agent_self_report: dict[str, Any]) -> str:
|
||||
def _attempts(agent_self_report: dict[str, Any], manual_audit: dict[str, Any]) -> str:
|
||||
# Manual audits are authoritative for aggregate tables; the raw self-report
|
||||
# remains visible in final-report.md for traceability.
|
||||
audit_evidence = _dict(manual_audit.get("evidence"))
|
||||
audit_total = audit_evidence.get("attempts_total")
|
||||
audit_failed = audit_evidence.get("attempts_failed")
|
||||
if isinstance(audit_total, int) and isinstance(audit_failed, int):
|
||||
return f"{audit_failed}/{audit_total}"
|
||||
|
||||
attempts = _dict(agent_self_report.get("attempts"))
|
||||
total = attempts.get("total")
|
||||
failed = attempts.get("failed")
|
||||
@@ -86,7 +94,7 @@ def load_trial_summary(path: Path) -> TrialSummary:
|
||||
validity=_string(outcome.get("evaluation_validity")),
|
||||
duration_seconds=_float(outcome.get("duration_seconds")),
|
||||
tokens_total=_int(tokens.get("total")),
|
||||
attempts=_attempts(self_report),
|
||||
attempts=_attempts(self_report, manual_audit),
|
||||
read_flags=_read_flags(self_report),
|
||||
notes=" ".join(notes.split()),
|
||||
)
|
||||
|
||||
@@ -77,6 +77,24 @@ def test_load_trial_summary_uses_short_labels_and_manual_outcome(
|
||||
assert summary.notes == "Pass | with newline and spacing."
|
||||
|
||||
|
||||
def test_load_trial_summary_prefers_manual_attempt_evidence(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
from examples.agent_challenges.summarize_trials import load_trial_summary
|
||||
|
||||
report = _write_report(tmp_path / "results" / "trial.report.json")
|
||||
payload = json.loads(report.read_text(encoding="utf-8"))
|
||||
payload["manual_audit"]["evidence"] = {
|
||||
"attempts_total": 3,
|
||||
"attempts_failed": 2,
|
||||
}
|
||||
report.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
summary = load_trial_summary(report)
|
||||
|
||||
assert summary.attempts == "2/3"
|
||||
|
||||
|
||||
def test_render_markdown_escapes_table_cells(tmp_path: Path) -> None:
|
||||
from examples.agent_challenges.summarize_trials import (
|
||||
load_trial_summary,
|
||||
|
||||
Reference in New Issue
Block a user