docs: prepare agent challenge reruns

This commit is contained in:
lda
2026-06-28 01:39:54 +07:00 Verified
parent 74b66c9db7
commit 67ada5cbc4
8 changed files with 53 additions and 13 deletions
@@ -171,8 +171,8 @@ uv run wf --config wf.python.config.json cap call local.ops.echo --input '{"text
Turn it into a saved workflow:
```powershell
uv run wf --config wf.python.config.json draft create-from-capability `
python_echo_ws local.ops.echo --name python_echo
uv run wf --config wf.python.config.json draft create `
python_echo_ws --capability local.ops.echo --name python_echo
uv run wf --config wf.python.config.json draft save python_echo_ws `
--artifact python_echo `
+2 -2
View File
@@ -240,8 +240,8 @@ uv run wf --config wf.python.config.json cap call local.ops.echo --input '{"text
Turn it into a saved workflow:
```powershell
uv run wf --config wf.python.config.json draft create-from-capability `
python_echo_ws local.ops.echo --name python_echo
uv run wf --config wf.python.config.json draft create `
python_echo_ws --capability local.ops.echo --name python_echo
uv run wf --config wf.python.config.json draft save python_echo_ws `
--artifact python_echo `
+3 -3
View File
@@ -1018,7 +1018,7 @@ The draft path demonstrates agent-oriented authoring. A draft workspace can be
seeded from one capability's input and output schemas:
```powershell
wf draft create-from-capability report_ws local.report.extract_report
wf draft create report_ws --capability local.report.extract_report
```
That command is intentionally a best-effort bootstrap, not a complete workflow
@@ -1550,13 +1550,13 @@ cap call local.report.extract_report `
## Draft Bootstrap And Focused Edits
`create-from-capability` is a best-effort bootstrap. It creates a one-step
`wf draft create --capability` is a best-effort bootstrap. It creates a one-step
draft from the selected capability's wrapper hints. Focused commands then cover
common edits without requiring the agent to write RFC 6902 patches by hand.
```powershell
uv run wf --config examples/report_workflow/wf.config.json `
draft create-from-capability report_ws local.report.extract_report `
draft create report_ws --capability local.report.extract_report `
--name report_case_study --title "Report Case Study"
uv run wf --config examples/report_workflow/wf.config.json `
+1 -1
View File
@@ -411,7 +411,7 @@ wf config validate
-> wf source list
-> wf source resources / prompts
-> wf cap list / inspect / call
-> wf draft create-from-capability
-> wf draft create --capability
-> wf draft save
-> wf deploy save / validate
-> wf run start
+3 -3
View File
@@ -64,9 +64,9 @@ clear operator feedback before adding more architecture.
compact JSON outlines for agent discovery, and emits valid self-contained
JSON Schema with `--verbose`.
- Completed: `wf draft bind --from ... --to ...` composes input/state/output
schema projection with step binding merge, replacing the narrower
`bind-output-to-state` helper and reducing manual draft patch repairs in
agent challenge runs.
schema projection with step binding merge, replacing the prior narrower
output-to-state helper and reducing manual draft patch repairs in agent
challenge runs.
- Completed: draft CLI vocabulary now uses `wf draft create --capability` and
`wf draft add-step --capability`, replacing the longer
`*-from-capability` commands that agents repeatedly guessed around.
+14
View File
@@ -0,0 +1,14 @@
# Agent Challenge Review Notes
Before reviewing, grading, or summarizing challenge runs, read:
- [`docs/runbooks/agent-challenge-evaluation.md`](../../docs/runbooks/agent-challenge-evaluation.md)
Before changing or running a specific challenge, also read that challenge's
local docs:
- `browser_click_challenge/README.md`
- `report_workflow_challenge/README.md`
Keep challenge-specific instructions in each challenge's `challenge-prompt.md`.
Keep shared harness behavior in the root `examples/agent_challenges` modules.
+10 -2
View File
@@ -44,7 +44,15 @@ def _dict(value: object) -> dict[str, Any]:
return value if isinstance(value, dict) else {}
def _attempts(agent_self_report: dict[str, Any]) -> str:
def _attempts(agent_self_report: dict[str, Any], manual_audit: dict[str, Any]) -> str:
# Manual audits are authoritative for aggregate tables; the raw self-report
# remains visible in final-report.md for traceability.
audit_evidence = _dict(manual_audit.get("evidence"))
audit_total = audit_evidence.get("attempts_total")
audit_failed = audit_evidence.get("attempts_failed")
if isinstance(audit_total, int) and isinstance(audit_failed, int):
return f"{audit_failed}/{audit_total}"
attempts = _dict(agent_self_report.get("attempts"))
total = attempts.get("total")
failed = attempts.get("failed")
@@ -86,7 +94,7 @@ def load_trial_summary(path: Path) -> TrialSummary:
validity=_string(outcome.get("evaluation_validity")),
duration_seconds=_float(outcome.get("duration_seconds")),
tokens_total=_int(tokens.get("total")),
attempts=_attempts(self_report),
attempts=_attempts(self_report, manual_audit),
read_flags=_read_flags(self_report),
notes=" ".join(notes.split()),
)
@@ -77,6 +77,24 @@ def test_load_trial_summary_uses_short_labels_and_manual_outcome(
assert summary.notes == "Pass | with newline and spacing."
def test_load_trial_summary_prefers_manual_attempt_evidence(
tmp_path: Path,
) -> None:
from examples.agent_challenges.summarize_trials import load_trial_summary
report = _write_report(tmp_path / "results" / "trial.report.json")
payload = json.loads(report.read_text(encoding="utf-8"))
payload["manual_audit"]["evidence"] = {
"attempts_total": 3,
"attempts_failed": 2,
}
report.write_text(json.dumps(payload), encoding="utf-8")
summary = load_trial_summary(report)
assert summary.attempts == "2/3"
def test_render_markdown_escapes_table_cells(tmp_path: Path) -> None:
from examples.agent_challenges.summarize_trials import (
load_trial_summary,