lda.chat/lda-wf
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: auto-save browser challenge reports

Browse Source
This commit is contained in:
lda
2026-06-15 21:15:31 +07:00 Verified
parent 66eaa4a67d
commit 008103ac15
3 changed files with 220 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/agent_challenges/browser_click_challenge/reports.py
+42 -5
View File
@@ -8,7 +8,10 @@ from pathlib import Path
from examples.agent_challenges.browser_click_challenge.classification import (
extract_challenge_report,
)
from examples.agent_challenges.browser_click_challenge.opencode_io import _result_text
from examples.agent_challenges.browser_click_challenge.opencode_io import (
_result_text,
parse_opencode_output,
)
def save_report(
@@ -35,14 +38,44 @@ def report_from_result(result_path: Path) -> tuple[Path, str]:
result = json.loads(result_path.read_text(encoding="utf-8"))
if not isinstance(result, dict):
raise ValueError("result file must contain a JSON object")
parsed = result.get("parsed")
if not isinstance(parsed, dict):
raise ValueError("result file is missing parsed output")
report_text = _result_text(parsed)
report_text = _report_text_from_result(result)
workspace = _workspace_from_result(result, report_text)
return workspace, report_text
def save_report_from_result_payload(
result: dict[str, object],
*,
output_name: str = "final-report.md",
) -> Path:
"""Save final report text from an in-memory harness result payload."""
report_text = _report_text_from_result(result)
workspace = _workspace_from_result(result, report_text)
return save_report(
workspace=workspace,
report_text=report_text,
output_name=output_name,
)
def _report_text_from_result(result: dict[str, object]) -> str:
parsed = result.get("parsed")
if isinstance(parsed, dict):
return _result_text(parsed)
stdout = result.get("stdout")
if isinstance(stdout, str) and stdout.strip():
try:
recovered = parse_opencode_output(stdout)
except ValueError as exc:
raise ValueError(
"result file is missing parsed output and stdout has no report text"
) from exc
return _result_text(recovered)
raise ValueError("result file is missing parsed output")
def _workspace_from_result(result: dict[str, object], report_text: str) -> Path:
config = result.get("config")
if isinstance(config, dict):
@@ -61,6 +94,10 @@ def _workspace_from_result(result: dict[str, object], report_text: str) -> Path:
def _read_report_text(input_file: Path | None) -> str:
if input_file is None:
if sys.stdin.isatty():
raise ValueError(
"manual report mode needs piped stdin, --input-file, or --from-result"
)
return sys.stdin.read()
return input_file.read_text(encoding="utf-8")
examples/agent_challenges/browser_click_challenge/run_opencode_trials.py
+17
View File
@@ -50,6 +50,9 @@ from examples.agent_challenges.browser_click_challenge.opencode_io import ( # n
build_opencode_command,
parse_opencode_output,
)
from examples.agent_challenges.browser_click_challenge.reports import ( # noqa: E402
save_report_from_result_payload,
)
__all__ = [
"CHALLENGE_DIR",
@@ -85,6 +88,7 @@ __all__ = [
"render_prompt",
"rpc_url_for_port",
"run_trial",
"save_report_from_result_payload",
"server_command",
"start_server",
"starting_trial_index",
@@ -282,6 +286,7 @@ def run_trial(config: TrialConfig, *, index: int, results_dir: Path) -> dict[str
"stderr": exc.stderr or "",
"parsed": None,
}
_write_trial_report(payload)
_write_trial_result(results_dir, config=config, index=index, payload=payload)
return payload
@@ -305,6 +310,7 @@ def run_trial(config: TrialConfig, *, index: int, results_dir: Path) -> dict[str
"stderr": completed.stderr,
"parsed": parsed,
}
_write_trial_report(payload)
_write_trial_result(results_dir, config=config, index=index, payload=payload)
return payload
@@ -327,6 +333,15 @@ def _write_trial_result(
path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
def _write_trial_report(payload: dict[str, Any]) -> None:
try:
report_path = save_report_from_result_payload(payload)
except ValueError as exc:
payload["report_save_error"] = str(exc)
return
payload["report_path"] = report_path.as_posix()
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", default="opencode/mimo-v2.5-free")
@@ -422,6 +437,8 @@ def main(argv: list[str] | None = None) -> int:
"classification": result["classification"],
"returncode": result["returncode"],
"duration_seconds": round(float(result["duration_seconds"]), 3),
"report_path": result.get("report_path"),
"report_save_error": result.get("report_save_error"),
}
)
print(json.dumps(summaries[-1], sort_keys=True))
tests/examples/test_opencode_browser_click_challenge.py
+161
View File
@@ -1,8 +1,13 @@
from __future__ import annotations
import json
import subprocess
from pathlib import Path
from examples.agent_challenges.browser_click_challenge import (
reports,
run_opencode_trials,
)
from examples.agent_challenges.browser_click_challenge.challenge import (
LOCAL_WF_COMMAND_PREFIX,
TrialConfig,
@@ -25,6 +30,7 @@ from examples.agent_challenges.browser_click_challenge.reports import (
)
from examples.agent_challenges.browser_click_challenge.run_opencode_trials import (
prepare_trial_workspace,
run_trial,
starting_trial_index,
trial_output_path,
wf_command_prefix_for_config,
@@ -104,6 +110,115 @@ def test_build_opencode_command_with_attach(tmp_path: Path) -> None:
assert "http://127.0.0.1:4096" in command
def test_run_trial_saves_final_report_from_successful_result(
tmp_path: Path,
monkeypatch,
) -> None:
workspace = tmp_path / "trial"
workspace.mkdir()
prompt = workspace / "prompt.md"
prompt.write_text("hello", encoding="utf-8")
workflow_file = workspace / "workflow.plan.json"
report = "\n".join(
[
"## Report",
"",
"```yaml",
"challenge_report:",
" used_product_path: true",
" used_helper_script: false",
f' workflow_file: "{workflow_file.as_posix()}"',
' deployment_id: "browser_click_case_study.default"',
' run_id: "run_123"',
" before_clicked: false",
" after_clicked: true",
" run_failed: false",
" leftover_processes: false",
" read:",
" skills: true",
" docs: true",
" product_code: false",
" adjacent_attempts: false",
" prior_store: false",
" existing_solution: false",
" attempts:",
" total: 1",
" failed: 0",
" missed_requirements:",
' - "none"',
' notes: "ok"',
"```",
"",
]
)
def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]:
return subprocess.CompletedProcess(
args=["opencode"],
returncode=0,
stdout=json.dumps({"text": report}),
stderr="",
)
monkeypatch.setattr(run_opencode_trials.subprocess, "run", fake_run)
config = TrialConfig(
model="opencode/mimo-v2.5-free",
variant="high",
prompt_path=prompt,
attach_url=None,
timeout_seconds=120,
wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
server_context="Use local CLI mode.",
)
result = run_trial(config, index=1, results_dir=tmp_path / "results")
assert result["classification"] == "success"
assert result["report_path"] == (workspace / "final-report.md").as_posix()
assert (workspace / "final-report.md").read_text(encoding="utf-8") == (
report.rstrip() + "\n"
)
saved_result = json.loads(
(tmp_path / "results" / "opencode_mimo-v2.5-free-trial-001.json").read_text(
encoding="utf-8"
)
)
assert saved_result["report_path"] == (workspace / "final-report.md").as_posix()
def test_run_trial_records_report_save_error_for_timeout(
tmp_path: Path,
monkeypatch,
) -> None:
prompt = tmp_path / "prompt.md"
prompt.write_text("hello", encoding="utf-8")
def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]:
raise subprocess.TimeoutExpired(cmd=["opencode"], timeout=120)
monkeypatch.setattr(run_opencode_trials.subprocess, "run", fake_run)
config = TrialConfig(
model="opencode/mimo-v2.5-free",
variant="high",
prompt_path=prompt,
attach_url=None,
timeout_seconds=120,
wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
server_context="Use local CLI mode.",
)
result = run_trial(config, index=1, results_dir=tmp_path / "results")
assert result["classification"] == "timeout"
assert result["report_save_error"] == "result file is missing parsed output"
saved_result = json.loads(
(tmp_path / "results" / "opencode_mimo-v2.5-free-trial-001.json").read_text(
encoding="utf-8"
)
)
assert saved_result["report_save_error"] == "result file is missing parsed output"
def test_parse_opencode_output_reads_json_object() -> None:
payload = {
"text": "wf run start demo.default\nbefore.clicked false\nafter.clicked true"
@@ -374,6 +489,29 @@ def test_report_from_result_infers_workspace_from_prompt_path(tmp_path: Path) ->
assert report_text == "# Report\n\nok"
def test_report_from_result_recovers_text_from_stdout_when_parsed_is_null(
tmp_path: Path,
) -> None:
workspace = tmp_path / "trial"
workspace.mkdir()
result_path = tmp_path / "result.json"
result_path.write_text(
json.dumps(
{
"config": {"prompt_path": str(workspace / "prompt.md")},
"parsed": None,
"stdout": json.dumps({"type": "message", "text": "# Partial report"}),
}
),
encoding="utf-8",
)
inferred_workspace, report_text = report_from_result(result_path)
assert inferred_workspace == workspace
assert report_text == "# Partial report"
def test_save_trial_report_from_result_writes_inferred_workspace(
tmp_path: Path,
capsys,
@@ -399,6 +537,29 @@ def test_save_trial_report_from_result_writes_inferred_workspace(
assert (workspace / "final-report.md").as_posix() in capsys.readouterr().out
def test_save_trial_report_requires_input_when_workspace_only(
tmp_path: Path,
monkeypatch,
) -> None:
class _InteractiveStdin:
def isatty(self) -> bool:
return True
def read(self) -> str:
raise AssertionError("should not block on interactive stdin")
workspace = tmp_path / "trial"
workspace.mkdir()
monkeypatch.setattr(reports.sys, "stdin", _InteractiveStdin())
try:
save_trial_report_main([str(workspace)])
except SystemExit as exc:
assert exc.code == 2
else:
raise AssertionError("expected argparse failure")
def test_prepare_trial_workspace_uses_next_available_directory(
tmp_path: Path,
) -> None: