feat: auto-save browser challenge reports

examples/agent_challenges/browser_click_challenge/reports.py

@@ -8,7 +8,10 @@ from pathlib import Path
 from examples.agent_challenges.browser_click_challenge.classification import (
     extract_challenge_report,
 )
-from examples.agent_challenges.browser_click_challenge.opencode_io import _result_text
+from examples.agent_challenges.browser_click_challenge.opencode_io import (
+    _result_text,
+    parse_opencode_output,
+)
 
 
 def save_report(
@@ -35,14 +38,44 @@ def report_from_result(result_path: Path) -> tuple[Path, str]:
     result = json.loads(result_path.read_text(encoding="utf-8"))
     if not isinstance(result, dict):
         raise ValueError("result file must contain a JSON object")
-    parsed = result.get("parsed")
+    report_text = _report_text_from_result(result)
-    if not isinstance(parsed, dict):
-        raise ValueError("result file is missing parsed output")
-    report_text = _result_text(parsed)
     workspace = _workspace_from_result(result, report_text)
     return workspace, report_text
 
 
+def save_report_from_result_payload(
+    result: dict[str, object],
+    *,
+    output_name: str = "final-report.md",
+) -> Path:
+    """Save final report text from an in-memory harness result payload."""
+    report_text = _report_text_from_result(result)
+    workspace = _workspace_from_result(result, report_text)
+    return save_report(
+        workspace=workspace,
+        report_text=report_text,
+        output_name=output_name,
+    )
+
+
+def _report_text_from_result(result: dict[str, object]) -> str:
+    parsed = result.get("parsed")
+    if isinstance(parsed, dict):
+        return _result_text(parsed)
+
+    stdout = result.get("stdout")
+    if isinstance(stdout, str) and stdout.strip():
+        try:
+            recovered = parse_opencode_output(stdout)
+        except ValueError as exc:
+            raise ValueError(
+                "result file is missing parsed output and stdout has no report text"
+            ) from exc
+        return _result_text(recovered)
+
+    raise ValueError("result file is missing parsed output")
+
+
 def _workspace_from_result(result: dict[str, object], report_text: str) -> Path:
     config = result.get("config")
     if isinstance(config, dict):
@@ -61,6 +94,10 @@ def _workspace_from_result(result: dict[str, object], report_text: str) -> Path:
 
 def _read_report_text(input_file: Path | None) -> str:
     if input_file is None:
+        if sys.stdin.isatty():
+            raise ValueError(
+                "manual report mode needs piped stdin, --input-file, or --from-result"
+            )
         return sys.stdin.read()
     return input_file.read_text(encoding="utf-8")
 

examples/agent_challenges/browser_click_challenge/run_opencode_trials.py

@@ -50,6 +50,9 @@ from examples.agent_challenges.browser_click_challenge.opencode_io import (  # n
     build_opencode_command,
     parse_opencode_output,
 )
+from examples.agent_challenges.browser_click_challenge.reports import (  # noqa: E402
+    save_report_from_result_payload,
+)
 
 __all__ = [
     "CHALLENGE_DIR",
@@ -85,6 +88,7 @@ __all__ = [
     "render_prompt",
     "rpc_url_for_port",
     "run_trial",
+    "save_report_from_result_payload",
     "server_command",
     "start_server",
     "starting_trial_index",
@@ -282,6 +286,7 @@ def run_trial(config: TrialConfig, *, index: int, results_dir: Path) -> dict[str
             "stderr": exc.stderr or "",
             "parsed": None,
         }
+        _write_trial_report(payload)
         _write_trial_result(results_dir, config=config, index=index, payload=payload)
         return payload
 
@@ -305,6 +310,7 @@ def run_trial(config: TrialConfig, *, index: int, results_dir: Path) -> dict[str
         "stderr": completed.stderr,
         "parsed": parsed,
     }
+    _write_trial_report(payload)
     _write_trial_result(results_dir, config=config, index=index, payload=payload)
     return payload
 
@@ -327,6 +333,15 @@ def _write_trial_result(
     path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
 
 
+def _write_trial_report(payload: dict[str, Any]) -> None:
+    try:
+        report_path = save_report_from_result_payload(payload)
+    except ValueError as exc:
+        payload["report_save_error"] = str(exc)
+        return
+    payload["report_path"] = report_path.as_posix()
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--model", default="opencode/mimo-v2.5-free")
@@ -422,6 +437,8 @@ def main(argv: list[str] | None = None) -> int:
                     "classification": result["classification"],
                     "returncode": result["returncode"],
                     "duration_seconds": round(float(result["duration_seconds"]), 3),
+                    "report_path": result.get("report_path"),
+                    "report_save_error": result.get("report_save_error"),
                 }
             )
             print(json.dumps(summaries[-1], sort_keys=True))

tests/examples/test_opencode_browser_click_challenge.py

@@ -1,8 +1,13 @@
 from __future__ import annotations
 
 import json
+import subprocess
 from pathlib import Path
 
+from examples.agent_challenges.browser_click_challenge import (
+    reports,
+    run_opencode_trials,
+)
 from examples.agent_challenges.browser_click_challenge.challenge import (
     LOCAL_WF_COMMAND_PREFIX,
     TrialConfig,
@@ -25,6 +30,7 @@ from examples.agent_challenges.browser_click_challenge.reports import (
 )
 from examples.agent_challenges.browser_click_challenge.run_opencode_trials import (
     prepare_trial_workspace,
+    run_trial,
     starting_trial_index,
     trial_output_path,
     wf_command_prefix_for_config,
@@ -104,6 +110,115 @@ def test_build_opencode_command_with_attach(tmp_path: Path) -> None:
     assert "http://127.0.0.1:4096" in command
 
 
+def test_run_trial_saves_final_report_from_successful_result(
+    tmp_path: Path,
+    monkeypatch,
+) -> None:
+    workspace = tmp_path / "trial"
+    workspace.mkdir()
+    prompt = workspace / "prompt.md"
+    prompt.write_text("hello", encoding="utf-8")
+    workflow_file = workspace / "workflow.plan.json"
+    report = "\n".join(
+        [
+            "## Report",
+            "",
+            "```yaml",
+            "challenge_report:",
+            "  used_product_path: true",
+            "  used_helper_script: false",
+            f'  workflow_file: "{workflow_file.as_posix()}"',
+            '  deployment_id: "browser_click_case_study.default"',
+            '  run_id: "run_123"',
+            "  before_clicked: false",
+            "  after_clicked: true",
+            "  run_failed: false",
+            "  leftover_processes: false",
+            "  read:",
+            "    skills: true",
+            "    docs: true",
+            "    product_code: false",
+            "    adjacent_attempts: false",
+            "    prior_store: false",
+            "    existing_solution: false",
+            "  attempts:",
+            "    total: 1",
+            "    failed: 0",
+            "  missed_requirements:",
+            '    - "none"',
+            '  notes: "ok"',
+            "```",
+            "",
+        ]
+    )
+
+    def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]:
+        return subprocess.CompletedProcess(
+            args=["opencode"],
+            returncode=0,
+            stdout=json.dumps({"text": report}),
+            stderr="",
+        )
+
+    monkeypatch.setattr(run_opencode_trials.subprocess, "run", fake_run)
+    config = TrialConfig(
+        model="opencode/mimo-v2.5-free",
+        variant="high",
+        prompt_path=prompt,
+        attach_url=None,
+        timeout_seconds=120,
+        wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
+        server_context="Use local CLI mode.",
+    )
+
+    result = run_trial(config, index=1, results_dir=tmp_path / "results")
+
+    assert result["classification"] == "success"
+    assert result["report_path"] == (workspace / "final-report.md").as_posix()
+    assert (workspace / "final-report.md").read_text(encoding="utf-8") == (
+        report.rstrip() + "\n"
+    )
+    saved_result = json.loads(
+        (tmp_path / "results" / "opencode_mimo-v2.5-free-trial-001.json").read_text(
+            encoding="utf-8"
+        )
+    )
+    assert saved_result["report_path"] == (workspace / "final-report.md").as_posix()
+
+
+def test_run_trial_records_report_save_error_for_timeout(
+    tmp_path: Path,
+    monkeypatch,
+) -> None:
+    prompt = tmp_path / "prompt.md"
+    prompt.write_text("hello", encoding="utf-8")
+
+    def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]:
+        raise subprocess.TimeoutExpired(cmd=["opencode"], timeout=120)
+
+    monkeypatch.setattr(run_opencode_trials.subprocess, "run", fake_run)
+    config = TrialConfig(
+        model="opencode/mimo-v2.5-free",
+        variant="high",
+        prompt_path=prompt,
+        attach_url=None,
+        timeout_seconds=120,
+        wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
+        server_context="Use local CLI mode.",
+    )
+
+    result = run_trial(config, index=1, results_dir=tmp_path / "results")
+
+    assert result["classification"] == "timeout"
+    assert result["report_save_error"] == "result file is missing parsed output"
+    saved_result = json.loads(
+        (tmp_path / "results" / "opencode_mimo-v2.5-free-trial-001.json").read_text(
+            encoding="utf-8"
+        )
+    )
+    assert saved_result["report_save_error"] == "result file is missing parsed output"
+
+
 def test_parse_opencode_output_reads_json_object() -> None:
     payload = {
         "text": "wf run start demo.default\nbefore.clicked false\nafter.clicked true"
@@ -374,6 +489,29 @@ def test_report_from_result_infers_workspace_from_prompt_path(tmp_path: Path) ->
     assert report_text == "# Report\n\nok"
 
 
+def test_report_from_result_recovers_text_from_stdout_when_parsed_is_null(
+    tmp_path: Path,
+) -> None:
+    workspace = tmp_path / "trial"
+    workspace.mkdir()
+    result_path = tmp_path / "result.json"
+    result_path.write_text(
+        json.dumps(
+            {
+                "config": {"prompt_path": str(workspace / "prompt.md")},
+                "parsed": None,
+                "stdout": json.dumps({"type": "message", "text": "# Partial report"}),
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    inferred_workspace, report_text = report_from_result(result_path)
+
+    assert inferred_workspace == workspace
+    assert report_text == "# Partial report"
+
+
 def test_save_trial_report_from_result_writes_inferred_workspace(
     tmp_path: Path,
     capsys,
@@ -399,6 +537,29 @@ def test_save_trial_report_from_result_writes_inferred_workspace(
     assert (workspace / "final-report.md").as_posix() in capsys.readouterr().out
 
 
+def test_save_trial_report_requires_input_when_workspace_only(
+    tmp_path: Path,
+    monkeypatch,
+) -> None:
+    class _InteractiveStdin:
+        def isatty(self) -> bool:
+            return True
+
+        def read(self) -> str:
+            raise AssertionError("should not block on interactive stdin")
+
+    workspace = tmp_path / "trial"
+    workspace.mkdir()
+    monkeypatch.setattr(reports.sys, "stdin", _InteractiveStdin())
+
+    try:
+        save_trial_report_main([str(workspace)])
+    except SystemExit as exc:
+        assert exc.code == 2
+    else:
+        raise AssertionError("expected argparse failure")
+
+
 def test_prepare_trial_workspace_uses_next_available_directory(
     tmp_path: Path,
 ) -> None: