from __future__ import annotations import json import subprocess import sys from pathlib import Path import yaml from examples.agent_challenges import reports as generic_reports from examples.agent_challenges.audit import ( main as save_manual_audit_main, ) from examples.agent_challenges.manifests import load_challenge_manifest from examples.agent_challenges.models import InstructionProfile from examples.agent_challenges.opencode_io import ( build_opencode_command, parse_opencode_output, ) from examples.agent_challenges.reports import ( main as save_trial_report_main, ) from examples.agent_challenges.reports import ( report_from_result, save_report, ) from examples.agent_challenges.runner import main as generic_runner_main from examples.agent_challenges.runner import run_trial from examples.agent_challenges.workspace import ( ChallengeDef, TrialConfig, prepare_trial_workspace, prepare_v2_trial_workspace, render_prompt, server_command, starting_trial_index, trial_output_path, wf_command_prefix_for_config, write_trial_config, ) ROOT = Path(__file__).resolve().parents[2] LOCAL_WF_COMMAND_PREFIX = ( "uv run wf --config examples/browser_click_workflow/wf.config.json --local" ) BROWSER_CLICK_DEF = ChallengeDef( name="browser_click", source_root=ROOT / "examples" / "browser_click_workflow", source_id="local.browser_click", source_module="ops", source_registry="registry", store_root=".wf_browser_click_store", default_workspace_template=ROOT / "examples" / "agent_challenges" / "browser_click_challenge" / "workspace_template", default_workspaces_dir=ROOT / "examples" / "agent_challenges" / "browser_click_challenge" / "workspaces", default_results_dir=ROOT / "examples" / "agent_challenges" / "browser_click_challenge" / "results", default_prompt=ROOT / "examples" / "agent_challenges" / "browser_click_challenge" / "challenge-prompt.md", default_server_port=8772, server_config_arg="examples/browser_click_workflow/wf.config.json", ) def test_build_opencode_command_without_attach(tmp_path: Path) -> None: prompt = tmp_path / "prompt.md" prompt.write_text("hello", encoding="utf-8") config = TrialConfig( model="opencode/mimo-v2.5-free", variant="high", prompt_path=prompt, attach_url=None, timeout_seconds=120, wf_command_prefix=LOCAL_WF_COMMAND_PREFIX, server_context="Use local CLI mode.", ) command = build_opencode_command(config) assert command[:2] == ["opencode", "run"] assert "--attach" not in command assert "--format" in command assert "json" in command assert "--model" in command assert "opencode/mimo-v2.5-free" in command assert "hello" in command def test_build_opencode_command_with_attach(tmp_path: Path) -> None: prompt = tmp_path / "prompt.md" prompt.write_text("hello", encoding="utf-8") config = TrialConfig( model="opencode/deepseek-v3.1-free", variant="high", prompt_path=prompt, attach_url="http://127.0.0.1:4096", timeout_seconds=120, wf_command_prefix=LOCAL_WF_COMMAND_PREFIX, server_context="Use local CLI mode.", ) command = build_opencode_command(config) assert "--attach" in command assert "http://127.0.0.1:4096" in command def test_run_trial_saves_final_report_from_successful_result( tmp_path: Path, monkeypatch, ) -> None: workspace = tmp_path / "trial" workspace.mkdir() prompt = workspace / "prompt.md" prompt.write_text("hello", encoding="utf-8") workflow_file = workspace / "workflow.plan.json" report = "\n".join( [ "## Report", "", "```yaml", "challenge_report:", " used_product_path: true", " used_helper_script: false", f' workflow_file: "{workflow_file.as_posix()}"', ' deployment_id: "browser_click_case_study.default"', ' run_id: "run_123"', " before_clicked: false", " after_clicked: true", " run_failed: false", " leftover_processes: false", " read:", " skills: true", " docs: true", " product_code: false", " adjacent_attempts: false", " prior_store: false", " existing_solution: false", " attempts:", " total: 1", " failed: 0", " missed_requirements:", ' - "none"', ' notes: "ok"', "```", "", ] ) def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]: return subprocess.CompletedProcess( args=["opencode"], returncode=0, stdout=json.dumps({"text": report}), stderr="", ) from examples.agent_challenges import runner as generic_runner monkeypatch.setattr(generic_runner.subprocess, "run", fake_run) config = TrialConfig( model="opencode/mimo-v2.5-free", variant="high", prompt_path=prompt, attach_url=None, timeout_seconds=120, wf_command_prefix=LOCAL_WF_COMMAND_PREFIX, server_context="Use local CLI mode.", ) result = run_trial( config, index=1, results_dir=tmp_path / "results", classify_fn=lambda _text: "success", ) assert result["classification"] == "success" assert result["report_path"] == (workspace / "final-report.md").as_posix() assert (workspace / "final-report.md").read_text(encoding="utf-8") == ( report.rstrip() + "\n" ) saved_result = json.loads( (tmp_path / "results" / "opencode_mimo-v2.5-free-trial-001.json").read_text( encoding="utf-8" ) ) assert saved_result["report_path"] == (workspace / "final-report.md").as_posix() def test_run_trial_records_report_save_error_for_timeout( tmp_path: Path, monkeypatch, ) -> None: prompt = tmp_path / "prompt.md" prompt.write_text("hello", encoding="utf-8") def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]: raise subprocess.TimeoutExpired(cmd=["opencode"], timeout=120) from examples.agent_challenges import runner as generic_runner monkeypatch.setattr(generic_runner.subprocess, "run", fake_run) config = TrialConfig( model="opencode/mimo-v2.5-free", variant="high", prompt_path=prompt, attach_url=None, timeout_seconds=120, wf_command_prefix=LOCAL_WF_COMMAND_PREFIX, server_context="Use local CLI mode.", ) result = run_trial( config, index=1, results_dir=tmp_path / "results", classify_fn=lambda _text: "unknown", ) assert result["classification"] == "timeout" assert result["report_save_error"] == "result file is missing parsed output" saved_result = json.loads( (tmp_path / "results" / "opencode_mimo-v2.5-free-trial-001.json").read_text( encoding="utf-8" ) ) assert saved_result["report_save_error"] == "result file is missing parsed output" def test_run_trial_records_parse_error_details( tmp_path: Path, monkeypatch, ) -> None: prompt = tmp_path / "prompt.md" prompt.write_text("hello", encoding="utf-8") def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]: return subprocess.CompletedProcess( args=["opencode"], returncode=0, stdout="not-json", stderr="", ) from examples.agent_challenges import runner as generic_runner monkeypatch.setattr(generic_runner.subprocess, "run", fake_run) config = TrialConfig( model="opencode/mimo-v2.5-free", variant="high", prompt_path=prompt, attach_url=None, timeout_seconds=120, wf_command_prefix=LOCAL_WF_COMMAND_PREFIX, server_context="Use local CLI mode.", ) result = run_trial( config, index=1, results_dir=tmp_path / "results", classify_fn=lambda _text: "unknown", ) assert result["classification"] == "parse_error" assert result["parse_error"]["type"] == "JSONDecodeError" assert "Expecting value" in result["parse_error"]["message"] saved_result = json.loads( (tmp_path / "results" / "opencode_mimo-v2.5-free-trial-001.json").read_text( encoding="utf-8" ) ) assert saved_result["parse_error"] == result["parse_error"] def test_parse_opencode_output_reads_json_object() -> None: payload = { "text": "wf run start demo.default\nbefore.clicked false\nafter.clicked true" } parsed = parse_opencode_output(json.dumps(payload)) assert parsed["text"] == payload["text"] def test_parse_opencode_output_reads_last_jsonl_object() -> None: payload = "\n".join( [ json.dumps({"type": "log", "text": "starting"}), json.dumps({"type": "message", "text": "final"}), ] ) parsed = parse_opencode_output(payload) assert parsed["text"] == "final" def test_parse_opencode_output_prefers_text_event_before_step_finish() -> None: payload = "\n".join( [ json.dumps( { "type": "text", "part": { "type": "text", "text": "deployment id: demo.default\nbefore.clicked is false", }, } ), json.dumps({"type": "step_finish", "part": {"type": "step-finish"}}), ] ) parsed = parse_opencode_output(payload) assert parsed["text"] == "deployment id: demo.default\nbefore.clicked is false" def test_trial_output_path_is_zero_padded(tmp_path: Path) -> None: path = trial_output_path(tmp_path, model="opencode/mimo-v2.5-free", index=3) assert path.name == "opencode_mimo-v2.5-free-trial-003.json" def test_prepare_trial_workspace_copies_template_to_model_trial_dir( tmp_path: Path, ) -> None: template = tmp_path / "template" template.mkdir() (template / "prompt.md").write_text("prompt", encoding="utf-8") (template / ".gitignore").write_text(".wf_store/\n", encoding="utf-8") workspaces = tmp_path / "workspaces" source_root = tmp_path / "browser_click_workflow" source_root.mkdir() defn = ChallengeDef( name="browser_click", source_root=source_root, source_id="local.browser_click", source_module="ops", source_registry="registry", store_root=".wf_browser_click_store", default_workspace_template=template, default_workspaces_dir=workspaces, default_results_dir=tmp_path / "results", default_prompt=template / "prompt.md", default_server_port=8772, server_config_arg="examples/browser_click_workflow/wf.config.json", ) prepared = prepare_trial_workspace( defn, model="opencode/mimo-v2.5-free", index=7, workspaces_dir=workspaces, template_dir=template, source_root=source_root, ) assert prepared.root == workspaces / "opencode_mimo-v2.5-free-trial-007" config = json.loads(prepared.config_path.read_text(encoding="utf-8")) assert config["client"]["target"] == {"kind": "local"} assert config["server"]["store"] == { "kind": "filesystem", "root": ".wf_browser_click_store", } assert config["server"]["sources"][0] == { "kind": "python", "id": "local.browser_click", "path": "../../browser_click_workflow", "module": "ops", "registry": "registry", } assert prepared.prompt_path.read_text(encoding="utf-8") == "prompt" assert (prepared.root / ".gitignore").read_text(encoding="utf-8") == ".wf_store/\n" def test_save_trial_report_copies_report_into_workspace(tmp_path: Path) -> None: workspace = tmp_path / "trial" workspace.mkdir() output = save_report(workspace=workspace, report_text="# Report\n\nok\n") assert output == workspace / "final-report.md" assert output.read_text(encoding="utf-8") == "# Report\n\nok\n" def test_report_from_result_infers_workspace_from_prompt_path(tmp_path: Path) -> None: workspace = tmp_path / "trial" workspace.mkdir() result_path = tmp_path / "result.json" result_path.write_text( json.dumps( { "config": {"prompt_path": str(workspace / "prompt.md")}, "parsed": {"text": "# Report\n\nok"}, } ), encoding="utf-8", ) inferred_workspace, report_text = report_from_result(result_path) assert inferred_workspace == workspace assert report_text == "# Report\n\nok" def test_report_from_result_recovers_text_from_stdout_when_parsed_is_null( tmp_path: Path, ) -> None: workspace = tmp_path / "trial" workspace.mkdir() result_path = tmp_path / "result.json" result_path.write_text( json.dumps( { "config": {"prompt_path": str(workspace / "prompt.md")}, "parsed": None, "stdout": json.dumps({"type": "message", "text": "# Partial report"}), } ), encoding="utf-8", ) inferred_workspace, report_text = report_from_result(result_path) assert inferred_workspace == workspace assert report_text == "# Partial report" def test_save_trial_report_from_result_writes_inferred_workspace( tmp_path: Path, capsys, ) -> None: workspace = tmp_path / "trial" workspace.mkdir() result_path = tmp_path / "result.json" result_path.write_text( json.dumps( { "config": {"prompt_path": str(workspace / "prompt.md")}, "parsed": {"text": "# Report\n\nok"}, } ), encoding="utf-8", ) assert save_trial_report_main(["--from-result", str(result_path)]) == 0 assert (workspace / "final-report.md").read_text(encoding="utf-8") == ( "# Report\n\nok\n" ) assert (workspace / "final-report.md").as_posix() in capsys.readouterr().out def test_save_trial_report_requires_input_when_workspace_only( tmp_path: Path, monkeypatch, ) -> None: class _InteractiveStdin: def isatty(self) -> bool: return True def read(self) -> str: raise AssertionError("should not block on interactive stdin") workspace = tmp_path / "trial" workspace.mkdir() monkeypatch.setattr(generic_reports.sys, "stdin", _InteractiveStdin()) try: save_trial_report_main([str(workspace)]) except SystemExit as exc: assert exc.code == 2 else: raise AssertionError("expected argparse failure") def test_save_manual_audit_from_result_infers_report_and_applies_overrides( tmp_path: Path, capsys, ) -> None: workspace = tmp_path / "trial" workspace.mkdir() report_text = "\n".join( [ "## Report", "", "```yaml", "challenge_report:", " used_product_path: true", " used_helper_script: false", ' workflow_file: "workflow.plan.json"', ' deployment_id: "browser_click_case_study.default"', ' run_id: "run_123"', " before_clicked: false", " after_clicked: true", " run_failed: false", " leftover_processes: false", " read:", " skills: true", " docs: true", " product_code: false", " adjacent_attempts: false", " prior_store: false", " existing_solution: false", " attempts:", " total: 1", " failed: 0", " missed_requirements:", ' - "none"', ' notes: "agent said clean"', "```", "", ] ) result_path = tmp_path / "result.json" result_path.write_text( json.dumps( { "classification": "success", "duration_seconds": 42.5, "returncode": 0, "config": {"prompt_path": str(workspace / "prompt.md")}, "parsed": {"text": report_text}, } ), encoding="utf-8", ) assert ( save_manual_audit_main( [ "--from-result", str(result_path), "--manual-classification", "success_code_assisted", "--audited-at", "2026-06-16T00:00:00Z", "--set-read", "product_code=true", "--set-evidence", "trace_count=3", "--correction", "read.product_code: agent reported false, audited true", "--notes", "Valid product run, but code-assisted.", ] ) == 0 ) audit_path = workspace / "manual-audit.yaml" audit = yaml.safe_load(audit_path.read_text(encoding="utf-8"))["manual_audit"] assert audit["auto_classification"] == "success" assert audit["manual_classification"] == "success_code_assisted" assert audit["audited_at"] == "2026-06-16T00:00:00Z" assert audit["valid_product_run"] is True assert audit["product_path_used"] is True assert audit["helper_script_used"] is False assert audit["run_succeeded"] is True assert audit["evidence"]["deployment_id"] == "browser_click_case_study.default" assert audit["evidence"]["run_id"] == "run_123" assert audit["evidence"]["before_clicked"] is False assert audit["evidence"]["after_clicked"] is True assert audit["evidence"]["trace_count"] == 3 assert audit["read_flags"]["product_code"] is True assert audit["read_flags"]["adjacent_attempts"] is False assert audit["attempts"] == {"total": 1, "failed": 0} assert audit["corrections"] == [ "read.product_code: agent reported false, audited true" ] assert audit["notes"] == "Valid product run, but code-assisted." assert audit_path.as_posix() in capsys.readouterr().out def test_save_manual_audit_from_report_overrides_timeout_result_stream( tmp_path: Path, capsys, ) -> None: workspace = tmp_path / "trial" workspace.mkdir() report_path = workspace / "final-report.md" report_path.write_text( "\n".join( [ "```yaml", "challenge_report:", " used_product_path: true", " used_helper_script: false", ' workflow_file: "workflow.plan.json"', ' deployment_id: "browser_click_deployment"', ' run_id: "run_123"', " before_clicked: false", " after_clicked: true", " run_failed: false", " leftover_processes: false", " read:", " skills: false", " docs: true", " product_code: true", " adjacent_attempts: false", " prior_store: false", " existing_solution: true", " attempts:", " total: 1", " failed: 0", " missed_requirements:", ' - "none"', ' notes: "manual UI recovery"', "```", "", ] ), encoding="utf-8", ) result_path = tmp_path / "result.json" result_path.write_text( json.dumps( { "classification": "timeout", "duration_seconds": 1000, "returncode": None, "config": {"prompt_path": str(workspace / "prompt.md")}, "parsed": {"text": "stale event stream, no final report"}, } ), encoding="utf-8", ) assert ( save_manual_audit_main( [ "--from-result", str(result_path), "--from-report", str(report_path), "--manual-classification", "success_code_assisted", "--audited-at", "2026-06-16T00:00:00Z", ] ) == 0 ) audit_path = workspace / "manual-audit.yaml" audit = yaml.safe_load(audit_path.read_text(encoding="utf-8"))["manual_audit"] assert audit["auto_classification"] == "timeout" assert audit["manual_classification"] == "success_code_assisted" assert audit["valid_product_run"] is True assert audit["product_path_used"] is True assert audit["run_succeeded"] is True assert audit["evidence"]["deployment_id"] == "browser_click_deployment" assert audit["evidence"]["run_id"] == "run_123" assert audit["read_flags"]["product_code"] is True assert audit["agent_notes"] == "manual UI recovery" assert audit_path.as_posix() in capsys.readouterr().out def test_save_manual_audit_rejects_non_boolean_read_override(tmp_path: Path) -> None: workspace = tmp_path / "trial" workspace.mkdir() result_path = tmp_path / "result.json" result_path.write_text( json.dumps( { "classification": "unknown", "config": {"prompt_path": str(workspace / "prompt.md")}, "parsed": {"text": "no yaml"}, } ), encoding="utf-8", ) try: save_manual_audit_main( [ "--from-result", str(result_path), "--manual-classification", "invalid", "--set-read", "product_code=maybe", ] ) except SystemExit as exc: assert exc.code == 2 else: raise AssertionError("expected argparse failure") def test_prepare_trial_workspace_uses_next_available_directory( tmp_path: Path, ) -> None: template = tmp_path / "template" template.mkdir() (template / "prompt.md").write_text("prompt", encoding="utf-8") source_root = tmp_path / "browser_click_workflow" source_root.mkdir() workspaces = tmp_path / "workspaces" first = workspaces / "opencode_mimo-v2.5-free-trial-001" stale = first / "old-answer.json" first.mkdir(parents=True) stale.write_text("stale", encoding="utf-8") defn = ChallengeDef( name="browser_click", source_root=source_root, source_id="local.browser_click", source_module="ops", source_registry="registry", store_root=".wf_browser_click_store", default_workspace_template=template, default_workspaces_dir=workspaces, default_results_dir=tmp_path / "results", default_prompt=template / "prompt.md", default_server_port=8772, server_config_arg="examples/browser_click_workflow/wf.config.json", ) next_index = starting_trial_index( model="opencode/mimo-v2.5-free", results_dir=tmp_path / "results", workspaces_dir=workspaces, ) prepared = prepare_trial_workspace( defn, model="opencode/mimo-v2.5-free", index=next_index, workspaces_dir=workspaces, template_dir=template, source_root=source_root, ) assert prepared.root == workspaces / "opencode_mimo-v2.5-free-trial-002" assert prepared.root.exists() assert stale.read_text(encoding="utf-8") == "stale" def test_main_uses_custom_workspace_template_and_source_root( tmp_path: Path, monkeypatch, capsys, ) -> None: template = tmp_path / "template" template.mkdir() (template / "prompt.md").write_text("custom prompt", encoding="utf-8") source_root = tmp_path / "source" source_root.mkdir() workspaces = tmp_path / "workspaces" results = tmp_path / "results" def fake_run_trial( config: TrialConfig, *, index: int, results_dir: Path, classify_fn: object = None, ) -> dict[str, object]: return { "index": index, "classification": "success", "returncode": 0, "duration_seconds": 1.0, "report_path": config.prompt_path.parent / "final-report.md", } from examples.agent_challenges import runner as generic_runner monkeypatch.setattr(generic_runner, "run_trial", fake_run_trial) assert ( generic_runner_main( BROWSER_CLICK_DEF, lambda _text: "success", [ "--model", "check/model", "--trials", "1", "--workspace-template", str(template), "--source-root", str(source_root), "--workspaces-dir", str(workspaces), "--results-dir", str(results), ], ) == 0 ) workspace = workspaces / "check_model-trial-001" assert workspace.exists() assert (workspace / "prompt.md").read_text(encoding="utf-8") == "custom prompt" config = json.loads((workspace / "wf.config.json").read_text(encoding="utf-8")) assert config["server"]["sources"][0]["path"] == "../../source" assert '"success_count": 1' in capsys.readouterr().out def test_starting_trial_index_accounts_for_existing_results_and_workspaces( tmp_path: Path, ) -> None: results = tmp_path / "results" workspaces = tmp_path / "workspaces" results.mkdir() workspaces.mkdir() (results / "opencode_mimo-v2.5-free-trial-003.json").write_text( "{}", encoding="utf-8", ) (workspaces / "opencode_mimo-v2.5-free-trial-005").mkdir() (workspaces / "other_model-trial-099").mkdir() assert ( starting_trial_index( model="opencode/mimo-v2.5-free", results_dir=results, workspaces_dir=workspaces, ) == 6 ) def test_wf_command_prefix_for_config_uses_repo_relative_path() -> None: config_path = ( Path("examples") / "agent_challenges" / "browser_click_challenge" / "workspaces" / "opencode_mimo-v2.5-free-trial-001" / "wf.config.json" ) prefix = wf_command_prefix_for_config(config_path) assert prefix == ( "uv run wf --config " "examples/agent_challenges/browser_click_challenge/workspaces/" "opencode_mimo-v2.5-free-trial-001/wf.config.json --local" ) def test_render_prompt_injects_command_prefix_and_server_context( tmp_path: Path, ) -> None: prompt = tmp_path / "prompt.md" prompt.write_text( "Use {{wf_command_prefix}}. {{server_context}}", encoding="utf-8", ) rendered = render_prompt( prompt, wf_command_prefix=LOCAL_WF_COMMAND_PREFIX, server_context="No RPC server is staged.", ) assert LOCAL_WF_COMMAND_PREFIX in rendered assert "No RPC server is staged." in rendered def test_server_command_uses_example_config_and_requested_port() -> None: command = server_command( port=8765, config_arg="examples/browser_click_workflow/wf.config.json" ) assert command[:3] == ["uv", "run", "wf-rpc-server"] assert "--config" in command assert "examples/browser_click_workflow/wf.config.json" in command assert "--port" in command assert "8765" in command # --- New generic-module tests --- def test_generic_workspace_preparation_writes_config_for_arbitrary_challenge_def( tmp_path: Path, ) -> None: template = tmp_path / "template" template.mkdir() (template / "prompt.md").write_text("test prompt", encoding="utf-8") workspaces = tmp_path / "workspaces" source_root = tmp_path / "my_source" source_root.mkdir() defn = ChallengeDef( name="custom_challenge", source_root=source_root, source_id="local.custom", source_module="custom_ops", source_registry="custom_registry", store_root=".custom_store", default_workspace_template=template, default_workspaces_dir=workspaces, default_results_dir=tmp_path / "results", default_prompt=template / "prompt.md", default_server_port=9999, server_config_arg="examples/custom/wf.config.json", ) ws = prepare_trial_workspace( defn, model="test-model", index=1, ) assert ws.root == workspaces / "test-model-trial-001" config = json.loads(ws.config_path.read_text(encoding="utf-8")) assert config["client"]["target"] == {"kind": "local"} assert config["server"]["store"] == {"kind": "filesystem", "root": ".custom_store"} assert config["server"]["sources"][0] == { "kind": "python", "id": "local.custom", "path": "../../my_source", "module": "custom_ops", "registry": "custom_registry", } assert (ws.root / "prompt.md").read_text(encoding="utf-8") == "test prompt" def test_browser_click_wrapper_produces_expected_paths_and_command_prefix() -> None: assert BROWSER_CLICK_DEF.name == "browser_click" assert BROWSER_CLICK_DEF.source_id == "local.browser_click" assert BROWSER_CLICK_DEF.server_config_arg == ( "examples/browser_click_workflow/wf.config.json" ) assert LOCAL_WF_COMMAND_PREFIX == ( "uv run wf --config examples/browser_click_workflow/wf.config.json --local" ) assert BROWSER_CLICK_DEF.default_prompt.name == "challenge-prompt.md" assert BROWSER_CLICK_DEF.default_prompt.parent.name == "browser_click_challenge" prompt = BROWSER_CLICK_DEF.default_prompt.read_text(encoding="utf-8") assert "Debug profile only: include ux_issues_found here" in prompt def test_generic_runner_can_be_configured_with_fake_challenge_and_fake_opencode( tmp_path: Path, monkeypatch, ) -> None: from examples.agent_challenges.runner import main as generic_main template = tmp_path / "template" template.mkdir() (template / "prompt.md").write_text("fake {{wf_command_prefix}}", encoding="utf-8") source_root = tmp_path / "source" source_root.mkdir() workspaces = tmp_path / "workspaces" results = tmp_path / "results" defn = ChallengeDef( name="fake_challenge", source_root=source_root, source_id="local.fake", source_module="fake_ops", source_registry="fake_registry", store_root=".fake_store", default_workspace_template=template, default_workspaces_dir=workspaces, default_results_dir=results, default_prompt=template / "prompt.md", default_server_port=9000, server_config_arg="fake/config.json", ) def classify_fn(text: str) -> str: if "success" in text.lower(): return "success" return "unknown" sent_text = json.dumps({"text": "success!"}) def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]: return subprocess.CompletedProcess( args=["opencode"], returncode=0, stdout=sent_text, stderr="", ) from examples.agent_challenges import runner as generic_runner monkeypatch.setattr(generic_runner.subprocess, "run", fake_run) exit_code = generic_main( defn, classify_fn, [ "--model", "fake/model", "--trials", "1", ], ) assert exit_code == 0 assert (workspaces / "fake_model-trial-001").exists() assert (results / "fake_model-trial-001.json").exists() def test_generic_write_trial_config_with_custom_source_root( tmp_path: Path, ) -> None: config_path = tmp_path / "wf.config.json" source_root = tmp_path / "custom_root" source_root.mkdir() defn = ChallengeDef( name="test", source_root=source_root, source_id="local.test", source_module="test_mod", source_registry="test_reg", store_root=".test_store", default_workspace_template=tmp_path / "template", default_workspaces_dir=tmp_path / "workspaces", default_results_dir=tmp_path / "results", default_prompt=tmp_path / "template" / "prompt.md", default_server_port=8000, server_config_arg="test/config.json", ) write_trial_config(config_path, defn=defn) config = json.loads(config_path.read_text(encoding="utf-8")) assert config["server"]["sources"][0] == { "kind": "python", "id": "local.test", "path": "custom_root", "module": "test_mod", "registry": "test_reg", } BROWSER_CHALLENGE = ( ROOT / "examples" / "agent_challenges" / "browser_click_challenge" / "challenge.yaml" ) def test_browser_click_manifest_declares_task_success_contract() -> None: loaded = load_challenge_manifest(BROWSER_CHALLENGE) assert loaded.manifest.id == "browser_click" assert loaded.manifest.source.id == "local.browser_click" assert loaded.manifest.report.success_assertions == { "before_clicked": False, "after_clicked": True, "run_failed": False, "leftover_processes": False, } def test_browser_click_workspace_uses_generic_profile_copy(tmp_path: Path) -> None: loaded = load_challenge_manifest(BROWSER_CHALLENGE) bundle = ROOT / "examples/agent_challenges/instruction_bundles/workflow_cli.yaml" workspace = prepare_v2_trial_workspace( loaded, profile=InstructionProfile.SKILLS, model="opencode/test", index=1, workspaces_dir=tmp_path, instruction_bundle=bundle, ) assert workspace.config_path.is_file() assert (workspace.root / ".agent/skills/wf-cli/SKILL.md").is_file() run_input = json.loads((workspace.root / "run-input.json").read_text()) assert run_input == { "button_label": "Launch Workflow", "open_browser": False, "simulate": True, "timeout_seconds": 2, } assert "safe fixture inputs" in (workspace.root / "TASK_FILES.md").read_text( encoding="utf-8" ) def test_central_runner_accepts_browser_challenge() -> None: result = subprocess.run( [ sys.executable, "examples/agent_challenges/run_trials.py", "--help", ], cwd=ROOT, text=True, capture_output=True, check=False, ) assert result.returncode == 0, result.stderr assert "--challenge" in result.stdout assert "--instruction-profile" in result.stdout