fix: retain agent challenge parse errors
This commit is contained in:
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
# Support direct execution as `python examples/.../run_opencode_trials.py`.
|
# Support direct execution as `python examples/.../run_opencode_trials.py`.
|
||||||
# ruff: noqa: I001 - imports must stay after sys.path.insert
|
# ruff: noqa: I001 - imports must stay after sys.path.insert
|
||||||
@@ -135,7 +136,7 @@ def run_trial(
|
|||||||
*,
|
*,
|
||||||
index: int,
|
index: int,
|
||||||
results_dir: Path,
|
results_dir: Path,
|
||||||
) -> dict:
|
) -> dict[str, Any]:
|
||||||
return _generic_run_trial(
|
return _generic_run_trial(
|
||||||
config, index=index, results_dir=results_dir, classify_fn=classify_output
|
config, index=index, results_dir=results_dir, classify_fn=classify_output
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -139,13 +139,18 @@ def run_trial(
|
|||||||
return payload
|
return payload
|
||||||
|
|
||||||
parsed: dict[str, Any] | None
|
parsed: dict[str, Any] | None
|
||||||
|
parse_error: dict[str, str] | None = None
|
||||||
try:
|
try:
|
||||||
parsed = parse_opencode_output(completed.stdout)
|
parsed = parse_opencode_output(completed.stdout)
|
||||||
text = result_text(parsed)
|
text = result_text(parsed)
|
||||||
classification = classify_fn(text)
|
classification = classify_fn(text)
|
||||||
except Exception:
|
except Exception as exc:
|
||||||
parsed = None
|
parsed = None
|
||||||
classification = "parse_error"
|
classification = "parse_error"
|
||||||
|
parse_error = {
|
||||||
|
"type": type(exc).__name__,
|
||||||
|
"message": str(exc),
|
||||||
|
}
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"index": index,
|
"index": index,
|
||||||
@@ -158,6 +163,8 @@ def run_trial(
|
|||||||
"stderr": completed.stderr,
|
"stderr": completed.stderr,
|
||||||
"parsed": parsed,
|
"parsed": parsed,
|
||||||
}
|
}
|
||||||
|
if parse_error is not None:
|
||||||
|
payload["parse_error"] = parse_error
|
||||||
_write_trial_report(payload)
|
_write_trial_report(payload)
|
||||||
_write_trial_result(results_dir, config=config, index=index, payload=payload)
|
_write_trial_result(results_dir, config=config, index=index, payload=payload)
|
||||||
return payload
|
return payload
|
||||||
|
|||||||
@@ -254,6 +254,47 @@ def test_run_trial_records_report_save_error_for_timeout(
|
|||||||
assert saved_result["report_save_error"] == "result file is missing parsed output"
|
assert saved_result["report_save_error"] == "result file is missing parsed output"
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_trial_records_parse_error_details(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
) -> None:
|
||||||
|
prompt = tmp_path / "prompt.md"
|
||||||
|
prompt.write_text("hello", encoding="utf-8")
|
||||||
|
|
||||||
|
def fake_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]:
|
||||||
|
return subprocess.CompletedProcess(
|
||||||
|
args=["opencode"],
|
||||||
|
returncode=0,
|
||||||
|
stdout="not-json",
|
||||||
|
stderr="",
|
||||||
|
)
|
||||||
|
|
||||||
|
from examples.agent_challenges import runner as generic_runner
|
||||||
|
|
||||||
|
monkeypatch.setattr(generic_runner.subprocess, "run", fake_run)
|
||||||
|
config = TrialConfig(
|
||||||
|
model="opencode/mimo-v2.5-free",
|
||||||
|
variant="high",
|
||||||
|
prompt_path=prompt,
|
||||||
|
attach_url=None,
|
||||||
|
timeout_seconds=120,
|
||||||
|
wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
|
||||||
|
server_context="Use local CLI mode.",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = run_trial(config, index=1, results_dir=tmp_path / "results")
|
||||||
|
|
||||||
|
assert result["classification"] == "parse_error"
|
||||||
|
assert result["parse_error"]["type"] == "JSONDecodeError"
|
||||||
|
assert "Expecting value" in result["parse_error"]["message"]
|
||||||
|
saved_result = json.loads(
|
||||||
|
(tmp_path / "results" / "opencode_mimo-v2.5-free-trial-001.json").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert saved_result["parse_error"] == result["parse_error"]
|
||||||
|
|
||||||
|
|
||||||
def test_parse_opencode_output_reads_json_object() -> None:
|
def test_parse_opencode_output_reads_json_object() -> None:
|
||||||
payload = {
|
payload = {
|
||||||
"text": "wf run start demo.default\nbefore.clicked false\nafter.clicked true"
|
"text": "wf run start demo.default\nbefore.clicked false\nafter.clicked true"
|
||||||
|
|||||||
Reference in New Issue
Block a user