lda.chat/lda-wf
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: record opencode resume metadata

Browse Source
This commit is contained in:
lda
2026-06-30 15:11:50 +07:00 Verified
parent 30ad99fca1
commit 04b1cf30f9
11 changed files with 1639 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/examples/test_agent_challenge_harness_v2.py
+146
View File
@@ -1942,3 +1942,149 @@ def test_base_prompt_mentions_self_report_rules() -> None:
assert "read.product_code" in text
assert "read.existing_solution" in text
assert "read.adjacent_attempts" in text
def test_v2_runner_stores_opencode_resume_metadata(tmp_path: Path) -> None:
from examples.agent_challenges.models import InstructionProfile
from examples.agent_challenges.runner import run_v2_trial
challenge = load_challenge_manifest(_write_manifest(tmp_path / "challenge"))
bundle = ROOT / "examples/agent_challenges/instruction_bundles/workflow_cli.yaml"
stdout = (
'{"type":"step_start","sessionID":"ses_runner"}\n'
'{"type":"text","sessionID":"ses_runner","text":"```yaml\\n'
"challenge_report:\\n"
" used_product_path: true\\n"
" used_helper_script: false\\n"
" before_clicked: false\\n"
" after_clicked: true\\n"
" run_failed: false\\n"
" leftover_processes: false\\n"
'```"}\n'
)
def fake_run(
command: list[str],
*,
cwd: str,
text: bool,
capture_output: bool,
timeout: float | None,
check: bool,
) -> object:
return type(
"Result",
(),
{"returncode": 0, "stdout": stdout, "stderr": ""},
)()
result = run_v2_trial(
challenge,
profile=InstructionProfile.NONE,
model="opencode/deepseek-v4-flash-free",
variant="max",
index=1,
results_dir=tmp_path / "results",
workspaces_dir=tmp_path / "workspaces",
instruction_bundle=bundle,
attach_url="http://127.0.0.1:8192/",
run_fn=fake_run,
)
opencode = result["opencode"]
assert opencode["session_id"] == "ses_runner"
assert opencode["attach_url"] == "http://127.0.0.1:8192/"
assert opencode["model"] == "opencode/deepseek-v4-flash-free"
assert opencode["variant"] == "max"
assert opencode["resume_command"][0:4] == [
"opencode",
"run",
"--session",
"ses_runner",
]
assert "challenge_report" in opencode["resume_prompt"]
def test_v2_runner_resume_prompt_sees_assertion_failures(tmp_path: Path) -> None:
from examples.agent_challenges.models import InstructionProfile
from examples.agent_challenges.runner import run_v2_trial
challenge = load_challenge_manifest(_write_manifest(tmp_path / "challenge"))
bundle = ROOT / "examples/agent_challenges/instruction_bundles/workflow_cli.yaml"
stdout = (
'{"type":"step_start","sessionID":"ses_runner"}\n'
'{"type":"text","sessionID":"ses_runner","text":"run completed"}\n'
)
def fake_run(
command: list[str],
*,
cwd: str,
text: bool,
capture_output: bool,
timeout: float | None,
check: bool,
) -> object:
return type(
"Result",
(),
{"returncode": 0, "stdout": stdout, "stderr": ""},
)()
result = run_v2_trial(
challenge,
profile=InstructionProfile.NONE,
model="opencode/deepseek-v4-flash-free",
variant="max",
index=1,
results_dir=tmp_path / "results",
workspaces_dir=tmp_path / "workspaces",
instruction_bundle=bundle,
run_fn=fake_run,
)
opencode = result["opencode"]
assert result["task_outcome"] == "failed"
assert "assertion_failures" in result
assert "do not continue coding" in opencode["resume_prompt"].lower()
def test_v2_runner_stores_null_session_when_stdout_has_no_session(
tmp_path: Path,
) -> None:
from examples.agent_challenges.models import InstructionProfile
from examples.agent_challenges.runner import run_v2_trial
challenge = load_challenge_manifest(_write_manifest(tmp_path / "challenge"))
bundle = ROOT / "examples/agent_challenges/instruction_bundles/workflow_cli.yaml"
def fake_run(
command: list[str],
*,
cwd: str,
text: bool,
capture_output: bool,
timeout: float | None,
check: bool,
) -> object:
return type(
"Result",
(),
{"returncode": 1, "stdout": "", "stderr": "boom"},
)()
result = run_v2_trial(
challenge,
profile=InstructionProfile.NONE,
model="opencode/deepseek-v4-flash-free",
variant="max",
index=1,
results_dir=tmp_path / "results",
workspaces_dir=tmp_path / "workspaces",
instruction_bundle=bundle,
run_fn=fake_run,
)
opencode = result["opencode"]
assert opencode["session_id"] is None
assert opencode["resume_command"] is None
tests/examples/test_agent_challenge_reports.py
+39
View File
@@ -577,3 +577,42 @@ def test_build_report_raises_on_missing_paths(tmp_path: Path) -> None:
with pytest.raises(ValueError, match="raw_result_path"):
build_trial_report(result, audit=None)
def test_trial_report_renders_opencode_resume_metadata(tmp_path: Path) -> None:
from examples.agent_challenges.report_models import build_trial_report
from examples.agent_challenges.reports import render_trial_report_markdown
result = _raw_result(tmp_path)
result["opencode"] = {
"attach_url": "http://127.0.0.1:8192/",
"command": ["opencode", "run", "--format", "json", "prompt"],
"model": "opencode/deepseek-v4-flash-free",
"variant": "max",
"session_id": "ses_report",
"resume_prompt": "continue?",
"resume_command": [
"opencode",
"run",
"--session",
"ses_report",
"--attach",
"http://127.0.0.1:8192/",
"--format",
"json",
"--model",
"opencode/deepseek-v4-flash-free",
"--variant",
"max",
"continue?",
],
}
report = build_trial_report(result, audit=None)
rendered = render_trial_report_markdown(report)
machine = report.model_dump(mode="json")
assert machine["opencode"]["session_id"] == "ses_report"
assert "## OpenCode Resume" in rendered
assert "ses_report" in rendered
assert "opencode run --session ses_report" in rendered
tests/examples/test_agent_challenge_resume.py
+205
View File
@@ -0,0 +1,205 @@
from __future__ import annotations
import json
from pathlib import Path
import pytest
from examples.agent_challenges.opencode_resume import (
build_resume_command,
display_resume_command,
extract_session_id,
resume_prompt_for_result,
resume_result_path,
)
def _event(**payload: object) -> str:
return json.dumps(payload, separators=(",", ":"))
def test_extract_session_id_reads_top_level_session_id() -> None:
stdout = "\n".join(
[
_event(type="step_start", sessionID="ses_abc"),
_event(type="text", sessionID="ses_def", text="later"),
]
)
assert extract_session_id(stdout) == "ses_abc"
def test_extract_session_id_reads_nested_part_session_id() -> None:
stdout = _event(type="step_start", part={"sessionID": "ses_nested"})
assert extract_session_id(stdout) == "ses_nested"
def test_extract_session_id_returns_none_for_empty_or_malformed_stdout() -> None:
assert extract_session_id("") is None
assert extract_session_id("not json\n{}") is None
def test_resume_prompt_asks_continue_for_timeout_with_partial_stdout() -> None:
prompt = resume_prompt_for_result(
{
"task_outcome": "timeout",
"stdout": _event(type="step_start", sessionID="ses_abc"),
}
)
assert "continue" in prompt.lower()
assert "do not restart" in prompt.lower()
def test_resume_prompt_asks_for_final_report_when_work_is_done_but_report_missing() -> (
None
):
prompt = resume_prompt_for_result(
{
"task_outcome": "failed",
"assertion_failures": [
"could not extract challenge report for required_fields evaluation"
],
"stdout": _event(type="text", sessionID="ses_abc", text="run completed"),
}
)
assert "do not continue coding" in prompt.lower()
assert "challenge_report" in prompt
def test_build_resume_command_includes_attach_session_model_variant_and_prompt() -> (
None
):
command = build_resume_command(
session_id="ses_abc",
attach_url="http://127.0.0.1:8192/",
model="opencode/deepseek-v4-flash-free",
variant="max",
prompt="continue?",
)
assert command == [
"opencode",
"run",
"--session",
"ses_abc",
"--attach",
"http://127.0.0.1:8192/",
"--format",
"json",
"--model",
"opencode/deepseek-v4-flash-free",
"--variant",
"max",
"continue?",
]
def test_display_resume_command_quotes_prompt_with_spaces() -> None:
rendered = display_resume_command(
["opencode", "run", "--session", "ses_cli", "continue this trial"]
)
assert rendered == 'opencode run --session ses_cli "continue this trial"'
def test_resume_result_path_uses_next_resume_index(tmp_path: Path) -> None:
original = tmp_path / "trial.json"
original.write_text("{}", encoding="utf-8")
(tmp_path / "trial.resume-001.json").write_text("{}", encoding="utf-8")
assert resume_result_path(original).name == "trial.resume-002.json"
def test_resume_trial_prints_resume_command(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
from examples.agent_challenges.resume_trial import main
result_path = tmp_path / "trial.json"
result_path.write_text(
json.dumps(
{
"model": "opencode/deepseek-v4-flash-free",
"variant": "max",
"opencode": {
"attach_url": "http://127.0.0.1:8192/",
"session_id": "ses_cli",
"resume_prompt": "continue?",
"resume_command": [
"opencode",
"run",
"--session",
"ses_cli",
"--attach",
"http://127.0.0.1:8192/",
"--format",
"json",
"--model",
"opencode/deepseek-v4-flash-free",
"--variant",
"max",
"continue?",
],
},
}
),
encoding="utf-8",
)
assert main(["--from-result", str(result_path), "--print-command"]) == 0
output = capsys.readouterr().out
assert "opencode run --session ses_cli" in output
def test_resume_trial_run_writes_resume_result(tmp_path: Path) -> None:
from subprocess import CompletedProcess
from examples.agent_challenges.resume_trial import resume_from_result
result_path = tmp_path / "trial.json"
result_path.write_text(
json.dumps(
{
"model": "opencode/deepseek-v4-flash-free",
"variant": "max",
"opencode": {
"attach_url": None,
"session_id": "ses_cli",
"resume_prompt": "continue?",
"resume_command": [
"opencode",
"run",
"--session",
"ses_cli",
"--format",
"json",
"--model",
"opencode/deepseek-v4-flash-free",
"--variant",
"max",
"continue?",
],
},
}
),
encoding="utf-8",
)
calls: list[list[str]] = []
def fake_run(command: list[str], **kwargs: object) -> CompletedProcess[str]:
calls.append(command)
return CompletedProcess(
command, 0, stdout='{"type":"text","text":"done"}\n', stderr=""
)
output_path = resume_from_result(result_path, run_fn=fake_run)
assert output_path.name == "trial.resume-001.json"
assert calls[0][0:4] == ["opencode", "run", "--session", "ses_cli"]
payload = json.loads(output_path.read_text(encoding="utf-8"))
assert payload["source_result_path"] == str(result_path.resolve())
assert payload["stdout"].strip()