feat: record opencode resume metadata
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
FINAL_REPORT_PROMPT = (
|
||||
"Your workflow attempt is over. Do not continue coding. Return only the "
|
||||
"final challenge report using the required challenge_report YAML schema. "
|
||||
"Include run_id, evidence, failed attempts, read flags, missed requirements, "
|
||||
"and whether the run succeeded."
|
||||
)
|
||||
|
||||
CONTINUE_PROMPT = (
|
||||
"Continue this same trial from the current session. Do not restart in a new "
|
||||
"workspace. If the workflow is already complete, stop and return only the "
|
||||
"final challenge_report YAML using the required schema."
|
||||
)
|
||||
|
||||
|
||||
def _event_session_id(event: dict[str, Any]) -> str | None:
|
||||
session_id = event.get("sessionID")
|
||||
if isinstance(session_id, str) and session_id:
|
||||
return session_id
|
||||
part = event.get("part")
|
||||
if isinstance(part, dict):
|
||||
nested = part.get("sessionID")
|
||||
if isinstance(nested, str) and nested:
|
||||
return nested
|
||||
return None
|
||||
|
||||
|
||||
def extract_session_id(stdout: str) -> str | None:
|
||||
"""Return the first OpenCode session id found in JSONL stdout."""
|
||||
for line in stdout.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
session_id = _event_session_id(event)
|
||||
if session_id is not None:
|
||||
return session_id
|
||||
return None
|
||||
|
||||
|
||||
def resume_prompt_for_result(result: dict[str, object]) -> str:
|
||||
"""Choose a continuation prompt from the result failure shape."""
|
||||
task_outcome = result.get("task_outcome")
|
||||
assertion_failures = result.get("assertion_failures")
|
||||
failures = assertion_failures if isinstance(assertion_failures, list) else []
|
||||
if task_outcome == "timeout":
|
||||
return CONTINUE_PROMPT
|
||||
if any("could not extract challenge report" in str(item) for item in failures):
|
||||
return FINAL_REPORT_PROMPT
|
||||
if result.get("parsed") is None and result.get("stdout"):
|
||||
return FINAL_REPORT_PROMPT
|
||||
return CONTINUE_PROMPT
|
||||
|
||||
|
||||
def build_resume_command(
|
||||
*,
|
||||
session_id: str,
|
||||
attach_url: str | None,
|
||||
model: str,
|
||||
variant: str,
|
||||
prompt: str,
|
||||
) -> list[str]:
|
||||
command = ["opencode", "run", "--session", session_id]
|
||||
if attach_url is not None:
|
||||
command.extend(["--attach", attach_url])
|
||||
command.extend(["--format", "json", "--model", model, "--variant", variant, prompt])
|
||||
return command
|
||||
|
||||
|
||||
def display_resume_command(command: list[str]) -> str:
|
||||
"""Render an argv list for copy-paste display without changing execution."""
|
||||
return subprocess.list2cmdline(command)
|
||||
|
||||
|
||||
def resume_result_path(result_path: Path) -> Path:
|
||||
stem = result_path.with_suffix("")
|
||||
index = 1
|
||||
while True:
|
||||
candidate = stem.with_name(f"{stem.name}.resume-{index:03d}.json")
|
||||
if not candidate.exists():
|
||||
return candidate
|
||||
index += 1
|
||||
@@ -22,6 +22,16 @@ class TrialIdentity(StrictReportModel):
|
||||
workspace_path: str
|
||||
|
||||
|
||||
class OpenCodeRunMetadata(StrictReportModel):
|
||||
attach_url: str | None = None
|
||||
command: list[str] = Field(default_factory=list)
|
||||
model: str = ""
|
||||
variant: str = ""
|
||||
session_id: str | None = None
|
||||
resume_prompt: str = ""
|
||||
resume_command: list[str] | None = None
|
||||
|
||||
|
||||
class TrialOutcome(StrictReportModel):
|
||||
task_outcome: str
|
||||
evaluation_validity: str
|
||||
@@ -85,6 +95,7 @@ class TrialReport(StrictReportModel):
|
||||
final_agent_answer: str | None = None
|
||||
commands_and_tools: list[CommandToolBrief] = Field(default_factory=list)
|
||||
automatic_evidence: AutomaticEvidence
|
||||
opencode: OpenCodeRunMetadata | None = None
|
||||
policy_findings: list[str] = Field(default_factory=list)
|
||||
self_report_discrepancies: list[str] = Field(default_factory=list)
|
||||
manual_audit: ManualAuditSummary = Field(default_factory=ManualAuditSummary)
|
||||
@@ -233,6 +244,24 @@ def _build_automatic_evidence(result: dict[str, object]) -> AutomaticEvidence:
|
||||
)
|
||||
|
||||
|
||||
def _build_opencode_metadata(result: dict[str, object]) -> OpenCodeRunMetadata | None:
|
||||
raw = result.get("opencode")
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
resume_command = raw.get("resume_command")
|
||||
return OpenCodeRunMetadata(
|
||||
attach_url=_str_none(raw.get("attach_url")),
|
||||
command=_list_str(raw.get("command")),
|
||||
model=_str(raw.get("model")),
|
||||
variant=_str(raw.get("variant")),
|
||||
session_id=_str_none(raw.get("session_id")),
|
||||
resume_prompt=_str(raw.get("resume_prompt")),
|
||||
resume_command=_list_str(resume_command)
|
||||
if isinstance(resume_command, list)
|
||||
else None,
|
||||
)
|
||||
|
||||
|
||||
def _build_trial_report(
|
||||
result: dict[str, object],
|
||||
*,
|
||||
@@ -271,6 +300,7 @@ def _build_trial_report(
|
||||
final_agent_answer=final_agent_answer,
|
||||
commands_and_tools=commands_and_tools,
|
||||
automatic_evidence=automatic_evidence,
|
||||
opencode=_build_opencode_metadata(result),
|
||||
policy_findings=policy_findings,
|
||||
self_report_discrepancies=self_report_discrepancies,
|
||||
follow_up_notes=follow_up_notes,
|
||||
|
||||
@@ -12,6 +12,7 @@ import yaml
|
||||
|
||||
from examples.agent_challenges.classification import extract_challenge_report
|
||||
from examples.agent_challenges.opencode_io import parse_opencode_output, result_text
|
||||
from examples.agent_challenges.opencode_resume import display_resume_command
|
||||
from examples.agent_challenges.report_models import TrialReport
|
||||
|
||||
|
||||
@@ -219,6 +220,10 @@ def _atomic_write_text(path: Path, text: str) -> None:
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def _shell_join(command: list[str]) -> str:
|
||||
return display_resume_command(command)
|
||||
|
||||
|
||||
def render_trial_report_markdown(report: TrialReport) -> str:
|
||||
lines: list[str] = []
|
||||
|
||||
@@ -244,6 +249,29 @@ def render_trial_report_markdown(report: TrialReport) -> str:
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
if report.opencode is not None:
|
||||
lines.append("## OpenCode Resume")
|
||||
lines.append("")
|
||||
if report.opencode.session_id:
|
||||
lines.append(f"- Session: `{report.opencode.session_id}`")
|
||||
else:
|
||||
lines.append("- Session: not captured")
|
||||
if report.opencode.attach_url:
|
||||
lines.append(f"- Attach URL: `{report.opencode.attach_url}`")
|
||||
if report.opencode.resume_command:
|
||||
lines.append("")
|
||||
lines.append("```powershell")
|
||||
lines.append(_shell_join(report.opencode.resume_command))
|
||||
lines.append("```")
|
||||
if report.opencode.resume_prompt:
|
||||
lines.append("")
|
||||
lines.append("Resume prompt:")
|
||||
lines.append("")
|
||||
lines.append("```text")
|
||||
lines.append(report.opencode.resume_prompt)
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Agent Self-Report")
|
||||
lines.append("")
|
||||
if report.agent_self_report is not None:
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from .opencode_resume import display_resume_command, resume_result_path
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from examples.agent_challenges.opencode_resume import (
|
||||
display_resume_command,
|
||||
resume_result_path,
|
||||
)
|
||||
|
||||
|
||||
RunFn = Callable[..., subprocess.CompletedProcess[str]]
|
||||
|
||||
|
||||
def _utf8_subprocess_env() -> dict[str, str]:
|
||||
env = dict(os.environ)
|
||||
env.setdefault("PYTHONUTF8", "1")
|
||||
env.setdefault("PYTHONIOENCODING", "utf-8")
|
||||
return env
|
||||
|
||||
|
||||
def _load_command(result: dict[str, Any]) -> list[str]:
|
||||
opencode = result.get("opencode")
|
||||
if not isinstance(opencode, dict):
|
||||
raise ValueError("result has no opencode metadata")
|
||||
command = opencode.get("resume_command")
|
||||
if not isinstance(command, list) or not all(
|
||||
isinstance(part, str) for part in command
|
||||
):
|
||||
raise ValueError("result has no resume_command; session id may be missing")
|
||||
return command
|
||||
|
||||
|
||||
def _display_command(command: list[str]) -> str:
|
||||
return display_resume_command(command)
|
||||
|
||||
|
||||
def resume_from_result(
|
||||
result_path: Path,
|
||||
*,
|
||||
run_fn: RunFn = subprocess.run,
|
||||
) -> Path:
|
||||
result = json.loads(result_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(result, dict):
|
||||
raise ValueError("result file must contain a JSON object")
|
||||
command = _load_command(result)
|
||||
workspace_path = result.get("workspace_path")
|
||||
cwd = str(workspace_path) if isinstance(workspace_path, str) else None
|
||||
started = time.monotonic()
|
||||
completed = run_fn(
|
||||
command,
|
||||
cwd=cwd,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
env=_utf8_subprocess_env(),
|
||||
)
|
||||
payload = {
|
||||
"harness_version": "v2-resume",
|
||||
"source_result_path": str(result_path.resolve()),
|
||||
"command": command,
|
||||
"duration_seconds": round(time.monotonic() - started, 3),
|
||||
"returncode": completed.returncode,
|
||||
"stdout": completed.stdout or "",
|
||||
"stderr": completed.stderr or "",
|
||||
}
|
||||
output_path = resume_result_path(result_path)
|
||||
output_path.write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8"
|
||||
)
|
||||
return output_path
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Print or run an OpenCode trial resume command."
|
||||
)
|
||||
parser.add_argument("--from-result", type=Path, required=True)
|
||||
parser.add_argument("--print-command", action="store_true")
|
||||
parser.add_argument("--run", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
result = json.loads(args.from_result.read_text(encoding="utf-8"))
|
||||
if not isinstance(result, dict):
|
||||
raise ValueError("result file must contain a JSON object")
|
||||
command = _load_command(result)
|
||||
if args.run:
|
||||
print(resume_from_result(args.from_result).as_posix())
|
||||
else:
|
||||
print(_display_command(command))
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -40,6 +40,11 @@ from examples.agent_challenges.opencode_io import ( # noqa: E402
|
||||
parse_opencode_output,
|
||||
result_text,
|
||||
)
|
||||
from examples.agent_challenges.opencode_resume import ( # noqa: E402
|
||||
build_resume_command,
|
||||
extract_session_id,
|
||||
resume_prompt_for_result,
|
||||
)
|
||||
from examples.agent_challenges.report_models import ( # noqa: E402
|
||||
build_trial_report,
|
||||
)
|
||||
@@ -648,6 +653,8 @@ def run_v2_trial(
|
||||
"machine": str(machine_report_path.resolve()),
|
||||
}
|
||||
|
||||
opencode_session_id = extract_session_id(stdout)
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"challenge_id": challenge.manifest.id,
|
||||
"instruction_profile": profile.value,
|
||||
@@ -696,6 +703,27 @@ def run_v2_trial(
|
||||
if challenge_report is not None:
|
||||
result["challenge_report"] = challenge_report
|
||||
|
||||
resume_prompt = resume_prompt_for_result(result)
|
||||
result["opencode"] = {
|
||||
"attach_url": attach_url,
|
||||
"command": command,
|
||||
"model": model,
|
||||
"variant": variant,
|
||||
"session_id": opencode_session_id,
|
||||
"resume_prompt": resume_prompt,
|
||||
"resume_command": (
|
||||
build_resume_command(
|
||||
session_id=opencode_session_id,
|
||||
attach_url=attach_url,
|
||||
model=model,
|
||||
variant=variant,
|
||||
prompt=resume_prompt,
|
||||
)
|
||||
if opencode_session_id is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
result_path.write_text(
|
||||
json.dumps(result, indent=2, sort_keys=True), encoding="utf-8"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user