130 lines
4.4 KiB
Python
130 lines
4.4 KiB
Python
"""Central CLI for running profiled agent challenge trials."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from .manifests import load_challenge_manifest
|
|
from .models import InstructionProfile
|
|
from .runner import run_v2_trial
|
|
from .workspace import starting_trial_index
|
|
except ImportError:
|
|
_project_root = Path(__file__).resolve().parents[2]
|
|
if str(_project_root) not in sys.path:
|
|
sys.path.insert(0, str(_project_root))
|
|
from examples.agent_challenges.manifests import load_challenge_manifest
|
|
from examples.agent_challenges.models import InstructionProfile
|
|
from examples.agent_challenges.runner import run_v2_trial
|
|
from examples.agent_challenges.workspace import starting_trial_index
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
)
|
|
parser.add_argument(
|
|
"--challenge",
|
|
type=Path,
|
|
required=True,
|
|
help="Path to challenge.yaml",
|
|
)
|
|
parser.add_argument(
|
|
"--instruction-profile",
|
|
"--profile",
|
|
dest="instruction_profile",
|
|
type=str,
|
|
choices=[p.value for p in InstructionProfile],
|
|
required=True,
|
|
help="Instruction profile for this trial",
|
|
)
|
|
parser.add_argument("--model", default="opencode/mimo-v2.5-free")
|
|
parser.add_argument("--variant", default="high")
|
|
parser.add_argument("--trials", type=int, default=1)
|
|
parser.add_argument(
|
|
"--concurrency",
|
|
type=int,
|
|
default=1,
|
|
help="Maximum number of trials for this model/profile/challenge to run at once.",
|
|
)
|
|
parser.add_argument("--timeout-seconds", type=int, default=3600)
|
|
parser.add_argument(
|
|
"--attach",
|
|
dest="attach_url",
|
|
default=None,
|
|
help="Attach to a running opencode server URL.",
|
|
)
|
|
parser.add_argument("--results-dir", type=Path, default=None)
|
|
parser.add_argument("--workspaces-dir", type=Path, default=None)
|
|
parser.add_argument(
|
|
"--instruction-bundle",
|
|
type=Path,
|
|
default=Path(__file__).resolve().parents[2]
|
|
/ "examples"
|
|
/ "agent_challenges"
|
|
/ "instruction_bundles"
|
|
/ "workflow_cli.yaml",
|
|
)
|
|
args = parser.parse_args(argv)
|
|
|
|
if args.trials < 1:
|
|
parser.error("--trials must be >= 1")
|
|
if args.concurrency < 1:
|
|
parser.error("--concurrency must be >= 1")
|
|
|
|
challenge = load_challenge_manifest(args.challenge)
|
|
profile = InstructionProfile(args.instruction_profile)
|
|
|
|
results_dir = args.results_dir or challenge.root / "results"
|
|
workspaces_dir = args.workspaces_dir or challenge.root / "workspaces"
|
|
|
|
first_index = starting_trial_index(
|
|
model=args.model,
|
|
results_dir=results_dir,
|
|
workspaces_dir=workspaces_dir,
|
|
)
|
|
|
|
def _run_one(index: int) -> dict[str, object]:
|
|
result = run_v2_trial(
|
|
challenge,
|
|
profile=profile,
|
|
model=args.model,
|
|
variant=args.variant,
|
|
index=index,
|
|
workspaces_dir=workspaces_dir,
|
|
results_dir=results_dir,
|
|
instruction_bundle=args.instruction_bundle,
|
|
timeout_seconds=args.timeout_seconds,
|
|
attach_url=args.attach_url,
|
|
)
|
|
return {
|
|
"index": index,
|
|
"task_outcome": result["task_outcome"],
|
|
"evaluation_validity": result["evaluation_validity"],
|
|
"duration_seconds": result["duration_seconds"],
|
|
"result_path": result.get("result_path"),
|
|
"report_paths": result.get("report_paths"),
|
|
}
|
|
|
|
summaries: list[dict[str, object]] = []
|
|
indices = list(range(first_index, first_index + args.trials))
|
|
with ThreadPoolExecutor(max_workers=min(args.concurrency, args.trials)) as pool:
|
|
futures = [pool.submit(_run_one, index) for index in indices]
|
|
for future in as_completed(futures):
|
|
summary = future.result()
|
|
summaries.append(summary)
|
|
print(json.dumps(summary, sort_keys=True))
|
|
|
|
success_count = sum(1 for s in summaries if s["task_outcome"] == "success")
|
|
print(json.dumps({"success_count": success_count, "trial_count": len(summaries)}))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|