254 lines
8.4 KiB
Python
254 lines
8.4 KiB
Python
"""Run a challenge/profile/model matrix with bounded global concurrency."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
try:
|
|
from .manifests import load_challenge_manifest
|
|
from .models import InstructionProfile, LoadedChallenge
|
|
from .runner import run_v2_trial
|
|
from .workspace import PROJECT_ROOT, starting_trial_index
|
|
except ImportError:
|
|
_project_root = Path(__file__).resolve().parents[2]
|
|
if str(_project_root) not in sys.path:
|
|
sys.path.insert(0, str(_project_root))
|
|
from examples.agent_challenges.manifests import load_challenge_manifest
|
|
from examples.agent_challenges.models import InstructionProfile, LoadedChallenge
|
|
from examples.agent_challenges.runner import run_v2_trial
|
|
from examples.agent_challenges.workspace import PROJECT_ROOT, starting_trial_index
|
|
|
|
|
|
DEFAULT_CHALLENGES = (
|
|
PROJECT_ROOT / "examples/agent_challenges/browser_click_challenge/challenge.yaml",
|
|
PROJECT_ROOT / "examples/agent_challenges/report_workflow_challenge/challenge.yaml",
|
|
)
|
|
DEFAULT_PROFILES = (
|
|
InstructionProfile.NONE,
|
|
InstructionProfile.SKILLS,
|
|
InstructionProfile.ALL,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ModelProfile:
|
|
model: str
|
|
variant: str
|
|
|
|
|
|
DEFAULT_MODELS = (
|
|
ModelProfile("opencode/deepseek-v4-flash-free", "max"),
|
|
ModelProfile("opencode/mimo-v2.5-free", "high"),
|
|
# Disabled after repeated provider timeouts and resource-exhausted failures.
|
|
# ModelProfile("opencode/nemotron-3-ultra-free", "high"),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class MatrixTask:
|
|
challenge: LoadedChallenge
|
|
profile: InstructionProfile
|
|
model: str
|
|
variant: str
|
|
index: int
|
|
workspaces_dir: Path
|
|
results_dir: Path
|
|
|
|
|
|
def parse_model_profile(raw: str) -> ModelProfile:
|
|
"""Parse MODEL or MODEL=VARIANT values from CLI flags."""
|
|
model, separator, variant = raw.partition("=")
|
|
if not model:
|
|
raise ValueError("model cannot be empty")
|
|
if separator and not variant:
|
|
raise ValueError("variant cannot be empty when using MODEL=VARIANT")
|
|
return ModelProfile(model=model, variant=variant if separator else "high")
|
|
|
|
|
|
def build_matrix_tasks(
|
|
*,
|
|
challenges: list[LoadedChallenge],
|
|
profiles: list[InstructionProfile],
|
|
models: list[ModelProfile],
|
|
trials: int,
|
|
) -> list[MatrixTask]:
|
|
"""Allocate unique trial indices for each challenge/model across profiles."""
|
|
tasks: list[MatrixTask] = []
|
|
for challenge in challenges:
|
|
results_dir = challenge.root / "results"
|
|
workspaces_dir = challenge.root / "workspaces"
|
|
next_indices: dict[str, int] = {}
|
|
for model in models:
|
|
next_index = next_indices.setdefault(
|
|
model.model,
|
|
starting_trial_index(
|
|
model=model.model,
|
|
results_dir=results_dir,
|
|
workspaces_dir=workspaces_dir,
|
|
),
|
|
)
|
|
for profile in profiles:
|
|
for _ in range(trials):
|
|
tasks.append(
|
|
MatrixTask(
|
|
challenge=challenge,
|
|
profile=profile,
|
|
model=model.model,
|
|
variant=model.variant,
|
|
index=next_index,
|
|
workspaces_dir=workspaces_dir,
|
|
results_dir=results_dir,
|
|
)
|
|
)
|
|
next_index += 1
|
|
next_indices[model.model] = next_index
|
|
return tasks
|
|
|
|
|
|
def _summary_from_result(task: MatrixTask, result: dict[str, Any]) -> dict[str, object]:
|
|
return {
|
|
"challenge": task.challenge.manifest.id,
|
|
"instruction_profile": task.profile.value,
|
|
"model": task.model,
|
|
"variant": task.variant,
|
|
"index": task.index,
|
|
"task_outcome": result["task_outcome"],
|
|
"evaluation_validity": result["evaluation_validity"],
|
|
"duration_seconds": result["duration_seconds"],
|
|
"result_path": result.get("result_path"),
|
|
"report_paths": result.get("report_paths"),
|
|
}
|
|
|
|
|
|
def _summary_from_exception(task: MatrixTask, exc: BaseException) -> dict[str, object]:
|
|
"""Return an auditable summary when a worker fails before writing a result."""
|
|
return {
|
|
"challenge": task.challenge.manifest.id,
|
|
"instruction_profile": task.profile.value,
|
|
"model": task.model,
|
|
"variant": task.variant,
|
|
"index": task.index,
|
|
"task_outcome": "runner_error",
|
|
"evaluation_validity": "unauditable",
|
|
"duration_seconds": 0.0,
|
|
"result_path": None,
|
|
"report_paths": None,
|
|
"error": {"type": type(exc).__name__, "message": str(exc)},
|
|
}
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--challenge",
|
|
action="append",
|
|
type=Path,
|
|
default=None,
|
|
help="Path to challenge.yaml. May be repeated. Defaults to all bundled challenges.",
|
|
)
|
|
parser.add_argument(
|
|
"--instruction-profile",
|
|
"--profile",
|
|
dest="instruction_profile",
|
|
action="append",
|
|
choices=[p.value for p in InstructionProfile],
|
|
default=None,
|
|
help=(
|
|
"Instruction profile. May be repeated. Defaults to none, skills, "
|
|
"all; use debug explicitly for UX issue mining."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--model",
|
|
action="append",
|
|
default=None,
|
|
help=(
|
|
"Model profile as MODEL or MODEL=VARIANT. May be repeated. "
|
|
"Defaults to the standard free OpenCode model set."
|
|
),
|
|
)
|
|
parser.add_argument("--trials", type=int, default=5)
|
|
parser.add_argument("--concurrency", type=int, default=2)
|
|
parser.add_argument("--timeout-seconds", type=int, default=3600)
|
|
parser.add_argument("--attach", dest="attach_url", default=None)
|
|
parser.add_argument(
|
|
"--instruction-bundle",
|
|
type=Path,
|
|
default=PROJECT_ROOT
|
|
/ "examples"
|
|
/ "agent_challenges"
|
|
/ "instruction_bundles"
|
|
/ "workflow_cli.yaml",
|
|
)
|
|
args = parser.parse_args(argv)
|
|
|
|
if args.trials < 1:
|
|
parser.error("--trials must be >= 1")
|
|
if args.concurrency < 1:
|
|
parser.error("--concurrency must be >= 1")
|
|
|
|
try:
|
|
challenges = [
|
|
load_challenge_manifest(path)
|
|
for path in (args.challenge or DEFAULT_CHALLENGES)
|
|
]
|
|
profiles = [
|
|
InstructionProfile(value)
|
|
for value in (
|
|
args.instruction_profile or [p.value for p in DEFAULT_PROFILES]
|
|
)
|
|
]
|
|
models = [parse_model_profile(raw) for raw in (args.model or [])] or list(
|
|
DEFAULT_MODELS
|
|
)
|
|
except ValueError as exc:
|
|
parser.error(str(exc))
|
|
|
|
tasks = build_matrix_tasks(
|
|
challenges=challenges,
|
|
profiles=profiles,
|
|
models=models,
|
|
trials=args.trials,
|
|
)
|
|
|
|
def _run_task(task: MatrixTask) -> dict[str, object]:
|
|
result = run_v2_trial(
|
|
task.challenge,
|
|
profile=task.profile,
|
|
model=task.model,
|
|
variant=task.variant,
|
|
index=task.index,
|
|
workspaces_dir=task.workspaces_dir,
|
|
results_dir=task.results_dir,
|
|
instruction_bundle=args.instruction_bundle,
|
|
timeout_seconds=args.timeout_seconds,
|
|
attach_url=args.attach_url,
|
|
)
|
|
return _summary_from_result(task, result)
|
|
|
|
summaries: list[dict[str, object]] = []
|
|
with ThreadPoolExecutor(max_workers=min(args.concurrency, len(tasks))) as pool:
|
|
futures = {pool.submit(_run_task, task): task for task in tasks}
|
|
for future in as_completed(futures):
|
|
task = futures[future]
|
|
try:
|
|
summary = future.result()
|
|
except Exception as exc:
|
|
summary = _summary_from_exception(task, exc)
|
|
summaries.append(summary)
|
|
print(json.dumps(summary, sort_keys=True))
|
|
|
|
success_count = sum(1 for item in summaries if item["task_outcome"] == "success")
|
|
print(json.dumps({"success_count": success_count, "trial_count": len(summaries)}))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|