from __future__ import annotations import argparse import json import sys from dataclasses import asdict, dataclass from pathlib import Path from typing import Any try: from .names import short_challenge_name, short_model_name except ImportError: _project_root = Path(__file__).resolve().parents[2] if str(_project_root) not in sys.path: sys.path.insert(0, str(_project_root)) from examples.agent_challenges.names import short_challenge_name, short_model_name ROOT = Path(__file__).resolve().parents[2] DEFAULT_CHALLENGES_ROOT = ROOT / "examples" / "agent_challenges" @dataclass(frozen=True, slots=True) class TrialSummary: challenge: str model: str profile: str trial: int manual: str task: str validity: str duration_seconds: float tokens_total: int attempts: str read_flags: str notes: str def _string(value: object, default: str = "") -> str: return value if isinstance(value, str) else default def _int(value: object, default: int = 0) -> int: return value if isinstance(value, int) else default def _float(value: object, default: float = 0.0) -> float: return float(value) if isinstance(value, int | float) else default def _dict(value: object) -> dict[str, Any]: return value if isinstance(value, dict) else {} def _attempts(agent_self_report: dict[str, Any], manual_audit: dict[str, Any]) -> str: # Manual audits are authoritative for aggregate tables; the raw self-report # remains visible in final-report.md for traceability. audit_evidence = _dict(manual_audit.get("evidence")) audit_total = audit_evidence.get("attempts_total") audit_failed = audit_evidence.get("attempts_failed") if isinstance(audit_total, int) and isinstance(audit_failed, int): return f"{audit_failed}/{audit_total}" attempts = _dict(agent_self_report.get("attempts")) total = attempts.get("total") failed = attempts.get("failed") if isinstance(total, int) and isinstance(failed, int): return f"{failed}/{total}" return "" def _read_flags(agent_self_report: dict[str, Any]) -> str: reads = _dict(agent_self_report.get("read")) enabled = [key for key, value in sorted(reads.items()) if value is True] return ",".join(enabled) def load_trial_summary(path: Path) -> TrialSummary: data = json.loads(path.read_text(encoding="utf-8")) if not isinstance(data, dict): raise ValueError(f"report must be a JSON object: {path}") identity = _dict(data.get("identity")) outcome = _dict(data.get("outcome")) manual_audit = _dict(data.get("manual_audit")) evidence = _dict(data.get("automatic_evidence")) tokens = _dict(evidence.get("tokens")) self_report = _dict(data.get("agent_self_report")) challenge = short_challenge_name(_string(identity.get("challenge_id"))) model = short_model_name(_string(identity.get("model"))) manual = _string(manual_audit.get("official_outcome"), "pending") notes = _string(manual_audit.get("notes")) return TrialSummary( challenge=challenge, model=model, profile=_string(identity.get("instruction_profile")), trial=_int(identity.get("trial_index")), manual=manual or "pending", task=_string(outcome.get("task_outcome")), validity=_string(outcome.get("evaluation_validity")), duration_seconds=_float(outcome.get("duration_seconds")), tokens_total=_int(tokens.get("total")), attempts=_attempts(self_report, manual_audit), read_flags=_read_flags(self_report), notes=" ".join(notes.split()), ) def _raw_result_path_for_report(report_path: Path) -> Path: name = report_path.name if name.endswith(".report.json"): return report_path.with_name(name.removesuffix(".report.json") + ".json") return report_path def _recorded_raw_result_path(report_path: Path) -> Path | None: """Read the raw result identity recorded in a machine report.""" try: payload = json.loads(report_path.read_text(encoding="utf-8")) except OSError, json.JSONDecodeError: return None if not isinstance(payload, dict): return None identity = payload.get("identity") if not isinstance(identity, dict): return None raw_result_path = identity.get("raw_result_path") if not isinstance(raw_result_path, str) or not raw_result_path: return None return Path(raw_result_path) def _sort_mtime(path: Path, *, sort_by: str) -> float: if sort_by == "result": raw_result = _recorded_raw_result_path(path) or _raw_result_path_for_report( path ) if raw_result.exists(): return raw_result.stat().st_mtime return path.stat().st_mtime def find_report_files( paths: list[Path], *, last: int | None = None, over_list: int = 0, sort_by: str = "result", ) -> list[Path]: roots = paths or sorted(DEFAULT_CHALLENGES_ROOT.glob("*_challenge")) reports: list[Path] = [] for root in roots: if root.is_file(): reports.append(root) continue results_dir = root / "results" if results_dir.is_dir(): reports.extend(sorted(results_dir.glob("*.report.json"))) continue reports.extend(sorted(root.glob("*.report.json"))) unique_reports = sorted(set(reports)) if last is None: return unique_reports limit = last + over_list # Active matrix review wants newest completed trials. Manual audits rewrite # .report.json projections, so default to raw result mtimes when present. newest = sorted( unique_reports, key=lambda path: (_sort_mtime(path, sort_by=sort_by), path.as_posix()), reverse=True, )[:limit] return sorted(newest) def _markdown_escape(value: object) -> str: text = str(value) return text.replace("|", "\\|").replace("\n", " ") def render_markdown(summaries: list[TrialSummary]) -> str: headers = [ "challenge", "model", "profile", "trial", "manual", "task", "validity", "minutes", "tokens", "attempts", "reads", "notes", ] rows = [ [ item.challenge, item.model, item.profile, f"{item.trial:03d}", item.manual, item.task, item.validity, f"{item.duration_seconds / 60:.1f}", str(item.tokens_total), item.attempts, item.read_flags, item.notes, ] for item in summaries ] lines = [ "| " + " | ".join(headers) + " |", "| " + " | ".join("---" for _ in headers) + " |", ] lines.extend( "| " + " | ".join(_markdown_escape(cell) for cell in row) + " |" for row in rows ) return "\n".join(lines) + "\n" def render_json(summaries: list[TrialSummary]) -> str: return json.dumps( [asdict(item) for item in summaries], indent=2, sort_keys=True, ) def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description="Summarize agent challenge report projections." ) parser.add_argument( "paths", nargs="*", type=Path, help="Challenge directories, results directories, or .report.json files.", ) parser.add_argument( "--format", choices=("markdown", "json"), default="markdown", help="Output format.", ) parser.add_argument( "--last", type=int, default=None, help="Summarize the newest N trials by raw result modification time.", ) parser.add_argument( "--over-list", type=int, default=0, help="With --last, include this many extra newest reports as padding.", ) parser.add_argument( "--sort-by", choices=("result", "report"), default="result", help=( "When using --last, choose newest by raw result mtime or report " "projection mtime. Defaults to result so manual audits do not " "change batch selection." ), ) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) if args.last is not None and args.last < 1: raise SystemExit("--last must be >= 1") if args.over_list < 0: raise SystemExit("--over-list must be >= 0") reports = find_report_files( args.paths, last=args.last, over_list=args.over_list, sort_by=args.sort_by, ) summaries = [load_trial_summary(path) for path in reports] if args.format == "json": print(render_json(summaries)) else: print(render_markdown(summaries), end="") return 0 if __name__ == "__main__": raise SystemExit(main())