294 lines
8.8 KiB
Python
294 lines
8.8 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from dataclasses import asdict, dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
try:
|
|
from .names import short_challenge_name, short_model_name
|
|
except ImportError:
|
|
_project_root = Path(__file__).resolve().parents[2]
|
|
if str(_project_root) not in sys.path:
|
|
sys.path.insert(0, str(_project_root))
|
|
from examples.agent_challenges.names import short_challenge_name, short_model_name
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
DEFAULT_CHALLENGES_ROOT = ROOT / "examples" / "agent_challenges"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class TrialSummary:
|
|
challenge: str
|
|
model: str
|
|
profile: str
|
|
trial: int
|
|
manual: str
|
|
task: str
|
|
validity: str
|
|
duration_seconds: float
|
|
tokens_total: int
|
|
attempts: str
|
|
read_flags: str
|
|
notes: str
|
|
|
|
|
|
def _string(value: object, default: str = "") -> str:
|
|
return value if isinstance(value, str) else default
|
|
|
|
|
|
def _int(value: object, default: int = 0) -> int:
|
|
return value if isinstance(value, int) else default
|
|
|
|
|
|
def _float(value: object, default: float = 0.0) -> float:
|
|
return float(value) if isinstance(value, int | float) else default
|
|
|
|
|
|
def _dict(value: object) -> dict[str, Any]:
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def _attempts(agent_self_report: dict[str, Any], manual_audit: dict[str, Any]) -> str:
|
|
# Manual audits are authoritative for aggregate tables; the raw self-report
|
|
# remains visible in final-report.md for traceability.
|
|
audit_evidence = _dict(manual_audit.get("evidence"))
|
|
audit_total = audit_evidence.get("attempts_total")
|
|
audit_failed = audit_evidence.get("attempts_failed")
|
|
if isinstance(audit_total, int) and isinstance(audit_failed, int):
|
|
return f"{audit_failed}/{audit_total}"
|
|
|
|
attempts = _dict(agent_self_report.get("attempts"))
|
|
total = attempts.get("total")
|
|
failed = attempts.get("failed")
|
|
if isinstance(total, int) and isinstance(failed, int):
|
|
return f"{failed}/{total}"
|
|
return ""
|
|
|
|
|
|
def _read_flags(agent_self_report: dict[str, Any]) -> str:
|
|
reads = _dict(agent_self_report.get("read"))
|
|
enabled = [key for key, value in sorted(reads.items()) if value is True]
|
|
return ",".join(enabled)
|
|
|
|
|
|
def load_trial_summary(path: Path) -> TrialSummary:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(data, dict):
|
|
raise ValueError(f"report must be a JSON object: {path}")
|
|
|
|
identity = _dict(data.get("identity"))
|
|
outcome = _dict(data.get("outcome"))
|
|
manual_audit = _dict(data.get("manual_audit"))
|
|
evidence = _dict(data.get("automatic_evidence"))
|
|
tokens = _dict(evidence.get("tokens"))
|
|
self_report = _dict(data.get("agent_self_report"))
|
|
|
|
challenge = short_challenge_name(_string(identity.get("challenge_id")))
|
|
model = short_model_name(_string(identity.get("model")))
|
|
manual = _string(manual_audit.get("official_outcome"), "pending")
|
|
notes = _string(manual_audit.get("notes"))
|
|
|
|
return TrialSummary(
|
|
challenge=challenge,
|
|
model=model,
|
|
profile=_string(identity.get("instruction_profile")),
|
|
trial=_int(identity.get("trial_index")),
|
|
manual=manual or "pending",
|
|
task=_string(outcome.get("task_outcome")),
|
|
validity=_string(outcome.get("evaluation_validity")),
|
|
duration_seconds=_float(outcome.get("duration_seconds")),
|
|
tokens_total=_int(tokens.get("total")),
|
|
attempts=_attempts(self_report, manual_audit),
|
|
read_flags=_read_flags(self_report),
|
|
notes=" ".join(notes.split()),
|
|
)
|
|
|
|
|
|
def _raw_result_path_for_report(report_path: Path) -> Path:
|
|
name = report_path.name
|
|
if name.endswith(".report.json"):
|
|
return report_path.with_name(name.removesuffix(".report.json") + ".json")
|
|
return report_path
|
|
|
|
|
|
def _recorded_raw_result_path(report_path: Path) -> Path | None:
|
|
"""Read the raw result identity recorded in a machine report."""
|
|
try:
|
|
payload = json.loads(report_path.read_text(encoding="utf-8"))
|
|
except OSError, json.JSONDecodeError:
|
|
return None
|
|
if not isinstance(payload, dict):
|
|
return None
|
|
identity = payload.get("identity")
|
|
if not isinstance(identity, dict):
|
|
return None
|
|
raw_result_path = identity.get("raw_result_path")
|
|
if not isinstance(raw_result_path, str) or not raw_result_path:
|
|
return None
|
|
return Path(raw_result_path)
|
|
|
|
|
|
def _sort_mtime(path: Path, *, sort_by: str) -> float:
|
|
if sort_by == "result":
|
|
raw_result = _recorded_raw_result_path(path) or _raw_result_path_for_report(
|
|
path
|
|
)
|
|
if raw_result.exists():
|
|
return raw_result.stat().st_mtime
|
|
return path.stat().st_mtime
|
|
|
|
|
|
def find_report_files(
|
|
paths: list[Path],
|
|
*,
|
|
last: int | None = None,
|
|
over_list: int = 0,
|
|
sort_by: str = "result",
|
|
) -> list[Path]:
|
|
roots = paths or sorted(DEFAULT_CHALLENGES_ROOT.glob("*_challenge"))
|
|
reports: list[Path] = []
|
|
for root in roots:
|
|
if root.is_file():
|
|
reports.append(root)
|
|
continue
|
|
results_dir = root / "results"
|
|
if results_dir.is_dir():
|
|
reports.extend(sorted(results_dir.glob("*.report.json")))
|
|
continue
|
|
reports.extend(sorted(root.glob("*.report.json")))
|
|
unique_reports = sorted(set(reports))
|
|
if last is None:
|
|
return unique_reports
|
|
limit = last + over_list
|
|
# Active matrix review wants newest completed trials. Manual audits rewrite
|
|
# .report.json projections, so default to raw result mtimes when present.
|
|
newest = sorted(
|
|
unique_reports,
|
|
key=lambda path: (_sort_mtime(path, sort_by=sort_by), path.as_posix()),
|
|
reverse=True,
|
|
)[:limit]
|
|
return sorted(newest)
|
|
|
|
|
|
def _markdown_escape(value: object) -> str:
|
|
text = str(value)
|
|
return text.replace("|", "\\|").replace("\n", " ")
|
|
|
|
|
|
def render_markdown(summaries: list[TrialSummary]) -> str:
|
|
headers = [
|
|
"challenge",
|
|
"model",
|
|
"profile",
|
|
"trial",
|
|
"manual",
|
|
"task",
|
|
"validity",
|
|
"minutes",
|
|
"tokens",
|
|
"attempts",
|
|
"reads",
|
|
"notes",
|
|
]
|
|
rows = [
|
|
[
|
|
item.challenge,
|
|
item.model,
|
|
item.profile,
|
|
f"{item.trial:03d}",
|
|
item.manual,
|
|
item.task,
|
|
item.validity,
|
|
f"{item.duration_seconds / 60:.1f}",
|
|
str(item.tokens_total),
|
|
item.attempts,
|
|
item.read_flags,
|
|
item.notes,
|
|
]
|
|
for item in summaries
|
|
]
|
|
lines = [
|
|
"| " + " | ".join(headers) + " |",
|
|
"| " + " | ".join("---" for _ in headers) + " |",
|
|
]
|
|
lines.extend(
|
|
"| " + " | ".join(_markdown_escape(cell) for cell in row) + " |" for row in rows
|
|
)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def render_json(summaries: list[TrialSummary]) -> str:
|
|
return json.dumps(
|
|
[asdict(item) for item in summaries],
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
|
|
|
|
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Summarize agent challenge report projections."
|
|
)
|
|
parser.add_argument(
|
|
"paths",
|
|
nargs="*",
|
|
type=Path,
|
|
help="Challenge directories, results directories, or .report.json files.",
|
|
)
|
|
parser.add_argument(
|
|
"--format",
|
|
choices=("markdown", "json"),
|
|
default="markdown",
|
|
help="Output format.",
|
|
)
|
|
parser.add_argument(
|
|
"--last",
|
|
type=int,
|
|
default=None,
|
|
help="Summarize the newest N trials by raw result modification time.",
|
|
)
|
|
parser.add_argument(
|
|
"--over-list",
|
|
type=int,
|
|
default=0,
|
|
help="With --last, include this many extra newest reports as padding.",
|
|
)
|
|
parser.add_argument(
|
|
"--sort-by",
|
|
choices=("result", "report"),
|
|
default="result",
|
|
help=(
|
|
"When using --last, choose newest by raw result mtime or report "
|
|
"projection mtime. Defaults to result so manual audits do not "
|
|
"change batch selection."
|
|
),
|
|
)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = _parse_args(argv)
|
|
if args.last is not None and args.last < 1:
|
|
raise SystemExit("--last must be >= 1")
|
|
if args.over_list < 0:
|
|
raise SystemExit("--over-list must be >= 0")
|
|
reports = find_report_files(
|
|
args.paths,
|
|
last=args.last,
|
|
over_list=args.over_list,
|
|
sort_by=args.sort_by,
|
|
)
|
|
summaries = [load_trial_summary(path) for path in reports]
|
|
if args.format == "json":
|
|
print(render_json(summaries))
|
|
else:
|
|
print(render_markdown(summaries), end="")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|