lda.chat/lda-wf
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs: publish thesis evaluation bundle

Browse Source
This commit is contained in:
lda
2026-06-30 22:46:17 +07:00 Verified
parent 5e9b76eb94
commit e12ccdf18a
28 changed files with 11555 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/agent_challenges/evaluation_figures.py
+386
View File
@@ -0,0 +1,386 @@
from __future__ import annotations
from collections import Counter
from pathlib import Path
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from matplotlib.axes import Axes
from matplotlib.figure import Figure
from .evaluation import EvaluationCohort, EvaluationTrial
_OUTCOMES = ("pass", "invalid", "fail")
_OUTCOME_COLORS = {
"pass": "#00796B",
"invalid": "#E07A1F",
"fail": "#9E2A2B",
}
_OUTCOME_HATCHES = {"pass": "", "invalid": "///", "fail": "xx"}
_MODEL_STYLES = {
"deepseek": {"color": "#0067A5", "marker": "o", "offset": -0.13},
"mimo": {"color": "#D55E00", "marker": "s", "offset": 0.13},
}
_PROFILE_ORDER = {"none": 0, "skills": 1, "all": 2}
_CHALLENGE_ORDER = {"browser": 0, "report": 1}
_MODEL_ORDER = {"deepseek": 0, "mimo": 1}
def _configure_matplotlib() -> Any:
"""Import Matplotlib with a headless backend suitable for docs builds."""
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot as plt
plt.rcParams.update(
{
"font.family": "DejaVu Sans",
"font.size": 9,
"svg.hashsalt": "lda-chat-agent-challenge-evaluation",
"axes.titleweight": "bold",
"axes.titlesize": 12,
"axes.labelcolor": "#28323c",
"axes.edgecolor": "#8b949e",
"axes.spines.top": False,
"axes.spines.right": False,
"figure.facecolor": "white",
"axes.facecolor": "#F3F6F7",
"grid.color": "#C9D1D5",
"grid.linewidth": 0.6,
}
)
return plt
def _trial_sort_key(trial: EvaluationTrial) -> tuple[int, int, int, int]:
return (
_CHALLENGE_ORDER.get(trial.challenge, 99),
_MODEL_ORDER.get(trial.model, 99),
_PROFILE_ORDER.get(trial.profile, 99),
trial.wave,
)
def _cell_key(trial: EvaluationTrial) -> tuple[str, str, str]:
return trial.challenge, trial.model, trial.profile
def _ordered_cells(cohort: EvaluationCohort) -> list[tuple[str, str, str]]:
return sorted(
{_cell_key(trial) for trial in cohort.trials},
key=lambda cell: (
_CHALLENGE_ORDER.get(cell[0], 99),
_MODEL_ORDER.get(cell[1], 99),
_PROFILE_ORDER.get(cell[2], 99),
),
)
def _save_figure(
figure: Figure, output_dir: Path, stem: str, plt: Any
) -> tuple[Path, Path]:
"""Save one source figure in web-native SVG and print-native PDF."""
output_dir.mkdir(parents=True, exist_ok=True)
written: list[Path] = []
for suffix in ("svg", "pdf"):
path = output_dir / f"{stem}.{suffix}"
# Matplotlib otherwise embeds current timestamps, and SVG element IDs
# use a random salt. Stable metadata makes checked-in figures reproducible.
metadata = (
{"Creator": "lda.chat thesis evaluation", "Date": None}
if suffix == "svg"
else {
"Creator": "lda.chat thesis evaluation",
"CreationDate": None,
"ModDate": None,
}
)
figure.savefig(
path,
bbox_inches="tight",
pad_inches=0.12,
metadata=metadata,
)
if suffix == "svg":
# Matplotlib writes SVG path data across lines with trailing spaces.
# Normalize it so generated figures can pass git whitespace checks.
lines = path.read_text(encoding="utf-8").splitlines()
path.write_text(
"\n".join(line.rstrip() for line in lines) + "\n",
encoding="utf-8",
newline="\n",
)
written.append(path)
plt.close(figure)
return written[0], written[1]
def _outcomes_by_cell(cohort: EvaluationCohort, plt: Any) -> Figure:
cells = _ordered_cells(cohort)
counts = {
cell: Counter(
trial.manual_outcome for trial in cohort.trials if _cell_key(trial) == cell
)
for cell in cells
}
labels = [
f"{challenge} | {model} | {profile}" for challenge, model, profile in cells
]
figure, axis = plt.subplots(figsize=(9.2, 6.4))
left = [0] * len(cells)
for outcome in _OUTCOMES:
values = [counts[cell][outcome] for cell in cells]
bars = axis.barh(
range(len(cells)),
values,
left=left,
label=outcome.capitalize(),
color=_OUTCOME_COLORS[outcome],
edgecolor="#263238",
linewidth=0.55,
hatch=_OUTCOME_HATCHES[outcome],
height=0.68,
)
for bar, value in zip(bars, values, strict=True):
if value:
axis.text(
bar.get_x() + bar.get_width() / 2,
bar.get_y() + bar.get_height() / 2,
str(value),
ha="center",
va="center",
color="#17212B" if outcome == "invalid" else "white",
fontweight="bold",
)
left = [current + value for current, value in zip(left, values, strict=True)]
axis.set_yticks(range(len(cells)), labels)
axis.invert_yaxis()
axis.set_xticks((0, 1, 2, 3))
axis.set_xlim(0, 3)
axis.set_xlabel("Manually audited trials (n=3 per cell)")
axis.set_title("Audited outcomes by challenge, model, and instruction profile")
axis.grid(axis="x")
axis.legend(loc="lower right", frameon=False, ncol=3)
figure.tight_layout()
return figure
def _automatic_vs_manual(cohort: EvaluationCohort, plt: Any) -> Figure:
from matplotlib.colors import LinearSegmentedColormap
task_labels = ("success", "failed")
manual_labels = ("pass", "invalid", "fail")
matrix = [
[
sum(
trial.task_outcome == task and trial.manual_outcome == manual
for trial in cohort.trials
)
for manual in manual_labels
]
for task in task_labels
]
figure, axis = plt.subplots(figsize=(6.8, 3.8))
count_cmap = LinearSegmentedColormap.from_list(
"lda_count", ("#F3F6F7", "#79B8B3", "#005F73")
)
maximum = max(map(max, matrix))
image = axis.imshow(matrix, cmap=count_cmap, vmin=0, vmax=maximum)
for row, values in enumerate(matrix):
for column, value in enumerate(values):
axis.text(
column,
row,
str(value),
ha="center",
va="center",
color="white" if value > maximum / 2 else "#17212B",
fontsize=13,
fontweight="bold",
)
axis.set_xticks(
range(len(manual_labels)), [label.capitalize() for label in manual_labels]
)
axis.set_yticks(
range(len(task_labels)), [label.capitalize() for label in task_labels]
)
axis.set_xlabel("Manual official outcome")
axis.set_ylabel("Automatic task outcome")
axis.set_title("Automatic completion does not imply clean evaluation evidence")
axis.set_xticks([value - 0.5 for value in range(1, len(manual_labels))], minor=True)
axis.set_yticks([0.5], minor=True)
axis.grid(which="minor", color="white", linewidth=2)
axis.tick_params(which="minor", bottom=False, left=False)
figure.colorbar(image, ax=axis, label="Trial count", shrink=0.82)
figure.tight_layout()
return figure
def _longitudinal_outcomes(cohort: EvaluationCohort, plt: Any) -> Figure:
waves = sorted({trial.wave for trial in cohort.trials})
counts = {
wave: Counter(
trial.manual_outcome for trial in cohort.trials if trial.wave == wave
)
for wave in waves
}
figure, axis = plt.subplots(figsize=(6.8, 4.0))
bottom = [0] * len(waves)
for outcome in _OUTCOMES:
values = [counts[wave][outcome] for wave in waves]
bars = axis.bar(
waves,
values,
bottom=bottom,
label=outcome.capitalize(),
color=_OUTCOME_COLORS[outcome],
edgecolor="#263238",
linewidth=0.55,
hatch=_OUTCOME_HATCHES[outcome],
width=0.62,
)
for bar, value in zip(bars, values, strict=True):
if value:
axis.text(
bar.get_x() + bar.get_width() / 2,
bar.get_y() + bar.get_height() / 2,
str(value),
ha="center",
va="center",
color="#17212B" if outcome == "invalid" else "white",
fontweight="bold",
)
bottom = [
current + value for current, value in zip(bottom, values, strict=True)
]
axis.set_xticks(waves, [f"Wave {wave}" for wave in waves])
axis.set_ylim(0, max(bottom) + 1)
axis.set_ylabel("Manually audited trials")
axis.set_title("Outcomes across three evolving product and prompt snapshots")
axis.grid(axis="y")
axis.legend(frameon=False, ncol=3, loc="upper center")
figure.tight_layout()
return figure
def _scatter_metric(
axis: Axes,
trials: list[EvaluationTrial],
*,
metric: str,
) -> None:
profiles = ("none", "skills", "all")
wave_offsets = {1: -0.055, 2: 0.0, 3: 0.055}
for trial in sorted(trials, key=_trial_sort_key):
style = _MODEL_STYLES[trial.model]
x = (
profiles.index(trial.profile)
+ float(style["offset"])
+ wave_offsets.get(trial.wave, 0.0)
)
if metric == "duration":
value = trial.duration_seconds / 60
else:
value = trial.tokens_total / 1_000_000
axis.scatter(
x,
value,
color=str(style["color"]),
marker=str(style["marker"]),
edgecolor="#17212B",
linewidth=0.65,
s=86,
zorder=3,
)
axis.annotate(
str(trial.wave),
(x, value),
ha="center",
va="center",
color="white",
fontsize=6,
fontweight="bold",
zorder=4,
)
axis.set_xticks(
range(len(profiles)), [profile.capitalize() for profile in profiles]
)
axis.set_xlim(-0.42, 2.42)
axis.grid(axis="y")
def _duration_and_tokens(cohort: EvaluationCohort, plt: Any) -> Figure:
from matplotlib.lines import Line2D
figure, axes = plt.subplots(2, 2, figsize=(9.6, 6.8), sharex=True)
challenges = (("browser", "Browser click"), ("report", "Report workflow"))
for row, (challenge, challenge_label) in enumerate(challenges):
trials = [trial for trial in cohort.trials if trial.challenge == challenge]
duration_axis, token_axis = axes[row]
_scatter_metric(duration_axis, trials, metric="duration")
_scatter_metric(token_axis, trials, metric="tokens")
duration_axis.set_ylabel(f"{challenge_label}\nMinutes")
token_axis.set_ylabel(f"{challenge_label}\nMillion tokens")
axes[0, 0].set_title("Wall-clock duration")
axes[0, 1].set_title("Recorded token volume")
axes[1, 0].set_xlabel("Instruction profile")
axes[1, 1].set_xlabel("Instruction profile")
legend_handles = [
Line2D(
[],
[],
color=str(style["color"]),
marker=str(style["marker"]),
linestyle="None",
markeredgecolor="#17212B",
markersize=8,
label=f"{model.capitalize()} ({'circle' if model == 'deepseek' else 'square'})",
)
for model, style in _MODEL_STYLES.items()
]
figure.suptitle(
"Runtime evidence by challenge, profile, model, and wave",
fontsize=13,
fontweight="bold",
)
figure.legend(
handles=legend_handles,
loc="upper center",
bbox_to_anchor=(0.5, 0.94),
frameon=False,
ncol=2,
)
figure.text(
0.5,
0.015,
"Point labels 1–3 identify waves; token totals include OpenCode cache-read accounting.",
ha="center",
color="#4C5961",
fontsize=8,
)
figure.tight_layout(rect=(0.0, 0.05, 1.0, 0.88))
return figure
def render_evaluation_figures(
cohort: EvaluationCohort, output_dir: Path
) -> tuple[Path, ...]:
"""Write all named evaluation figures as SVG and PDF pairs."""
plt = _configure_matplotlib()
figures = (
("agent-challenge-audited-outcomes-by-cell", _outcomes_by_cell(cohort, plt)),
(
"agent-challenge-automatic-vs-manual-outcomes",
_automatic_vs_manual(cohort, plt),
),
("agent-challenge-longitudinal-outcomes", _longitudinal_outcomes(cohort, plt)),
("agent-challenge-duration-and-tokens", _duration_and_tokens(cohort, plt)),
)
written: list[Path] = []
for stem, figure in figures:
written.extend(_save_figure(figure, output_dir, stem, plt))
return tuple(written)