docs: publish thesis evaluation bundle

2026-06-30 22:46:17 +07:00
parent 5e9b76eb94
commit e12ccdf18a
28 changed files with 11555 additions and 103 deletions
@@ -1,8 +1,13 @@
 from __future__ import annotations

+import json
 import re
+import shutil
+import subprocess
 from pathlib import Path

+import pytest
+
 ROOT = Path(__file__).resolve().parents[2]


@@ -12,7 +17,7 @@ def markdown_links(text: str) -> set[str]:


 def test_big_doc_links_case_study_and_embeds_evidence_index() -> None:
-    doc = (ROOT / "docs" / "add" / "system-design-implementation.md").read_text(
+    doc = (ROOT / "docs" / "thesis" / "system-design-implementation.md").read_text(
        encoding="utf-8"
    )
    links = markdown_links(doc)
@@ -34,10 +39,111 @@ def test_project_map_links_big_doc() -> None:


 def test_big_doc_keeps_mcp_as_source_family() -> None:
-    doc = (ROOT / "docs" / "add" / "system-design-implementation.md").read_text(
+    doc = (ROOT / "docs" / "thesis" / "system-design-implementation.md").read_text(
        encoding="utf-8"
    )

    assert "MCP" in doc
    assert "source family" in doc
    assert "product identity" in doc
+
+
+def test_thesis_has_no_placeholder_author_and_keeps_appendix_evidence_together() -> (
+    None
+):
+    doc = (ROOT / "docs" / "thesis" / "system-design-implementation.md").read_text(
+        encoding="utf-8"
+    )
+    appendix = doc.split("# Agent Challenge Harness", maxsplit=1)[1]
+    figure = appendix.index("#fig:agent-challenge-audit")
+    evidence = appendix.index("Evidence:")
+    page_break = appendix.index("\\clearpage")
+
+    assert 'author: "draft"' not in doc
+    assert "This draft includes" not in doc
+    assert page_break < evidence < figure
+    assert figure > evidence
+    assert 'latex-placement="H"' in appendix[figure : figure + 300]
+
+
+def test_thesis_bundle_has_reproducible_agent_evaluation_assets() -> None:
+    thesis = ROOT / "docs" / "thesis"
+    doc = (thesis / "system-design-implementation.md").read_text(encoding="utf-8")
+    results = (thesis / "agent-challenge-results.md").read_text(encoding="utf-8")
+    generate_script = (thesis / "generate.ps1").read_text(encoding="utf-8")
+    combined_build_script = (thesis / "gengen.ps1").read_text(encoding="utf-8")
+    figure_stems = (
+        "agent-challenge-audited-outcomes-by-cell",
+        "agent-challenge-automatic-vs-manual-outcomes",
+        "agent-challenge-longitudinal-outcomes",
+        "agent-challenge-duration-and-tokens",
+    )
+
+    assert (thesis / "agent-challenge-cohort.json").is_file()
+    assert (thesis / "agent-challenge-results.md").is_file()
+    assert "include-agent-challenge-results" in doc
+    assert "include-markdown.lua" in generate_script
+    assert "figure-format.lua" in generate_script
+    assert "thesisFigureFormat" in generate_script
+    assert (
+        generate_script.index("$include_markdown_filter `")
+        < generate_script.index("$diagram_filter `")
+        < generate_script.index("--filter=pandoc-crossref")
+    )
+    assert "generate_agent_challenge_evaluation.py" in combined_build_script
+    assert "--resource-path" in combined_build_script
+    for stem in figure_stems:
+        assert f"figures/{stem}.svg" in results
+        assert (thesis / "figures" / f"{stem}.svg").is_file()
+        assert (thesis / "figures" / f"{stem}.pdf").is_file()
+
+
+@pytest.mark.skipif(shutil.which("pandoc") is None, reason="pandoc is not installed")
+def test_thesis_lua_filters_include_results_before_rewriting_figures(
+    tmp_path: Path,
+) -> None:
+    thesis = ROOT / "docs" / "thesis"
+    source = tmp_path / "source.md"
+    included = tmp_path / "included.md"
+    source.write_text("::: {#include-agent-challenge-results}\n:::\n", encoding="utf-8")
+    included.write_text(
+        "## Included result\n\n![Result](figures/result.svg)\n", encoding="utf-8"
+    )
+
+    completed = subprocess.run(
+        [
+            "pandoc",
+            str(source),
+            "--lua-filter",
+            str(thesis / "include-markdown.lua"),
+            "--lua-filter",
+            str(thesis / "figure-format.lua"),
+            "--metadata",
+            f"thesisAgentResults={included}",
+            "--metadata",
+            "thesisFigureFormat=pdf",
+            "--to",
+            "json",
+        ],
+        capture_output=True,
+        check=False,
+        text=True,
+        encoding="utf-8",
+    )
+
+    assert completed.returncode == 0, completed.stderr
+    document = json.loads(completed.stdout)
+    assert document["blocks"][0]["t"] == "Header"
+
+    def image_targets(value: object) -> list[str]:
+        if isinstance(value, dict):
+            if value.get("t") == "Image":
+                content = value.get("c")
+                if isinstance(content, list):
+                    return [content[2][0]]
+            return [target for item in value.values() for target in image_targets(item)]
+        if isinstance(value, list):
+            return [target for item in value for target in image_targets(item)]
+        return []
+
+    assert image_targets(document) == ["figures/result.pdf"]
@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+COHORT_PATH = ROOT / "docs" / "thesis" / "agent-challenge-cohort.json"
+
+
+def test_primary_agent_challenge_cohort_is_complete_and_audited() -> None:
+    from examples.agent_challenges.evaluation import load_evaluation_cohort
+
+    cohort = load_evaluation_cohort(COHORT_PATH, repository_root=ROOT)
+
+    assert len(cohort.trials) == 36
+    assert {trial.wave for trial in cohort.trials} == {1, 2, 3}
+    assert Counter(trial.manual_outcome for trial in cohort.trials) == {
+        "pass": 27,
+        "invalid": 8,
+        "fail": 1,
+    }
+
+    cells = Counter(
+        (trial.challenge, trial.model, trial.profile) for trial in cohort.trials
+    )
+    assert len(cells) == 12
+    assert set(cells.values()) == {3}
+
+
+def test_primary_cohort_preserves_automatic_and_manual_outcomes() -> None:
+    from examples.agent_challenges.evaluation import load_evaluation_cohort
+
+    cohort = load_evaluation_cohort(COHORT_PATH, repository_root=ROOT)
+    outcome_pairs = Counter(
+        (trial.task_outcome, trial.manual_outcome) for trial in cohort.trials
+    )
+
+    assert outcome_pairs == {
+        ("success", "pass"): 24,
+        ("success", "invalid"): 7,
+        ("failed", "pass"): 3,
+        ("failed", "invalid"): 1,
+        ("failed", "fail"): 1,
+    }
+
+
+def test_primary_cohort_snapshot_loads_without_local_report_files(
+    tmp_path: Path,
+) -> None:
+    from examples.agent_challenges.evaluation import load_evaluation_cohort
+
+    manifest = json.loads(COHORT_PATH.read_text(encoding="utf-8"))
+    assert all(len(run["report_sha256"]) == 64 for run in manifest["runs"])
+    snapshot = tmp_path / "agent-challenge-cohort.json"
+    snapshot.write_text(json.dumps(manifest), encoding="utf-8")
+
+    cohort = load_evaluation_cohort(snapshot, repository_root=tmp_path)
+
+    assert len(cohort.trials) == 36
+
+
+def test_evaluation_renderer_writes_stable_svg_and_pdf_names(tmp_path: Path) -> None:
+    from examples.agent_challenges.evaluation import (
+        FIGURE_STEMS,
+        load_evaluation_cohort,
+        render_evaluation_figures,
+    )
+
+    cohort = load_evaluation_cohort(COHORT_PATH, repository_root=ROOT)
+    written = render_evaluation_figures(cohort, tmp_path)
+
+    expected = {
+        tmp_path / f"{stem}.{suffix}"
+        for stem in FIGURE_STEMS
+        for suffix in ("svg", "pdf")
+    }
+    assert set(written) == expected
+    assert all(path.stat().st_size > 0 for path in written)
+
+
+def test_evaluation_figures_are_byte_stable_across_regeneration(
+    tmp_path: Path,
+) -> None:
+    from examples.agent_challenges.evaluation import (
+        load_evaluation_cohort,
+        render_evaluation_figures,
+    )
+
+    cohort = load_evaluation_cohort(COHORT_PATH, repository_root=ROOT)
+    first = render_evaluation_figures(cohort, tmp_path)
+    first_bytes = {path.name: path.read_bytes() for path in first}
+
+    second = render_evaluation_figures(cohort, tmp_path)
+
+    assert {path.name: path.read_bytes() for path in second} == first_bytes
+
+
+def test_evaluation_markdown_states_counts_and_longitudinal_limit() -> None:
+    from examples.agent_challenges.evaluation import (
+        load_evaluation_cohort,
+        render_evaluation_markdown,
+    )
+
+    cohort = load_evaluation_cohort(COHORT_PATH, repository_root=ROOT)
+    markdown = render_evaluation_markdown(cohort)
+
+    assert "36 audited trials" in markdown
+    assert "27 passes, 8 invalid samples, and 1 failure" in markdown
+    assert "not a controlled model comparison" in markdown
+    assert "agent-challenge-audited-outcomes-by-cell.svg" in markdown
+    assert "agent-challenge-longitudinal-outcomes.svg" in markdown
+    assert "Figure [@fig:" not in markdown