fix: harden workflow demo and evaluation tooling

tests/examples/test_agent_challenge_evaluation.py

@@ -2,8 +2,11 @@ from __future__ import annotations
 
 import json
 from collections import Counter
+from dataclasses import replace
 from pathlib import Path
 
+import pytest
+
 ROOT = Path(__file__).resolve().parents[2]
 COHORT_PATH = ROOT / "docs" / "thesis" / "agent-challenge-cohort.json"
 
@@ -60,6 +63,80 @@ def test_primary_cohort_snapshot_loads_without_local_report_files(
     assert len(cohort.trials) == 36
 
 
+@pytest.mark.parametrize(
+    ("field", "value", "message"),
+    [
+        ("profile", "debug", "unsupported evaluation profile"),
+        ("task_outcome", "unknown", "unsupported task outcome"),
+        ("audit_notes", ["not", "text"], "audit_notes must be a string"),
+    ],
+)
+def test_evaluation_cohort_rejects_unknown_or_mistyped_run_values(
+    tmp_path: Path, field: str, value: object, message: str
+) -> None:
+    from examples.agent_challenges.evaluation import load_evaluation_cohort
+
+    manifest = json.loads(COHORT_PATH.read_text(encoding="utf-8"))
+    manifest["runs"][0][field] = value
+    snapshot = tmp_path / "agent-challenge-cohort.json"
+    snapshot.write_text(json.dumps(manifest), encoding="utf-8")
+
+    with pytest.raises(ValueError, match=message):
+        load_evaluation_cohort(snapshot, repository_root=tmp_path)
+
+
+def test_evaluation_figures_reject_unknown_direct_trial_values() -> None:
+    import matplotlib.pyplot as plt
+
+    from examples.agent_challenges.evaluation import load_evaluation_cohort
+    from examples.agent_challenges.evaluation_figures import (
+        _automatic_vs_manual,
+        _scatter_metric,
+    )
+
+    cohort = load_evaluation_cohort(COHORT_PATH, repository_root=ROOT)
+    figure, axis = plt.subplots()
+    try:
+        with pytest.raises(ValueError, match="unsupported evaluation model"):
+            _scatter_metric(
+                axis,
+                [replace(cohort.trials[0], model="unknown-model")],
+                metric="duration",
+            )
+        with pytest.raises(ValueError, match="unsupported evaluation profile"):
+            _scatter_metric(
+                axis,
+                [replace(cohort.trials[0], profile="unknown-profile")],
+                metric="duration",
+            )
+        invalid_cohort = replace(
+            cohort,
+            trials=(replace(cohort.trials[0], task_outcome="unknown"),),
+        )
+        with pytest.raises(ValueError, match="unsupported task outcomes"):
+            _automatic_vs_manual(invalid_cohort, plt)
+    finally:
+        plt.close(figure)
+
+
+def test_evaluation_generator_prints_paths_outside_repository(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    from docs.thesis import generate_agent_challenge_evaluation as generator
+
+    generated = tmp_path / "agent-challenge-results.md"
+    monkeypatch.setattr(
+        generator,
+        "generate",
+        lambda **_kwargs: (generated,),
+    )
+
+    assert generator.main(["--output-dir", str(tmp_path)]) == 0
+    assert capsys.readouterr().out.strip() == "agent-challenge-results.md"
+
+
 def test_evaluation_renderer_writes_stable_svg_and_pdf_names(tmp_path: Path) -> None:
     from examples.agent_challenges.evaluation import (
         FIGURE_STEMS,

tests/examples/test_agent_challenge_harness_v2.py

@@ -129,6 +129,13 @@ def test_challenge_prompt_is_identical_across_profiles(tmp_path: Path) -> None:
     assert "genuinely blocked" in rendered[InstructionProfile.DEBUG].text
     assert "debug profile only" in rendered[InstructionProfile.DEBUG].text
     assert "ux_issues_found" in rendered[InstructionProfile.DEBUG].text
+    for profile in (
+        InstructionProfile.NONE,
+        InstructionProfile.SKILLS,
+        InstructionProfile.ALL,
+    ):
+        assert "debug profile only" not in rendered[profile].text
+        assert "ux_issues_found" not in rendered[profile].text
     none_prompt = rendered[InstructionProfile.NONE].text.replace("\n", " ")
     assert "inline in your" in none_prompt
     assert "without an inline self-report is invalid" in none_prompt

tests/examples/test_agent_challenge_reports.py

@@ -616,3 +616,20 @@ def test_trial_report_renders_opencode_resume_metadata(tmp_path: Path) -> None:
     assert "## OpenCode Resume" in rendered
     assert "ses_report" in rendered
     assert "opencode run --session ses_report" in rendered
+
+
+def test_trial_report_skips_resume_section_without_session_id(tmp_path: Path) -> None:
+    from examples.agent_challenges.report_models import build_trial_report
+    from examples.agent_challenges.reports import render_trial_report_markdown
+
+    result = _raw_result(tmp_path)
+    result["opencode"] = {
+        "model": "opencode/deepseek-v4-flash-free",
+        "variant": "max",
+        "session_id": None,
+    }
+
+    report = build_trial_report(result, audit=None)
+    rendered = render_trial_report_markdown(report)
+
+    assert "## OpenCode Resume" not in rendered

tests/examples/test_agent_challenge_resume.py

@@ -183,6 +183,25 @@ def test_resume_command_prompt_mode_overrides_stored_command() -> None:
     assert any("Continue this same trial" in part for part in command)
 
 
+def test_resume_command_never_executes_persisted_argv() -> None:
+    command = resume_command_from_result(
+        {
+            "task_outcome": "timeout",
+            "stdout": "",
+            "opencode": {
+                "model": "opencode/mimo-v2.5-free",
+                "variant": "high",
+                "session_id": "ses_metadata",
+                "resume_command": ["powershell", "-Command", "Write-Host tampered"],
+            },
+        }
+    )
+
+    assert command[0:4] == ["opencode", "run", "--session", "ses_metadata"]
+    assert "powershell" not in command
+    assert "Write-Host tampered" not in command
+
+
 def test_resume_result_path_uses_next_resume_index(tmp_path: Path) -> None:
     original = tmp_path / "trial.json"
     original.write_text("{}", encoding="utf-8")
@@ -232,6 +251,20 @@ def test_resume_trial_prints_resume_command(
     assert "opencode run --session ses_cli" in output
 
 
+def test_resume_trial_reports_missing_result_as_cli_error(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    from examples.agent_challenges.resume_trial import main
+
+    missing = tmp_path / "missing.json"
+
+    with pytest.raises(SystemExit) as exc_info:
+        main(["--from-result", str(missing), "--print-command"])
+
+    assert exc_info.value.code == 2
+    assert "missing.json" in capsys.readouterr().err
+
+
 def test_resume_trial_prints_command_for_old_raw_result(
     tmp_path: Path, capsys: pytest.CaptureFixture[str]
 ) -> None:

tests/examples/test_lda_report_workflow_example.py

@@ -181,6 +181,7 @@ def test_lda_report_workflow_builder_generates_committed_raw_plan() -> None:
 
     assert workflow.name == "lda_report_case_study"
     assert validated.name == "lda_report_case_study"
+    assert validated.start == "reset_board"
     assert any(node.id == "review_issues" for node in validated.nodes)
     assert payload == committed
 
@@ -219,7 +220,20 @@ async def test_lda_report_workflow_artifact_interrupt_resume_path(
         }
     )
     run_input = json.loads((EXAMPLE_DIR / "run-input.json").read_text(encoding="utf-8"))
-    run_input["board_path"] = str(tmp_path / "issue-board.json")
+    board_path = tmp_path / "issue-board.json"
+    board_path.write_text(
+        json.dumps(
+            [
+                {
+                    "id": "ISSUE-099",
+                    "title": "Stale demo issue",
+                    "url": "local://issue-board/ISSUE-099",
+                }
+            ]
+        ),
+        encoding="utf-8",
+    )
+    run_input["board_path"] = str(board_path)
     started = await server.api.run_deployment(
         deployment_id="lda_report_case_study.default",
         workflow_input=run_input,
@@ -228,14 +242,14 @@ async def test_lda_report_workflow_artifact_interrupt_resume_path(
     assert started["status"] == "interrupted"
     assert started["interrupt"]["kind"] == "issue_review"
     assert started["interrupt"]["typed"] is True
-    assert started["interrupt"]["request_schema"]["required"] == [
+    assert set(started["interrupt"]["request_schema"]["required"]) == {
         "report_markdown",
         "proposed_issues",
-    ]
-    assert started["interrupt"]["resume_schema"]["required"] == [
+    }
+    assert set(started["interrupt"]["resume_schema"]["required"]) == {
         "approved",
         "selected_issue_ids",
-    ]
+    }
     proposed_ids = [
         issue["id"] for issue in started["interrupt"]["payload"]["proposed_issues"]
     ]
@@ -255,6 +269,7 @@ async def test_lda_report_workflow_artifact_interrupt_resume_path(
     assert resumed["outcome"] == "completed"
     assert resumed["output"]["approved"] is True
     assert resumed["output"]["created_issues"]
+    assert resumed["output"]["created_issues"][0]["id"] == "ISSUE-001"
     assert resumed["output"]["markdown"].startswith(
         "# lda.chat Thesis And Project Readiness Report"
     )