chore: tune challenge matrix and debug prompts

This commit is contained in:
lda
2026-06-29 19:25:35 +07:00 Verified
parent 5f520d74bd
commit e0315f295c
10 changed files with 27 additions and 18 deletions
+8
View File
@@ -84,6 +84,14 @@ this line ends the list, because it has no indents.
```
2. otherwise this will be "fixed" and renumbered to 1.
<!-- in case -->
9. we have come far
- notice the indents
<!-- a blank line here is standard -->
10. after this point
- the indent increases
- since the list marker is longer
````
### why not prettier? it does much more
@@ -92,7 +92,7 @@ challenge_report:
failed: 0
missed_requirements:
- "none"
ux_issues_found: []
# Debug profile only: include ux_issues_found here. See the debug profile instructions.
notes: "short explanation"
```
@@ -123,8 +123,6 @@ Reporting rules:
abandoned workflow plans.
- `attempts.failed` should count attempts that failed validation, failed to run,
produced wrong output, or were abandoned.
- `ux_issues_found` should always be present. Use an empty list when you found
no UX issues; otherwise include concrete command/error/workaround notes.
Spawned subagents count as you. If a subagent reads product code, set
`read.product_code: true`. If a subagent reads prior attempts, set
+6 -3
View File
@@ -4,9 +4,12 @@ from __future__ import annotations
def short_model_name(model: str) -> str:
"""Return compact model labels for dense trial tables and session titles."""
name = model.rsplit("/", 1)[-1]
for suffix in ("-v4-flash-free", "-v2.5-free", "-3-ultra-free"):
name = name.replace(suffix, "")
return name
return {
"deepseek-v4-flash-free": "deepseek",
"mimo-v2.5-free": "mimo",
"nemotron-3-ultra-free": "nemotron",
"north-mini-code-free": "north",
}.get(name, name)
def short_challenge_name(challenge: str) -> str:
@@ -23,8 +23,10 @@ honestly in the `read` flags and explain why. Never read or copy adjacent trial
answers, prior result files, prior stores, or complete ready-made solution plans
for the same challenge.
In the final `challenge_report`, include a `ux_issues_found` list. Use an empty
list if there were no issues. Each issue should be concrete and evidence-backed:
For this debug profile only, extend the final `challenge_report` YAML with a
top-level `ux_issues_found` list next to `missed_requirements` and `notes`. Use
an empty list if there were no issues. Each issue should be concrete and
evidence-backed:
```yaml
ux_issues_found:
@@ -92,7 +92,7 @@ challenge_report:
failed: 0
missed_requirements:
- "none"
ux_issues_found: []
# Debug profile only: include ux_issues_found here. See the debug profile instructions.
notes: "short explanation"
```
@@ -127,8 +127,6 @@ Reporting rules:
abandoned workflow plans.
- `attempts.failed` should count attempts that failed validation, failed to run,
produced wrong output, or were abandoned.
- `ux_issues_found` should always be present. Use an empty list when you found
no UX issues; otherwise include concrete command/error/workaround notes.
Spawned subagents count as you. If a subagent reads product code, set
`read.product_code: true`. If a subagent reads prior attempts, set
+2 -2
View File
@@ -5,8 +5,8 @@ param(
[int]$TimeoutSeconds = 3600,
[object[]]$models = @( # object[] because ModelProfile is not known yet
[ModelProfile]::new("opencode/deepseek-v4-flash-free", "max"),
[ModelProfile]::new("opencode/mimo-v2.5-free", "high"),
[ModelProfile]::new("opencode/nemotron-3-ultra-free", "high")
[ModelProfile]::new("opencode/mimo-v2.5-free", "high")
# [ModelProfile]::new("opencode/nemotron-3-ultra-free", "high")
)
)
+1 -1
View File
@@ -45,7 +45,7 @@ class ModelProfile:
DEFAULT_MODELS = (
ModelProfile("opencode/deepseek-v4-flash-free", "max"),
ModelProfile("opencode/mimo-v2.5-free", "high"),
ModelProfile("opencode/nemotron-3-ultra-free", "high"),
# ModelProfile("opencode/nemotron-3-ultra-free", "high"),
)
@@ -127,6 +127,7 @@ def test_challenge_prompt_is_identical_across_profiles(tmp_path: Path) -> None:
assert ".agent/skills" in rendered[InstructionProfile.SKILLS].text
assert "inspect broader repository" in rendered[InstructionProfile.ALL].text
assert "genuinely blocked" in rendered[InstructionProfile.DEBUG].text
assert "debug profile only" in rendered[InstructionProfile.DEBUG].text
assert "ux_issues_found" in rendered[InstructionProfile.DEBUG].text
@@ -950,9 +950,8 @@ def test_browser_click_wrapper_produces_expected_paths_and_command_prefix() -> N
)
assert BROWSER_CLICK_DEF.default_prompt.name == "challenge-prompt.md"
assert BROWSER_CLICK_DEF.default_prompt.parent.name == "browser_click_challenge"
assert "ux_issues_found: []" in BROWSER_CLICK_DEF.default_prompt.read_text(
encoding="utf-8"
)
prompt = BROWSER_CLICK_DEF.default_prompt.read_text(encoding="utf-8")
assert "Debug profile only: include ux_issues_found here" in prompt
def test_generic_runner_can_be_configured_with_fake_challenge_and_fake_opencode(
@@ -40,7 +40,7 @@ def test_report_challenge_prompt_requires_full_product_lifecycle() -> None:
assert "render_markdown_report" in prompt
assert "deployment" in prompt.lower()
assert "run_id" in prompt
assert "ux_issues_found: []" in prompt
assert "Debug profile only: include ux_issues_found here" in prompt
def test_report_challenge_workspace_template_contains_safe_input_files(