chore: tune challenge matrix and debug prompts

This commit is contained in:
lda
2026-06-29 19:25:35 +07:00 Verified
parent 5f520d74bd
commit e0315f295c
10 changed files with 27 additions and 18 deletions
@@ -92,7 +92,7 @@ challenge_report:
failed: 0
missed_requirements:
- "none"
ux_issues_found: []
# Debug profile only: include ux_issues_found here. See the debug profile instructions.
notes: "short explanation"
```
@@ -123,8 +123,6 @@ Reporting rules:
abandoned workflow plans.
- `attempts.failed` should count attempts that failed validation, failed to run,
produced wrong output, or were abandoned.
- `ux_issues_found` should always be present. Use an empty list when you found
no UX issues; otherwise include concrete command/error/workaround notes.
Spawned subagents count as you. If a subagent reads product code, set
`read.product_code: true`. If a subagent reads prior attempts, set
+6 -3
View File
@@ -4,9 +4,12 @@ from __future__ import annotations
def short_model_name(model: str) -> str:
"""Return compact model labels for dense trial tables and session titles."""
name = model.rsplit("/", 1)[-1]
for suffix in ("-v4-flash-free", "-v2.5-free", "-3-ultra-free"):
name = name.replace(suffix, "")
return name
return {
"deepseek-v4-flash-free": "deepseek",
"mimo-v2.5-free": "mimo",
"nemotron-3-ultra-free": "nemotron",
"north-mini-code-free": "north",
}.get(name, name)
def short_challenge_name(challenge: str) -> str:
@@ -23,8 +23,10 @@ honestly in the `read` flags and explain why. Never read or copy adjacent trial
answers, prior result files, prior stores, or complete ready-made solution plans
for the same challenge.
In the final `challenge_report`, include a `ux_issues_found` list. Use an empty
list if there were no issues. Each issue should be concrete and evidence-backed:
For this debug profile only, extend the final `challenge_report` YAML with a
top-level `ux_issues_found` list next to `missed_requirements` and `notes`. Use
an empty list if there were no issues. Each issue should be concrete and
evidence-backed:
```yaml
ux_issues_found:
@@ -92,7 +92,7 @@ challenge_report:
failed: 0
missed_requirements:
- "none"
ux_issues_found: []
# Debug profile only: include ux_issues_found here. See the debug profile instructions.
notes: "short explanation"
```
@@ -127,8 +127,6 @@ Reporting rules:
abandoned workflow plans.
- `attempts.failed` should count attempts that failed validation, failed to run,
produced wrong output, or were abandoned.
- `ux_issues_found` should always be present. Use an empty list when you found
no UX issues; otherwise include concrete command/error/workaround notes.
Spawned subagents count as you. If a subagent reads product code, set
`read.product_code: true`. If a subagent reads prior attempts, set
+2 -2
View File
@@ -5,8 +5,8 @@ param(
[int]$TimeoutSeconds = 3600,
[object[]]$models = @( # object[] because ModelProfile is not known yet
[ModelProfile]::new("opencode/deepseek-v4-flash-free", "max"),
[ModelProfile]::new("opencode/mimo-v2.5-free", "high"),
[ModelProfile]::new("opencode/nemotron-3-ultra-free", "high")
[ModelProfile]::new("opencode/mimo-v2.5-free", "high")
# [ModelProfile]::new("opencode/nemotron-3-ultra-free", "high")
)
)
+1 -1
View File
@@ -45,7 +45,7 @@ class ModelProfile:
DEFAULT_MODELS = (
ModelProfile("opencode/deepseek-v4-flash-free", "max"),
ModelProfile("opencode/mimo-v2.5-free", "high"),
ModelProfile("opencode/nemotron-3-ultra-free", "high"),
# ModelProfile("opencode/nemotron-3-ultra-free", "high"),
)