chore: tune challenge matrix and debug prompts
This commit is contained in:
@@ -92,7 +92,7 @@ challenge_report:
|
||||
failed: 0
|
||||
missed_requirements:
|
||||
- "none"
|
||||
ux_issues_found: []
|
||||
# Debug profile only: include ux_issues_found here. See the debug profile instructions.
|
||||
notes: "short explanation"
|
||||
```
|
||||
|
||||
@@ -123,8 +123,6 @@ Reporting rules:
|
||||
abandoned workflow plans.
|
||||
- `attempts.failed` should count attempts that failed validation, failed to run,
|
||||
produced wrong output, or were abandoned.
|
||||
- `ux_issues_found` should always be present. Use an empty list when you found
|
||||
no UX issues; otherwise include concrete command/error/workaround notes.
|
||||
|
||||
Spawned subagents count as you. If a subagent reads product code, set
|
||||
`read.product_code: true`. If a subagent reads prior attempts, set
|
||||
|
||||
@@ -4,9 +4,12 @@ from __future__ import annotations
|
||||
def short_model_name(model: str) -> str:
|
||||
"""Return compact model labels for dense trial tables and session titles."""
|
||||
name = model.rsplit("/", 1)[-1]
|
||||
for suffix in ("-v4-flash-free", "-v2.5-free", "-3-ultra-free"):
|
||||
name = name.replace(suffix, "")
|
||||
return name
|
||||
return {
|
||||
"deepseek-v4-flash-free": "deepseek",
|
||||
"mimo-v2.5-free": "mimo",
|
||||
"nemotron-3-ultra-free": "nemotron",
|
||||
"north-mini-code-free": "north",
|
||||
}.get(name, name)
|
||||
|
||||
|
||||
def short_challenge_name(challenge: str) -> str:
|
||||
|
||||
@@ -23,8 +23,10 @@ honestly in the `read` flags and explain why. Never read or copy adjacent trial
|
||||
answers, prior result files, prior stores, or complete ready-made solution plans
|
||||
for the same challenge.
|
||||
|
||||
In the final `challenge_report`, include a `ux_issues_found` list. Use an empty
|
||||
list if there were no issues. Each issue should be concrete and evidence-backed:
|
||||
For this debug profile only, extend the final `challenge_report` YAML with a
|
||||
top-level `ux_issues_found` list next to `missed_requirements` and `notes`. Use
|
||||
an empty list if there were no issues. Each issue should be concrete and
|
||||
evidence-backed:
|
||||
|
||||
```yaml
|
||||
ux_issues_found:
|
||||
|
||||
@@ -92,7 +92,7 @@ challenge_report:
|
||||
failed: 0
|
||||
missed_requirements:
|
||||
- "none"
|
||||
ux_issues_found: []
|
||||
# Debug profile only: include ux_issues_found here. See the debug profile instructions.
|
||||
notes: "short explanation"
|
||||
```
|
||||
|
||||
@@ -127,8 +127,6 @@ Reporting rules:
|
||||
abandoned workflow plans.
|
||||
- `attempts.failed` should count attempts that failed validation, failed to run,
|
||||
produced wrong output, or were abandoned.
|
||||
- `ux_issues_found` should always be present. Use an empty list when you found
|
||||
no UX issues; otherwise include concrete command/error/workaround notes.
|
||||
|
||||
Spawned subagents count as you. If a subagent reads product code, set
|
||||
`read.product_code: true`. If a subagent reads prior attempts, set
|
||||
|
||||
@@ -5,8 +5,8 @@ param(
|
||||
[int]$TimeoutSeconds = 3600,
|
||||
[object[]]$models = @( # object[] because ModelProfile is not known yet
|
||||
[ModelProfile]::new("opencode/deepseek-v4-flash-free", "max"),
|
||||
[ModelProfile]::new("opencode/mimo-v2.5-free", "high"),
|
||||
[ModelProfile]::new("opencode/nemotron-3-ultra-free", "high")
|
||||
[ModelProfile]::new("opencode/mimo-v2.5-free", "high")
|
||||
# [ModelProfile]::new("opencode/nemotron-3-ultra-free", "high")
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ class ModelProfile:
|
||||
DEFAULT_MODELS = (
|
||||
ModelProfile("opencode/deepseek-v4-flash-free", "max"),
|
||||
ModelProfile("opencode/mimo-v2.5-free", "high"),
|
||||
ModelProfile("opencode/nemotron-3-ultra-free", "high"),
|
||||
# ModelProfile("opencode/nemotron-3-ultra-free", "high"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user