fix: generate browser challenge workspace config

This commit is contained in:
lda
2026-06-15 06:29:25 +07:00 Verified
parent 73dc1d958a
commit 3b19884919
4 changed files with 59 additions and 35 deletions
@@ -19,10 +19,11 @@ uv run wf --config examples/agent_challenges/browser_click_challenge/workspaces/
``` ```
For each local-mode trial, the harness copies `workspace_template/` into For each local-mode trial, the harness copies `workspace_template/` into
`workspaces/<model>-trial-<n>/` and injects that config path into the prompt. `workspaces/<model>-trial-<n>/`, generates a config whose Python source path is
This builds the configured workflow server in the CLI process for each command relative to that copied config, and injects the config path into the prompt. This
and uses the copied workspace's durable store. It does not reuse in-memory builds the configured workflow server in the CLI process for each command and
source sessions across CLI invocations. uses the copied workspace's durable store. It does not reuse in-memory source
sessions across CLI invocations.
Use `--start-server` when the trial should exercise the JSON-RPC server path. Use `--start-server` when the trial should exercise the JSON-RPC server path.
With `--start-server`, the harness starts: With `--start-server`, the harness starts:
@@ -60,10 +61,10 @@ For manual authoring trials, use:
examples/agent_challenges/browser_click_challenge/workspace_template/ examples/agent_challenges/browser_click_challenge/workspace_template/
``` ```
It contains a local workflow config and prompt template that point at the It contains the prompt template and local store ignore rules without exposing a
browser-click Python source without exposing a generated draft patch answer generated draft patch answer file. The harness copies it automatically for
file. The harness copies it automatically for normal local-mode trials. For normal local-mode trials and writes `wf.config.json` into the copied workspace.
manual experiments, copy it under: For manual experiments, copy it under:
```text ```text
examples/agent_challenges/browser_click_challenge/workspaces/ examples/agent_challenges/browser_click_challenge/workspaces/
@@ -2,6 +2,7 @@ from __future__ import annotations
import argparse import argparse
import json import json
import os
import shutil import shutil
import subprocess import subprocess
import sys import sys
@@ -30,6 +31,7 @@ DEFAULT_WORKSPACES_DIR = CHALLENGE_DIR / "workspaces"
DEFAULT_WORKSPACE_TEMPLATE = CHALLENGE_DIR / "workspace_template" DEFAULT_WORKSPACE_TEMPLATE = CHALLENGE_DIR / "workspace_template"
DEFAULT_SERVER_PORT = 8772 DEFAULT_SERVER_PORT = 8772
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json" EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
EXAMPLE_SOURCE_ROOT = ROOT / "examples" / "browser_click_workflow"
EXAMPLE_CONFIG_ARG = "examples/browser_click_workflow/wf.config.json" EXAMPLE_CONFIG_ARG = "examples/browser_click_workflow/wf.config.json"
LOCAL_WF_COMMAND_PREFIX = f"uv run wf --config {EXAMPLE_CONFIG_ARG} --local" LOCAL_WF_COMMAND_PREFIX = f"uv run wf --config {EXAMPLE_CONFIG_ARG} --local"
@@ -127,6 +129,7 @@ def prepare_trial_workspace(
index: int, index: int,
workspaces_dir: Path = DEFAULT_WORKSPACES_DIR, workspaces_dir: Path = DEFAULT_WORKSPACES_DIR,
template_dir: Path = DEFAULT_WORKSPACE_TEMPLATE, template_dir: Path = DEFAULT_WORKSPACE_TEMPLATE,
source_root: Path = EXAMPLE_SOURCE_ROOT,
) -> TrialWorkspace: ) -> TrialWorkspace:
"""Copy the authoring template into a clean ignored per-trial directory.""" """Copy the authoring template into a clean ignored per-trial directory."""
root = workspaces_dir / f"{_safe_model_name(model)}-trial-{index:03d}" root = workspaces_dir / f"{_safe_model_name(model)}-trial-{index:03d}"
@@ -135,11 +138,38 @@ def prepare_trial_workspace(
# the ignored per-trial directory before copying the template. # the ignored per-trial directory before copying the template.
shutil.rmtree(root) shutil.rmtree(root)
shutil.copytree(template_dir, root) shutil.copytree(template_dir, root)
return TrialWorkspace( workspace = TrialWorkspace(
root=root, root=root,
config_path=root / "wf.config.json", config_path=root / "wf.config.json",
prompt_path=root / "prompt.md", prompt_path=root / "prompt.md",
) )
write_trial_config(workspace.config_path, source_root=source_root)
return workspace
def write_trial_config(config_path: Path, *, source_root: Path) -> None:
"""Write a per-trial config with Python source path relative to config."""
relative_source = Path(os.path.relpath(source_root, config_path.parent)).as_posix()
config = {
"version": 1,
"client": {"target": {"kind": "local"}},
"server": {
"store": {"kind": "filesystem", "root": ".wf_browser_click_store"},
"sources": [
{
"kind": "python",
"id": "local.browser_click",
"path": relative_source,
"module": "ops",
"registry": "registry",
}
],
},
}
config_path.write_text(
json.dumps(config, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def wf_command_prefix_for_config(config_path: Path) -> str: def wf_command_prefix_for_config(config_path: Path) -> str:
@@ -1,23 +0,0 @@
{
"version": 1,
"client": {
"target": {
"kind": "local"
}
},
"server": {
"store": {
"kind": "filesystem",
"root": ".wf_browser_click_store"
},
"sources": [
{
"kind": "python",
"id": "local.browser_click",
"path": "../../../../browser_click_workflow",
"module": "ops",
"registry": "registry"
}
]
}
}
@@ -254,20 +254,34 @@ def test_prepare_trial_workspace_copies_template_to_model_trial_dir(
) -> None: ) -> None:
template = tmp_path / "template" template = tmp_path / "template"
template.mkdir() template.mkdir()
(template / "wf.config.json").write_text('{"version": 1}', encoding="utf-8")
(template / "prompt.md").write_text("prompt", encoding="utf-8") (template / "prompt.md").write_text("prompt", encoding="utf-8")
(template / ".gitignore").write_text(".wf_store/\n", encoding="utf-8") (template / ".gitignore").write_text(".wf_store/\n", encoding="utf-8")
workspaces = tmp_path / "workspaces" workspaces = tmp_path / "workspaces"
source_root = tmp_path / "browser_click_workflow"
source_root.mkdir()
prepared = prepare_trial_workspace( prepared = prepare_trial_workspace(
model="opencode/mimo-v2.5-free", model="opencode/mimo-v2.5-free",
index=7, index=7,
workspaces_dir=workspaces, workspaces_dir=workspaces,
template_dir=template, template_dir=template,
source_root=source_root,
) )
assert prepared.root == workspaces / "opencode_mimo-v2.5-free-trial-007" assert prepared.root == workspaces / "opencode_mimo-v2.5-free-trial-007"
assert prepared.config_path.read_text(encoding="utf-8") == '{"version": 1}' config = json.loads(prepared.config_path.read_text(encoding="utf-8"))
assert config["client"]["target"] == {"kind": "local"}
assert config["server"]["store"] == {
"kind": "filesystem",
"root": ".wf_browser_click_store",
}
assert config["server"]["sources"][0] == {
"kind": "python",
"id": "local.browser_click",
"path": "../../browser_click_workflow",
"module": "ops",
"registry": "registry",
}
assert prepared.prompt_path.read_text(encoding="utf-8") == "prompt" assert prepared.prompt_path.read_text(encoding="utf-8") == "prompt"
assert (prepared.root / ".gitignore").read_text(encoding="utf-8") == ".wf_store/\n" assert (prepared.root / ".gitignore").read_text(encoding="utf-8") == ".wf_store/\n"
@@ -277,8 +291,9 @@ def test_prepare_trial_workspace_removes_stale_previous_attempt(
) -> None: ) -> None:
template = tmp_path / "template" template = tmp_path / "template"
template.mkdir() template.mkdir()
(template / "wf.config.json").write_text('{"version": 1}', encoding="utf-8")
(template / "prompt.md").write_text("prompt", encoding="utf-8") (template / "prompt.md").write_text("prompt", encoding="utf-8")
source_root = tmp_path / "browser_click_workflow"
source_root.mkdir()
workspaces = tmp_path / "workspaces" workspaces = tmp_path / "workspaces"
stale = workspaces / "opencode_mimo-v2.5-free-trial-001" / "old-answer.json" stale = workspaces / "opencode_mimo-v2.5-free-trial-001" / "old-answer.json"
stale.parent.mkdir(parents=True) stale.parent.mkdir(parents=True)
@@ -289,6 +304,7 @@ def test_prepare_trial_workspace_removes_stale_previous_attempt(
index=1, index=1,
workspaces_dir=workspaces, workspaces_dir=workspaces,
template_dir=template, template_dir=template,
source_root=source_root,
) )
assert prepared.root.exists() assert prepared.root.exists()