fix: generate browser challenge workspace config
This commit is contained in:
@@ -19,10 +19,11 @@ uv run wf --config examples/agent_challenges/browser_click_challenge/workspaces/
|
|||||||
```
|
```
|
||||||
|
|
||||||
For each local-mode trial, the harness copies `workspace_template/` into
|
For each local-mode trial, the harness copies `workspace_template/` into
|
||||||
`workspaces/<model>-trial-<n>/` and injects that config path into the prompt.
|
`workspaces/<model>-trial-<n>/`, generates a config whose Python source path is
|
||||||
This builds the configured workflow server in the CLI process for each command
|
relative to that copied config, and injects the config path into the prompt. This
|
||||||
and uses the copied workspace's durable store. It does not reuse in-memory
|
builds the configured workflow server in the CLI process for each command and
|
||||||
source sessions across CLI invocations.
|
uses the copied workspace's durable store. It does not reuse in-memory source
|
||||||
|
sessions across CLI invocations.
|
||||||
|
|
||||||
Use `--start-server` when the trial should exercise the JSON-RPC server path.
|
Use `--start-server` when the trial should exercise the JSON-RPC server path.
|
||||||
With `--start-server`, the harness starts:
|
With `--start-server`, the harness starts:
|
||||||
@@ -60,10 +61,10 @@ For manual authoring trials, use:
|
|||||||
examples/agent_challenges/browser_click_challenge/workspace_template/
|
examples/agent_challenges/browser_click_challenge/workspace_template/
|
||||||
```
|
```
|
||||||
|
|
||||||
It contains a local workflow config and prompt template that point at the
|
It contains the prompt template and local store ignore rules without exposing a
|
||||||
browser-click Python source without exposing a generated draft patch answer
|
generated draft patch answer file. The harness copies it automatically for
|
||||||
file. The harness copies it automatically for normal local-mode trials. For
|
normal local-mode trials and writes `wf.config.json` into the copied workspace.
|
||||||
manual experiments, copy it under:
|
For manual experiments, copy it under:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
examples/agent_challenges/browser_click_challenge/workspaces/
|
examples/agent_challenges/browser_click_challenge/workspaces/
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
@@ -30,6 +31,7 @@ DEFAULT_WORKSPACES_DIR = CHALLENGE_DIR / "workspaces"
|
|||||||
DEFAULT_WORKSPACE_TEMPLATE = CHALLENGE_DIR / "workspace_template"
|
DEFAULT_WORKSPACE_TEMPLATE = CHALLENGE_DIR / "workspace_template"
|
||||||
DEFAULT_SERVER_PORT = 8772
|
DEFAULT_SERVER_PORT = 8772
|
||||||
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
|
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
|
||||||
|
EXAMPLE_SOURCE_ROOT = ROOT / "examples" / "browser_click_workflow"
|
||||||
EXAMPLE_CONFIG_ARG = "examples/browser_click_workflow/wf.config.json"
|
EXAMPLE_CONFIG_ARG = "examples/browser_click_workflow/wf.config.json"
|
||||||
LOCAL_WF_COMMAND_PREFIX = f"uv run wf --config {EXAMPLE_CONFIG_ARG} --local"
|
LOCAL_WF_COMMAND_PREFIX = f"uv run wf --config {EXAMPLE_CONFIG_ARG} --local"
|
||||||
|
|
||||||
@@ -127,6 +129,7 @@ def prepare_trial_workspace(
|
|||||||
index: int,
|
index: int,
|
||||||
workspaces_dir: Path = DEFAULT_WORKSPACES_DIR,
|
workspaces_dir: Path = DEFAULT_WORKSPACES_DIR,
|
||||||
template_dir: Path = DEFAULT_WORKSPACE_TEMPLATE,
|
template_dir: Path = DEFAULT_WORKSPACE_TEMPLATE,
|
||||||
|
source_root: Path = EXAMPLE_SOURCE_ROOT,
|
||||||
) -> TrialWorkspace:
|
) -> TrialWorkspace:
|
||||||
"""Copy the authoring template into a clean ignored per-trial directory."""
|
"""Copy the authoring template into a clean ignored per-trial directory."""
|
||||||
root = workspaces_dir / f"{_safe_model_name(model)}-trial-{index:03d}"
|
root = workspaces_dir / f"{_safe_model_name(model)}-trial-{index:03d}"
|
||||||
@@ -135,11 +138,38 @@ def prepare_trial_workspace(
|
|||||||
# the ignored per-trial directory before copying the template.
|
# the ignored per-trial directory before copying the template.
|
||||||
shutil.rmtree(root)
|
shutil.rmtree(root)
|
||||||
shutil.copytree(template_dir, root)
|
shutil.copytree(template_dir, root)
|
||||||
return TrialWorkspace(
|
workspace = TrialWorkspace(
|
||||||
root=root,
|
root=root,
|
||||||
config_path=root / "wf.config.json",
|
config_path=root / "wf.config.json",
|
||||||
prompt_path=root / "prompt.md",
|
prompt_path=root / "prompt.md",
|
||||||
)
|
)
|
||||||
|
write_trial_config(workspace.config_path, source_root=source_root)
|
||||||
|
return workspace
|
||||||
|
|
||||||
|
|
||||||
|
def write_trial_config(config_path: Path, *, source_root: Path) -> None:
|
||||||
|
"""Write a per-trial config with Python source path relative to config."""
|
||||||
|
relative_source = Path(os.path.relpath(source_root, config_path.parent)).as_posix()
|
||||||
|
config = {
|
||||||
|
"version": 1,
|
||||||
|
"client": {"target": {"kind": "local"}},
|
||||||
|
"server": {
|
||||||
|
"store": {"kind": "filesystem", "root": ".wf_browser_click_store"},
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"kind": "python",
|
||||||
|
"id": "local.browser_click",
|
||||||
|
"path": relative_source,
|
||||||
|
"module": "ops",
|
||||||
|
"registry": "registry",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
config_path.write_text(
|
||||||
|
json.dumps(config, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def wf_command_prefix_for_config(config_path: Path) -> str:
|
def wf_command_prefix_for_config(config_path: Path) -> str:
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
{
|
|
||||||
"version": 1,
|
|
||||||
"client": {
|
|
||||||
"target": {
|
|
||||||
"kind": "local"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"server": {
|
|
||||||
"store": {
|
|
||||||
"kind": "filesystem",
|
|
||||||
"root": ".wf_browser_click_store"
|
|
||||||
},
|
|
||||||
"sources": [
|
|
||||||
{
|
|
||||||
"kind": "python",
|
|
||||||
"id": "local.browser_click",
|
|
||||||
"path": "../../../../browser_click_workflow",
|
|
||||||
"module": "ops",
|
|
||||||
"registry": "registry"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -254,20 +254,34 @@ def test_prepare_trial_workspace_copies_template_to_model_trial_dir(
|
|||||||
) -> None:
|
) -> None:
|
||||||
template = tmp_path / "template"
|
template = tmp_path / "template"
|
||||||
template.mkdir()
|
template.mkdir()
|
||||||
(template / "wf.config.json").write_text('{"version": 1}', encoding="utf-8")
|
|
||||||
(template / "prompt.md").write_text("prompt", encoding="utf-8")
|
(template / "prompt.md").write_text("prompt", encoding="utf-8")
|
||||||
(template / ".gitignore").write_text(".wf_store/\n", encoding="utf-8")
|
(template / ".gitignore").write_text(".wf_store/\n", encoding="utf-8")
|
||||||
workspaces = tmp_path / "workspaces"
|
workspaces = tmp_path / "workspaces"
|
||||||
|
source_root = tmp_path / "browser_click_workflow"
|
||||||
|
source_root.mkdir()
|
||||||
|
|
||||||
prepared = prepare_trial_workspace(
|
prepared = prepare_trial_workspace(
|
||||||
model="opencode/mimo-v2.5-free",
|
model="opencode/mimo-v2.5-free",
|
||||||
index=7,
|
index=7,
|
||||||
workspaces_dir=workspaces,
|
workspaces_dir=workspaces,
|
||||||
template_dir=template,
|
template_dir=template,
|
||||||
|
source_root=source_root,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert prepared.root == workspaces / "opencode_mimo-v2.5-free-trial-007"
|
assert prepared.root == workspaces / "opencode_mimo-v2.5-free-trial-007"
|
||||||
assert prepared.config_path.read_text(encoding="utf-8") == '{"version": 1}'
|
config = json.loads(prepared.config_path.read_text(encoding="utf-8"))
|
||||||
|
assert config["client"]["target"] == {"kind": "local"}
|
||||||
|
assert config["server"]["store"] == {
|
||||||
|
"kind": "filesystem",
|
||||||
|
"root": ".wf_browser_click_store",
|
||||||
|
}
|
||||||
|
assert config["server"]["sources"][0] == {
|
||||||
|
"kind": "python",
|
||||||
|
"id": "local.browser_click",
|
||||||
|
"path": "../../browser_click_workflow",
|
||||||
|
"module": "ops",
|
||||||
|
"registry": "registry",
|
||||||
|
}
|
||||||
assert prepared.prompt_path.read_text(encoding="utf-8") == "prompt"
|
assert prepared.prompt_path.read_text(encoding="utf-8") == "prompt"
|
||||||
assert (prepared.root / ".gitignore").read_text(encoding="utf-8") == ".wf_store/\n"
|
assert (prepared.root / ".gitignore").read_text(encoding="utf-8") == ".wf_store/\n"
|
||||||
|
|
||||||
@@ -277,8 +291,9 @@ def test_prepare_trial_workspace_removes_stale_previous_attempt(
|
|||||||
) -> None:
|
) -> None:
|
||||||
template = tmp_path / "template"
|
template = tmp_path / "template"
|
||||||
template.mkdir()
|
template.mkdir()
|
||||||
(template / "wf.config.json").write_text('{"version": 1}', encoding="utf-8")
|
|
||||||
(template / "prompt.md").write_text("prompt", encoding="utf-8")
|
(template / "prompt.md").write_text("prompt", encoding="utf-8")
|
||||||
|
source_root = tmp_path / "browser_click_workflow"
|
||||||
|
source_root.mkdir()
|
||||||
workspaces = tmp_path / "workspaces"
|
workspaces = tmp_path / "workspaces"
|
||||||
stale = workspaces / "opencode_mimo-v2.5-free-trial-001" / "old-answer.json"
|
stale = workspaces / "opencode_mimo-v2.5-free-trial-001" / "old-answer.json"
|
||||||
stale.parent.mkdir(parents=True)
|
stale.parent.mkdir(parents=True)
|
||||||
@@ -289,6 +304,7 @@ def test_prepare_trial_workspace_removes_stale_previous_attempt(
|
|||||||
index=1,
|
index=1,
|
||||||
workspaces_dir=workspaces,
|
workspaces_dir=workspaces,
|
||||||
template_dir=template,
|
template_dir=template,
|
||||||
|
source_root=source_root,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert prepared.root.exists()
|
assert prepared.root.exists()
|
||||||
|
|||||||
Reference in New Issue
Block a user