lda.chat/lda-wf
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stage per-trial browser challenge workspaces

Browse Source
This commit is contained in:
lda
2026-06-15 06:15:30 +07:00 Verified
parent 10f6d7d1bc
commit 6e2c3b8879
4 changed files with 158 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/agent_challenges/browser_click_challenge/README.md
+8 -5
View File
@@ -12,15 +12,17 @@ examples/browser_click_workflow/
## Default Behavior
By default the harness does not start a `wf-rpc-server`. It prompts agents to
use the configured local CLI path:
use a per-trial configured local CLI path:
```powershell
uv run wf --config examples/browser_click_workflow/wf.config.json --local
uv run wf --config examples/agent_challenges/browser_click_challenge/workspaces/<trial>/wf.config.json --local
```
For each local-mode trial, the harness copies `workspace_template/` into
`workspaces/<model>-trial-<n>/` and injects that config path into the prompt.
This builds the configured workflow server in the CLI process for each command
and reuses the configured durable store. It does not reuse in-memory source
sessions across CLI invocations.
and uses the copied workspace's durable store. It does not reuse in-memory
source sessions across CLI invocations.
Use `--start-server` when the trial should exercise the JSON-RPC server path.
With `--start-server`, the harness starts:
@@ -60,7 +62,8 @@ examples/agent_challenges/browser_click_challenge/workspace_template/
It contains a local workflow config and prompt template that point at the
browser-click Python source without exposing a generated draft patch answer
file. Use it as the starting context for an agent trial, or copy it under:
file. The harness copies it automatically for normal local-mode trials. For
manual experiments, copy it under:
```text
examples/agent_challenges/browser_click_challenge/workspaces/
examples/agent_challenges/browser_click_challenge/run_opencode_trials.py
+75 -8
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import argparse
import json
import shutil
import subprocess
import sys
import time
@@ -25,6 +26,8 @@ ROOT = Path(__file__).resolve().parents[3]
CHALLENGE_DIR = Path(__file__).resolve().parent
DEFAULT_PROMPT = CHALLENGE_DIR / "prompt.md"
DEFAULT_RESULTS_DIR = CHALLENGE_DIR / "results"
DEFAULT_WORKSPACES_DIR = CHALLENGE_DIR / "workspaces"
DEFAULT_WORKSPACE_TEMPLATE = CHALLENGE_DIR / "workspace_template"
DEFAULT_SERVER_PORT = 8772
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
EXAMPLE_CONFIG_ARG = "examples/browser_click_workflow/wf.config.json"
@@ -68,6 +71,15 @@ class ManagedServer:
rpc_url: str
@dataclass(frozen=True, slots=True)
class TrialWorkspace:
"""Per-trial scratch area copied from the challenge workspace template."""
root: Path
config_path: Path
prompt_path: Path
def start_server(*, port: int, timeout_seconds: int = 30) -> ManagedServer:
command = server_command(port=port)
process = subprocess.Popen(
@@ -105,6 +117,42 @@ def wait_for_status(*, rpc_url: str, timeout_seconds: int) -> None:
raise RuntimeError(f"wf status did not become ready: {last_stderr}")
def _safe_model_name(model: str) -> str:
return model.replace("/", "_").replace(":", "_")
def prepare_trial_workspace(
*,
model: str,
index: int,
workspaces_dir: Path = DEFAULT_WORKSPACES_DIR,
template_dir: Path = DEFAULT_WORKSPACE_TEMPLATE,
) -> TrialWorkspace:
"""Copy the authoring template into a clean ignored per-trial directory."""
root = workspaces_dir / f"{_safe_model_name(model)}-trial-{index:03d}"
if root.exists():
# Stale scratch files can leak answers between trials, so reset only
# the ignored per-trial directory before copying the template.
shutil.rmtree(root)
shutil.copytree(template_dir, root)
return TrialWorkspace(
root=root,
config_path=root / "wf.config.json",
prompt_path=root / "prompt.md",
)
def wf_command_prefix_for_config(config_path: Path) -> str:
path = config_path
if not path.is_absolute():
path = ROOT / path
try:
path_arg = path.resolve().relative_to(ROOT.resolve()).as_posix()
except ValueError:
path_arg = str(path.resolve())
return f"uv run wf --config {path_arg} --local"
def stop_server(process: subprocess.Popen[str]) -> None:
if process.poll() is not None:
return
@@ -334,8 +382,7 @@ def _contains_bool_marker(text: str, marker: str, value: str) -> bool:
def trial_output_path(results_dir: Path, *, model: str, index: int) -> Path:
safe_model = model.replace("/", "_").replace(":", "_")
return results_dir / f"{safe_model}-trial-{index:03d}.json"
return results_dir / f"{_safe_model_name(model)}-trial-{index:03d}.json"
def run_trial(config: TrialConfig, *, index: int, results_dir: Path) -> dict[str, Any]:
@@ -433,6 +480,7 @@ def main(argv: list[str] | None = None) -> int:
)
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
parser.add_argument("--workspaces-dir", type=Path, default=DEFAULT_WORKSPACES_DIR)
parser.add_argument("--server-url", default=None)
parser.add_argument("--start-server", action="store_true", default=False)
parser.add_argument("--no-start-server", action="store_false", dest="start_server")
@@ -464,18 +512,37 @@ def main(argv: list[str] | None = None) -> int:
)
try:
use_trial_workspace = args.server_url is None and not args.start_server
summaries: list[dict[str, Any]] = []
for index in range(1, args.trials + 1):
prompt_path = args.prompt
trial_wf_command_prefix = wf_command_prefix
trial_server_context = server_context
if use_trial_workspace:
workspace = prepare_trial_workspace(
model=args.model,
index=index,
workspaces_dir=args.workspaces_dir,
)
if args.prompt == DEFAULT_PROMPT:
prompt_path = workspace.prompt_path
trial_wf_command_prefix = wf_command_prefix_for_config(
workspace.config_path
)
trial_server_context = (
"No external workflow RPC server is staged. Use the "
"per-trial workspace config copied to "
f"`{workspace.config_path.relative_to(ROOT).as_posix()}`."
)
config = TrialConfig(
model=args.model,
variant=args.variant,
prompt_path=args.prompt,
prompt_path=prompt_path,
attach_url=args.attach_url,
timeout_seconds=args.timeout_seconds,
wf_command_prefix=wf_command_prefix,
server_context=server_context,
wf_command_prefix=trial_wf_command_prefix,
server_context=trial_server_context,
)
summaries: list[dict[str, Any]] = []
for index in range(1, args.trials + 1):
result = run_trial(config, index=index, results_dir=args.results_dir)
summaries.append(
{
examples/agent_challenges/browser_click_challenge/workspace_template/prompt.md
+3 -2
View File
@@ -10,9 +10,11 @@ Build and successfully run a workflow that:
Use this command prefix for product-facing operations:
```powershell
uv run wf --config examples/agent_challenges/browser_click_challenge/workspace_template/wf.config.json --local
{{wf_command_prefix}}
```
{{server_context}}
Start from `skills/`, especially the workflow/CLI skill references, before doing
broad docs or code search. Do not read outside this repository. Do not solve the
challenge with only a standalone Playwright/Python script.
@@ -70,4 +72,3 @@ the `wf` CLI, either in local same-process mode or through `wf-rpc-server`. Set
`WorkflowApi` directly.
If something fails, report the exact command and error instead of hiding it.
tests/examples/test_opencode_browser_click_challenge.py
+67
View File
@@ -11,9 +11,11 @@ from examples.agent_challenges.browser_click_challenge.run_opencode_trials impor
classify_output,
extract_challenge_report,
parse_opencode_output,
prepare_trial_workspace,
render_prompt,
server_command,
trial_output_path,
wf_command_prefix_for_config,
)
@@ -247,6 +249,71 @@ def test_trial_output_path_is_zero_padded(tmp_path: Path) -> None:
assert path.name == "opencode_mimo-v2.5-free-trial-003.json"
def test_prepare_trial_workspace_copies_template_to_model_trial_dir(
tmp_path: Path,
) -> None:
template = tmp_path / "template"
template.mkdir()
(template / "wf.config.json").write_text('{"version": 1}', encoding="utf-8")
(template / "prompt.md").write_text("prompt", encoding="utf-8")
(template / ".gitignore").write_text(".wf_store/\n", encoding="utf-8")
workspaces = tmp_path / "workspaces"
prepared = prepare_trial_workspace(
model="opencode/mimo-v2.5-free",
index=7,
workspaces_dir=workspaces,
template_dir=template,
)
assert prepared.root == workspaces / "opencode_mimo-v2.5-free-trial-007"
assert prepared.config_path.read_text(encoding="utf-8") == '{"version": 1}'
assert prepared.prompt_path.read_text(encoding="utf-8") == "prompt"
assert (prepared.root / ".gitignore").read_text(encoding="utf-8") == ".wf_store/\n"
def test_prepare_trial_workspace_removes_stale_previous_attempt(
tmp_path: Path,
) -> None:
template = tmp_path / "template"
template.mkdir()
(template / "wf.config.json").write_text('{"version": 1}', encoding="utf-8")
(template / "prompt.md").write_text("prompt", encoding="utf-8")
workspaces = tmp_path / "workspaces"
stale = workspaces / "opencode_mimo-v2.5-free-trial-001" / "old-answer.json"
stale.parent.mkdir(parents=True)
stale.write_text("stale", encoding="utf-8")
prepared = prepare_trial_workspace(
model="opencode/mimo-v2.5-free",
index=1,
workspaces_dir=workspaces,
template_dir=template,
)
assert prepared.root.exists()
assert not stale.exists()
def test_wf_command_prefix_for_config_uses_repo_relative_path() -> None:
config_path = (
Path("examples")
/ "agent_challenges"
/ "browser_click_challenge"
/ "workspaces"
/ "opencode_mimo-v2.5-free-trial-001"
/ "wf.config.json"
)
prefix = wf_command_prefix_for_config(config_path)
assert prefix == (
"uv run wf --config "
"examples/agent_challenges/browser_click_challenge/workspaces/"
"opencode_mimo-v2.5-free-trial-001/wf.config.json --local"
)
def test_render_prompt_injects_command_prefix_and_server_context(
tmp_path: Path,
) -> None: