feat: make browser challenge harness local first

This commit is contained in:
lda
2026-06-15 05:05:24 +07:00 Verified
parent a6487225fa
commit bee5ead63e
5 changed files with 76 additions and 42 deletions
+3 -3
View File
@@ -51,9 +51,9 @@ clear operator feedback before adding more architecture.
- Completed: raw JSON/YAML workflow plans can be turned into artifacts through - Completed: raw JSON/YAML workflow plans can be turned into artifacts through
JSON-RPC and `wf artifact create-from-plan`, allowing agent/evidence harnesses JSON-RPC and `wf artifact create-from-plan`, allowing agent/evidence harnesses
to use the product-facing CLI path. to use the product-facing CLI path.
- Completed: the opencode browser-click challenge harness stages a real - Completed: the opencode browser-click challenge harness is local-first via
`wf-rpc-server`, injects the RPC URL into the prompt, and tears the server `wf --config examples/browser_click_workflow/wf.config.json --local`, with
down after trials. optional `--start-server` / `--server-url` modes for JSON-RPC-path trials.
- Keep status read-only; do not mutate registry, auth, config, or stores. - Keep status read-only; do not mutate registry, auth, config, or stores.
## Priority 2: Durable Run/Resume Hardening ## Priority 2: Durable Run/Resume Hardening
@@ -11,21 +11,27 @@ examples/browser_click_workflow/
## Default Behavior ## Default Behavior
By default the harness starts: By default the harness does not start a `wf-rpc-server`. It prompts agents to
use the configured local CLI path:
```powershell
uv run wf --config examples/browser_click_workflow/wf.config.json --local
```
This builds the configured workflow server in the CLI process for each command
and reuses the configured durable store. It does not reuse in-memory source
sessions across CLI invocations.
Use `--start-server` when the trial should exercise the JSON-RPC server path.
With `--start-server`, the harness starts:
```powershell ```powershell
uv run wf-rpc-server --config examples/browser_click_workflow/wf.config.json --host 127.0.0.1 --port 8772 uv run wf-rpc-server --config examples/browser_click_workflow/wf.config.json --host 127.0.0.1 --port 8772
``` ```
It waits until: It waits until `uv run wf --url http://127.0.0.1:8772/rpc status` passes,
injects that URL into the prompt, runs opencode, then stops the server. Use
```powershell `--server-url` to target an already-running server.
uv run wf --url http://127.0.0.1:8772/rpc status
```
passes, injects that URL into the prompt, runs opencode, then stops the server.
Use `--server-url` to target an already-running server or `--no-start-server` to
skip lifecycle management.
## One Trial ## One Trial
@@ -7,22 +7,20 @@ Build and successfully run a workflow that:
3. Captures a before snapshot and an after snapshot. 3. Captures a before snapshot and an after snapshot.
4. Returns both snapshots as workflow output. 4. Returns both snapshots as workflow output.
A workflow RPC server is already running at:
```text
{{rpc_url}}
```
Use this command prefix for product-facing operations: Use this command prefix for product-facing operations:
```powershell ```powershell
{{wf_command_prefix}} {{wf_command_prefix}}
``` ```
Use this repository's workflow product path. That means you should use the {{server_context}}
`wf` CLI against the provided URL, create or reuse a workflow deployment, and
run the deployment through the workflow API. Do not solve the challenge with Use this repository's workflow product path. Start by checking the relevant
only a standalone Playwright/Python script. agent skills under `skills/`, especially the workflow/CLI skill references, and
use broad docs/code search only after those pointers are not enough. You should
use the `wf` CLI with the command prefix above, create or reuse a workflow
deployment, and run the deployment through the workflow API. Do not solve the
challenge with only a standalone Playwright/Python script.
Do not create a new helper script whose only job is to drive `WorkflowApi` Do not create a new helper script whose only job is to drive `WorkflowApi`
directly. The challenge is about whether the product-facing CLI/server workflow directly. The challenge is about whether the product-facing CLI/server workflow
@@ -64,7 +62,8 @@ challenge_report:
``` ```
Set `used_product_path` to true only if the workflow was applied and run through Set `used_product_path` to true only if the workflow was applied and run through
the `wf` CLI or `wf-rpc-server` path. Set `used_helper_script` to true if you the `wf` CLI, either in local same-process mode or through `wf-rpc-server`. Set
created a Python script to drive `WorkflowApi` directly. `used_helper_script` to true if you created a Python script to drive
`WorkflowApi` directly.
If something fails, report the exact command and error instead of hiding it. If something fails, report the exact command and error instead of hiding it.
@@ -27,6 +27,8 @@ DEFAULT_PROMPT = CHALLENGE_DIR / "prompt.md"
DEFAULT_RESULTS_DIR = CHALLENGE_DIR / "results" DEFAULT_RESULTS_DIR = CHALLENGE_DIR / "results"
DEFAULT_SERVER_PORT = 8772 DEFAULT_SERVER_PORT = 8772
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json" EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
EXAMPLE_CONFIG_ARG = "examples/browser_click_workflow/wf.config.json"
LOCAL_WF_COMMAND_PREFIX = f"uv run wf --config {EXAMPLE_CONFIG_ARG} --local"
def rpc_url_for_port(port: int) -> str: def rpc_url_for_port(port: int) -> str:
@@ -39,7 +41,7 @@ def server_command(*, port: int) -> list[str]:
"run", "run",
"wf-rpc-server", "wf-rpc-server",
"--config", "--config",
str(EXAMPLE_CONFIG.relative_to(ROOT)).replace("\\", "/"), EXAMPLE_CONFIG_ARG,
"--host", "--host",
"127.0.0.1", "127.0.0.1",
"--port", "--port",
@@ -47,12 +49,16 @@ def server_command(*, port: int) -> list[str]:
] ]
def render_prompt(prompt_path: Path, *, rpc_url: str) -> str: def render_prompt(
command_prefix = f"uv run wf --url {rpc_url}" prompt_path: Path,
*,
wf_command_prefix: str,
server_context: str,
) -> str:
return ( return (
prompt_path.read_text(encoding="utf-8") prompt_path.read_text(encoding="utf-8")
.replace("{{rpc_url}}", rpc_url) .replace("{{wf_command_prefix}}", wf_command_prefix)
.replace("{{wf_command_prefix}}", command_prefix) .replace("{{server_context}}", server_context)
) )
@@ -128,11 +134,16 @@ class TrialConfig:
prompt_path: Path prompt_path: Path
attach_url: str | None attach_url: str | None
timeout_seconds: int timeout_seconds: int
rpc_url: str wf_command_prefix: str
server_context: str
def build_opencode_command(config: TrialConfig) -> list[str]: def build_opencode_command(config: TrialConfig) -> list[str]:
prompt_text = render_prompt(config.prompt_path, rpc_url=config.rpc_url) prompt_text = render_prompt(
config.prompt_path,
wf_command_prefix=config.wf_command_prefix,
server_context=config.server_context,
)
command = [ command = [
"opencode", "opencode",
"run", "run",
@@ -423,7 +434,7 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT) parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR) parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
parser.add_argument("--server-url", default=None) parser.add_argument("--server-url", default=None)
parser.add_argument("--start-server", action="store_true", default=True) parser.add_argument("--start-server", action="store_true", default=False)
parser.add_argument("--no-start-server", action="store_false", dest="start_server") parser.add_argument("--no-start-server", action="store_false", dest="start_server")
parser.add_argument("--server-port", type=int, default=DEFAULT_SERVER_PORT) parser.add_argument("--server-port", type=int, default=DEFAULT_SERVER_PORT)
args = parser.parse_args(argv) args = parser.parse_args(argv)
@@ -434,12 +445,23 @@ def main(argv: list[str] | None = None) -> int:
if args.server_url is not None: if args.server_url is not None:
rpc_url = args.server_url rpc_url = args.server_url
managed_server: ManagedServer | None = None managed_server: ManagedServer | None = None
wf_command_prefix = f"uv run wf --url {rpc_url}"
server_context = f"A workflow RPC server is available at `{rpc_url}`."
elif args.start_server: elif args.start_server:
managed_server = start_server(port=args.server_port) managed_server = start_server(port=args.server_port)
rpc_url = managed_server.rpc_url rpc_url = managed_server.rpc_url
wf_command_prefix = f"uv run wf --url {rpc_url}"
server_context = (
f"The harness started a workflow RPC server at `{rpc_url}` for this trial."
)
else: else:
rpc_url = rpc_url_for_port(args.server_port)
managed_server = None managed_server = None
wf_command_prefix = LOCAL_WF_COMMAND_PREFIX
server_context = (
"No external workflow RPC server is staged. The command prefix uses "
"`--local`, which builds the configured workflow server in the CLI "
"process for each command."
)
try: try:
config = TrialConfig( config = TrialConfig(
@@ -448,7 +470,8 @@ def main(argv: list[str] | None = None) -> int:
prompt_path=args.prompt, prompt_path=args.prompt,
attach_url=args.attach_url, attach_url=args.attach_url,
timeout_seconds=args.timeout_seconds, timeout_seconds=args.timeout_seconds,
rpc_url=rpc_url, wf_command_prefix=wf_command_prefix,
server_context=server_context,
) )
summaries: list[dict[str, Any]] = [] summaries: list[dict[str, Any]] = []
@@ -4,6 +4,7 @@ import json
from pathlib import Path from pathlib import Path
from examples.agent_challenges.browser_click_challenge.run_opencode_trials import ( from examples.agent_challenges.browser_click_challenge.run_opencode_trials import (
LOCAL_WF_COMMAND_PREFIX,
TrialConfig, TrialConfig,
build_opencode_command, build_opencode_command,
classify_challenge_report, classify_challenge_report,
@@ -25,7 +26,8 @@ def test_build_opencode_command_without_attach(tmp_path: Path) -> None:
prompt_path=prompt, prompt_path=prompt,
attach_url=None, attach_url=None,
timeout_seconds=120, timeout_seconds=120,
rpc_url="http://127.0.0.1:8772/rpc", wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
server_context="Use local CLI mode.",
) )
command = build_opencode_command(config) command = build_opencode_command(config)
@@ -48,7 +50,8 @@ def test_build_opencode_command_with_attach(tmp_path: Path) -> None:
prompt_path=prompt, prompt_path=prompt,
attach_url="http://127.0.0.1:4096", attach_url="http://127.0.0.1:4096",
timeout_seconds=120, timeout_seconds=120,
rpc_url="http://127.0.0.1:8772/rpc", wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
server_context="Use local CLI mode.",
) )
command = build_opencode_command(config) command = build_opencode_command(config)
@@ -244,20 +247,23 @@ def test_trial_output_path_is_zero_padded(tmp_path: Path) -> None:
assert path.name == "opencode_mimo-v2.5-free-trial-003.json" assert path.name == "opencode_mimo-v2.5-free-trial-003.json"
def test_render_prompt_injects_rpc_url_and_command_prefix(tmp_path: Path) -> None: def test_render_prompt_injects_command_prefix_and_server_context(
tmp_path: Path,
) -> None:
prompt = tmp_path / "prompt.md" prompt = tmp_path / "prompt.md"
prompt.write_text( prompt.write_text(
"Use {{rpc_url}} with {{wf_command_prefix}}", "Use {{wf_command_prefix}}. {{server_context}}",
encoding="utf-8", encoding="utf-8",
) )
rendered = render_prompt( rendered = render_prompt(
prompt, prompt,
rpc_url="http://127.0.0.1:8765/rpc", wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
server_context="No RPC server is staged.",
) )
assert "http://127.0.0.1:8765/rpc" in rendered assert LOCAL_WF_COMMAND_PREFIX in rendered
assert "uv run wf --url http://127.0.0.1:8765/rpc" in rendered assert "No RPC server is staged." in rendered
def test_server_command_uses_example_config_and_requested_port() -> None: def test_server_command_uses_example_config_and_requested_port() -> None: