feat: make browser challenge harness local first

This commit is contained in:
lda
2026-06-15 05:05:24 +07:00 Verified
parent a6487225fa
commit bee5ead63e
5 changed files with 76 additions and 42 deletions
@@ -11,21 +11,27 @@ examples/browser_click_workflow/
## Default Behavior
By default the harness starts:
By default the harness does not start a `wf-rpc-server`. It prompts agents to
use the configured local CLI path:
```powershell
uv run wf --config examples/browser_click_workflow/wf.config.json --local
```
This builds the configured workflow server in the CLI process for each command
and reuses the configured durable store. It does not reuse in-memory source
sessions across CLI invocations.
Use `--start-server` when the trial should exercise the JSON-RPC server path.
With `--start-server`, the harness starts:
```powershell
uv run wf-rpc-server --config examples/browser_click_workflow/wf.config.json --host 127.0.0.1 --port 8772
```
It waits until:
```powershell
uv run wf --url http://127.0.0.1:8772/rpc status
```
passes, injects that URL into the prompt, runs opencode, then stops the server.
Use `--server-url` to target an already-running server or `--no-start-server` to
skip lifecycle management.
It waits until `uv run wf --url http://127.0.0.1:8772/rpc status` passes,
injects that URL into the prompt, runs opencode, then stops the server. Use
`--server-url` to target an already-running server.
## One Trial
@@ -7,22 +7,20 @@ Build and successfully run a workflow that:
3. Captures a before snapshot and an after snapshot.
4. Returns both snapshots as workflow output.
A workflow RPC server is already running at:
```text
{{rpc_url}}
```
Use this command prefix for product-facing operations:
```powershell
{{wf_command_prefix}}
```
Use this repository's workflow product path. That means you should use the
`wf` CLI against the provided URL, create or reuse a workflow deployment, and
run the deployment through the workflow API. Do not solve the challenge with
only a standalone Playwright/Python script.
{{server_context}}
Use this repository's workflow product path. Start by checking the relevant
agent skills under `skills/`, especially the workflow/CLI skill references, and
use broad docs/code search only after those pointers are not enough. You should
use the `wf` CLI with the command prefix above, create or reuse a workflow
deployment, and run the deployment through the workflow API. Do not solve the
challenge with only a standalone Playwright/Python script.
Do not create a new helper script whose only job is to drive `WorkflowApi`
directly. The challenge is about whether the product-facing CLI/server workflow
@@ -64,7 +62,8 @@ challenge_report:
```
Set `used_product_path` to true only if the workflow was applied and run through
the `wf` CLI or `wf-rpc-server` path. Set `used_helper_script` to true if you
created a Python script to drive `WorkflowApi` directly.
the `wf` CLI, either in local same-process mode or through `wf-rpc-server`. Set
`used_helper_script` to true if you created a Python script to drive
`WorkflowApi` directly.
If something fails, report the exact command and error instead of hiding it.
@@ -27,6 +27,8 @@ DEFAULT_PROMPT = CHALLENGE_DIR / "prompt.md"
DEFAULT_RESULTS_DIR = CHALLENGE_DIR / "results"
DEFAULT_SERVER_PORT = 8772
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
EXAMPLE_CONFIG_ARG = "examples/browser_click_workflow/wf.config.json"
LOCAL_WF_COMMAND_PREFIX = f"uv run wf --config {EXAMPLE_CONFIG_ARG} --local"
def rpc_url_for_port(port: int) -> str:
@@ -39,7 +41,7 @@ def server_command(*, port: int) -> list[str]:
"run",
"wf-rpc-server",
"--config",
str(EXAMPLE_CONFIG.relative_to(ROOT)).replace("\\", "/"),
EXAMPLE_CONFIG_ARG,
"--host",
"127.0.0.1",
"--port",
@@ -47,12 +49,16 @@ def server_command(*, port: int) -> list[str]:
]
def render_prompt(prompt_path: Path, *, rpc_url: str) -> str:
command_prefix = f"uv run wf --url {rpc_url}"
def render_prompt(
prompt_path: Path,
*,
wf_command_prefix: str,
server_context: str,
) -> str:
return (
prompt_path.read_text(encoding="utf-8")
.replace("{{rpc_url}}", rpc_url)
.replace("{{wf_command_prefix}}", command_prefix)
.replace("{{wf_command_prefix}}", wf_command_prefix)
.replace("{{server_context}}", server_context)
)
@@ -128,11 +134,16 @@ class TrialConfig:
prompt_path: Path
attach_url: str | None
timeout_seconds: int
rpc_url: str
wf_command_prefix: str
server_context: str
def build_opencode_command(config: TrialConfig) -> list[str]:
prompt_text = render_prompt(config.prompt_path, rpc_url=config.rpc_url)
prompt_text = render_prompt(
config.prompt_path,
wf_command_prefix=config.wf_command_prefix,
server_context=config.server_context,
)
command = [
"opencode",
"run",
@@ -423,7 +434,7 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
parser.add_argument("--server-url", default=None)
parser.add_argument("--start-server", action="store_true", default=True)
parser.add_argument("--start-server", action="store_true", default=False)
parser.add_argument("--no-start-server", action="store_false", dest="start_server")
parser.add_argument("--server-port", type=int, default=DEFAULT_SERVER_PORT)
args = parser.parse_args(argv)
@@ -434,12 +445,23 @@ def main(argv: list[str] | None = None) -> int:
if args.server_url is not None:
rpc_url = args.server_url
managed_server: ManagedServer | None = None
wf_command_prefix = f"uv run wf --url {rpc_url}"
server_context = f"A workflow RPC server is available at `{rpc_url}`."
elif args.start_server:
managed_server = start_server(port=args.server_port)
rpc_url = managed_server.rpc_url
wf_command_prefix = f"uv run wf --url {rpc_url}"
server_context = (
f"The harness started a workflow RPC server at `{rpc_url}` for this trial."
)
else:
rpc_url = rpc_url_for_port(args.server_port)
managed_server = None
wf_command_prefix = LOCAL_WF_COMMAND_PREFIX
server_context = (
"No external workflow RPC server is staged. The command prefix uses "
"`--local`, which builds the configured workflow server in the CLI "
"process for each command."
)
try:
config = TrialConfig(
@@ -448,7 +470,8 @@ def main(argv: list[str] | None = None) -> int:
prompt_path=args.prompt,
attach_url=args.attach_url,
timeout_seconds=args.timeout_seconds,
rpc_url=rpc_url,
wf_command_prefix=wf_command_prefix,
server_context=server_context,
)
summaries: list[dict[str, Any]] = []