lda.chat/lda-wf
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: make browser challenge harness local first

Browse Source
This commit is contained in:
lda
2026-06-15 05:05:24 +07:00 Verified
parent a6487225fa
commit bee5ead63e
5 changed files with 76 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/current_roadmap.md
+3 -3
View File
@@ -51,9 +51,9 @@ clear operator feedback before adding more architecture.
- Completed: raw JSON/YAML workflow plans can be turned into artifacts through
JSON-RPC and `wf artifact create-from-plan`, allowing agent/evidence harnesses
to use the product-facing CLI path.
- Completed: the opencode browser-click challenge harness stages a real
`wf-rpc-server`, injects the RPC URL into the prompt, and tears the server
down after trials.
- Completed: the opencode browser-click challenge harness is local-first via
`wf --config examples/browser_click_workflow/wf.config.json --local`, with
optional `--start-server` / `--server-url` modes for JSON-RPC-path trials.
- Keep status read-only; do not mutate registry, auth, config, or stores.
## Priority 2: Durable Run/Resume Hardening
examples/agent_challenges/browser_click_challenge/README.md
+16 -10
View File
@@ -11,21 +11,27 @@ examples/browser_click_workflow/
## Default Behavior
By default the harness starts:
By default the harness does not start a `wf-rpc-server`. It prompts agents to
use the configured local CLI path:
```powershell
uv run wf --config examples/browser_click_workflow/wf.config.json --local
```
This builds the configured workflow server in the CLI process for each command
and reuses the configured durable store. It does not reuse in-memory source
sessions across CLI invocations.
Use `--start-server` when the trial should exercise the JSON-RPC server path.
With `--start-server`, the harness starts:
```powershell
uv run wf-rpc-server --config examples/browser_click_workflow/wf.config.json --host 127.0.0.1 --port 8772
```
It waits until:
```powershell
uv run wf --url http://127.0.0.1:8772/rpc status
```
passes, injects that URL into the prompt, runs opencode, then stops the server.
Use `--server-url` to target an already-running server or `--no-start-server` to
skip lifecycle management.
It waits until `uv run wf --url http://127.0.0.1:8772/rpc status` passes,
injects that URL into the prompt, runs opencode, then stops the server. Use
`--server-url` to target an already-running server.
## One Trial
examples/agent_challenges/browser_click_challenge/prompt.md
+11 -12
View File
@@ -7,22 +7,20 @@ Build and successfully run a workflow that:
3. Captures a before snapshot and an after snapshot.
4. Returns both snapshots as workflow output.
A workflow RPC server is already running at:
```text
{{rpc_url}}
```
Use this command prefix for product-facing operations:
```powershell
{{wf_command_prefix}}
```
Use this repository's workflow product path. That means you should use the
`wf` CLI against the provided URL, create or reuse a workflow deployment, and
run the deployment through the workflow API. Do not solve the challenge with
only a standalone Playwright/Python script.
{{server_context}}
Use this repository's workflow product path. Start by checking the relevant
agent skills under `skills/`, especially the workflow/CLI skill references, and
use broad docs/code search only after those pointers are not enough. You should
use the `wf` CLI with the command prefix above, create or reuse a workflow
deployment, and run the deployment through the workflow API. Do not solve the
challenge with only a standalone Playwright/Python script.
Do not create a new helper script whose only job is to drive `WorkflowApi`
directly. The challenge is about whether the product-facing CLI/server workflow
@@ -64,7 +62,8 @@ challenge_report:
```
Set `used_product_path` to true only if the workflow was applied and run through
the `wf` CLI or `wf-rpc-server` path. Set `used_helper_script` to true if you
created a Python script to drive `WorkflowApi` directly.
the `wf` CLI, either in local same-process mode or through `wf-rpc-server`. Set
`used_helper_script` to true if you created a Python script to drive
`WorkflowApi` directly.
If something fails, report the exact command and error instead of hiding it.
examples/agent_challenges/browser_click_challenge/run_opencode_trials.py
+33 -10
View File
@@ -27,6 +27,8 @@ DEFAULT_PROMPT = CHALLENGE_DIR / "prompt.md"
DEFAULT_RESULTS_DIR = CHALLENGE_DIR / "results"
DEFAULT_SERVER_PORT = 8772
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
EXAMPLE_CONFIG_ARG = "examples/browser_click_workflow/wf.config.json"
LOCAL_WF_COMMAND_PREFIX = f"uv run wf --config {EXAMPLE_CONFIG_ARG} --local"
def rpc_url_for_port(port: int) -> str:
@@ -39,7 +41,7 @@ def server_command(*, port: int) -> list[str]:
"run",
"wf-rpc-server",
"--config",
str(EXAMPLE_CONFIG.relative_to(ROOT)).replace("\\", "/"),
EXAMPLE_CONFIG_ARG,
"--host",
"127.0.0.1",
"--port",
@@ -47,12 +49,16 @@ def server_command(*, port: int) -> list[str]:
]
def render_prompt(prompt_path: Path, *, rpc_url: str) -> str:
command_prefix = f"uv run wf --url {rpc_url}"
def render_prompt(
prompt_path: Path,
*,
wf_command_prefix: str,
server_context: str,
) -> str:
return (
prompt_path.read_text(encoding="utf-8")
.replace("{{rpc_url}}", rpc_url)
.replace("{{wf_command_prefix}}", command_prefix)
.replace("{{wf_command_prefix}}", wf_command_prefix)
.replace("{{server_context}}", server_context)
)
@@ -128,11 +134,16 @@ class TrialConfig:
prompt_path: Path
attach_url: str | None
timeout_seconds: int
rpc_url: str
wf_command_prefix: str
server_context: str
def build_opencode_command(config: TrialConfig) -> list[str]:
prompt_text = render_prompt(config.prompt_path, rpc_url=config.rpc_url)
prompt_text = render_prompt(
config.prompt_path,
wf_command_prefix=config.wf_command_prefix,
server_context=config.server_context,
)
command = [
"opencode",
"run",
@@ -423,7 +434,7 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
parser.add_argument("--server-url", default=None)
parser.add_argument("--start-server", action="store_true", default=True)
parser.add_argument("--start-server", action="store_true", default=False)
parser.add_argument("--no-start-server", action="store_false", dest="start_server")
parser.add_argument("--server-port", type=int, default=DEFAULT_SERVER_PORT)
args = parser.parse_args(argv)
@@ -434,12 +445,23 @@ def main(argv: list[str] | None = None) -> int:
if args.server_url is not None:
rpc_url = args.server_url
managed_server: ManagedServer | None = None
wf_command_prefix = f"uv run wf --url {rpc_url}"
server_context = f"A workflow RPC server is available at `{rpc_url}`."
elif args.start_server:
managed_server = start_server(port=args.server_port)
rpc_url = managed_server.rpc_url
wf_command_prefix = f"uv run wf --url {rpc_url}"
server_context = (
f"The harness started a workflow RPC server at `{rpc_url}` for this trial."
)
else:
rpc_url = rpc_url_for_port(args.server_port)
managed_server = None
wf_command_prefix = LOCAL_WF_COMMAND_PREFIX
server_context = (
"No external workflow RPC server is staged. The command prefix uses "
"`--local`, which builds the configured workflow server in the CLI "
"process for each command."
)
try:
config = TrialConfig(
@@ -448,7 +470,8 @@ def main(argv: list[str] | None = None) -> int:
prompt_path=args.prompt,
attach_url=args.attach_url,
timeout_seconds=args.timeout_seconds,
rpc_url=rpc_url,
wf_command_prefix=wf_command_prefix,
server_context=server_context,
)
summaries: list[dict[str, Any]] = []
tests/examples/test_opencode_browser_click_challenge.py
+13 -7
View File
@@ -4,6 +4,7 @@ import json
from pathlib import Path
from examples.agent_challenges.browser_click_challenge.run_opencode_trials import (
LOCAL_WF_COMMAND_PREFIX,
TrialConfig,
build_opencode_command,
classify_challenge_report,
@@ -25,7 +26,8 @@ def test_build_opencode_command_without_attach(tmp_path: Path) -> None:
prompt_path=prompt,
attach_url=None,
timeout_seconds=120,
rpc_url="http://127.0.0.1:8772/rpc",
wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
server_context="Use local CLI mode.",
)
command = build_opencode_command(config)
@@ -48,7 +50,8 @@ def test_build_opencode_command_with_attach(tmp_path: Path) -> None:
prompt_path=prompt,
attach_url="http://127.0.0.1:4096",
timeout_seconds=120,
rpc_url="http://127.0.0.1:8772/rpc",
wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
server_context="Use local CLI mode.",
)
command = build_opencode_command(config)
@@ -244,20 +247,23 @@ def test_trial_output_path_is_zero_padded(tmp_path: Path) -> None:
assert path.name == "opencode_mimo-v2.5-free-trial-003.json"
def test_render_prompt_injects_rpc_url_and_command_prefix(tmp_path: Path) -> None:
def test_render_prompt_injects_command_prefix_and_server_context(
tmp_path: Path,
) -> None:
prompt = tmp_path / "prompt.md"
prompt.write_text(
"Use {{rpc_url}} with {{wf_command_prefix}}",
"Use {{wf_command_prefix}}. {{server_context}}",
encoding="utf-8",
)
rendered = render_prompt(
prompt,
rpc_url="http://127.0.0.1:8765/rpc",
wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
server_context="No RPC server is staged.",
)
assert "http://127.0.0.1:8765/rpc" in rendered
assert "uv run wf --url http://127.0.0.1:8765/rpc" in rendered
assert LOCAL_WF_COMMAND_PREFIX in rendered
assert "No RPC server is staged." in rendered
def test_server_command_uses_example_config_and_requested_port() -> None: