feat: make browser challenge harness local first
This commit is contained in:
@@ -51,9 +51,9 @@ clear operator feedback before adding more architecture.
|
|||||||
- Completed: raw JSON/YAML workflow plans can be turned into artifacts through
|
- Completed: raw JSON/YAML workflow plans can be turned into artifacts through
|
||||||
JSON-RPC and `wf artifact create-from-plan`, allowing agent/evidence harnesses
|
JSON-RPC and `wf artifact create-from-plan`, allowing agent/evidence harnesses
|
||||||
to use the product-facing CLI path.
|
to use the product-facing CLI path.
|
||||||
- Completed: the opencode browser-click challenge harness stages a real
|
- Completed: the opencode browser-click challenge harness is local-first via
|
||||||
`wf-rpc-server`, injects the RPC URL into the prompt, and tears the server
|
`wf --config examples/browser_click_workflow/wf.config.json --local`, with
|
||||||
down after trials.
|
optional `--start-server` / `--server-url` modes for JSON-RPC-path trials.
|
||||||
- Keep status read-only; do not mutate registry, auth, config, or stores.
|
- Keep status read-only; do not mutate registry, auth, config, or stores.
|
||||||
|
|
||||||
## Priority 2: Durable Run/Resume Hardening
|
## Priority 2: Durable Run/Resume Hardening
|
||||||
|
|||||||
@@ -11,21 +11,27 @@ examples/browser_click_workflow/
|
|||||||
|
|
||||||
## Default Behavior
|
## Default Behavior
|
||||||
|
|
||||||
By default the harness starts:
|
By default the harness does not start a `wf-rpc-server`. It prompts agents to
|
||||||
|
use the configured local CLI path:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
uv run wf --config examples/browser_click_workflow/wf.config.json --local
|
||||||
|
```
|
||||||
|
|
||||||
|
This builds the configured workflow server in the CLI process for each command
|
||||||
|
and reuses the configured durable store. It does not reuse in-memory source
|
||||||
|
sessions across CLI invocations.
|
||||||
|
|
||||||
|
Use `--start-server` when the trial should exercise the JSON-RPC server path.
|
||||||
|
With `--start-server`, the harness starts:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
uv run wf-rpc-server --config examples/browser_click_workflow/wf.config.json --host 127.0.0.1 --port 8772
|
uv run wf-rpc-server --config examples/browser_click_workflow/wf.config.json --host 127.0.0.1 --port 8772
|
||||||
```
|
```
|
||||||
|
|
||||||
It waits until:
|
It waits until `uv run wf --url http://127.0.0.1:8772/rpc status` passes,
|
||||||
|
injects that URL into the prompt, runs opencode, then stops the server. Use
|
||||||
```powershell
|
`--server-url` to target an already-running server.
|
||||||
uv run wf --url http://127.0.0.1:8772/rpc status
|
|
||||||
```
|
|
||||||
|
|
||||||
passes, injects that URL into the prompt, runs opencode, then stops the server.
|
|
||||||
Use `--server-url` to target an already-running server or `--no-start-server` to
|
|
||||||
skip lifecycle management.
|
|
||||||
|
|
||||||
## One Trial
|
## One Trial
|
||||||
|
|
||||||
|
|||||||
@@ -7,22 +7,20 @@ Build and successfully run a workflow that:
|
|||||||
3. Captures a before snapshot and an after snapshot.
|
3. Captures a before snapshot and an after snapshot.
|
||||||
4. Returns both snapshots as workflow output.
|
4. Returns both snapshots as workflow output.
|
||||||
|
|
||||||
A workflow RPC server is already running at:
|
|
||||||
|
|
||||||
```text
|
|
||||||
{{rpc_url}}
|
|
||||||
```
|
|
||||||
|
|
||||||
Use this command prefix for product-facing operations:
|
Use this command prefix for product-facing operations:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
{{wf_command_prefix}}
|
{{wf_command_prefix}}
|
||||||
```
|
```
|
||||||
|
|
||||||
Use this repository's workflow product path. That means you should use the
|
{{server_context}}
|
||||||
`wf` CLI against the provided URL, create or reuse a workflow deployment, and
|
|
||||||
run the deployment through the workflow API. Do not solve the challenge with
|
Use this repository's workflow product path. Start by checking the relevant
|
||||||
only a standalone Playwright/Python script.
|
agent skills under `skills/`, especially the workflow/CLI skill references, and
|
||||||
|
use broad docs/code search only after those pointers are not enough. You should
|
||||||
|
use the `wf` CLI with the command prefix above, create or reuse a workflow
|
||||||
|
deployment, and run the deployment through the workflow API. Do not solve the
|
||||||
|
challenge with only a standalone Playwright/Python script.
|
||||||
|
|
||||||
Do not create a new helper script whose only job is to drive `WorkflowApi`
|
Do not create a new helper script whose only job is to drive `WorkflowApi`
|
||||||
directly. The challenge is about whether the product-facing CLI/server workflow
|
directly. The challenge is about whether the product-facing CLI/server workflow
|
||||||
@@ -64,7 +62,8 @@ challenge_report:
|
|||||||
```
|
```
|
||||||
|
|
||||||
Set `used_product_path` to true only if the workflow was applied and run through
|
Set `used_product_path` to true only if the workflow was applied and run through
|
||||||
the `wf` CLI or `wf-rpc-server` path. Set `used_helper_script` to true if you
|
the `wf` CLI, either in local same-process mode or through `wf-rpc-server`. Set
|
||||||
created a Python script to drive `WorkflowApi` directly.
|
`used_helper_script` to true if you created a Python script to drive
|
||||||
|
`WorkflowApi` directly.
|
||||||
|
|
||||||
If something fails, report the exact command and error instead of hiding it.
|
If something fails, report the exact command and error instead of hiding it.
|
||||||
|
|||||||
@@ -27,6 +27,8 @@ DEFAULT_PROMPT = CHALLENGE_DIR / "prompt.md"
|
|||||||
DEFAULT_RESULTS_DIR = CHALLENGE_DIR / "results"
|
DEFAULT_RESULTS_DIR = CHALLENGE_DIR / "results"
|
||||||
DEFAULT_SERVER_PORT = 8772
|
DEFAULT_SERVER_PORT = 8772
|
||||||
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
|
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
|
||||||
|
EXAMPLE_CONFIG_ARG = "examples/browser_click_workflow/wf.config.json"
|
||||||
|
LOCAL_WF_COMMAND_PREFIX = f"uv run wf --config {EXAMPLE_CONFIG_ARG} --local"
|
||||||
|
|
||||||
|
|
||||||
def rpc_url_for_port(port: int) -> str:
|
def rpc_url_for_port(port: int) -> str:
|
||||||
@@ -39,7 +41,7 @@ def server_command(*, port: int) -> list[str]:
|
|||||||
"run",
|
"run",
|
||||||
"wf-rpc-server",
|
"wf-rpc-server",
|
||||||
"--config",
|
"--config",
|
||||||
str(EXAMPLE_CONFIG.relative_to(ROOT)).replace("\\", "/"),
|
EXAMPLE_CONFIG_ARG,
|
||||||
"--host",
|
"--host",
|
||||||
"127.0.0.1",
|
"127.0.0.1",
|
||||||
"--port",
|
"--port",
|
||||||
@@ -47,12 +49,16 @@ def server_command(*, port: int) -> list[str]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def render_prompt(prompt_path: Path, *, rpc_url: str) -> str:
|
def render_prompt(
|
||||||
command_prefix = f"uv run wf --url {rpc_url}"
|
prompt_path: Path,
|
||||||
|
*,
|
||||||
|
wf_command_prefix: str,
|
||||||
|
server_context: str,
|
||||||
|
) -> str:
|
||||||
return (
|
return (
|
||||||
prompt_path.read_text(encoding="utf-8")
|
prompt_path.read_text(encoding="utf-8")
|
||||||
.replace("{{rpc_url}}", rpc_url)
|
.replace("{{wf_command_prefix}}", wf_command_prefix)
|
||||||
.replace("{{wf_command_prefix}}", command_prefix)
|
.replace("{{server_context}}", server_context)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -128,11 +134,16 @@ class TrialConfig:
|
|||||||
prompt_path: Path
|
prompt_path: Path
|
||||||
attach_url: str | None
|
attach_url: str | None
|
||||||
timeout_seconds: int
|
timeout_seconds: int
|
||||||
rpc_url: str
|
wf_command_prefix: str
|
||||||
|
server_context: str
|
||||||
|
|
||||||
|
|
||||||
def build_opencode_command(config: TrialConfig) -> list[str]:
|
def build_opencode_command(config: TrialConfig) -> list[str]:
|
||||||
prompt_text = render_prompt(config.prompt_path, rpc_url=config.rpc_url)
|
prompt_text = render_prompt(
|
||||||
|
config.prompt_path,
|
||||||
|
wf_command_prefix=config.wf_command_prefix,
|
||||||
|
server_context=config.server_context,
|
||||||
|
)
|
||||||
command = [
|
command = [
|
||||||
"opencode",
|
"opencode",
|
||||||
"run",
|
"run",
|
||||||
@@ -423,7 +434,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
|
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
|
||||||
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
|
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
|
||||||
parser.add_argument("--server-url", default=None)
|
parser.add_argument("--server-url", default=None)
|
||||||
parser.add_argument("--start-server", action="store_true", default=True)
|
parser.add_argument("--start-server", action="store_true", default=False)
|
||||||
parser.add_argument("--no-start-server", action="store_false", dest="start_server")
|
parser.add_argument("--no-start-server", action="store_false", dest="start_server")
|
||||||
parser.add_argument("--server-port", type=int, default=DEFAULT_SERVER_PORT)
|
parser.add_argument("--server-port", type=int, default=DEFAULT_SERVER_PORT)
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
@@ -434,12 +445,23 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
if args.server_url is not None:
|
if args.server_url is not None:
|
||||||
rpc_url = args.server_url
|
rpc_url = args.server_url
|
||||||
managed_server: ManagedServer | None = None
|
managed_server: ManagedServer | None = None
|
||||||
|
wf_command_prefix = f"uv run wf --url {rpc_url}"
|
||||||
|
server_context = f"A workflow RPC server is available at `{rpc_url}`."
|
||||||
elif args.start_server:
|
elif args.start_server:
|
||||||
managed_server = start_server(port=args.server_port)
|
managed_server = start_server(port=args.server_port)
|
||||||
rpc_url = managed_server.rpc_url
|
rpc_url = managed_server.rpc_url
|
||||||
|
wf_command_prefix = f"uv run wf --url {rpc_url}"
|
||||||
|
server_context = (
|
||||||
|
f"The harness started a workflow RPC server at `{rpc_url}` for this trial."
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
rpc_url = rpc_url_for_port(args.server_port)
|
|
||||||
managed_server = None
|
managed_server = None
|
||||||
|
wf_command_prefix = LOCAL_WF_COMMAND_PREFIX
|
||||||
|
server_context = (
|
||||||
|
"No external workflow RPC server is staged. The command prefix uses "
|
||||||
|
"`--local`, which builds the configured workflow server in the CLI "
|
||||||
|
"process for each command."
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
config = TrialConfig(
|
config = TrialConfig(
|
||||||
@@ -448,7 +470,8 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
prompt_path=args.prompt,
|
prompt_path=args.prompt,
|
||||||
attach_url=args.attach_url,
|
attach_url=args.attach_url,
|
||||||
timeout_seconds=args.timeout_seconds,
|
timeout_seconds=args.timeout_seconds,
|
||||||
rpc_url=rpc_url,
|
wf_command_prefix=wf_command_prefix,
|
||||||
|
server_context=server_context,
|
||||||
)
|
)
|
||||||
|
|
||||||
summaries: list[dict[str, Any]] = []
|
summaries: list[dict[str, Any]] = []
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import json
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from examples.agent_challenges.browser_click_challenge.run_opencode_trials import (
|
from examples.agent_challenges.browser_click_challenge.run_opencode_trials import (
|
||||||
|
LOCAL_WF_COMMAND_PREFIX,
|
||||||
TrialConfig,
|
TrialConfig,
|
||||||
build_opencode_command,
|
build_opencode_command,
|
||||||
classify_challenge_report,
|
classify_challenge_report,
|
||||||
@@ -25,7 +26,8 @@ def test_build_opencode_command_without_attach(tmp_path: Path) -> None:
|
|||||||
prompt_path=prompt,
|
prompt_path=prompt,
|
||||||
attach_url=None,
|
attach_url=None,
|
||||||
timeout_seconds=120,
|
timeout_seconds=120,
|
||||||
rpc_url="http://127.0.0.1:8772/rpc",
|
wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
|
||||||
|
server_context="Use local CLI mode.",
|
||||||
)
|
)
|
||||||
|
|
||||||
command = build_opencode_command(config)
|
command = build_opencode_command(config)
|
||||||
@@ -48,7 +50,8 @@ def test_build_opencode_command_with_attach(tmp_path: Path) -> None:
|
|||||||
prompt_path=prompt,
|
prompt_path=prompt,
|
||||||
attach_url="http://127.0.0.1:4096",
|
attach_url="http://127.0.0.1:4096",
|
||||||
timeout_seconds=120,
|
timeout_seconds=120,
|
||||||
rpc_url="http://127.0.0.1:8772/rpc",
|
wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
|
||||||
|
server_context="Use local CLI mode.",
|
||||||
)
|
)
|
||||||
|
|
||||||
command = build_opencode_command(config)
|
command = build_opencode_command(config)
|
||||||
@@ -244,20 +247,23 @@ def test_trial_output_path_is_zero_padded(tmp_path: Path) -> None:
|
|||||||
assert path.name == "opencode_mimo-v2.5-free-trial-003.json"
|
assert path.name == "opencode_mimo-v2.5-free-trial-003.json"
|
||||||
|
|
||||||
|
|
||||||
def test_render_prompt_injects_rpc_url_and_command_prefix(tmp_path: Path) -> None:
|
def test_render_prompt_injects_command_prefix_and_server_context(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
prompt = tmp_path / "prompt.md"
|
prompt = tmp_path / "prompt.md"
|
||||||
prompt.write_text(
|
prompt.write_text(
|
||||||
"Use {{rpc_url}} with {{wf_command_prefix}}",
|
"Use {{wf_command_prefix}}. {{server_context}}",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
|
||||||
rendered = render_prompt(
|
rendered = render_prompt(
|
||||||
prompt,
|
prompt,
|
||||||
rpc_url="http://127.0.0.1:8765/rpc",
|
wf_command_prefix=LOCAL_WF_COMMAND_PREFIX,
|
||||||
|
server_context="No RPC server is staged.",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert "http://127.0.0.1:8765/rpc" in rendered
|
assert LOCAL_WF_COMMAND_PREFIX in rendered
|
||||||
assert "uv run wf --url http://127.0.0.1:8765/rpc" in rendered
|
assert "No RPC server is staged." in rendered
|
||||||
|
|
||||||
|
|
||||||
def test_server_command_uses_example_config_and_requested_port() -> None:
|
def test_server_command_uses_example_config_and_requested_port() -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user