feat: stage rpc server for opencode challenge

This commit is contained in:
lda
2026-06-15 03:51:39 +07:00 Verified
parent acd599d09f
commit 552fd24d69
4 changed files with 199 additions and 25 deletions
@@ -9,6 +9,24 @@ The deterministic workflow example is:
examples/browser_click_workflow/ examples/browser_click_workflow/
``` ```
## Default Behavior
By default the harness starts:
```powershell
uv run wf-rpc-server --config examples/browser_click_workflow/wf.config.json --host 127.0.0.1 --port 8772
```
It waits until:
```powershell
uv run wf --url http://127.0.0.1:8772/rpc status
```
passes, injects that URL into the prompt, runs opencode, then stops the server.
Use `--server-url` to target an already-running server or `--no-start-server` to
skip lifecycle management.
## One Trial ## One Trial
From the repository root: From the repository root:
@@ -7,10 +7,22 @@ Build and successfully run a workflow that:
3. Captures a before snapshot and an after snapshot. 3. Captures a before snapshot and an after snapshot.
4. Returns both snapshots as workflow output. 4. Returns both snapshots as workflow output.
A workflow RPC server is already running at:
```text
{{rpc_url}}
```
Use this command prefix for product-facing operations:
```powershell
{{wf_command_prefix}}
```
Use this repository's workflow product path. That means you should use the Use this repository's workflow product path. That means you should use the
`wf` CLI and/or `wf-rpc-server`, create or reuse a workflow deployment, and run `wf` CLI against the provided URL, create or reuse a workflow deployment, and
the deployment through the workflow API. Do not solve the challenge with only a run the deployment through the workflow API. Do not solve the challenge with
standalone Playwright/Python script. only a standalone Playwright/Python script.
Do not create a new helper script whose only job is to drive `WorkflowApi` Do not create a new helper script whose only job is to drive `WorkflowApi`
directly. The challenge is about whether the product-facing CLI/server workflow directly. The challenge is about whether the product-facing CLI/server workflow
@@ -3,6 +3,7 @@ from __future__ import annotations
import argparse import argparse
import json import json
import subprocess import subprocess
import sys
import time import time
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from pathlib import Path from pathlib import Path
@@ -24,6 +25,97 @@ ROOT = Path(__file__).resolve().parents[3]
CHALLENGE_DIR = Path(__file__).resolve().parent CHALLENGE_DIR = Path(__file__).resolve().parent
DEFAULT_PROMPT = CHALLENGE_DIR / "prompt.md" DEFAULT_PROMPT = CHALLENGE_DIR / "prompt.md"
DEFAULT_RESULTS_DIR = CHALLENGE_DIR / "results" DEFAULT_RESULTS_DIR = CHALLENGE_DIR / "results"
DEFAULT_SERVER_PORT = 8772
EXAMPLE_CONFIG = ROOT / "examples" / "browser_click_workflow" / "wf.config.json"
def rpc_url_for_port(port: int) -> str:
return f"http://127.0.0.1:{port}/rpc"
def server_command(*, port: int) -> list[str]:
return [
"uv",
"run",
"wf-rpc-server",
"--config",
str(EXAMPLE_CONFIG.relative_to(ROOT)).replace("\\", "/"),
"--host",
"127.0.0.1",
"--port",
str(port),
]
def render_prompt(prompt_path: Path, *, rpc_url: str) -> str:
command_prefix = f"uv run wf --url {rpc_url}"
return (
prompt_path.read_text(encoding="utf-8")
.replace("{{rpc_url}}", rpc_url)
.replace("{{wf_command_prefix}}", command_prefix)
)
@dataclass(slots=True)
class ManagedServer:
process: subprocess.Popen[str]
rpc_url: str
def start_server(*, port: int, timeout_seconds: int = 30) -> ManagedServer:
command = server_command(port=port)
process = subprocess.Popen(
command,
cwd=ROOT,
text=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
rpc_url = rpc_url_for_port(port)
try:
wait_for_status(rpc_url=rpc_url, timeout_seconds=timeout_seconds)
except Exception:
stop_server(process)
raise
return ManagedServer(process=process, rpc_url=rpc_url)
def wait_for_status(*, rpc_url: str, timeout_seconds: int) -> None:
deadline = time.monotonic() + timeout_seconds
command = ["uv", "run", "wf", "--url", rpc_url, "status"]
last_stderr = ""
while time.monotonic() < deadline:
completed = subprocess.run(
command,
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
if completed.returncode == 0:
return
last_stderr = completed.stderr
time.sleep(0.5)
raise RuntimeError(f"wf status did not become ready: {last_stderr}")
def stop_server(process: subprocess.Popen[str]) -> None:
if process.poll() is not None:
return
if sys.platform == "win32":
subprocess.run(
["taskkill", "/F", "/T", "/PID", str(process.pid)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
else:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=10)
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
@@ -33,10 +125,11 @@ class TrialConfig:
prompt_path: Path prompt_path: Path
attach_url: str | None attach_url: str | None
timeout_seconds: int timeout_seconds: int
rpc_url: str
def build_opencode_command(config: TrialConfig) -> list[str]: def build_opencode_command(config: TrialConfig) -> list[str]:
prompt_text = config.prompt_path.read_text(encoding="utf-8") prompt_text = render_prompt(config.prompt_path, rpc_url=config.rpc_url)
command = [ command = [
"opencode", "opencode",
"run", "run",
@@ -326,35 +419,56 @@ def main(argv: list[str] | None = None) -> int:
) )
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT) parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR) parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
parser.add_argument("--server-url", default=None)
parser.add_argument("--start-server", action="store_true", default=True)
parser.add_argument("--no-start-server", action="store_false", dest="start_server")
parser.add_argument("--server-port", type=int, default=DEFAULT_SERVER_PORT)
args = parser.parse_args(argv) args = parser.parse_args(argv)
if args.trials < 1: if args.trials < 1:
parser.error("--trials must be >= 1") parser.error("--trials must be >= 1")
config = TrialConfig( if args.server_url is not None:
model=args.model, rpc_url = args.server_url
variant=args.variant, managed_server: ManagedServer | None = None
prompt_path=args.prompt, elif args.start_server:
attach_url=args.attach_url, managed_server = start_server(port=args.server_port)
timeout_seconds=args.timeout_seconds, rpc_url = managed_server.rpc_url
) else:
rpc_url = rpc_url_for_port(args.server_port)
managed_server = None
summaries: list[dict[str, Any]] = [] try:
for index in range(1, args.trials + 1): config = TrialConfig(
result = run_trial(config, index=index, results_dir=args.results_dir) model=args.model,
summaries.append( variant=args.variant,
{ prompt_path=args.prompt,
"index": index, attach_url=args.attach_url,
"classification": result["classification"], timeout_seconds=args.timeout_seconds,
"returncode": result["returncode"], rpc_url=rpc_url,
"duration_seconds": round(float(result["duration_seconds"]), 3),
}
) )
print(json.dumps(summaries[-1], sort_keys=True))
success_count = sum(1 for item in summaries if item["classification"] == "success") summaries: list[dict[str, Any]] = []
print(json.dumps({"success_count": success_count, "trial_count": len(summaries)})) for index in range(1, args.trials + 1):
return 0 result = run_trial(config, index=index, results_dir=args.results_dir)
summaries.append(
{
"index": index,
"classification": result["classification"],
"returncode": result["returncode"],
"duration_seconds": round(float(result["duration_seconds"]), 3),
}
)
print(json.dumps(summaries[-1], sort_keys=True))
success_count = sum(
1 for item in summaries if item["classification"] == "success"
)
print(json.dumps({"success_count": success_count, "trial_count": len(summaries)}))
return 0
finally:
if managed_server is not None:
stop_server(managed_server.process)
if __name__ == "__main__": if __name__ == "__main__":
@@ -10,6 +10,8 @@ from examples.agent_challenges.browser_click_challenge.run_opencode_trials impor
classify_output, classify_output,
extract_challenge_report, extract_challenge_report,
parse_opencode_output, parse_opencode_output,
render_prompt,
server_command,
trial_output_path, trial_output_path,
) )
@@ -23,6 +25,7 @@ def test_build_opencode_command_without_attach(tmp_path: Path) -> None:
prompt_path=prompt, prompt_path=prompt,
attach_url=None, attach_url=None,
timeout_seconds=120, timeout_seconds=120,
rpc_url="http://127.0.0.1:8772/rpc",
) )
command = build_opencode_command(config) command = build_opencode_command(config)
@@ -45,6 +48,7 @@ def test_build_opencode_command_with_attach(tmp_path: Path) -> None:
prompt_path=prompt, prompt_path=prompt,
attach_url="http://127.0.0.1:4096", attach_url="http://127.0.0.1:4096",
timeout_seconds=120, timeout_seconds=120,
rpc_url="http://127.0.0.1:8772/rpc",
) )
command = build_opencode_command(config) command = build_opencode_command(config)
@@ -238,3 +242,29 @@ def test_trial_output_path_is_zero_padded(tmp_path: Path) -> None:
path = trial_output_path(tmp_path, model="opencode/mimo-v2.5-free", index=3) path = trial_output_path(tmp_path, model="opencode/mimo-v2.5-free", index=3)
assert path.name == "opencode_mimo-v2.5-free-trial-003.json" assert path.name == "opencode_mimo-v2.5-free-trial-003.json"
def test_render_prompt_injects_rpc_url_and_command_prefix(tmp_path: Path) -> None:
prompt = tmp_path / "prompt.md"
prompt.write_text(
"Use {{rpc_url}} with {{wf_command_prefix}}",
encoding="utf-8",
)
rendered = render_prompt(
prompt,
rpc_url="http://127.0.0.1:8765/rpc",
)
assert "http://127.0.0.1:8765/rpc" in rendered
assert "uv run wf --url http://127.0.0.1:8765/rpc" in rendered
def test_server_command_uses_example_config_and_requested_port() -> None:
command = server_command(port=8765)
assert command[:3] == ["uv", "run", "wf-rpc-server"]
assert "--config" in command
assert "examples/browser_click_workflow/wf.config.json" in command
assert "--port" in command
assert "8765" in command