feat: smooth browser challenge harness runs
This commit is contained in:
@@ -25,6 +25,10 @@ builds the configured workflow server in the CLI process for each command and
|
||||
uses the copied workspace's durable store. It does not reuse in-memory source
|
||||
sessions across CLI invocations.
|
||||
|
||||
Use `--workspace-template` and `--source-root` to run a same-shape challenge with
|
||||
a different prompt template or Python source root. Both default to the bundled
|
||||
browser-click example settings.
|
||||
|
||||
Use `--start-server` when the trial should exercise the JSON-RPC server path.
|
||||
With `--start-server`, the harness starts:
|
||||
|
||||
@@ -47,6 +51,16 @@ uv run python examples/agent_challenges/browser_click_challenge/run_opencode_tri
|
||||
--trials 1
|
||||
```
|
||||
|
||||
Variant challenge example:
|
||||
|
||||
```powershell
|
||||
uv run python examples/agent_challenges/browser_click_challenge/run_opencode_trials.py `
|
||||
--workspace-template examples/agent_challenges/browser_click_challenge/workspace_template `
|
||||
--source-root examples/browser_click_workflow `
|
||||
--workspaces-dir examples/agent_challenges/browser_click_challenge/workspaces_alt `
|
||||
--results-dir examples/agent_challenges/browser_click_challenge/results_alt
|
||||
```
|
||||
|
||||
Results are written to:
|
||||
|
||||
```text
|
||||
|
||||
@@ -235,6 +235,13 @@ def wf_command_prefix_for_config(config_path: Path) -> str:
|
||||
return f"uv run wf --config {path_arg} --local"
|
||||
|
||||
|
||||
def _display_path(path: Path) -> str:
|
||||
try:
|
||||
return path.resolve().relative_to(ROOT.resolve()).as_posix()
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def stop_server(process: subprocess.Popen[str]) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
@@ -360,6 +367,18 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
|
||||
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
|
||||
parser.add_argument("--workspaces-dir", type=Path, default=DEFAULT_WORKSPACES_DIR)
|
||||
parser.add_argument(
|
||||
"--workspace-template",
|
||||
type=Path,
|
||||
default=DEFAULT_WORKSPACE_TEMPLATE,
|
||||
help="Template directory copied for each local-mode trial workspace.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source-root",
|
||||
type=Path,
|
||||
default=EXAMPLE_SOURCE_ROOT,
|
||||
help="Python source root written into each generated trial config.",
|
||||
)
|
||||
parser.add_argument("--server-url", default=None)
|
||||
parser.add_argument("--start-server", action="store_true", default=False)
|
||||
parser.add_argument("--no-start-server", action="store_false", dest="start_server")
|
||||
@@ -407,14 +426,16 @@ def main(argv: list[str] | None = None) -> int:
|
||||
model=args.model,
|
||||
index=index,
|
||||
workspaces_dir=args.workspaces_dir,
|
||||
template_dir=args.workspace_template,
|
||||
source_root=args.source_root,
|
||||
)
|
||||
if args.prompt == DEFAULT_PROMPT:
|
||||
prompt_path = workspace.prompt_path
|
||||
trial_wf_command_prefix = wf_command_prefix_for_config(
|
||||
workspace.config_path
|
||||
)
|
||||
workspace_path = workspace.root.relative_to(ROOT).as_posix()
|
||||
config_path = workspace.config_path.relative_to(ROOT).as_posix()
|
||||
workspace_path = _display_path(workspace.root)
|
||||
config_path = _display_path(workspace.config_path)
|
||||
trial_server_context = (
|
||||
"No external workflow RPC server is staged. Use the "
|
||||
"per-trial workspace config copied to "
|
||||
@@ -437,7 +458,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"classification": result["classification"],
|
||||
"returncode": result["returncode"],
|
||||
"duration_seconds": round(float(result["duration_seconds"]), 3),
|
||||
"report_path": result.get("report_path"),
|
||||
"report_path": _optional_string(result.get("report_path")),
|
||||
"report_save_error": result.get("report_save_error"),
|
||||
}
|
||||
)
|
||||
@@ -453,5 +474,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
stop_server(managed_server.process)
|
||||
|
||||
|
||||
def _optional_string(value: object) -> str | None:
|
||||
return None if value is None else str(value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
Reference in New Issue
Block a user