feat: smooth browser challenge harness runs
This commit is contained in:
@@ -25,6 +25,10 @@ builds the configured workflow server in the CLI process for each command and
|
|||||||
uses the copied workspace's durable store. It does not reuse in-memory source
|
uses the copied workspace's durable store. It does not reuse in-memory source
|
||||||
sessions across CLI invocations.
|
sessions across CLI invocations.
|
||||||
|
|
||||||
|
Use `--workspace-template` and `--source-root` to run a same-shape challenge with
|
||||||
|
a different prompt template or Python source root. Both default to the bundled
|
||||||
|
browser-click example settings.
|
||||||
|
|
||||||
Use `--start-server` when the trial should exercise the JSON-RPC server path.
|
Use `--start-server` when the trial should exercise the JSON-RPC server path.
|
||||||
With `--start-server`, the harness starts:
|
With `--start-server`, the harness starts:
|
||||||
|
|
||||||
@@ -47,6 +51,16 @@ uv run python examples/agent_challenges/browser_click_challenge/run_opencode_tri
|
|||||||
--trials 1
|
--trials 1
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Variant challenge example:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
uv run python examples/agent_challenges/browser_click_challenge/run_opencode_trials.py `
|
||||||
|
--workspace-template examples/agent_challenges/browser_click_challenge/workspace_template `
|
||||||
|
--source-root examples/browser_click_workflow `
|
||||||
|
--workspaces-dir examples/agent_challenges/browser_click_challenge/workspaces_alt `
|
||||||
|
--results-dir examples/agent_challenges/browser_click_challenge/results_alt
|
||||||
|
```
|
||||||
|
|
||||||
Results are written to:
|
Results are written to:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
|||||||
@@ -235,6 +235,13 @@ def wf_command_prefix_for_config(config_path: Path) -> str:
|
|||||||
return f"uv run wf --config {path_arg} --local"
|
return f"uv run wf --config {path_arg} --local"
|
||||||
|
|
||||||
|
|
||||||
|
def _display_path(path: Path) -> str:
|
||||||
|
try:
|
||||||
|
return path.resolve().relative_to(ROOT.resolve()).as_posix()
|
||||||
|
except ValueError:
|
||||||
|
return str(path.resolve())
|
||||||
|
|
||||||
|
|
||||||
def stop_server(process: subprocess.Popen[str]) -> None:
|
def stop_server(process: subprocess.Popen[str]) -> None:
|
||||||
if process.poll() is not None:
|
if process.poll() is not None:
|
||||||
return
|
return
|
||||||
@@ -360,6 +367,18 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
|
parser.add_argument("--prompt", type=Path, default=DEFAULT_PROMPT)
|
||||||
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
|
parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR)
|
||||||
parser.add_argument("--workspaces-dir", type=Path, default=DEFAULT_WORKSPACES_DIR)
|
parser.add_argument("--workspaces-dir", type=Path, default=DEFAULT_WORKSPACES_DIR)
|
||||||
|
parser.add_argument(
|
||||||
|
"--workspace-template",
|
||||||
|
type=Path,
|
||||||
|
default=DEFAULT_WORKSPACE_TEMPLATE,
|
||||||
|
help="Template directory copied for each local-mode trial workspace.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--source-root",
|
||||||
|
type=Path,
|
||||||
|
default=EXAMPLE_SOURCE_ROOT,
|
||||||
|
help="Python source root written into each generated trial config.",
|
||||||
|
)
|
||||||
parser.add_argument("--server-url", default=None)
|
parser.add_argument("--server-url", default=None)
|
||||||
parser.add_argument("--start-server", action="store_true", default=False)
|
parser.add_argument("--start-server", action="store_true", default=False)
|
||||||
parser.add_argument("--no-start-server", action="store_false", dest="start_server")
|
parser.add_argument("--no-start-server", action="store_false", dest="start_server")
|
||||||
@@ -407,14 +426,16 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
model=args.model,
|
model=args.model,
|
||||||
index=index,
|
index=index,
|
||||||
workspaces_dir=args.workspaces_dir,
|
workspaces_dir=args.workspaces_dir,
|
||||||
|
template_dir=args.workspace_template,
|
||||||
|
source_root=args.source_root,
|
||||||
)
|
)
|
||||||
if args.prompt == DEFAULT_PROMPT:
|
if args.prompt == DEFAULT_PROMPT:
|
||||||
prompt_path = workspace.prompt_path
|
prompt_path = workspace.prompt_path
|
||||||
trial_wf_command_prefix = wf_command_prefix_for_config(
|
trial_wf_command_prefix = wf_command_prefix_for_config(
|
||||||
workspace.config_path
|
workspace.config_path
|
||||||
)
|
)
|
||||||
workspace_path = workspace.root.relative_to(ROOT).as_posix()
|
workspace_path = _display_path(workspace.root)
|
||||||
config_path = workspace.config_path.relative_to(ROOT).as_posix()
|
config_path = _display_path(workspace.config_path)
|
||||||
trial_server_context = (
|
trial_server_context = (
|
||||||
"No external workflow RPC server is staged. Use the "
|
"No external workflow RPC server is staged. Use the "
|
||||||
"per-trial workspace config copied to "
|
"per-trial workspace config copied to "
|
||||||
@@ -437,7 +458,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
"classification": result["classification"],
|
"classification": result["classification"],
|
||||||
"returncode": result["returncode"],
|
"returncode": result["returncode"],
|
||||||
"duration_seconds": round(float(result["duration_seconds"]), 3),
|
"duration_seconds": round(float(result["duration_seconds"]), 3),
|
||||||
"report_path": result.get("report_path"),
|
"report_path": _optional_string(result.get("report_path")),
|
||||||
"report_save_error": result.get("report_save_error"),
|
"report_save_error": result.get("report_save_error"),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -453,5 +474,9 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
stop_server(managed_server.process)
|
stop_server(managed_server.process)
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_string(value: object) -> str | None:
|
||||||
|
return None if value is None else str(value)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
raise SystemExit(main())
|
raise SystemExit(main())
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from typing import Annotated
|
|||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
|
from wf_cli.context import config_path_from_context
|
||||||
from wf_cli.io import emit_json
|
from wf_cli.io import emit_json
|
||||||
from wf_config import McpSourceConfig, PythonSourceConfig, StdlibSourceConfig
|
from wf_config import McpSourceConfig, PythonSourceConfig, StdlibSourceConfig
|
||||||
from wf_config.loader import load_workflow_config
|
from wf_config.loader import load_workflow_config
|
||||||
@@ -44,21 +45,25 @@ def migrate_mcp_config(
|
|||||||
|
|
||||||
@app.command("validate")
|
@app.command("validate")
|
||||||
def validate_config(
|
def validate_config(
|
||||||
|
ctx: typer.Context,
|
||||||
config_path: Annotated[
|
config_path: Annotated[
|
||||||
Path,
|
Path | None,
|
||||||
typer.Argument(help="Neutral workflow config JSON path."),
|
typer.Argument(
|
||||||
],
|
help="Neutral workflow config JSON path. Defaults to global --config.",
|
||||||
|
),
|
||||||
|
] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Validate config shape and trusted static source imports."""
|
"""Validate config shape and trusted static source imports."""
|
||||||
|
resolved_config_path = config_path or Path(config_path_from_context(ctx))
|
||||||
try:
|
try:
|
||||||
config = load_workflow_config(config_path)
|
config = load_workflow_config(resolved_config_path)
|
||||||
sources = [_validate_source(source) for source in config.server.sources]
|
sources = [_validate_source(source) for source in config.server.sources]
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise typer.BadParameter(f"invalid workflow config: {exc}") from exc
|
raise typer.BadParameter(f"invalid workflow config: {exc}") from exc
|
||||||
emit_json(
|
emit_json(
|
||||||
{
|
{
|
||||||
"valid": True,
|
"valid": True,
|
||||||
"path": str(config_path),
|
"path": str(resolved_config_path),
|
||||||
"sources": sources,
|
"sources": sources,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -592,6 +592,63 @@ def test_prepare_trial_workspace_uses_next_available_directory(
|
|||||||
assert stale.read_text(encoding="utf-8") == "stale"
|
assert stale.read_text(encoding="utf-8") == "stale"
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_uses_custom_workspace_template_and_source_root(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
capsys,
|
||||||
|
) -> None:
|
||||||
|
template = tmp_path / "template"
|
||||||
|
template.mkdir()
|
||||||
|
(template / "prompt.md").write_text("custom prompt", encoding="utf-8")
|
||||||
|
source_root = tmp_path / "source"
|
||||||
|
source_root.mkdir()
|
||||||
|
workspaces = tmp_path / "workspaces"
|
||||||
|
results = tmp_path / "results"
|
||||||
|
|
||||||
|
def fake_run_trial(
|
||||||
|
config: TrialConfig,
|
||||||
|
*,
|
||||||
|
index: int,
|
||||||
|
results_dir: Path,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"index": index,
|
||||||
|
"classification": "success",
|
||||||
|
"returncode": 0,
|
||||||
|
"duration_seconds": 1.0,
|
||||||
|
"report_path": config.prompt_path.parent / "final-report.md",
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(run_opencode_trials, "run_trial", fake_run_trial)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
run_opencode_trials.main(
|
||||||
|
[
|
||||||
|
"--model",
|
||||||
|
"check/model",
|
||||||
|
"--trials",
|
||||||
|
"1",
|
||||||
|
"--workspace-template",
|
||||||
|
str(template),
|
||||||
|
"--source-root",
|
||||||
|
str(source_root),
|
||||||
|
"--workspaces-dir",
|
||||||
|
str(workspaces),
|
||||||
|
"--results-dir",
|
||||||
|
str(results),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
== 0
|
||||||
|
)
|
||||||
|
|
||||||
|
workspace = workspaces / "check_model-trial-001"
|
||||||
|
assert workspace.exists()
|
||||||
|
assert (workspace / "prompt.md").read_text(encoding="utf-8") == "custom prompt"
|
||||||
|
config = json.loads((workspace / "wf.config.json").read_text(encoding="utf-8"))
|
||||||
|
assert config["server"]["sources"][0]["path"] == "../../source"
|
||||||
|
assert '"success_count": 1' in capsys.readouterr().out
|
||||||
|
|
||||||
|
|
||||||
def test_starting_trial_index_accounts_for_existing_results_and_workspaces(
|
def test_starting_trial_index_accounts_for_existing_results_and_workspaces(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -144,6 +144,37 @@ registry = [echo]
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_wf_config_validate_uses_global_config_when_path_omitted(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
config_path = tmp_path / "wf.config.json"
|
||||||
|
config_path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"version": 1,
|
||||||
|
"server": {
|
||||||
|
"store": {"kind": "filesystem", "root": "store"},
|
||||||
|
"sources": [{"kind": "stdlib", "id": "wf.std"}],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
app,
|
||||||
|
["--config", str(config_path), "config", "validate"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
payload = json.loads(result.output)
|
||||||
|
assert payload["valid"] is True
|
||||||
|
assert payload["path"] == str(config_path)
|
||||||
|
assert payload["sources"] == [
|
||||||
|
{"id": "wf.std", "kind": "stdlib", "status": "ok"}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_wf_config_validate_reports_python_source_import_failure(tmp_path: Path) -> None:
|
def test_wf_config_validate_reports_python_source_import_failure(tmp_path: Path) -> None:
|
||||||
config_path = tmp_path / "wf.config.json"
|
config_path = tmp_path / "wf.config.json"
|
||||||
config_path.write_text(
|
config_path.write_text(
|
||||||
|
|||||||
Reference in New Issue
Block a user