fix: allocate browser challenge trial numbers globally
This commit is contained in:
@@ -132,11 +132,9 @@ def prepare_trial_workspace(
|
|||||||
source_root: Path = EXAMPLE_SOURCE_ROOT,
|
source_root: Path = EXAMPLE_SOURCE_ROOT,
|
||||||
) -> TrialWorkspace:
|
) -> TrialWorkspace:
|
||||||
"""Copy the authoring template into a clean ignored per-trial directory."""
|
"""Copy the authoring template into a clean ignored per-trial directory."""
|
||||||
root = _next_trial_workspace_root(
|
root = workspaces_dir / f"{_safe_model_name(model)}-trial-{index:03d}"
|
||||||
workspaces_dir=workspaces_dir,
|
if root.exists():
|
||||||
model=model,
|
raise FileExistsError(f"trial workspace already exists: {root}")
|
||||||
index=index,
|
|
||||||
)
|
|
||||||
shutil.copytree(template_dir, root)
|
shutil.copytree(template_dir, root)
|
||||||
workspace = TrialWorkspace(
|
workspace = TrialWorkspace(
|
||||||
root=root,
|
root=root,
|
||||||
@@ -147,22 +145,35 @@ def prepare_trial_workspace(
|
|||||||
return workspace
|
return workspace
|
||||||
|
|
||||||
|
|
||||||
def _next_trial_workspace_root(
|
def starting_trial_index(
|
||||||
*,
|
*,
|
||||||
workspaces_dir: Path,
|
|
||||||
model: str,
|
model: str,
|
||||||
index: int,
|
results_dir: Path,
|
||||||
) -> Path:
|
workspaces_dir: Path,
|
||||||
stem = f"{_safe_model_name(model)}-trial-{index:03d}"
|
) -> int:
|
||||||
first = workspaces_dir / stem
|
"""Return the next global trial number for this model across invocations."""
|
||||||
if not first.exists():
|
safe_model = _safe_model_name(model)
|
||||||
return first
|
highest = 0
|
||||||
suffix = 2
|
for directory in (results_dir, workspaces_dir):
|
||||||
while True:
|
if not directory.exists():
|
||||||
candidate = workspaces_dir / f"{stem}-r{suffix:03d}"
|
continue
|
||||||
if not candidate.exists():
|
for path in directory.iterdir():
|
||||||
return candidate
|
index = _trial_index_from_name(path.name, safe_model=safe_model)
|
||||||
suffix += 1
|
if index is not None:
|
||||||
|
highest = max(highest, index)
|
||||||
|
return highest + 1
|
||||||
|
|
||||||
|
|
||||||
|
def _trial_index_from_name(name: str, *, safe_model: str) -> int | None:
|
||||||
|
prefix = f"{safe_model}-trial-"
|
||||||
|
if not name.startswith(prefix):
|
||||||
|
return None
|
||||||
|
suffix = name.removeprefix(prefix)
|
||||||
|
if "." in suffix:
|
||||||
|
suffix = suffix.split(".", 1)[0]
|
||||||
|
if not suffix.isdigit():
|
||||||
|
return None
|
||||||
|
return int(suffix)
|
||||||
|
|
||||||
|
|
||||||
def write_trial_config(config_path: Path, *, source_root: Path) -> None:
|
def write_trial_config(config_path: Path, *, source_root: Path) -> None:
|
||||||
@@ -561,8 +572,13 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
use_trial_workspace = args.server_url is None and not args.start_server
|
use_trial_workspace = args.server_url is None and not args.start_server
|
||||||
|
first_index = starting_trial_index(
|
||||||
|
model=args.model,
|
||||||
|
results_dir=args.results_dir,
|
||||||
|
workspaces_dir=args.workspaces_dir,
|
||||||
|
)
|
||||||
summaries: list[dict[str, Any]] = []
|
summaries: list[dict[str, Any]] = []
|
||||||
for index in range(1, args.trials + 1):
|
for index in range(first_index, first_index + args.trials):
|
||||||
prompt_path = args.prompt
|
prompt_path = args.prompt
|
||||||
trial_wf_command_prefix = wf_command_prefix
|
trial_wf_command_prefix = wf_command_prefix
|
||||||
trial_server_context = server_context
|
trial_server_context = server_context
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from examples.agent_challenges.browser_click_challenge.run_opencode_trials impor
|
|||||||
prepare_trial_workspace,
|
prepare_trial_workspace,
|
||||||
render_prompt,
|
render_prompt,
|
||||||
server_command,
|
server_command,
|
||||||
|
starting_trial_index,
|
||||||
trial_output_path,
|
trial_output_path,
|
||||||
wf_command_prefix_for_config,
|
wf_command_prefix_for_config,
|
||||||
)
|
)
|
||||||
@@ -300,19 +301,48 @@ def test_prepare_trial_workspace_uses_next_available_directory(
|
|||||||
first.mkdir(parents=True)
|
first.mkdir(parents=True)
|
||||||
stale.write_text("stale", encoding="utf-8")
|
stale.write_text("stale", encoding="utf-8")
|
||||||
|
|
||||||
|
next_index = starting_trial_index(
|
||||||
|
model="opencode/mimo-v2.5-free",
|
||||||
|
results_dir=tmp_path / "results",
|
||||||
|
workspaces_dir=workspaces,
|
||||||
|
)
|
||||||
prepared = prepare_trial_workspace(
|
prepared = prepare_trial_workspace(
|
||||||
model="opencode/mimo-v2.5-free",
|
model="opencode/mimo-v2.5-free",
|
||||||
index=1,
|
index=next_index,
|
||||||
workspaces_dir=workspaces,
|
workspaces_dir=workspaces,
|
||||||
template_dir=template,
|
template_dir=template,
|
||||||
source_root=source_root,
|
source_root=source_root,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert prepared.root == workspaces / "opencode_mimo-v2.5-free-trial-001-r002"
|
assert prepared.root == workspaces / "opencode_mimo-v2.5-free-trial-002"
|
||||||
assert prepared.root.exists()
|
assert prepared.root.exists()
|
||||||
assert stale.read_text(encoding="utf-8") == "stale"
|
assert stale.read_text(encoding="utf-8") == "stale"
|
||||||
|
|
||||||
|
|
||||||
|
def test_starting_trial_index_accounts_for_existing_results_and_workspaces(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
results = tmp_path / "results"
|
||||||
|
workspaces = tmp_path / "workspaces"
|
||||||
|
results.mkdir()
|
||||||
|
workspaces.mkdir()
|
||||||
|
(results / "opencode_mimo-v2.5-free-trial-003.json").write_text(
|
||||||
|
"{}",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(workspaces / "opencode_mimo-v2.5-free-trial-005").mkdir()
|
||||||
|
(workspaces / "other_model-trial-099").mkdir()
|
||||||
|
|
||||||
|
assert (
|
||||||
|
starting_trial_index(
|
||||||
|
model="opencode/mimo-v2.5-free",
|
||||||
|
results_dir=results,
|
||||||
|
workspaces_dir=workspaces,
|
||||||
|
)
|
||||||
|
== 6
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_wf_command_prefix_for_config_uses_repo_relative_path() -> None:
|
def test_wf_command_prefix_for_config_uses_repo_relative_path() -> None:
|
||||||
config_path = (
|
config_path = (
|
||||||
Path("examples")
|
Path("examples")
|
||||||
|
|||||||
Reference in New Issue
Block a user