628 lines
33 KiB
JSON
628 lines
33 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"cohort_id": "agent-operability-n3-2026-06-30",
|
|
"title": "Agent-operability longitudinal n=3 campaign",
|
|
"selection_rule": "The latest three completed, manually audited trials per challenge, model, and instruction profile as of 2026-06-30.",
|
|
"limitations": [
|
|
"The three waves span repository snapshots; they are longitudinal engineering evidence, not a controlled model comparison.",
|
|
"The base prompt changed before wave 3 to require the challenge report inline.",
|
|
"The models were free hosted OpenCode endpoints, so service load and latency were not controlled.",
|
|
"The campaign is limited to two tasks and two hosted models, so results may not transfer to other workflows, providers, models, or deployment conditions.",
|
|
"All manual audits were performed by the author; no second-rater reliability check was conducted."
|
|
],
|
|
"runs": [
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_deepseek-v4-flash-free-trial-034.report.json",
|
|
"report_sha256": "337cc7db84c1c95d58e8ab85525ed2739f8fab58acc69fcd6f74f041f7b463eb",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "none",
|
|
"trial_index": 34,
|
|
"repository_commit": "30ad99fca1e382d13fc1daacf9772ce5e709013f",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 305.975,
|
|
"tokens_total": 1267986,
|
|
"audited_at": "2026-06-29T21:19:49Z",
|
|
"audit_notes": "Valid product-path success. Draft authoring completed after repaired bindings and a deployment binding retry; no disqualifying reads observed."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_deepseek-v4-flash-free-trial-035.report.json",
|
|
"report_sha256": "0ba1212eae4303d066577f170a40e0a5892b9945eeb80005be9873772e562348",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "skills",
|
|
"trial_index": 35,
|
|
"repository_commit": "30ad99fca1e382d13fc1daacf9772ce5e709013f",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 255.076,
|
|
"tokens_total": 1198504,
|
|
"audited_at": "2026-06-29T21:19:49Z",
|
|
"audit_notes": "Valid product-path success. Used supplied skills and trial workspace files only; no disqualifying reads observed."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_deepseek-v4-flash-free-trial-036.report.json",
|
|
"report_sha256": "c523a041223ac83d68d4c72ec8b6d7668ee62dc3f8b84482d84858815a803dff",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "all",
|
|
"trial_index": 36,
|
|
"repository_commit": "30ad99fca1e382d13fc1daacf9772ce5e709013f",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 223.147,
|
|
"tokens_total": 1041287,
|
|
"audited_at": "2026-06-29T21:19:50Z",
|
|
"audit_notes": "Valid product-path success. All-profile reads were docs/skills/challenge context, not existing solution or implementation code."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_mimo-v2.5-free-trial-027.report.json",
|
|
"report_sha256": "b287a707f1f34abf1e3ef3110f2815b576e75f86f3c130d8913ff152d9a39a00",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "none",
|
|
"trial_index": 27,
|
|
"repository_commit": "30ad99fca1e382d13fc1daacf9772ce5e709013f",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "failed",
|
|
"duration_seconds": 1103.134,
|
|
"tokens_total": 1073828,
|
|
"audited_at": "2026-06-30T09:39:44Z",
|
|
"audit_notes": "Manual outcome pass with resume evidence. Automatic contamination came from broad repo/workspace reads, but sampled transcript did not show product-code, adjacent-solution, prior-store, or helper-script bypass."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_mimo-v2.5-free-trial-028.report.json",
|
|
"report_sha256": "ea400f221a58fd13dc29432b5b1857eabdc730e314777d09a07e0b265621759e",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "skills",
|
|
"trial_index": 28,
|
|
"repository_commit": "30ad99fca1e382d13fc1daacf9772ce5e709013f",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 164.709,
|
|
"tokens_total": 564051,
|
|
"audited_at": "2026-06-29T21:20:17Z",
|
|
"audit_notes": "Valid product-path success. Used supplied skill docs and raw-plan path; no existing solution, product code, prior store, or adjacent attempt reads observed."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_mimo-v2.5-free-trial-029.report.json",
|
|
"report_sha256": "a55e2658b71ef1dc17174a76a7a2304b4818becd7e8cef499616b8af6b8af3df",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "all",
|
|
"trial_index": 29,
|
|
"repository_commit": "30ad99fca1e382d13fc1daacf9772ce5e709013f",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 197.079,
|
|
"tokens_total": 641558,
|
|
"audited_at": "2026-06-29T21:20:18Z",
|
|
"audit_notes": "Valid product-path success. All-profile docs/skills reads did not include existing solution or implementation code."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_deepseek-v4-flash-free-trial-024.report.json",
|
|
"report_sha256": "5e40f75599c291808e1c378962989e749f80f4daeeeabce08777dee8c08b1173",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "none",
|
|
"trial_index": 24,
|
|
"repository_commit": "30ad99fca1e382d13fc1daacf9772ce5e709013f",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "failed",
|
|
"duration_seconds": 934.753,
|
|
"tokens_total": 326439,
|
|
"audited_at": "2026-06-30T09:38:52Z",
|
|
"audit_notes": "Manual outcome pass with resume evidence. Reads were limited to trial workspace fixtures and public CLI capability inspection in sampled report."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_deepseek-v4-flash-free-trial-025.report.json",
|
|
"report_sha256": "d7c5be60ec253c42667154299127d960e2afce23914d6b0e3aac4d13932e389e",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "skills",
|
|
"trial_index": 25,
|
|
"repository_commit": "30ad99fca1e382d13fc1daacf9772ce5e709013f",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "failed",
|
|
"duration_seconds": 854.959,
|
|
"tokens_total": 56915,
|
|
"audited_at": "2026-06-30T09:38:53Z",
|
|
"audit_notes": "Manual outcome pass with resume evidence. Automatic contamination came from supplied skills/example-directory reads; sampled evidence did not show reading an existing solution or product implementation code."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_deepseek-v4-flash-free-trial-026.report.json",
|
|
"report_sha256": "e08248b55ca9af2a6cf4163176cb29f22254261c1bbbe7257ea244956c53702d",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "all",
|
|
"trial_index": 26,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "invalid",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 249.318,
|
|
"tokens_total": 1346114,
|
|
"audited_at": "2026-06-30T11:33:50Z",
|
|
"audit_notes": "Invalid as clean benchmark. Product path completed, but the run used existing solution/example implementation as reference."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_mimo-v2.5-free-trial-024.report.json",
|
|
"report_sha256": "1fb02a19d0a3e0c7baec01e2d15df11f2016224d56e872edc910b796e224d4be",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "none",
|
|
"trial_index": 24,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 187.106,
|
|
"tokens_total": 612379,
|
|
"audited_at": "2026-06-30T11:33:49Z",
|
|
"audit_notes": "Clean report-workflow pass. Agent used none profile, workspace files only, raw-plan path, no helper script or disallowed reads observed."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_mimo-v2.5-free-trial-023.report.json",
|
|
"report_sha256": "a38ae6ced8eae200b713c23a4cf6ebd07ec3cf72f1991490cad52436a8df3cf2",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "skills",
|
|
"trial_index": 23,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 253.113,
|
|
"tokens_total": 823612,
|
|
"audited_at": "2026-06-30T11:33:49Z",
|
|
"audit_notes": "Clean report-workflow pass. Agent used supplied skills and workspace files only, completed through public wf commands, no helper script or disallowed reads observed."
|
|
},
|
|
{
|
|
"wave": 1,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_mimo-v2.5-free-trial-025.report.json",
|
|
"report_sha256": "08ca446fca8e708c89ee426d5e174949f398953b708e4212684337258fe3d68c",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "all",
|
|
"trial_index": 25,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 196.772,
|
|
"tokens_total": 690458,
|
|
"audited_at": "2026-06-30T11:33:49Z",
|
|
"audit_notes": "Clean report-workflow pass. Agent used all profile resources without reading product code, existing solution, prior store, or adjacent attempts; raw-plan path succeeded on first attempt."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_deepseek-v4-flash-free-trial-037.report.json",
|
|
"report_sha256": "52f429902bfd68b1b8893b5be03fa050d302968b2b6bb8906613e9ac5c6bde32",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "none",
|
|
"trial_index": 37,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 257.747,
|
|
"tokens_total": 1355428,
|
|
"audited_at": "2026-06-30T11:32:57Z",
|
|
"audit_notes": "Clean browser-click pass. Agent used draft path, public wf commands, no helper script, no product-code or existing-solution reads observed."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_deepseek-v4-flash-free-trial-038.report.json",
|
|
"report_sha256": "dc643e38440d6c644c4149be49b400deb021dfe2eb68f5ff605c54ef8491a26a",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "skills",
|
|
"trial_index": 38,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 210.851,
|
|
"tokens_total": 1009244,
|
|
"audited_at": "2026-06-30T11:32:57Z",
|
|
"audit_notes": "Manual outcome pass with policy note. Useful evidence that skills profile solved browser via draft path."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_deepseek-v4-flash-free-trial-039.report.json",
|
|
"report_sha256": "ab4cda9f59c9e101d551cbd3a3399e8f3b600b31bfa9f6a7e8321dbe1947ad88",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "all",
|
|
"trial_index": 39,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 240.817,
|
|
"tokens_total": 1271971,
|
|
"audited_at": "2026-06-30T11:32:58Z",
|
|
"audit_notes": "Clean browser-click pass. Agent used all-profile skills/workspace resources without disallowed reads."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_mimo-v2.5-free-trial-030.report.json",
|
|
"report_sha256": "01e19f100066e1e69609268086924dfeed9d9a1f396e28283c0e5b571796eb8b",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "none",
|
|
"trial_index": 30,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "fail",
|
|
"task_outcome": "failed",
|
|
"duration_seconds": 198.976,
|
|
"tokens_total": 575081,
|
|
"audited_at": "2026-06-30T11:32:58Z",
|
|
"audit_notes": "Manual outcome fail for challenge contract. Treat as useful UX/process-cleanup evidence; could be false-positive Chrome/process detection, but no contrary evidence was captured."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_mimo-v2.5-free-trial-031.report.json",
|
|
"report_sha256": "2de73f2b87182ae822a1a1cb0065c9f4fd243248c0293b7b7a0bacc8b523bcde",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "skills",
|
|
"trial_index": 31,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "invalid",
|
|
"task_outcome": "failed",
|
|
"duration_seconds": 373.645,
|
|
"tokens_total": 1659079,
|
|
"audited_at": "2026-06-30T11:32:58Z",
|
|
"audit_notes": "Invalid benchmark sample. Useful prompt finding: final self-report must be inline; prompt was updated after this run."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_mimo-v2.5-free-trial-032.report.json",
|
|
"report_sha256": "8c758fb289a1dd41b82ab181dda33bc52e7163f48002b641913666ddc0f2cf7f",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "all",
|
|
"trial_index": 32,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 283.632,
|
|
"tokens_total": 1152542,
|
|
"audited_at": "2026-06-30T11:32:59Z",
|
|
"audit_notes": "Clean browser-click pass. Agent used raw-plan path, completed with before=false and after=true, no leftover processes."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_deepseek-v4-flash-free-trial-027.report.json",
|
|
"report_sha256": "6e0fce88c196713a23fde3558df204ba5b8047b46c48f5e7b27249cd79d20a34",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "none",
|
|
"trial_index": 27,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 314.767,
|
|
"tokens_total": 1349666,
|
|
"audited_at": "2026-06-30T10:42:07Z",
|
|
"audit_notes": "Clean pass for report workflow. Agent used raw-plan path, public wf commands, no helper script, no product-code or existing-solution reads observed."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_deepseek-v4-flash-free-trial-028.report.json",
|
|
"report_sha256": "7d1db2a8b2974d2815856faffd8e4d2898eb90d663e670024e1e52eb54928852",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "skills",
|
|
"trial_index": 28,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 244.242,
|
|
"tokens_total": 1087828,
|
|
"audited_at": "2026-06-30T10:42:07Z",
|
|
"audit_notes": "Manual outcome pass. Keep a policy note: not as clean as mimo skills because the transcript read challenge directory/prompt artifacts, but it did not appear to copy a ready-made answer."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_deepseek-v4-flash-free-trial-029.report.json",
|
|
"report_sha256": "55c484089b471462132398fcb955afab8b36ea00723cd5ada19855ac6f7d0661",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "all",
|
|
"trial_index": 29,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "invalid",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 187.826,
|
|
"tokens_total": 749971,
|
|
"audited_at": "2026-06-30T10:42:08Z",
|
|
"audit_notes": "Invalid as clean benchmark. Product path completed, but the run used an existing solution/example implementation as reference."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_mimo-v2.5-free-trial-026.report.json",
|
|
"report_sha256": "6423d2f6fb918e33fca2f826532ee1b34dd8a208cbb29cdb988e26fcda191a8b",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "none",
|
|
"trial_index": 26,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "invalid",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 224.583,
|
|
"tokens_total": 757043,
|
|
"audited_at": "2026-06-30T10:42:08Z",
|
|
"audit_notes": "Invalid as strict none-profile benchmark. Useful UX evidence: missed source binding, then missing end edge, then succeeded."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_mimo-v2.5-free-trial-027.report.json",
|
|
"report_sha256": "97c9365feeb5401e91d0a0bd92d7a75e87745c08b1dbdaadd639e2a8a777136c",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "skills",
|
|
"trial_index": 27,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 258.463,
|
|
"tokens_total": 1043878,
|
|
"audited_at": "2026-06-30T10:42:08Z",
|
|
"audit_notes": "Clean pass. Agent used supplied skills and workspace files only, completed through public wf commands, no helper script or disallowed reads observed."
|
|
},
|
|
{
|
|
"wave": 2,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_mimo-v2.5-free-trial-028.report.json",
|
|
"report_sha256": "5305c841ea80b697b0f72eeedc693bb2fcb87623ceabfe88100606e46b65b79e",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "all",
|
|
"trial_index": 28,
|
|
"repository_commit": "04b1cf30f93f5946bfb7f0b001036319365aa237",
|
|
"base_prompt_hash": "ae06f961d3c64d4d53da259cc0f9492290a3ccb6c6984433fb0986a4ad5643fb",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 223.048,
|
|
"tokens_total": 899893,
|
|
"audited_at": "2026-06-30T10:42:09Z",
|
|
"audit_notes": "Clean pass. Agent used all profile resources without reading product code, existing solution, prior store, or adjacent attempt; raw-plan path succeeded on first attempt."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_deepseek-v4-flash-free-trial-040.report.json",
|
|
"report_sha256": "fc0a9d519bc32bc9721b394bb2f4c3b3d9773996f89aed77a7932351040e51a2",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "none",
|
|
"trial_index": 40,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "invalid",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 261.703,
|
|
"tokens_total": 1211918,
|
|
"audited_at": "2026-06-30T11:43:50Z",
|
|
"audit_notes": "Invalid for clean benchmark. Product path completed through draft authoring and run output showed before.clicked=false and after.clicked=true, but there was a disallowed repository-root read."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_deepseek-v4-flash-free-trial-041.report.json",
|
|
"report_sha256": "e4906b96c644040cf740485043b6e816b6d08975458ecb63cb33d20fdfe41abf",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "skills",
|
|
"trial_index": 41,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 160.444,
|
|
"tokens_total": 658537,
|
|
"audited_at": "2026-06-30T11:44:09Z",
|
|
"audit_notes": "Clean browser-click pass. Agent used supplied skills and workspace files, completed through public wf CLI draft/deploy/run path, returned inline report, and evidence shows before.clicked=false and after.clicked=true with no leftover process claim."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_deepseek-v4-flash-free-trial-042.report.json",
|
|
"report_sha256": "706c3b226ee197649a90524919c4b62f447c15fba6396dcf86ac8f6ef61a2d52",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "all",
|
|
"trial_index": 42,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "invalid",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 1663.355,
|
|
"tokens_total": 1088332,
|
|
"audited_at": "2026-06-30T12:06:41Z",
|
|
"audit_notes": "Invalid for clean benchmark. Product path completed through draft authoring and run output showed before.clicked=false and after.clicked=true, but there was a repository-root read and a very long recovery path."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_mimo-v2.5-free-trial-033.report.json",
|
|
"report_sha256": "ef4feb1fc5f6fe0324e4e74fd1ee19281832c11c1c1087888af4b8b858b3d54b",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "none",
|
|
"trial_index": 33,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 211.796,
|
|
"tokens_total": 759769,
|
|
"audited_at": "2026-06-30T11:45:06Z",
|
|
"audit_notes": "Browser-click pass with caveat. Product path completed via raw-plan import, deployment, and run; evidence shows before.clicked=false and after.clicked=true. Manifest read is recorded but not treated as answer leakage."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_mimo-v2.5-free-trial-034.report.json",
|
|
"report_sha256": "d8e81e671df9cc2cf421118ccdb9139d6bfee9dc307cb312ecb4b50f5a911675",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "skills",
|
|
"trial_index": 34,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 223.292,
|
|
"tokens_total": 742181,
|
|
"audited_at": "2026-06-30T11:47:29Z",
|
|
"audit_notes": "Clean browser-click pass. Agent used supplied skills and workspace files, completed through public wf CLI draft/deploy/run path, and evidence shows before.clicked=false and after.clicked=true."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/browser_click_challenge/results/opencode_mimo-v2.5-free-trial-035.report.json",
|
|
"report_sha256": "fd52890062cc309f82204e3b022d37a8494ec0a6bde880ad49b12f24bfbcf678",
|
|
"challenge": "browser_click",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "all",
|
|
"trial_index": 35,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 199.298,
|
|
"tokens_total": 674210,
|
|
"audited_at": "2026-06-30T11:47:45Z",
|
|
"audit_notes": "Browser-click pass with caveat. Product path completed via raw-plan import, deployment, and run; evidence shows before.clicked=false and after.clicked=true. Challenge README read is recorded as docs/example context, not answer leakage."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_deepseek-v4-flash-free-trial-030.report.json",
|
|
"report_sha256": "fd5f2c4f62c3997e749e233f15c70c1de48c4312ffaffd312e36666028824270",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "none",
|
|
"trial_index": 30,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 402.334,
|
|
"tokens_total": 2824556,
|
|
"audited_at": "2026-06-30T12:39:21Z",
|
|
"audit_notes": "Clean report-workflow pass. Agent used workspace files only, completed through public wf CLI draft/deploy/run path, and output markdown matched the expected title."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_deepseek-v4-flash-free-trial-031.report.json",
|
|
"report_sha256": "b44fbd5c6cbbc3031a40384fe3ae7b861a363a21f1a69326f0b55a01b8898029",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "skills",
|
|
"trial_index": 31,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "invalid",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 341.685,
|
|
"tokens_total": 1650195,
|
|
"audited_at": "2026-06-30T12:43:17Z",
|
|
"audit_notes": "Invalid for clean benchmark. Product path completed and output was correct, but the agent read the source example directory outside the supplied trial workspace."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_deepseek-v4-flash-free-trial-032.report.json",
|
|
"report_sha256": "a14f4467e4173b454ce139e3bf3d09d656e90f0ffc31ca66e83a1cc4015e6e8c",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/deepseek-v4-flash-free",
|
|
"profile": "all",
|
|
"trial_index": 32,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 217.551,
|
|
"tokens_total": 844578,
|
|
"audited_at": "2026-06-30T12:40:00Z",
|
|
"audit_notes": "Report-workflow pass with caveat. Agent used raw-plan import and completed through public wf CLI. Repo-index read is recorded but not treated as answer leakage."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_mimo-v2.5-free-trial-029.report.json",
|
|
"report_sha256": "ce2a905b9b9caca531cc93201c48bd4a85ceb5454321f4dc68da231d937af4b2",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "none",
|
|
"trial_index": 29,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "invalid",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 470.566,
|
|
"tokens_total": 2030961,
|
|
"audited_at": "2026-06-30T12:40:16Z",
|
|
"audit_notes": "Invalid for clean benchmark. Product path completed, but store/repo reads make the clean evidence unreliable."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_mimo-v2.5-free-trial-030.report.json",
|
|
"report_sha256": "b116ef6ba104c2ad0451e7a4412c0b70c121c8ce1e170f596189a6cd66710e47",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "skills",
|
|
"trial_index": 30,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 160.917,
|
|
"tokens_total": 521020,
|
|
"audited_at": "2026-06-30T12:40:35Z",
|
|
"audit_notes": "Clean report-workflow pass. Agent used supplied skills and workspace files, completed through raw-plan import/deploy/run, and output markdown matched the expected title."
|
|
},
|
|
{
|
|
"wave": 3,
|
|
"report": "examples/agent_challenges/report_workflow_challenge/results/opencode_mimo-v2.5-free-trial-031.report.json",
|
|
"report_sha256": "de86f4fec7577f2bfe3d99db8b8b82076d055ee66b13cb4ae8a419b6478c4001",
|
|
"challenge": "report_workflow",
|
|
"model": "opencode/mimo-v2.5-free",
|
|
"profile": "all",
|
|
"trial_index": 31,
|
|
"repository_commit": "5e9b76eb946d94838f6d933e0fe99c8d6030072f",
|
|
"base_prompt_hash": "b94659167706e5e5edc13429e1dd6c4f59a4f226bd9f926a4a4757c91becf160",
|
|
"manual_outcome": "pass",
|
|
"task_outcome": "success",
|
|
"duration_seconds": 144.907,
|
|
"tokens_total": 459046,
|
|
"audited_at": "2026-06-30T12:40:53Z",
|
|
"audit_notes": "Clean report-workflow pass. Agent used supplied skills/docs plus workspace files, completed through raw-plan import/deploy/run, and output markdown matched the expected title."
|
|
}
|
|
]
|
|
}
|