{ "schema": "agent-comparison-pilot.v1", "campaign": "2026-10-15", "sourceSha": "7496f8b0d2e5fc730911953294f3f212e188a61d", "generatedAt": "2026-10-16T03:41:48.348598+00:00", "deadlineAt": "2026-10-16T03:52:27Z", "state": "Exploratory pilot. Run-level limitations and all recorded attempts are disclosed.", "protocol": [ "Freeze synthetic tasks and independent acceptance checks before execution.", "Run each agent in a fresh disposable workspace with the same task inputs and limits.", "Separate matched-model harness comparisons from native products using different models.", "Keep every attempted run. Unavailable access, identity gaps, timeouts and apparatus errors are disclosed separately from code failures.", "A single trial is a feasibility observation, not a general product ranking.", "Blueprint planning checks do not establish generated-application execution.", "Runtime instances and apparatus revisions vary across attempts and are recorded per receipt. Shared hardware contention and different time limits prevent a pooled speed ranking.", "Status totals include apparatus diagnostics and access preflight. They are not coding-agent success rates." ], "conclusions": [ "All ten model configurations in the harness matrix were attempted with both Verdict Code and Aider. Attempt coverage is not a count of completed, comparable pairs; interrupted and incomplete runs remain visible.", "Devin SWE-2 Medium, Claude Code Sonnet 5.5 and Codex each produced patches passing both coding fixtures: nine independent behavior test methods plus a file-scope check. Independent replay confirmed all six patches. Codex provider-resolved identity remains unverified.", "Blueprint passed both narrow extraction fixtures without a model call. Semantic review found that all six native agent plans preserved required work and exclusions; Codex also matched the exact four-task and five-task counts. This does not establish a planning-quality winner.", "Verdict Code with GLM-5.3-Flash completed the repository-contract fixture with 10/10 checks, 37,151 accounted tokens and an estimated API cost of $0.005941. Its invoice candidate passed the code checks, but the run did not finish within its token budget. Neither observation is a general product score.", "The harness matrix contains genuine passes and misses alongside apparatus diagnostics. Different attempt budgets, protocol corrections and shared hardware contention prevent a pooled speed, cost-saving or overall superiority claim.", "Antigravity received access preflight only: no supported isolated headless invocation was available. Its existing active conversation was left untouched; no coding-quality result is inferred." ], "rows": [ { "id": "20261016T032540Z_aider_byok_zai_glm-5.3-flash_690836", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 2.763, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T032540Z_aider_byok_zai_glm-5.3-flash_690836.json" }, { "id": "20261016T032610Z_verdict-code_byok_zai_glm-5.3-flash_e5fbf7", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 2.299, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T032610Z_verdict-code_byok_zai_glm-5.3-flash_e5fbf7.json" }, { "id": "20261016T032505Z_verdict-code_byok_zai_glm-5.3-flash_d49eef", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 110.494, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "TIMEOUT", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T032505Z_verdict-code_byok_zai_glm-5.3-flash_d49eef.json" }, { "id": "20261016T032612Z_aider_byok_zai_glm-5.3-flash_0d7d04", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 110.024, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "TIMEOUT", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T032612Z_aider_byok_zai_glm-5.3-flash_0d7d04.json" }, { "id": "20261016T032723Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_89dba5", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": null, "wallSeconds": 52.661, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T032723Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_89dba5.json" }, { "id": "20261016T032635Z_verdict-code_byok_zai_glm-5.3-flash_9bb93a", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 107.073, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T032635Z_verdict-code_byok_zai_glm-5.3-flash_9bb93a.json" }, { "id": "20261016T032817Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_d00d50", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": null, "wallSeconds": 32.471, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Gateway injected an interactive tool unsupported by this third-party client; protocol-contaminated attempt excluded.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T032817Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_d00d50.json" }, { "id": "20261016T032929Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_ee1486", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "wallSeconds": 102.387, "inputTokens": 36649, "outputTokens": 3336, "totalTokens": 39985, "costUsd": 0.0, "energyWh": 7.774862, "assertionsPassed": 18, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T032929Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_ee1486.json" }, { "id": "20261016T033112Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_ce857f", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": null, "wallSeconds": 7.745, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Gateway injected an interactive tool unsupported by this third-party client; protocol-contaminated attempt excluded.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033112Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_ce857f.json" }, { "id": "20261016T033153Z_verdict-code_byok_openai_gpt-4.1-mini_a1738b", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": null, "wallSeconds": 18.781, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033153Z_verdict-code_byok_openai_gpt-4.1-mini_a1738b.json" }, { "id": "20261016T033213Z_aider_byok_openai_gpt-4.1-mini_0f6aae", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": null, "wallSeconds": 7.525, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033213Z_aider_byok_openai_gpt-4.1-mini_0f6aae.json" }, { "id": "20261016T033058Z_verdict-code_byok_zai_glm-5.3-flash_f3fb28", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 95.385, "inputTokens": 39072, "outputTokens": 3186, "totalTokens": 42258, "costUsd": 0.007453799999999999, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033058Z_verdict-code_byok_zai_glm-5.3-flash_f3fb28.json" }, { "id": "20261016T033306Z_aider_byok_openai_gpt-4.1-mini_929765", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 10.68, "inputTokens": 831, "outputTokens": 654, "totalTokens": 1485, "costUsd": 0.001379, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033306Z_aider_byok_openai_gpt-4.1-mini_929765.json" }, { "id": "20261016T033232Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_9d83fa", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "wallSeconds": 96.087, "inputTokens": 46893, "outputTokens": 4674, "totalTokens": 51567, "costUsd": 0.0, "energyWh": 10.026917000000001, "assertionsPassed": 14, "assertionsTotal": 20, "family": "matched-harness", "status": "TIMEOUT", "tier": "local/LAN", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033232Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_9d83fa.json" }, { "id": "20261016T033347Z_verdict-code_byok_google_models_gemini-2.5-flash_2868d1", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "wallSeconds": 22.707, "inputTokens": 22822, "outputTokens": 1345, "totalTokens": 24167, "costUsd": 0.0141841, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033347Z_verdict-code_byok_google_models_gemini-2.5-flash_2868d1.json" }, { "id": "20261016T033345Z_verdict-code_byok_deepseek_deepseek-flash_d98d72", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/deepseek/deepseek-flash", "resolvedModel": null, "wallSeconds": 32.798, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033345Z_verdict-code_byok_deepseek_deepseek-flash_d98d72.json" }, { "id": "20261016T033234Z_aider_byok_zai_glm-5.3-flash_77519d", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 110.021, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "TIMEOUT", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033234Z_aider_byok_zai_glm-5.3-flash_77519d.json" }, { "id": "20261016T033410Z_aider_byok_google_models_gemini-2.5-flash_297e5d", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "wallSeconds": 16.017, "inputTokens": 870, "outputTokens": 204, "totalTokens": 1074, "costUsd": 0.007741, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033410Z_aider_byok_google_models_gemini-2.5-flash_297e5d.json" }, { "id": "20261016T033408Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_cb33da", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "wallSeconds": 24.469, "inputTokens": 3280, "outputTokens": 960, "totalTokens": 4240, "costUsd": null, "energyWh": 0.824445, "assertionsPassed": 15, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033408Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_cb33da.json" }, { "id": "20261016T033346Z_verdict-code_byok_qwen_qwen3-coder-flash_402794", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3-coder-flash", "resolvedModel": "byok/qwen/qwen3-coder-flash", "wallSeconds": 46.87, "inputTokens": 48543, "outputTokens": 2563, "totalTokens": 51106, "costUsd": 0.0184074, "energyWh": null, "assertionsPassed": 15, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033346Z_verdict-code_byok_qwen_qwen3-coder-flash_402794.json" }, { "id": "20261016T033418Z_aider_byok_deepseek_deepseek-flash_54cf2e", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/deepseek/deepseek-flash", "resolvedModel": "byok/deepseek/deepseek-flash", "wallSeconds": 20.293, "inputTokens": 847, "outputTokens": 3000, "totalTokens": 3847, "costUsd": 0.003854, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033418Z_aider_byok_deepseek_deepseek-flash_54cf2e.json" }, { "id": "20261016T033434Z_aider_byok_qwen_qwen3-coder-flash_98e5e2", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3-coder-flash", "resolvedModel": "byok/qwen/qwen3-coder-flash", "wallSeconds": 11.655, "inputTokens": 847, "outputTokens": 470, "totalTokens": 1317, "costUsd": 0.0009591000000000001, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033434Z_aider_byok_qwen_qwen3-coder-flash_98e5e2.json" }, { "id": "20261016T033458Z_verdict-code_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_54d914", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "resolvedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "wallSeconds": 17.277, "inputTokens": 15834, "outputTokens": 437, "totalTokens": 16271, "costUsd": 0.0, "energyWh": 3.1638050000000004, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Agent completed without the required file mutation; observed task failure.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033458Z_verdict-code_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_54d914.json" }, { "id": "20261016T033516Z_aider_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_da844e", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "resolvedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "wallSeconds": 5.019, "inputTokens": 847, "outputTokens": 169, "totalTokens": 1016, "costUsd": 0.0, "energyWh": 0.197556, "assertionsPassed": 4, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033516Z_aider_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_da844e.json" }, { "id": "20261016T033457Z_aider_byok_zai_glm-5.3-flash_936356", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 45.569, "inputTokens": 822, "outputTokens": 3000, "totalTokens": 3822, "costUsd": 0.0016233, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033457Z_aider_byok_zai_glm-5.3-flash_936356.json" }, { "id": "20261016T033540Z_verdict-code_local_lmstudio_qwen_qwen3-coder-30b_5b01c0", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/qwen/qwen3-coder-30b", "resolvedModel": "local/lmstudio/qwen/qwen3-coder-30b", "wallSeconds": 75.46, "inputTokens": 48143, "outputTokens": 3049, "totalTokens": 51192, "costUsd": 0.0, "energyWh": 9.954, "assertionsPassed": 18, "assertionsTotal": 20, "family": "matched-harness", "status": "TIMEOUT", "tier": "local/LAN", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033540Z_verdict-code_local_lmstudio_qwen_qwen3-coder-30b_5b01c0.json" }, { "id": "20261016T033656Z_aider_local_lmstudio_qwen_qwen3-coder-30b_9c8512", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/qwen/qwen3-coder-30b", "resolvedModel": "local/lmstudio/qwen/qwen3-coder-30b", "wallSeconds": 23.156, "inputTokens": 847, "outputTokens": 544, "totalTokens": 1391, "costUsd": 0.0, "energyWh": 0.270472, "assertionsPassed": 18, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033656Z_aider_local_lmstudio_qwen_qwen3-coder-30b_9c8512.json" }, { "id": "20261016T033742Z_verdict-code_local_lmstudio_openai_gpt-oss-20b_8d73b8", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/openai/gpt-oss-20b", "resolvedModel": "local/lmstudio/openai/gpt-oss-20b", "wallSeconds": 40.555, "inputTokens": 9135, "outputTokens": 767, "totalTokens": 9902, "costUsd": 0.0, "energyWh": 1.925389, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "TIMEOUT", "tier": "local/LAN", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033742Z_verdict-code_local_lmstudio_openai_gpt-oss-20b_8d73b8.json" }, { "id": "20261016T033823Z_aider_local_lmstudio_openai_gpt-oss-20b_66c271", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/openai/gpt-oss-20b", "resolvedModel": "local/lmstudio/openai/gpt-oss-20b", "wallSeconds": 14.421, "inputTokens": 900, "outputTokens": 407, "totalTokens": 1307, "costUsd": 0.0, "energyWh": 0.254139, "assertionsPassed": 18, "assertionsTotal": 20, "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033823Z_aider_local_lmstudio_openai_gpt-oss-20b_66c271.json" }, { "id": "20261016T033837Z_verdict-code_local_lmstudio_google_gemma-4-26b-a4b_82bf22", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/google/gemma-4-26b-a4b", "resolvedModel": null, "wallSeconds": 4.26, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033837Z_verdict-code_local_lmstudio_google_gemma-4-26b-a4b_82bf22.json" }, { "id": "20261016T033842Z_aider_local_lmstudio_google_gemma-4-26b-a4b_bfbe9b", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/google/gemma-4-26b-a4b", "resolvedModel": null, "wallSeconds": 40.014, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "family": "matched-harness", "status": "TIMEOUT", "tier": "local/LAN", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033842Z_aider_local_lmstudio_google_gemma-4-26b-a4b_bfbe9b.json" }, { "id": "20261016T033800Z_verdict-code_byok_zai_glm-5.3-flash_700b65", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 90.239, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "family": "matched-harness", "status": "TIMEOUT", "tier": "cloud/BYOK", "limitations": [ "Not eligible for a matched-model comparison; see the retained attempt evidence.", "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "20261016T033800Z_verdict-code_byok_zai_glm-5.3-flash_700b65.json" }, { "id": "native-claude-extra-dependency-scope-isolated", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Claude Code", "requestedModel": "sonnet", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 9.828, "inputTokens": 62902, "outputTokens": 859, "totalTokens": 63761, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-claude-extra-dependency-scope-isolated.json" }, { "id": "native-claude-extra-regression-repair-isolated", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Claude Code", "requestedModel": "sonnet", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 11.195, "inputTokens": 84512, "outputTokens": 1155, "totalTokens": 85667, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-claude-extra-regression-repair-isolated.json" }, { "id": "native-codex-extra-dependency-scope-isolated", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 21.358, "inputTokens": 64758, "outputTokens": 659, "totalTokens": 65417, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "dependency-discipline", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-resolved model identity not captured; assertions shown separately from provenance status.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-codex-extra-dependency-scope-isolated.json" }, { "id": "native-codex-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 32.171, "inputTokens": 158673, "outputTokens": 1172, "totalTokens": 159845, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "regression-repair", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Apparatus diagnostic only; excluded from product comparisons.", "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; one attempt per cell; no statistical ranking.", "Provider-resolved model identity not captured; assertions shown separately from provenance status.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Diagnostic only: workspace inherited Verdict repository instructions; independent corrected attempt retained.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-codex-extra-regression-repair.json" }, { "id": "native-codex-extra-regression-repair-isolated", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 16.473, "inputTokens": 109353, "outputTokens": 289, "totalTokens": 109642, "costUsd": null, "energyWh": null, "assertionsPassed": 2, "assertionsTotal": 10, "family": "regression-repair", "status": "BLOCKED", "tier": "native subscription", "limitations": [ "Apparatus diagnostic only; excluded from product comparisons.", "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Provider-resolved model identity not captured; assertions shown separately from provenance status.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Nested OS sandbox rejected tool startup; diagnostic infrastructure outcome only.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-codex-extra-regression-repair-isolated.json" }, { "id": "native-codex-extra-regression-repair-native-sandbox", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 32.961, "inputTokens": 82153, "outputTokens": 1139, "totalTokens": 83292, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "regression-repair", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-resolved model identity not captured; assertions shown separately from provenance status.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-codex-extra-regression-repair-native-sandbox.json" }, { "id": "native-devin-extra-dependency-scope-isolated", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 7.74, "inputTokens": 55391, "outputTokens": 606, "totalTokens": 55997, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "CLI permission gate rejected at least one tool call; only independent oracle execution establishes behavior.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-devin-extra-dependency-scope-isolated.json" }, { "id": "native-devin-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 18.269, "inputTokens": 61429, "outputTokens": 704, "totalTokens": 62133, "costUsd": null, "energyWh": null, "assertionsPassed": 2, "assertionsTotal": 10, "family": "regression-repair", "status": "BLOCKED", "tier": "native subscription", "limitations": [ "Apparatus diagnostic only; excluded from product comparisons.", "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; one attempt per cell; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Diagnostic only: workspace inherited Verdict repository instructions; independent corrected attempt retained.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-devin-extra-regression-repair.json" }, { "id": "native-devin-extra-regression-repair-isolated", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 9.722, "inputTokens": 56559, "outputTokens": 894, "totalTokens": 57453, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "CLI rejected a tool call, but captured patch independently passed all frozen checks.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-devin-extra-regression-repair-isolated.json" }, { "id": "native-devin-blueprint-explicit-dependencies-isolated", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 5.479, "inputTokens": 27466, "outputTokens": 490, "totalTokens": 27956, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "family": "blueprint", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Planning aggregate withheld: the frozen keyword/exact-count rubric is not a valid semantic quality score. Original structural checks remain in the receipt. See the independent semantic review.", "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-devin-blueprint-explicit-dependencies-isolated.json" }, { "id": "native-codex-blueprint-explicit-dependencies-isolated", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 12.906, "inputTokens": 45281, "outputTokens": 396, "totalTokens": 45677, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "family": "blueprint", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Planning aggregate withheld: the frozen keyword/exact-count rubric is not a valid semantic quality score. Original structural checks remain in the receipt. See the independent semantic review.", "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-resolved model identity not captured; assertions shown separately from provenance status.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-codex-blueprint-explicit-dependencies-isolated.json" }, { "id": "native-devin-blueprint-prose-scope-isolated", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 18.027, "inputTokens": 27522, "outputTokens": 524, "totalTokens": 28046, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "family": "blueprint", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Planning aggregate withheld: the frozen keyword/exact-count rubric is not a valid semantic quality score. Original structural checks remain in the receipt. See the independent semantic review.", "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-devin-blueprint-prose-scope-isolated.json" }, { "id": "native-claude-blueprint-explicit-dependencies-isolated", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Claude Code", "requestedModel": "sonnet", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 8.905, "inputTokens": 42339, "outputTokens": 904, "totalTokens": 43243, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "family": "blueprint", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Planning aggregate withheld: the frozen keyword/exact-count rubric is not a valid semantic quality score. Original structural checks remain in the receipt. See the independent semantic review.", "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-claude-blueprint-explicit-dependencies-isolated.json" }, { "id": "native-codex-blueprint-prose-scope-isolated", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 17.002, "inputTokens": 48428, "outputTokens": 563, "totalTokens": 48991, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "family": "blueprint", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Planning aggregate withheld: the frozen keyword/exact-count rubric is not a valid semantic quality score. Original structural checks remain in the receipt. See the independent semantic review.", "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-resolved model identity not captured; assertions shown separately from provenance status.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-codex-blueprint-prose-scope-isolated.json" }, { "id": "native-claude-blueprint-prose-scope-isolated", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Claude Code", "requestedModel": "sonnet", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 8.509, "inputTokens": 42363, "outputTokens": 753, "totalTokens": 43116, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "family": "blueprint", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Planning aggregate withheld: the frozen keyword/exact-count rubric is not a valid semantic quality score. Original structural checks remain in the receipt. See the independent semantic review.", "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-claude-blueprint-prose-scope-isolated.json" }, { "id": "native-antigravity-preflight", "taskId": "native-access-preflight", "taskTitle": "Isolated native execution access preflight", "agent": "Antigravity", "requestedModel": null, "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "family": "native", "status": "BLOCKED", "tier": "native subscription", "limitations": [ "Apparatus diagnostic only; excluded from product comparisons.", "Preflight only; no model invocation and no coding attempt.", "Observed CLI supports IDE operations, with no isolated headless coding-agent command; native app contained another active conversation and was not disturbed.", "No measured wall time, tokens, billing or code quality result.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-antigravity-preflight.json" }, { "id": "native-claude-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Claude Code", "requestedModel": "sonnet", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": null, "inputTokens": 202266, "outputTokens": 1324, "totalTokens": 203590, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "regression-repair", "status": "UNVERIFIED", "tier": "native subscription", "limitations": [ "Apparatus diagnostic only; excluded from product comparisons.", "Diagnostic only: initial workspace inherited Verdict repository instructions; corrected external-workspace attempt retained separately.", "The initial receipt parser rejected the native JSON array after candidate and oracle artifacts were retained; full CLI wall clock is unavailable, so it remains null.", "The native internal duration is retained separately and is not substituted for missing external wall time.", "Reported API-equivalent list cost is not a subscription billing decrement.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "native-claude-extra-regression-repair.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T032826Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 3.889, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Synthetic fixture, one trial, 90-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Actual recorded terminal error: optimized_transport_error; Local Gateway optimized stream failed without retry or model failover: RemoteProtocolError (read_timeout=3600s)", "All token/cost metrics withheld: complete authoritative terminal accounting unavailable.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T032826Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T032831Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 3.609, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "family": "dependency-discipline", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Synthetic fixture, one trial, 90-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Actual recorded terminal error: optimized_transport_error; Local Gateway optimized stream failed without retry or model failover: RemoteProtocolError (read_timeout=3600s)", "All token/cost metrics withheld: complete authoritative terminal accounting unavailable.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T032831Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T032920Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 63.62, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "limitations": [ "Synthetic fixture, one trial, 90-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Actual recorded terminal error: optimized_total_token_budget_exhausted; provider-authoritative prior usage exhausted the cumulative token budget (remaining=22549, client_input_reservation=22834, sealed_output_ceiling=3000); exact current-request enforcement is required at Gateway", "Generated code independently passes all9 behavior tests and scope check, but CLI did not terminate successfully. Do not count as completed product PASS.", "All token/cost metrics withheld: complete authoritative terminal accounting unavailable.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T032920Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T033024Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 58.652, "inputTokens": 36099, "outputTokens": 1052, "totalTokens": 37151, "costUsd": 0.005941, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "family": "dependency-discipline", "status": "PASS", "tier": "cloud/BYOK", "limitations": [ "Synthetic fixture, one trial, 90-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Schema-validated BenchmarkResult stored with verified JSON/DuckDB/Neon readback, leaderboard publication disabled. Formal admission is false because full formal sampler/request/settlement provenance was not imported; no qualification or leaderboard claim.", "JSON/DuckDB/Neon diagnostic persistence verified. Formal benchmark eligibility and model qualification are not established." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T033024Z.json" }, { "id": "TEST-blueprint-planning-20261016T032613Z-blueprint-explicit-dependencies-blueprint", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain (synthetic PRD input)", "agent": "Blueprint", "requestedModel": null, "resolvedModel": null, "wallSeconds": 12.849, "inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "costUsd": 0, "energyWh": null, "assertionsPassed": 14, "assertionsTotal": 14, "family": "blueprint", "status": "PASS", "tier": "deterministic planning", "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Heuristic planning, no model invocation; zero tokens are actual estimation inference usage. Forecast tokens/cost/time are excluded from actual run metrics.", "Energy unmeasured.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "TEST-blueprint-planning-20261016T032613Z-blueprint-explicit-dependencies-blueprint.json" }, { "id": "TEST-blueprint-planning-20261016T032638Z-blueprint-prose-scope-blueprint", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria (synthetic PRD input)", "agent": "Blueprint", "requestedModel": null, "resolvedModel": null, "wallSeconds": 0.799, "inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "costUsd": 0, "energyWh": null, "assertionsPassed": 11, "assertionsTotal": 11, "family": "blueprint", "status": "PASS", "tier": "deterministic planning", "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Heuristic planning, no model invocation; zero tokens are actual estimation inference usage. Forecast tokens/cost/time are excluded from actual run metrics.", "Energy unmeasured.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "TEST-blueprint-planning-20261016T032638Z-blueprint-prose-scope-blueprint.json" }, { "id": "TEST-blueprint-planning-20261016T032646Z-blueprint-explicit-dependencies-raw-glm", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain (synthetic PRD input)", "agent": "Raw model planning baseline", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 15.084, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "family": "blueprint", "status": "UNVERIFIED", "tier": "cloud/BYOK", "limitations": [ "Planning aggregate withheld: the frozen keyword/exact-count rubric is not a valid semantic quality score. Original structural checks remain in the receipt. See the independent semantic review.", "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Raw model baseline, not a competing coding agent. Requested model held constant only within this baseline; Blueprint has no inference model.", "Cost is published-rate estimate at $0.15 input/$0.50 output per MTok; provider cash receipt not captured.", "Frozen excluded-word check has false positives: output states excluded features remain excluded. Original raw rubric preserved, not a valid semantic failure score.", "Independent review: required work and exclusions retained; extra packaging/handoff/sign-off tasks cause exact task count mismatch.", "Response model field is Gateway echo, not independent provider identity proof. Provider token accounting incomplete, so cost and tokens unknown.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "TEST-blueprint-planning-20261016T032646Z-blueprint-explicit-dependencies-raw-glm.json" }, { "id": "TEST-blueprint-planning-20261016T032646Z-blueprint-prose-scope-raw-glm", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria (synthetic PRD input)", "agent": "Raw model planning baseline", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 42.742, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "family": "blueprint", "status": "UNVERIFIED", "tier": "cloud/BYOK", "limitations": [ "Planning aggregate withheld: the frozen keyword/exact-count rubric is not a valid semantic quality score. Original structural checks remain in the receipt. See the independent semantic review.", "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Raw model baseline, not a competing coding agent. Requested model held constant only within this baseline; Blueprint has no inference model.", "Cost is published-rate estimate at $0.15 input/$0.50 output per MTok; provider cash receipt not captured.", "Frozen excluded-word check has false positives: output states excluded features remain excluded. Original raw rubric preserved, not a valid semantic failure score.", "Independent review: required work and exclusions retained; extra packaging/handoff/sign-off tasks cause exact task count mismatch.", "Response model field is Gateway echo, not independent provider identity proof. Provider token accounting incomplete, so cost and tokens unknown.", "Exploratory artifact evidence; JSON/DuckDB/Neon admission and model qualification are not established." ], "evidenceFile": "TEST-blueprint-planning-20261016T032646Z-blueprint-prose-scope-raw-glm.json" } ] }