{ "schema": "agent-comparison-harness-repair.v1", "campaign": "agent-comparison-harness-repair-20261016", "sourceSha": "ff964e31c9729254b0fca2c736323d01447b686b", "generatedAt": "2026-10-16T11:23:24.585006+00:00", "state": "Parser harness repair; other task families retained from the prior campaign.", "protocol": [ "The parser uses the unchanged frozen prompt, starter and 20-case hidden oracle. Both prior campaigns and every new attempt are preserved.", "New parser pairs explicitly request temperature zero. Earlier requests specified Verdict Code 0.7 and Aider zero, but the OpenAI streaming adapter omitted upstream temperature. Source and sampling controls both changed; their effects cannot be isolated.", "Exact model pins and existing process, token and reasoning limits are retained. GLM uses the previously disclosed low reasoning setting; other models keep their default.", "Verdict Code now allows a bounded behavioral self-check using the visible task only. Neither agent receives hidden oracle code, examples, expected outputs or judge feedback.", "The latest scheduled attempt is shown, including worse or incomplete results. Earlier attempts stay in audit history; there is no best-of selection.", "Other coding and planning families retain their previous measured receipts and were not rerun in this harness-repair campaign.", "Synthetic n=1 fixtures; shared hardware timing; no statistical ranking or general product superiority claim. Canonical benchmark-store admission remains unverified." ], "conclusions": [ "The harness had real defects: compile-only completion, stale mutation guards, unsafe context trimming and provider tool-call incompatibilities. Those are repaired; the remaining model errors and incomplete runs remain visible.", "Results are mixed: ToolTerse 35B now passes 20/20 versus Aider 18/20; GPT-4.1-mini ties 18/20; GLM scores 18/20 versus Aider 20/20. Neither agent dominates this narrow fixture.", "Qwen 3.8 now completes 20/20 with default thinking preserved after a provider compatibility repair. It takes 223 seconds; the Aider counterpart reaches 240 seconds without completing.", "Behavioral self-checks can repair a draft, but they add time and tokens. Some models still loop or emit malformed calls. Partial artifacts and capacity refusals are not completed coding scores.", "Both prior campaigns and every new attempt are preserved. Task/oracle hashes are unchanged, no hidden-test feedback reaches models, and latest attempts are shown rather than best-of results. Source and sampling controls both changed; their effects cannot be isolated.", "Other coding and planning families retain their earlier measured results and were not rerun in this parser harness repair." ], "rows": [ { "id": "20261016T103826Z_verdict-code_byok_openai_gpt-4.1-mini_55f7f8", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": null, "wallSeconds": 18.621, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T103826Z_verdict-code_byok_openai_gpt-4.1-mini_55f7f8.json", "supersededBy": "20261016T104437Z_verdict-code_byok_openai_gpt-4.1-mini_0ea7f1" }, { "id": "20261016T104437Z_verdict-code_byok_openai_gpt-4.1-mini_0ea7f1", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 22.65, "inputTokens": 20811, "outputTokens": 320, "totalTokens": 21131, "costUsd": 0.008832000000000003, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": "Model kept revising and reached the fixed request limit without finishing.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T104437Z_verdict-code_byok_openai_gpt-4.1-mini_0ea7f1.json", "supersededBy": "20261016T105121Z_verdict-code_byok_openai_gpt-4.1-mini_004008" }, { "id": "20261016T105121Z_verdict-code_byok_openai_gpt-4.1-mini_004008", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 23.501, "inputTokens": 12928, "outputTokens": 2055, "totalTokens": 14983, "costUsd": 0.008459, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105121Z_verdict-code_byok_openai_gpt-4.1-mini_004008.json" }, { "id": "20261016T105251Z_verdict-code_byok_zai_glm-5.3-flash_11ed55", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 0.908, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": "Inferred cloud capability incorrectly blocked startup; repaired in a later attempt.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "No algorithm was executed; original raw oracle observation remains retained. The cloud catalog negative was inferred, not an authoritative declaration.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T105251Z_verdict-code_byok_zai_glm-5.3-flash_11ed55.json", "supersededBy": "20261016T105748Z_verdict-code_byok_zai_glm-5.3-flash_c91e41" }, { "id": "20261016T105629Z_aider_byok_openai_gpt-4.1-mini_c9315b", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": null, "wallSeconds": 2.47, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T105629Z_aider_byok_openai_gpt-4.1-mini_c9315b.json", "supersededBy": "20261016T105732Z_aider_byok_openai_gpt-4.1-mini_a2f541" }, { "id": "20261016T105711Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_a35cdd", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "wallSeconds": 132.722, "inputTokens": 21757, "outputTokens": 5129, "totalTokens": 26886, "costUsd": 0.0, "energyWh": 5.227834, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105711Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_a35cdd.json" }, { "id": "20261016T105732Z_aider_byok_openai_gpt-4.1-mini_a2f541", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 7.627, "inputTokens": 831, "outputTokens": 663, "totalTokens": 1494, "costUsd": 0.001393, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105732Z_aider_byok_openai_gpt-4.1-mini_a2f541.json" }, { "id": "20261016T105748Z_verdict-code_byok_zai_glm-5.3-flash_c91e41", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 37.542, "inputTokens": 7458, "outputTokens": 941, "totalTokens": 8399, "costUsd": 0.0015891999999999998, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105748Z_verdict-code_byok_zai_glm-5.3-flash_c91e41.json" }, { "id": "20261016T105804Z_verdict-code_byok_deepseek_deepseek-flash_da59fa", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/deepseek/deepseek-flash", "resolvedModel": "byok/deepseek/deepseek-flash", "wallSeconds": 59.482, "inputTokens": 39504, "outputTokens": 9259, "totalTokens": 48763, "costUsd": 0.022961999999999996, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105804Z_verdict-code_byok_deepseek_deepseek-flash_da59fa.json" }, { "id": "20261016T105815Z_verdict-code_byok_qwen_qwen3-coder-flash_62d44a", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3-coder-flash", "resolvedModel": "byok/qwen/qwen3-coder-flash", "wallSeconds": 139.171, "inputTokens": 71329, "outputTokens": 10476, "totalTokens": 81805, "costUsd": 0.037112700000000005, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": "Model kept revising and reached the fixed request limit without finishing.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105815Z_verdict-code_byok_qwen_qwen3-coder-flash_62d44a.json" }, { "id": "20261016T105821Z_verdict-code_byok_google_models_gemini-2.5-flash_c88b1e", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "wallSeconds": 59.745, "inputTokens": 18362, "outputTokens": 11629, "totalTokens": 29991, "costUsd": 0.034581100000000004, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105821Z_verdict-code_byok_google_models_gemini-2.5-flash_c88b1e.json" }, { "id": "20261016T105826Z_aider_byok_zai_glm-5.3-flash_721cfa", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 9.302, "inputTokens": 822, "outputTokens": 388, "totalTokens": 1210, "costUsd": 0.0003173, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105826Z_aider_byok_zai_glm-5.3-flash_721cfa.json" }, { "id": "20261016T105859Z_verdict-code_byok_qwen_qwen3.8-flash_0f8043", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": null, "wallSeconds": 3.584, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T105859Z_verdict-code_byok_qwen_qwen3.8-flash_0f8043.json", "supersededBy": "20261016T110648Z_verdict-code_byok_qwen_qwen3.8-flash_b9235f" }, { "id": "20261016T105900Z_verdict-code_byok_google_models_gemini-3.8-flash_1771fb", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-3.8-flash", "resolvedModel": "byok/google/models/gemini-3.8-flash", "wallSeconds": 71.485, "inputTokens": 15349, "outputTokens": 7666, "totalTokens": 23015, "costUsd": 0.04025925000000001, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "A metadata probe disconnected; inference is assessed separately from captured POST completion and accounting.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105900Z_verdict-code_byok_google_models_gemini-3.8-flash_1771fb.json" }, { "id": "20261016T105904Z_aider_byok_deepseek_deepseek-flash_89ec1a", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/deepseek/deepseek-flash", "resolvedModel": "byok/deepseek/deepseek-flash", "wallSeconds": 18.975, "inputTokens": 847, "outputTokens": 2984, "totalTokens": 3831, "costUsd": 0.003835, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105904Z_aider_byok_deepseek_deepseek-flash_89ec1a.json" }, { "id": "20261016T105921Z_aider_byok_google_models_gemini-2.5-flash_83648c", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "wallSeconds": 17.441, "inputTokens": 870, "outputTokens": 3367, "totalTokens": 4237, "costUsd": 0.0086785, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105921Z_aider_byok_google_models_gemini-2.5-flash_83648c.json" }, { "id": "20261016T105924Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_c34dae", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "wallSeconds": 8.564, "inputTokens": 886, "outputTokens": 438, "totalTokens": 1324, "costUsd": 0.0, "energyWh": 0.257444, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T105924Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_c34dae.json" }, { "id": "20261016T105934Z_verdict-code_local_lmstudio_qwen_qwen3-coder-30b_2b81cb", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/qwen/qwen3-coder-30b", "resolvedModel": null, "wallSeconds": 240.415, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 12, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "TIMEOUT", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": "Process reached the fixed time limit without completing.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T105934Z_verdict-code_local_lmstudio_qwen_qwen3-coder-30b_2b81cb.json" }, { "id": "20261016T110059Z_aider_byok_qwen_qwen3.8-flash_2075c9", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": null, "wallSeconds": 240.03, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "TIMEOUT", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": "Process reached the fixed time limit without completing.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T110059Z_aider_byok_qwen_qwen3.8-flash_2075c9.json" }, { "id": "20261016T110130Z_aider_byok_google_models_gemini-3.8-flash_5008ab", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/google/models/gemini-3.8-flash", "resolvedModel": "byok/google/models/gemini-3.8-flash", "wallSeconds": 67.397, "inputTokens": 870, "outputTokens": 7870, "totalTokens": 8740, "costUsd": 0.030165, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T110130Z_aider_byok_google_models_gemini-3.8-flash_5008ab.json" }, { "id": "20261016T110203Z_aider_byok_qwen_qwen3-coder-flash_0357e0", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3-coder-flash", "resolvedModel": "byok/qwen/qwen3-coder-flash", "wallSeconds": 15.068, "inputTokens": 847, "outputTokens": 459, "totalTokens": 1306, "costUsd": 0.0009426, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T110203Z_aider_byok_qwen_qwen3-coder-flash_0357e0.json" }, { "id": "20261016T110426Z_aider_local_lmstudio_qwen_qwen3-coder-30b_cc1a24", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/qwen/qwen3-coder-30b", "resolvedModel": "local/lmstudio/qwen/qwen3-coder-30b", "wallSeconds": 21.158, "inputTokens": 847, "outputTokens": 574, "totalTokens": 1421, "costUsd": 0.0, "energyWh": 0.276306, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T110426Z_aider_local_lmstudio_qwen_qwen3-coder-30b_cc1a24.json" }, { "id": "20261016T110648Z_verdict-code_byok_qwen_qwen3.8-flash_b9235f", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": "byok/qwen/qwen3.8-flash", "wallSeconds": 222.887, "inputTokens": 32456, "outputTokens": 8762, "totalTokens": 41218, "costUsd": 0.008986540000000001, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T110648Z_verdict-code_byok_qwen_qwen3.8-flash_b9235f.json" }, { "id": "20261016T110725Z_verdict-code_local_lmstudio_openai_gpt-oss-20b_634d13", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/openai/gpt-oss-20b", "resolvedModel": "local/lmstudio/openai/gpt-oss-20b", "wallSeconds": 60.517, "inputTokens": 4131, "outputTokens": 2564, "totalTokens": 6695, "costUsd": 0.0, "energyWh": 1.301806, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": "Model returned malformed tool arguments; stopped before executing that call.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T110725Z_verdict-code_local_lmstudio_openai_gpt-oss-20b_634d13.json" }, { "id": "20261016T111001Z_aider_local_lmstudio_openai_gpt-oss-20b_df2cfb", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/openai/gpt-oss-20b", "resolvedModel": "local/lmstudio/openai/gpt-oss-20b", "wallSeconds": 7.156, "inputTokens": 900, "outputTokens": 340, "totalTokens": 1240, "costUsd": 0.0, "energyWh": 0.241111, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T111001Z_aider_local_lmstudio_openai_gpt-oss-20b_df2cfb.json" }, { "id": "20261016T111009Z_verdict-code_local_lmstudio_google_gemma-4-26b-a4b_e4f791", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/google/gemma-4-26b-a4b", "resolvedModel": null, "wallSeconds": 1.688, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": "Model load refused: insufficient available memory; no candidate executed.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T111009Z_verdict-code_local_lmstudio_google_gemma-4-26b-a4b_e4f791.json", "supersededBy": "20261016T111315Z_verdict-code_local_lmstudio_google_gemma-4-26b-a4b_4d87aa" }, { "id": "20261016T111131Z_aider_local_lmstudio_google_gemma-4-26b-a4b_c80df5", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/google/gemma-4-26b-a4b", "resolvedModel": "local/lmstudio/google/gemma-4-26b-a4b", "wallSeconds": 102.588, "inputTokens": 903, "outputTokens": 5288, "totalTokens": 6191, "costUsd": 0.0, "energyWh": 1.203806, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T111131Z_aider_local_lmstudio_google_gemma-4-26b-a4b_c80df5.json" }, { "id": "20261016T111315Z_verdict-code_local_lmstudio_google_gemma-4-26b-a4b_4d87aa", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/google/gemma-4-26b-a4b", "resolvedModel": null, "wallSeconds": 240.665, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "TIMEOUT", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": "Four-minute limit reached; no completed result.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "The runtime ledger settled after terminal hashing. Original terminal bytes were recovered with an exact hash match; both states and the raw result are retained. This attempt remains incomplete and comparison-ineligible.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T111315Z_verdict-code_local_lmstudio_google_gemma-4-26b-a4b_4d87aa.json" }, { "id": "20261016T111737Z_verdict-code_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_7bc25f", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "resolvedModel": null, "wallSeconds": 2.706, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": "Model advertises no native tool support; stopped before inference.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "No algorithm was executed; original raw oracle observation remains retained. Provider-declared unsupported native-tool mode.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T111737Z_verdict-code_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_7bc25f.json" }, { "id": "20261016T111744Z_aider_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_de63a5", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "resolvedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "wallSeconds": 13.846, "inputTokens": 847, "outputTokens": 169, "totalTokens": 1016, "costUsd": 0.0, "energyWh": 0.197556, "assertionsPassed": 4, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T111744Z_aider_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_de63a5.json" }, { "id": "native-devin-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 10.936, "inputTokens": 72162, "outputTokens": 1086, "totalTokens": 73248, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-extra-regression-repair.json" }, { "id": "native-claude-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 19.102, "inputTokens": 129952, "outputTokens": 1655, "totalTokens": 131607, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-claude-extra-regression-repair.json" }, { "id": "native-codex-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 37.556, "inputTokens": 114952, "outputTokens": 1175, "totalTokens": 116127, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-extra-regression-repair.json" }, { "id": "native-devin-blueprint-explicit-dependencies", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 5.329, "inputTokens": 27466, "outputTokens": 512, "totalTokens": 27978, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-blueprint-explicit-dependencies.json" }, { "id": "native-devin-extra-dependency-scope", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 7.391, "inputTokens": 69917, "outputTokens": 563, "totalTokens": 70480, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-extra-dependency-scope.json" }, { "id": "native-devin-blueprint-prose-scope", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 16.52, "inputTokens": 27503, "outputTokens": 781, "totalTokens": 28284, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-blueprint-prose-scope.json" }, { "id": "native-codex-extra-dependency-scope", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 25.571, "inputTokens": 91480, "outputTokens": 710, "totalTokens": 92190, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-extra-dependency-scope.json" }, { "id": "native-codex-blueprint-explicit-dependencies", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 17.165, "inputTokens": 48367, "outputTokens": 537, "totalTokens": 48904, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-blueprint-explicit-dependencies.json" }, { "id": "native-claude-extra-dependency-scope", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 16.218, "inputTokens": 85576, "outputTokens": 908, "totalTokens": 86484, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Original raw receipt retained FAIL9/10 because pytest runtime cache was counted as authored code. Independent replay corrected this oracle inconsistency: same candidate9/9 behavior plus source scope passes. No new inference or response replacement." ], "evidenceFile": "native-claude-extra-dependency-scope.json" }, { "id": "native-codex-blueprint-prose-scope", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 17.226, "inputTokens": 48800, "outputTokens": 530, "totalTokens": 49330, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-blueprint-prose-scope.json" }, { "id": "native-claude-blueprint-explicit-dependencies", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 8.83, "inputTokens": 42329, "outputTokens": 849, "totalTokens": 43178, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-claude-blueprint-explicit-dependencies.json" }, { "id": "native-claude-blueprint-prose-scope", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 8.677, "inputTokens": 42355, "outputTokens": 753, "totalTokens": 43108, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-claude-blueprint-prose-scope.json" }, { "id": "native-antigravity-preflight", "taskId": "native-access-preflight", "taskTitle": "Repeat isolated native execution access preflight", "agent": "Antigravity", "requestedModel": null, "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "observedNativeModel": null, "modelIdentityStatus": "NOT_RUN", "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "native", "status": "BLOCKED", "tier": "native subscription", "diagnosticOnly": true, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Access preflight only; no model invocation and no coding failure.", "Installed CLI advertises editor operations, with no isolated headless coding-agent command. Native UI conversations were not disturbed.", "No measured inference wall time, tokens, billing or quality result." ], "evidenceFile": "native-antigravity-preflight.json" }, { "id": "TEST-blueprint-planning-20261016T035726Z-blueprint-explicit-dependencies-blueprint", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain (synthetic PRD input)", "agent": "Blueprint", "requestedModel": null, "resolvedModel": null, "wallSeconds": 0.455, "inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "costUsd": 0, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "deterministic planning", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Heuristic planning, no model invocation; zero tokens are actual estimation inference usage. Forecast tokens/cost/time are excluded from actual run metrics.", "Energy unmeasured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035726Z-blueprint-explicit-dependencies-blueprint.json" }, { "id": "TEST-blueprint-planning-20261016T035733Z-blueprint-prose-scope-blueprint", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria (synthetic PRD input)", "agent": "Blueprint", "requestedModel": null, "resolvedModel": null, "wallSeconds": 0.958, "inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "costUsd": 0, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "deterministic planning", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Heuristic planning, no model invocation; zero tokens are actual estimation inference usage. Forecast tokens/cost/time are excluded from actual run metrics.", "Energy unmeasured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035733Z-blueprint-prose-scope-blueprint.json" }, { "id": "TEST-blueprint-planning-20261016T035735Z-blueprint-explicit-dependencies-raw-glm", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain (synthetic PRD input)", "agent": "Raw model planning baseline", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 13.895, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Raw model baseline, not a competing coding agent. Requested model held constant only within this baseline; Blueprint has no inference model.", "Cost is published-rate estimate at $0.15 input/$0.50 output per MTok; provider cash receipt not captured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035735Z-blueprint-explicit-dependencies-raw-glm.json" }, { "id": "TEST-blueprint-planning-20261016T035735Z-blueprint-prose-scope-raw-glm", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria (synthetic PRD input)", "agent": "Raw model planning baseline", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 34.303, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Raw model baseline, not a competing coding agent. Requested model held constant only within this baseline; Blueprint has no inference model.", "Cost is published-rate estimate at $0.15 input/$0.50 output per MTok; provider cash receipt not captured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035735Z-blueprint-prose-scope-raw-glm.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T040002Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Gateway health preflight raised ConnectError [Errno61] Connection refused before agent/provider invocation. No coding quality result or spend inferred.", "This preflight diagnostic was reconstructed immediately after wrapper failure; elapsed time was not durably captured and remains unknown." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T040002Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T040152Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 4.061, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "CLI rejected a non-private 0755 instrumentation directory before inference. Corrected runner uses 0700. Configuration diagnostic, not a coding failure.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T040152Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T040534Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 56.813, "inputTokens": 25815, "outputTokens": 691, "totalTokens": 26506, "costUsd": 0.004217, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Gateway lifecycle persistence failed for request 19908254-ebcc-4ad1-867e-7dfa65664468: terminal durability 500 ms deadline expired beside a 996 ms Gateway loop stall. No timeout or token-budget failure. Complete usage accounting survived.", "Generated code independently passes 9 behavior tests plus 1 scope check, but the CLI did not terminate successfully. Do not count as completed product PASS.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T040534Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T042131Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 30.96, "inputTokens": 7064, "outputTokens": 414, "totalTokens": 7478, "costUsd": 0.0012670000000000001, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Protocol update: public --task-file filenames.py --pin-task-files --no-ide metadata explicitly scopes context to the same one-file contract already stated in the unchanged prompt. Earlier full-context attempts remain retained; model/task/oracle are unchanged.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T042131Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T035725Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 156.373, "inputTokens": 83930, "outputTokens": 4190, "totalTokens": 88120, "costUsd": 0.014682999999999998, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Instrumentation diagnostic: candidate read provider traces inside workspace. Completion/code checks are real; timing and token usage are confounded. A mistaken interrupted reconstruction was corrected from original terminal stdout/journal; correction retained.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T035725Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T040029Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Gateway health preflight raised ConnectError [Errno61] Connection refused before agent/provider invocation. No coding quality result or spend inferred.", "This preflight diagnostic was reconstructed immediately after wrapper failure; elapsed time was not durably captured and remains unknown." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T040029Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T040146Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 3.768, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "CLI rejected a non-private 0755 instrumentation directory before inference. Corrected runner uses 0700. Configuration diagnostic, not a coding failure.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T040146Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T040304Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 146.71, "inputTokens": 76351, "outputTokens": 3337, "totalTokens": 79688, "costUsd": 0.01312, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T040304Z.json" } ], "evidenceRoot": "/benchmarks/agent-comparison-20261016-harness-repair" }