{ "schema": "agent-comparison-rerun.v1", "campaign": "agent-comparison-rerun-20261016", "sourceSha": "ab05d4f5067356e5089e99f3806d1a7a39182c29", "generatedAt": "2026-10-16T04:42:39.595079+00:00", "state": "Corrected rerun, originals retained separately.", "protocol": [ "Same original task prompts, starter files and hidden oracles. The planning rubric was repaired before rerunning and is versioned separately.", "Fixed per-cell limits and exact model pins are recorded in each invocation. Original inexpensive models and current-model appendix are reported separately.", "Fresh external workspaces; hidden checks kept outside candidate workspaces and not supplied to agents; serial local inference; incremental receipts. Apparatus retries remain in history.", "Task acceptance, model identity and accounting completeness are separate fields. Unknown measurements stay unknown. No leaderboard qualification claim.", "The original 57-record pilot remains unchanged. Revised protocol results are not pooled with the pilot or interpreted as a statistical improvement.", "Shared hardware, model loading and run order can affect local timings. No pooled speed ranking; inspect per-run settings and runtime identity." ], "conclusions": [ "The repairs preserve the original pilot and every rerun attempt. Incomplete execution and missing measurements are separate from code correctness; genuine misses remain in the tables.", "On this parser fixture with matched low reasoning effort, GLM-5.3-Flash passes 20/20 through Verdict Code and 18/20 through Aider. Qwen 3.8 Flash passes 20/20 through both. Other models include Verdict Code losses; these single trials do not establish a general winner.", "Both Verdict Code coding fixtures now finish with 10/10 checks and complete accounting: 146.710 seconds for invoice repair and 30.960 seconds for the filename utility. All six native coding runs pass; the native agents are substantially faster on the invoice fixture.", "Blueprint passes both narrow planning fixtures without inference. The native agents and raw GLM baseline also pass the corrected scope and dependency checks. This supports deterministic extraction, not a general planning-quality winner." ], "rows": [ { "id": "20261016T035808Z_verdict-code_byok_openai_gpt-4.1-mini_216d27", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 17.989, "inputTokens": 3451, "outputTokens": 1058, "totalTokens": 4509, "costUsd": 0.003073, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Early rerun apparatus: Verdict Code CLI cumulative cap only; actual complete usage is below shared bounds but proxy parity was not enforced.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T035808Z_verdict-code_byok_openai_gpt-4.1-mini_216d27.json" }, { "id": "20261016T035826Z_aider_byok_openai_gpt-4.1-mini_7667be", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": null, "wallSeconds": 131.412, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Early rerun apparatus: Verdict Code CLI cumulative cap only; actual complete usage is below shared bounds but proxy parity was not enforced.", "Gateway instance continuity was not proven; excluded from matched-model comparison.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T035826Z_aider_byok_openai_gpt-4.1-mini_7667be.json" }, { "id": "20261016T040038Z_aider_byok_openai_gpt-4.1-mini_6a0e95", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/openai/gpt-4.1-mini", "resolvedModel": "byok/openai/gpt-4.1-mini", "wallSeconds": 6.85, "inputTokens": 831, "outputTokens": 442, "totalTokens": 1273, "costUsd": 0.00104, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Early rerun apparatus: Verdict Code CLI cumulative cap only; actual complete usage is below shared bounds but proxy parity was not enforced.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T040038Z_aider_byok_openai_gpt-4.1-mini_6a0e95.json" }, { "id": "20261016T040140Z_verdict-code_byok_qwen_qwen3.8-flash_a8a325", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": null, "wallSeconds": 1.308, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Early rerun apparatus: Verdict Code CLI cumulative cap only; actual complete usage is below shared bounds but proxy parity was not enforced.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T040140Z_verdict-code_byok_qwen_qwen3.8-flash_a8a325.json" }, { "id": "20261016T040218Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_c3dc31", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": null, "wallSeconds": 4.221, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Early rerun apparatus: Verdict Code CLI cumulative cap only; actual complete usage is below shared bounds but proxy parity was not enforced.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T040218Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_c3dc31.json" }, { "id": "20261016T040254Z_verdict-code_byok_qwen_qwen3.8-flash_27aaaf", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": null, "wallSeconds": 6.25, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Early rerun apparatus: Verdict Code CLI cumulative cap only; actual complete usage is below shared bounds but proxy parity was not enforced.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T040254Z_verdict-code_byok_qwen_qwen3.8-flash_27aaaf.json" }, { "id": "20261016T040618Z_verdict-code_byok_qwen_qwen3.8-flash_d11016", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": "byok/qwen/qwen3.8-flash", "wallSeconds": 92.805, "inputTokens": 7919, "outputTokens": 5123, "totalTokens": 13042, "costUsd": 0.00359566, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T040618Z_verdict-code_byok_qwen_qwen3.8-flash_d11016.json" }, { "id": "20261016T040627Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_564389", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "wallSeconds": 85.16, "inputTokens": 6828, "outputTokens": 1269, "totalTokens": 8097, "costUsd": 0.0, "energyWh": 1.5744170000000002, "assertionsPassed": 15, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T040627Z_verdict-code_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_564389.json" }, { "id": "20261016T040752Z_aider_byok_qwen_qwen3.8-flash_f8a219", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": "byok/qwen/qwen3.8-flash", "wallSeconds": 182.927, "inputTokens": 934, "outputTokens": 10384, "totalTokens": 11318, "costUsd": 0.00502058, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T040752Z_aider_byok_qwen_qwen3.8-flash_f8a219.json" }, { "id": "20261016T040753Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_bec8ed", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "resolvedModel": "local/lmstudio/verdict/verdict-4-35b-toolterse-v5-mlx", "wallSeconds": 10.45, "inputTokens": 886, "outputTokens": 438, "totalTokens": 1324, "costUsd": 0.0, "energyWh": 0.257444, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T040753Z_aider_local_lmstudio_verdict_verdict-4-35b-toolterse-v5-mlx_bec8ed.json" }, { "id": "20261016T040837Z_verdict-code_byok_zai_glm-5.3-flash_ae606b", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 245.073, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "TIMEOUT", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T040837Z_verdict-code_byok_zai_glm-5.3-flash_ae606b.json" }, { "id": "20261016T040931Z_verdict-code_local_lmstudio_qwen_qwen3-coder-30b_354ac5", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/qwen/qwen3-coder-30b", "resolvedModel": "local/lmstudio/qwen/qwen3-coder-30b", "wallSeconds": 88.856, "inputTokens": 9831, "outputTokens": 1631, "totalTokens": 11462, "costUsd": 0.0, "energyWh": 2.228722, "assertionsPassed": 12, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T040931Z_verdict-code_local_lmstudio_qwen_qwen3-coder-30b_354ac5.json" }, { "id": "20261016T041100Z_aider_local_lmstudio_qwen_qwen3-coder-30b_ef1750", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/qwen/qwen3-coder-30b", "resolvedModel": "local/lmstudio/qwen/qwen3-coder-30b", "wallSeconds": 25.527, "inputTokens": 847, "outputTokens": 551, "totalTokens": 1398, "costUsd": 0.0, "energyWh": 0.271833, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T041100Z_aider_local_lmstudio_qwen_qwen3-coder-30b_ef1750.json" }, { "id": "20261016T041127Z_verdict-code_local_lmstudio_openai_gpt-oss-20b_716b87", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/openai/gpt-oss-20b", "resolvedModel": null, "wallSeconds": 67.613, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T041127Z_verdict-code_local_lmstudio_openai_gpt-oss-20b_716b87.json" }, { "id": "20261016T041441Z_verdict-code_byok_zai_glm-5.3-flash_1f510f", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 33.418, "inputTokens": 5768, "outputTokens": 980, "totalTokens": 6748, "costUsd": 0.0013552, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T041441Z_verdict-code_byok_zai_glm-5.3-flash_1f510f.json" }, { "id": "20261016T041447Z_verdict-code_byok_deepseek_deepseek-flash_730d93", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/deepseek/deepseek-flash", "resolvedModel": "byok/deepseek/deepseek-flash", "wallSeconds": 50.879, "inputTokens": 5953, "outputTokens": 8717, "totalTokens": 14670, "costUsd": 0.012246, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T041447Z_verdict-code_byok_deepseek_deepseek-flash_730d93.json" }, { "id": "20261016T041515Z_aider_byok_zai_glm-5.3-flash_946658", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 9.644, "inputTokens": 822, "outputTokens": 433, "totalTokens": 1255, "costUsd": 0.0003398, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "low", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T041515Z_aider_byok_zai_glm-5.3-flash_946658.json" }, { "id": "20261016T041538Z_aider_byok_deepseek_deepseek-flash_a1a76b", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/deepseek/deepseek-flash", "resolvedModel": "byok/deepseek/deepseek-flash", "wallSeconds": 21.53, "inputTokens": 847, "outputTokens": 3598, "totalTokens": 4445, "costUsd": 0.004572, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T041538Z_aider_byok_deepseek_deepseek-flash_a1a76b.json" }, { "id": "20261016T041600Z_verdict-code_byok_qwen_qwen3-coder-flash_905503", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3-coder-flash", "resolvedModel": "byok/qwen/qwen3-coder-flash", "wallSeconds": 61.698, "inputTokens": 9590, "outputTokens": 1375, "totalTokens": 10965, "costUsd": 0.0049394999999999994, "energyWh": null, "assertionsPassed": 14, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Coordinator controller-pause intervention invalidates clean timing comparison; observed code acceptance retained.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T041600Z_verdict-code_byok_qwen_qwen3-coder-flash_905503.json" }, { "id": "20261016T041703Z_aider_byok_qwen_qwen3-coder-flash_2db869", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3-coder-flash", "resolvedModel": "byok/qwen/qwen3-coder-flash", "wallSeconds": 11.727, "inputTokens": 847, "outputTokens": 562, "totalTokens": 1409, "costUsd": 0.0010971, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T041703Z_aider_byok_qwen_qwen3-coder-flash_2db869.json" }, { "id": "20261016T042115Z_verdict-code_byok_google_models_gemini-2.5-flash_358bd6", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": null, "wallSeconds": 1.341, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T042115Z_verdict-code_byok_google_models_gemini-2.5-flash_358bd6.json" }, { "id": "20261016T042301Z_verdict-code_byok_qwen_qwen3-coder-flash_6100ff", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/qwen/qwen3-coder-flash", "resolvedModel": "byok/qwen/qwen3-coder-flash", "wallSeconds": 14.615, "inputTokens": 7015, "outputTokens": 945, "totalTokens": 7960, "costUsd": 0.003522, "energyWh": null, "assertionsPassed": 12, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T042301Z_verdict-code_byok_qwen_qwen3-coder-flash_6100ff.json" }, { "id": "20261016T042302Z_verdict-code_local_lmstudio_openai_gpt-oss-20b_b0ad76", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/openai/gpt-oss-20b", "resolvedModel": "local/lmstudio/openai/gpt-oss-20b", "wallSeconds": 52.967, "inputTokens": 2428, "outputTokens": 829, "totalTokens": 3257, "costUsd": 0.0, "energyWh": 0.633306, "assertionsPassed": 15, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T042302Z_verdict-code_local_lmstudio_openai_gpt-oss-20b_b0ad76.json" }, { "id": "20261016T042356Z_aider_local_lmstudio_openai_gpt-oss-20b_af7381", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/openai/gpt-oss-20b", "resolvedModel": "local/lmstudio/openai/gpt-oss-20b", "wallSeconds": 20.714, "inputTokens": 900, "outputTokens": 340, "totalTokens": 1240, "costUsd": 0.0, "energyWh": 0.241111, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T042356Z_aider_local_lmstudio_openai_gpt-oss-20b_af7381.json" }, { "id": "20261016T042418Z_verdict-code_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_273b6b", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "resolvedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "wallSeconds": 58.828, "inputTokens": 6598, "outputTokens": 1323, "totalTokens": 7921, "costUsd": 0.0, "energyWh": 1.5401939999999998, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Agent completed without the required file mutation; observed task failure.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T042418Z_verdict-code_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_273b6b.json" }, { "id": "20261016T042501Z_aider_byok_google_models_gemini-2.5-flash_6dd880", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "wallSeconds": 31.571, "inputTokens": 870, "outputTokens": 3408, "totalTokens": 4278, "costUsd": 0.008781, "energyWh": null, "assertionsPassed": 16, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Separately reported additive reasoning is exposed and reconciled without double counting in knownTokensIncludingAdditiveReasoning. Shared cumulative output ledger used completion_tokens; it did not enforce a separate cumulative reasoning cap.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Displayed tokens include separately reported reasoning. The uncorrected Gateway total and original output count remain in this receipt; raw vendor metadata was not captured." ], "reportedTotalTokens": 1388, "reportedOutputTokens": 518, "tokenAccountingSource": "reconciled_separate_reasoning", "evidenceFile": "20261016T042501Z_aider_byok_google_models_gemini-2.5-flash_6dd880.json" }, { "id": "20261016T042519Z_aider_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_c3fe55", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "resolvedModel": "local/lmstudio/qwen/qwen2.5-coder-1.5b-instruct", "wallSeconds": 8.806, "inputTokens": 847, "outputTokens": 169, "totalTokens": 1016, "costUsd": 0.0, "energyWh": 0.197556, "assertionsPassed": 4, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T042519Z_aider_local_lmstudio_qwen_qwen2.5-coder-1.5b-instruct_c3fe55.json" }, { "id": "20261016T042529Z_verdict-code_local_lmstudio_google_gemma-4-26b-a4b_5845b3", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "local/lmstudio/google/gemma-4-26b-a4b", "resolvedModel": null, "wallSeconds": 240.274, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "TIMEOUT", "tier": "local/LAN", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": "Four-minute limit reached; no completed result.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T042529Z_verdict-code_local_lmstudio_google_gemma-4-26b-a4b_5845b3.json" }, { "id": "20261016T042551Z_verdict-code_byok_google_models_gemini-2.5-flash_26e7d0", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-2.5-flash", "resolvedModel": "byok/google/models/gemini-2.5-flash", "wallSeconds": 32.76, "inputTokens": 5798, "outputTokens": 5614, "totalTokens": 11412, "costUsd": 0.0157744, "energyWh": null, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Separately reported additive reasoning is exposed and reconciled without double counting in knownTokensIncludingAdditiveReasoning. Shared cumulative output ledger used completion_tokens; it did not enforce a separate cumulative reasoning cap.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Displayed tokens include separately reported reasoning. The uncorrected Gateway total and original output count remain in this receipt; raw vendor metadata was not captured." ], "reportedTotalTokens": 7001, "reportedOutputTokens": 1203, "tokenAccountingSource": "reconciled_separate_reasoning", "evidenceFile": "20261016T042551Z_verdict-code_byok_google_models_gemini-2.5-flash_26e7d0.json" }, { "id": "20261016T042625Z_verdict-code_byok_google_models_gemini-3.8-flash_31c6d9", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-3.8-flash", "resolvedModel": null, "wallSeconds": 7.285, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Incomplete whole-attempt accounting; captured provider-call subtotals are lower bounds only." ], "evidenceFile": "20261016T042625Z_verdict-code_byok_google_models_gemini-3.8-flash_31c6d9.json" }, { "id": "20261016T042739Z_aider_byok_google_models_gemini-3.8-flash_4a1bb4", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/google/models/gemini-3.8-flash", "resolvedModel": "byok/google/models/gemini-3.8-flash", "wallSeconds": 70.112, "inputTokens": 870, "outputTokens": 7887, "totalTokens": 8757, "costUsd": 0.03022875, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Separately reported additive reasoning is exposed and reconciled without double counting in knownTokensIncludingAdditiveReasoning. Shared cumulative output ledger used completion_tokens; it did not enforce a separate cumulative reasoning cap.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED.", "Displayed tokens include separately reported reasoning. The uncorrected Gateway total and original output count remain in this receipt; raw vendor metadata was not captured." ], "reportedTotalTokens": 896, "reportedOutputTokens": 26, "tokenAccountingSource": "reconciled_separate_reasoning", "evidenceFile": "20261016T042739Z_aider_byok_google_models_gemini-3.8-flash_4a1bb4.json" }, { "id": "20261016T042948Z_aider_local_lmstudio_google_gemma-4-26b-a4b_9be558", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "local/lmstudio/google/gemma-4-26b-a4b", "resolvedModel": "local/lmstudio/google/gemma-4-26b-a4b", "wallSeconds": 229.41, "inputTokens": 903, "outputTokens": 5288, "totalTokens": 6191, "costUsd": 0.0, "energyWh": 1.203806, "assertionsPassed": 18, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "FAIL", "tier": "local/LAN", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "original-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T042948Z_aider_local_lmstudio_google_gemma-4-26b-a4b_9be558.json" }, { "id": "20261016T043737Z_verdict-code_byok_google_models_gemini-3.8-flash_41ca30", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/google/models/gemini-3.8-flash", "resolvedModel": "byok/google/models/gemini-3.8-flash", "wallSeconds": 165.699, "inputTokens": 15369, "outputTokens": 23348, "totalTokens": 38717, "costUsd": 0.09908174999999998, "energyWh": null, "assertionsPassed": 0, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "UNVERIFIED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "current-model", "resultNote": "Tool loop; final synthesis reached output cap.", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "The agent repeated Read and git exploration in the non-Git workspace, reached its tool-loop limit, then exhausted the final-synthesis output ceiling without an accepted candidate. Tool continuations and accounting succeeded; task completion is incomplete and generation-censored.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T043737Z_verdict-code_byok_google_models_gemini-3.8-flash_41ca30.json" }, { "id": "native-devin-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 10.936, "inputTokens": 72162, "outputTokens": 1086, "totalTokens": 73248, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-extra-regression-repair.json" }, { "id": "native-claude-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 19.102, "inputTokens": 129952, "outputTokens": 1655, "totalTokens": 131607, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-claude-extra-regression-repair.json" }, { "id": "native-codex-extra-regression-repair", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 37.556, "inputTokens": 114952, "outputTokens": 1175, "totalTokens": 116127, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-extra-regression-repair.json" }, { "id": "native-devin-blueprint-explicit-dependencies", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 5.329, "inputTokens": 27466, "outputTokens": 512, "totalTokens": 27978, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-blueprint-explicit-dependencies.json" }, { "id": "native-devin-extra-dependency-scope", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 7.391, "inputTokens": 69917, "outputTokens": 563, "totalTokens": 70480, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-extra-dependency-scope.json" }, { "id": "native-devin-blueprint-prose-scope", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Devin CLI", "requestedModel": "swe-2-medium", "resolvedModel": "swe-2-medium", "wallSeconds": 16.52, "inputTokens": 27503, "outputTokens": 781, "totalTokens": 28284, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "swe-2-medium", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter generation_model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-devin-blueprint-prose-scope.json" }, { "id": "native-codex-extra-dependency-scope", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 25.571, "inputTokens": 91480, "outputTokens": 710, "totalTokens": 92190, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-extra-dependency-scope.json" }, { "id": "native-codex-blueprint-explicit-dependencies", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 17.165, "inputTokens": 48367, "outputTokens": 537, "totalTokens": 48904, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-blueprint-explicit-dependencies.json" }, { "id": "native-claude-extra-dependency-scope", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 16.218, "inputTokens": 85576, "outputTokens": 908, "totalTokens": 86484, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Original raw receipt retained FAIL9/10 because pytest runtime cache was counted as authored code. Independent replay corrected this oracle inconsistency: same candidate9/9 behavior plus source scope passes. No new inference or response replacement." ], "evidenceFile": "native-claude-extra-dependency-scope.json" }, { "id": "native-codex-blueprint-prose-scope", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Codex CLI", "requestedModel": "gpt-6.1-sol", "resolvedModel": null, "wallSeconds": 17.226, "inputTokens": 48800, "outputTokens": 530, "totalTokens": 49330, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "gpt-6.1-sol", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native-effective-config; provider response identity not exposed", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion.", "Provider-response model identity not exposed; native effective selection is recorded separately and does not change acceptance status." ], "evidenceFile": "native-codex-blueprint-prose-scope.json" }, { "id": "native-claude-blueprint-explicit-dependencies", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 8.83, "inputTokens": 42329, "outputTokens": 849, "totalTokens": 43178, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-claude-blueprint-explicit-dependencies.json" }, { "id": "native-claude-blueprint-prose-scope", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria", "agent": "Claude Code", "requestedModel": "claude-sonnet-5-5", "resolvedModel": "claude-sonnet-5-5", "wallSeconds": 8.677, "inputTokens": 42355, "outputTokens": 753, "totalTokens": 43108, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": "claude-sonnet-5-5", "modelIdentityStatus": "MATCHED_NATIVE_RECEIPT", "modelIdentityEvidence": "native exporter modelUsage canonical model", "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "native subscription", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Auth uses existing native subscription/prepaid account; billing decrement unavailable.", "Native models differ: this is a product pilot, not a same-model comparison.", "Synthetic authored fixture; no statistical ranking.", "Planning structure rubric only; exact task count and exclusion text checks require semantic review. Not code execution quality.", "Custom pilot receipt has not been admitted through BenchmarkResult to JSON/DuckDB/Neon; no leaderboard promotion." ], "evidenceFile": "native-claude-blueprint-prose-scope.json" }, { "id": "native-antigravity-preflight", "taskId": "native-access-preflight", "taskTitle": "Repeat isolated native execution access preflight", "agent": "Antigravity", "requestedModel": null, "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": null, "observedNativeModel": null, "modelIdentityStatus": "NOT_RUN", "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "native", "status": "BLOCKED", "tier": "native subscription", "diagnosticOnly": true, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Access preflight only; no model invocation and no coding failure.", "Installed CLI advertises editor operations, with no isolated headless coding-agent command. Native UI conversations were not disturbed.", "No measured inference wall time, tokens, billing or quality result." ], "evidenceFile": "native-antigravity-preflight.json" }, { "id": "TEST-blueprint-planning-20261016T035726Z-blueprint-explicit-dependencies-blueprint", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain (synthetic PRD input)", "agent": "Blueprint", "requestedModel": null, "resolvedModel": null, "wallSeconds": 0.455, "inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "costUsd": 0, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "deterministic planning", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Heuristic planning, no model invocation; zero tokens are actual estimation inference usage. Forecast tokens/cost/time are excluded from actual run metrics.", "Energy unmeasured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035726Z-blueprint-explicit-dependencies-blueprint.json" }, { "id": "TEST-blueprint-planning-20261016T035733Z-blueprint-prose-scope-blueprint", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria (synthetic PRD input)", "agent": "Blueprint", "requestedModel": null, "resolvedModel": null, "wallSeconds": 0.958, "inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "costUsd": 0, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "deterministic planning", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Heuristic planning, no model invocation; zero tokens are actual estimation inference usage. Forecast tokens/cost/time are excluded from actual run metrics.", "Energy unmeasured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035733Z-blueprint-prose-scope-blueprint.json" }, { "id": "TEST-blueprint-planning-20261016T035735Z-blueprint-explicit-dependencies-raw-glm", "taskId": "blueprint-explicit-dependencies", "taskTitle": "Structured PRD: inventory reservation dependency chain (synthetic PRD input)", "agent": "Raw model planning baseline", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 13.895, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 13, "assertionsTotal": 13, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Raw model baseline, not a competing coding agent. Requested model held constant only within this baseline; Blueprint has no inference model.", "Cost is published-rate estimate at $0.15 input/$0.50 output per MTok; provider cash receipt not captured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035735Z-blueprint-explicit-dependencies-raw-glm.json" }, { "id": "TEST-blueprint-planning-20261016T035735Z-blueprint-prose-scope-raw-glm", "taskId": "blueprint-prose-scope", "taskTitle": "Prose PRD: CSV import with exclusions and acceptance criteria (synthetic PRD input)", "agent": "Raw model planning baseline", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 34.303, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "blueprint", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic PRD input; planning-only: no generated-file, build, or code-execution proof.", "One trial per task and arm; exact rubric is deliberately narrow; no statistical or production claim.", "Raw model baseline, not a competing coding agent. Requested model held constant only within this baseline; Blueprint has no inference model.", "Cost is published-rate estimate at $0.15 input/$0.50 output per MTok; provider cash receipt not captured.", "Semantic rubric v2 excludes exact task count from acceptance. Count is diagnostic only; exclusion mentions are evaluated for explicit delivery versus reaffirmed non-delivery. Ambiguous wording remains UNVERIFIED.", "Semantic-v2 rerun uses originalunchanged synthetic PRDs. Originalv1 attempts remain published separately. No generated-code execution claim." ], "evidenceFile": "TEST-blueprint-planning-20261016T035735Z-blueprint-prose-scope-raw-glm.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T040002Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Gateway health preflight raised ConnectError [Errno61] Connection refused before agent/provider invocation. No coding quality result or spend inferred.", "This preflight diagnostic was reconstructed immediately after wrapper failure; elapsed time was not durably captured and remains unknown." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T040002Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T040152Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 4.061, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "CLI rejected a non-private 0755 instrumentation directory before inference. Corrected runner uses 0700. Configuration diagnostic, not a coding failure.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T040152Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T040534Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 56.813, "inputTokens": 25815, "outputTokens": 691, "totalTokens": 26506, "costUsd": 0.004217, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Gateway lifecycle persistence failed for request 19908254-ebcc-4ad1-867e-7dfa65664468: terminal durability 500 ms deadline expired beside a 996 ms Gateway loop stall. No timeout or token-budget failure. Complete usage accounting survived.", "Generated code independently passes 9 behavior tests plus 1 scope check, but the CLI did not terminate successfully. Do not count as completed product PASS.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T040534Z.json" }, { "id": "TEST-extra-dependency-scope-verdict-code-20261016T042131Z", "taskId": "extra-dependency-scope", "taskTitle": "Add a safe filename utility without dependency or scope drift (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 30.96, "inputTokens": 7064, "outputTokens": 414, "totalTokens": 7478, "costUsd": 0.0012670000000000001, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "dependency-discipline", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Protocol update: public --task-file filenames.py --pin-task-files --no-ide metadata explicitly scopes context to the same one-file contract already stated in the unchanged prompt. Earlier full-context attempts remain retained; model/task/oracle are unchanged.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-dependency-scope-verdict-code-20261016T042131Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T035725Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 156.373, "inputTokens": 83930, "outputTokens": 4190, "totalTokens": 88120, "costUsd": 0.014682999999999998, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Instrumentation diagnostic: candidate read provider traces inside workspace. Completion/code checks are real; timing and token usage are confounded. A mistaken interrupted reconstruction was corrected from original terminal stdout/journal; correction retained.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T035725Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T040029Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": null, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "Gateway health preflight raised ConnectError [Errno61] Connection refused before agent/provider invocation. No coding quality result or spend inferred.", "This preflight diagnostic was reconstructed immediately after wrapper failure; elapsed time was not durably captured and remains unknown." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T040029Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T040146Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": null, "wallSeconds": 3.768, "inputTokens": null, "outputTokens": null, "totalTokens": null, "costUsd": null, "energyWh": null, "assertionsPassed": null, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "BLOCKED", "tier": "cloud/BYOK", "diagnosticOnly": true, "comparisonEligible": false, "cohort": "original-model", "resultNote": null, "limitations": [ "CLI rejected a non-private 0755 instrumentation directory before inference. Corrected runner uses 0700. Configuration diagnostic, not a coding failure.", "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T040146Z.json" }, { "id": "TEST-extra-regression-repair-verdict-code-20261016T040304Z", "taskId": "extra-regression-repair", "taskTitle": "Repair money calculations across modules (synthetic test input)", "agent": "verdict-code", "requestedModel": "byok/zai/glm-5.3-flash", "resolvedModel": "byok/zai/glm-5.3-flash", "wallSeconds": 146.71, "inputTokens": 76351, "outputTokens": 3337, "totalTokens": 79688, "costUsd": 0.01312, "energyWh": null, "assertionsPassed": 10, "assertionsTotal": 10, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": null, "family": "regression-repair", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": null, "cohort": "original-model", "resultNote": null, "limitations": [ "Synthetic fixture, one trial, 240-second limit; not a general coding-agent ranking.", "9 independent behavior unittest methods plus 1 changed-file scope check.", "Model and token accounting require captured provider/usage review; absent values remain unknown.", "Timeout/transport failure is infrastructure evidence, not a model quality failure.", "Complete Gateway terminal usage proof; cost is estimated BYOK USD, not a provider cash receipt." ], "evidenceFile": "TEST-extra-regression-repair-verdict-code-20261016T040304Z.json" } ], "evidenceRoot": "/benchmarks/agent-comparison-20261016" }