{ "schema": "agent-rerun-receipt.v1", "record": { "id": "20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": "byok/qwen/qwen3.8-flash", "wallSeconds": 192.155, "inputTokens": 934, "outputTokens": 10861, "totalTokens": 11795, "costUsd": 0.00524477, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedNativeModel": null, "modelIdentityStatus": null, "modelIdentityEvidence": null, "providerResponseModel": null, "reasoningEffort": "default", "family": "matched-harness", "status": "PASS", "tier": "cloud/BYOK", "diagnosticOnly": false, "comparisonEligible": true, "cohort": "current-model", "resultNote": null, "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "evidenceFile": "20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683.json", "archived": false, "supersededBy": "20261016T124951Z_aider_byok_qwen_qwen3.8-flash_d64870" }, "sourceRecord": { "id": "20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683", "originalCampaign": "agent_comparison_20261015", "rerun": true, "correctionOf": "20261016T122901Z_aider_byok_qwen_qwen3.8-flash_c9d466", "cohort": "current-model", "family": "current-model-appendix", "taskId": "retry-after-v1", "taskTitle": "Strict HTTP Retry-After parser (synthetic test input)", "agent": "aider", "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": "byok/qwen/qwen3.8-flash", "status": "PASS", "wallSeconds": 192.155, "inputTokens": 934, "outputTokens": 10861, "totalTokens": 11795, "costUsd": 0.00524477, "energyWh": null, "assertionsPassed": 20, "assertionsTotal": 20, "observedCodePass": true, "exitCode": 0, "evidencePaths": [ "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683" ], "arm": "E", "admission": "NOT_VERIFIED", "limitations": [ "Exploratory n=1 synthetic task; not a general coding-agent ranking.", "Exploratory arm E; JSON/DuckDB/Neon canonical admission NOT_VERIFIED." ], "freeze": { "task_sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle_sha256": "3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76", "starter_sha256": "595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2", "frozen_before_execution_at": 1792154171.42372 }, "mutableRuntimeFiles": [ "budget_ledger.json", "daa05cea8f694c9da27dc77d097fb45a.http.json", "daa05cea8f694c9da27dc77d097fb45a.timing.json", "daa05cea8f694c9da27dc77d097fb45a.request.json", "daa05cea8f694c9da27dc77d097fb45a.response.txt" ], "terminalBudgetSnapshot": "budget_ledger_at_terminal.json", "terminalTransportSnapshot": "terminal_evidence/", "rawStatus": "PASS", "metadataTransportDiagnosticCount": 0, "diagnosticOnly": false, "tokenSource": "provider", "accountingComplete": true, "identityVerified": true, "providerCallCount": 1, "partialInputTokens": 934, "partialOutputTokens": 10861, "partialTotalTokens": 11795, "partialEstimatedCostUsd": 0.00524477, "reportedAdditiveReasoningTokens": null, "reportedReasoningTokens": 9997, "allCallsReportReasoning": true, "tokensPerSecond": 61.383, "knownTokensIncludingAdditiveReasoning": 11795, "knownTokensIncludingAdditiveReasoningPerSecond": 61.383, "generationCensored": false, "budgetControls": { "schema": "agent_comparison_rerun_policy.v1", "temperature": 0, "timeout_seconds": 240, "per_request_output_tokens": 8192, "proxy_shared_max_input_tokens": 160000, "proxy_shared_max_output_tokens": 32000, "proxy_reservation": "entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing", "maximum_input_tokens": 160000, "total_token_budget": 400000, "max_turns": 24, "proxy_read_timeout_seconds": 270, "verdict_stream_idle_timeout_seconds": 240, "aider_request_timeout_seconds": 240, "reasoning": { "mode": "default" }, "models": "exact pins, no failover", "task_file": "solution.py", "hidden_oracle_fed_to_agent": false, "producer_location": "outside candidate workspace; candidate read denied", "aider_context": "live exact catalog context metadata; history ceiling=min(inputbudget,context/2)", "appendix_model_changed": true }, "reasoningEffort": "default", "sharedProxyCapsEnforced": true, "actualBelowCommonCaps": true, "timeoutSeconds": 240, "terminalError": null, "terminalCompleted": true, "agentFailureReason": null, "sameGatewayInstance": true, "comparisonEligible": true, "publicEvidencePath": "artifacts/agent_comparison_completion_repair_20261016/matrix/public_evidence/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683.json", "publicEvidenceSha256": "39f5a0a27954154cd4ee3dfbc67c9901fc493431bc5a2d5028430f830fd2d565" }, "sourceFile": "artifacts/agent_comparison_completion_repair_20261016/matrix/reviewed-summary.json", "sourceFileSha256": "cfdb4bb21c1038588f0951f2cbe4774e077c01691d7e0e53434881d3b9a16419", "files": [ { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/budget_ledger.json", "sha256": "37c475f1ceae66146860635e6a0a025c9f121e34609253857bdd3aca96818bf2", "bytes": 118 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/budget_ledger_at_terminal.json", "sha256": "37c475f1ceae66146860635e6a0a025c9f121e34609253857bdd3aca96818bf2", "bytes": 118 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/daa05cea8f694c9da27dc77d097fb45a.http.json", "sha256": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "bytes": 138 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/daa05cea8f694c9da27dc77d097fb45a.request.json", "sha256": "dff63f4ce6de2d1b40003cfcbf795a85e7ef6f34ec0195c7b0649a09ccb5d3f0", "bytes": 4495 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/daa05cea8f694c9da27dc77d097fb45a.response.txt", "sha256": "13a8ae44c48e04b2b9aff4d07982d30720f88133f003fb589bd6a9be58deca4d", "bytes": 62901 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/daa05cea8f694c9da27dc77d097fb45a.timing.json", "sha256": "cf8cafc3db5a6ae36204ceb934abdc2d86f991bc03ee8438e536805c74f7735c", "bytes": 307 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/freeze.json", "sha256": "a77bc6da8729af28b89978942b6aa89180637426da24ba2965ecd86c20e8521a", "bytes": 312, "content": "{\n \"task_sha256\": \"6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe\",\n \"oracle_sha256\": \"3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76\",\n \"starter_sha256\": \"595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2\",\n \"frozen_before_execution_at\": 1792154171.42372\n}", "contentRedacted": false, "publishedContentSha256": "a77bc6da8729af28b89978942b6aa89180637426da24ba2965ecd86c20e8521a" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/health_after.json", "sha256": "d26c42476b712b225f253657577ece06f830d996893415fb29a9187694d5868e", "bytes": 1326 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/health_before.json", "sha256": "690fd7da42123d0193b35f7d1c3b0ae62808f63cbd763ed4dfaa9ce5361b336d", "bytes": 1326 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/invocation.json", "sha256": "ca269595ffd99d9c8dc18fe6c426e581e54b5be4465c842ddcdfff313dce160e", "bytes": 1998 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/launch.json", "sha256": "252c7557a554c320428c65ad2938e50ed670c497d02c46bb0e9eadfcb85d6e85", "bytes": 219 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/model_preflight.json", "sha256": "c8e5d62d5b3eed8644996e6c1966e98f4cc831859adce57036f115dd006141b9", "bytes": 1975 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/oracle.stderr.txt", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes": 0, "content": "", "contentRedacted": false, "publishedContentSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/oracle.stdout.txt", "sha256": "5b22a02f39f6be766ee28b0cdfca80c3244449283a14f0786cda512f6659a6fd", "bytes": 1582, "content": "{\"passed\": 20, \"total\": 20, \"cases\": [{\"input\": \"0\", \"expected\": 0, \"actual\": 0, \"pass\": true}, {\"input\": \" 120\\t\", \"expected\": 120, \"actual\": 120, \"pass\": true}, {\"input\": \"0007\", \"expected\": 7, \"actual\": 7, \"pass\": true}, {\"input\": \"-1\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"+1\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"1.5\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"\\uff11\\uff12\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"\\u0661\\u0662\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": null, \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": 4, \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 GMT\", \"expected\": 1, \"actual\": 1, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:01 GMT\", \"expected\": 2, \"actual\": 2, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:27:00 GMT\", \"expected\": 0, \"actual\": 0, \"pass\": true}, {\"input\": \"Tue, 21 Oct 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 UTC\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"wed, 21 Oct 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 31 Feb 2015 07:28:00 GMT\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 GMT trailing\", \"expected\": null, \"actual\": null, \"pass\": true}, {\"input\": \"Wed, 21 Oct 2015 07:28:00 +0000\", \"expected\": null, \"actual\": null, \"pass\": true}]}\n", "contentRedacted": false, "publishedContentSha256": "5b22a02f39f6be766ee28b0cdfca80c3244449283a14f0786cda512f6659a6fd" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/policy.json", "sha256": "3163bbcc22b8d5d499225c93837b6be7a119a160855c41ab97209f56c997537d", "bytes": 920, "content": "{\n \"schema\": \"agent_comparison_rerun_policy.v1\",\n \"temperature\": 0,\n \"timeout_seconds\": 240,\n \"per_request_output_tokens\": 8192,\n \"proxy_shared_max_input_tokens\": 160000,\n \"proxy_shared_max_output_tokens\": 32000,\n \"proxy_reservation\": \"entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing\",\n \"maximum_input_tokens\": 160000,\n \"total_token_budget\": 400000,\n \"max_turns\": 24,\n \"proxy_read_timeout_seconds\": 270,\n \"verdict_stream_idle_timeout_seconds\": 240,\n \"aider_request_timeout_seconds\": 240,\n \"reasoning\": {\n \"mode\": \"default\"\n },\n \"models\": \"exact pins, no failover\",\n \"task_file\": \"solution.py\",\n \"hidden_oracle_fed_to_agent\": false,\n \"producer_location\": \"outside candidate workspace; candidate read denied\",\n \"aider_context\": \"live exact catalog context metadata; history ceiling=min(inputbudget,context/2)\",\n \"appendix_model_changed\": true\n}", "contentRedacted": false, "publishedContentSha256": "3163bbcc22b8d5d499225c93837b6be7a119a160855c41ab97209f56c997537d" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/post-terminal-settlement.json", "sha256": "86f1196886643528d63589853daff4056c40e5373c13de3622c2c864036154ed", "bytes": 383 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/product_source.json", "sha256": "dbbdef3c8b1626c6e10404b9a184bf921e3b359c48f0e0fdd7c7dbaac0b9fbe4", "bytes": 846, "content": "{\n \"head\": \"409511b4c7a8ff3f29dff3ac2f029f5f1a469d41\",\n \"files\": {\n \"scripts/benchmarks/agent_comparison_rerun_20261016.py\": \"c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0\",\n \"tools/verdict-code/verdict_code/agent.py\": \"7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e\",\n \"tools/verdict-code/verdict_code/tools/__init__.py\": \"609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57\",\n \"tools/verdict-code/verdict_code/main.py\": \"af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab\",\n \"tools/verdict-code/VERSION\": \"98ddd8c64dd951f851d65ceaa4a8939727b829f146950a55439b56bd45e16208\",\n \"scripts/benchmarks/agent_comparison_completion_repair_20261016.py\": \"827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa\"\n },\n \"retainedSourceBytes\": \"source_snapshot/\"\n}", "contentRedacted": false, "publishedContentSha256": "dbbdef3c8b1626c6e10404b9a184bf921e3b359c48f0e0fdd7c7dbaac0b9fbe4" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/prompt.txt", "sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "bytes": 837, "content": "Synthetic benchmark task: Retry-After parsing. Implement `retry_after(value, now)` in solution.py using only Python standard library. now is an aware UTC datetime. Return a nonnegative integer number of seconds or None for invalid input. Strings containing only ASCII decimal digits after surrounding whitespace return that integer. Otherwise accept an RFC HTTP-date of form `Wed, 21 Oct 2015 07:28:00 GMT`, case sensitive, validated weekday must agree with date. For a valid HTTP date return max(0, ceil((date-now).total_seconds())). Reject non-string inputs, signs, fractions, non-ASCII digits, extra text, and non-GMT timezone names. Do not change any other source file. Hidden tests are unavailable. Write the implementation to disk and finish. Do not install dependencies, access network, use other models, or edit unrelated paths.\n", "contentRedacted": false, "publishedContentSha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe" }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/result.json", "sha256": "1daa94f5f316d9c03fdf0444d4b750a378c750b04f4c2496be57c98273509d9d", "bytes": 5343 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/sandbox.sb", "sha256": "f7c4de2de4511c9fae802b8e235ccccd138ed2d2b3fcfb32875278097434e4fd", "bytes": 5691 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/sandbox_preflight.json", "sha256": "8415d156075475eba2b82a04636f833e85484020de5e6424f6140cd0ac61390f", "bytes": 93 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/source_snapshot/scripts/benchmarks/agent_comparison_completion_repair_20261016.py", "sha256": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa", "bytes": 13274 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/source_snapshot/scripts/benchmarks/agent_comparison_rerun_20261016.py", "sha256": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "bytes": 38332 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/source_snapshot/tools/verdict-code/VERSION", "sha256": "98ddd8c64dd951f851d65ceaa4a8939727b829f146950a55439b56bd45e16208", "bytes": 8 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/source_snapshot/tools/verdict-code/verdict_code/agent.py", "sha256": "7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e", "bytes": 763436 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/source_snapshot/tools/verdict-code/verdict_code/main.py", "sha256": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "bytes": 625279 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/source_snapshot/tools/verdict-code/verdict_code/tools/__init__.py", "sha256": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "bytes": 269877 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/stderr.txt", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes": 0 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/stdout.txt", "sha256": "54c832ff163285f01ba43c52bf4a899834f8bb712263f4be5861205c85d632c5", "bytes": 3199 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/terminal_evidence/budget_ledger.json", "sha256": "37c475f1ceae66146860635e6a0a025c9f121e34609253857bdd3aca96818bf2", "bytes": 118 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/terminal_evidence/daa05cea8f694c9da27dc77d097fb45a.http.json", "sha256": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "bytes": 138 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/terminal_evidence/daa05cea8f694c9da27dc77d097fb45a.request.json", "sha256": "dff63f4ce6de2d1b40003cfcbf795a85e7ef6f34ec0195c7b0649a09ccb5d3f0", "bytes": 4495 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/terminal_evidence/daa05cea8f694c9da27dc77d097fb45a.response.txt", "sha256": "13a8ae44c48e04b2b9aff4d07982d30720f88133f003fb589bd6a9be58deca4d", "bytes": 62901 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/terminal_evidence/daa05cea8f694c9da27dc77d097fb45a.timing.json", "sha256": "cf8cafc3db5a6ae36204ceb934abdc2d86f991bc03ee8438e536805c74f7735c", "bytes": 307 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/workspace/aider_model_metadata.json", "sha256": "ca6e584ae0ec3f49c5fd105409e1df2658db69ea35dfa0e1f49ae965deca6dbb", "bytes": 185 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/workspace/aider_settings.yml", "sha256": "34d42aeca49ec38ee9e07ea615a92560d2430d5c460d3dd433745b1b99ac1a4a", "bytes": 181 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/workspace/chat_history.md", "sha256": "310c15da99b80f9245a3f32a6d6522be3d1a672a4bbad41f99193675be19c38c", "bytes": 5284 }, { "path": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/workspace/solution.py", "sha256": "b5826b089e9c72d39d58665b6692d0dc219ed619b81506a03c3e0a9c8bd015e1", "bytes": 2870, "content": "from datetime import datetime, timezone\n\n\n_WEEKDAYS = (\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\")\n_WEEKDAY_SET = set(_WEEKDAYS)\n\n_MONTHS = {\n \"Jan\": 1,\n \"Feb\": 2,\n \"Mar\": 3,\n \"Apr\": 4,\n \"May\": 5,\n \"Jun\": 6,\n \"Jul\": 7,\n \"Aug\": 8,\n \"Sep\": 9,\n \"Oct\": 10,\n \"Nov\": 11,\n \"Dec\": 12,\n}\n\n\ndef _is_ascii_digits(value, length=None):\n if length is not None and len(value) != length:\n return False\n\n if not value:\n return False\n\n for ch in value:\n if ord(ch) < ord(\"0\") or ord(ch) > ord(\"9\"):\n return False\n\n return True\n\n\ndef _parse_http_date(value):\n parts = value.split(\" \")\n if len(parts) != 6:\n return None\n\n weekday_part, day_part, month_part, year_part, time_part, tz_part = parts\n\n if len(weekday_part) != 4 or weekday_part[-1] != \",\":\n return None\n\n weekday = weekday_part[:-1]\n if weekday not in _WEEKDAY_SET:\n return None\n\n if not _is_ascii_digits(day_part, 2):\n return None\n\n month = _MONTHS.get(month_part)\n if month is None:\n return None\n\n if not _is_ascii_digits(year_part, 4):\n return None\n\n if tz_part != \"GMT\":\n return None\n\n time_parts = time_part.split(\":\")\n if len(time_parts) != 3:\n return None\n\n hour_part, minute_part, second_part = time_parts\n\n if (\n not _is_ascii_digits(hour_part, 2)\n or not _is_ascii_digits(minute_part, 2)\n or not _is_ascii_digits(second_part, 2)\n ):\n return None\n\n try:\n year = int(year_part)\n month = int(month)\n day = int(day_part)\n hour = int(hour_part)\n minute = int(minute_part)\n second = int(second_part)\n\n target = datetime(\n year,\n month,\n day,\n hour,\n minute,\n second,\n tzinfo=timezone.utc,\n )\n except (ValueError, OverflowError):\n return None\n\n if _WEEKDAYS[target.weekday()] != weekday:\n return None\n\n return target\n\n\ndef _ceil_delta_seconds(delta):\n numerator = (\n delta.days * 86_400_000_000\n + delta.seconds * 1_000_000\n + delta.microseconds\n )\n denominator = 1_000_000\n\n return -((-numerator) // denominator)\n\n\ndef retry_after(value, now):\n \"\"\"Parse an HTTP Retry-After header according to the task contract.\"\"\"\n if not isinstance(value, str):\n return None\n\n stripped = value.strip()\n\n if _is_ascii_digits(stripped):\n return int(stripped)\n\n target = _parse_http_date(stripped)\n if target is None:\n return None\n\n if not isinstance(now, datetime):\n return None\n\n if now.tzinfo is None:\n now = now.replace(tzinfo=timezone.utc)\n\n try:\n delta = target - now\n except TypeError:\n return None\n\n return max(0, _ceil_delta_seconds(delta))\n", "contentRedacted": false, "publishedContentSha256": "b5826b089e9c72d39d58665b6692d0dc219ed619b81506a03c3e0a9c8bd015e1" } ], "disclosure": "Synthetic inputs and observed execution. Original-byte hashes retained; selected source and oracle outputs reproduced. Private provider transcripts remain local.", "reviewedEvidence": { "provenance": "Synthetic test input; actual installed CLI execution; reviewed public-safe evidence fixture.", "runId": "20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683", "task": "Synthetic benchmark task: Retry-After parsing. Implement `retry_after(value, now)` in solution.py using only Python standard library. now is an aware UTC datetime. Return a nonnegative integer number of seconds or None for invalid input. Strings containing only ASCII decimal digits after surrounding whitespace return that integer. Otherwise accept an RFC HTTP-date of form `Wed, 21 Oct 2015 07:28:00 GMT`, case sensitive, validated weekday must agree with date. For a valid HTTP date return max(0, ceil((date-now).total_seconds())). Reject non-string inputs, signs, fractions, non-ASCII digits, extra text, and non-GMT timezone names. Do not change any other source file. Hidden tests are unavailable. Write the implementation to disk and finish. Do not install dependencies, access network, use other models, or edit unrelated paths.\n", "freeze": { "task_sha256": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle_sha256": "3675736119622cac168d711045426c6678bd08212b9bedf1aab71126a7ef1a76", "starter_sha256": "595a2a41e7bfd09ad28045cb3be95f0126d702eb25e6f53c723c0b77c46b44e2", "frozen_before_execution_at": 1792154171.42372 }, "candidateCode": "from datetime import datetime, timezone\n\n\n_WEEKDAYS = (\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\")\n_WEEKDAY_SET = set(_WEEKDAYS)\n\n_MONTHS = {\n \"Jan\": 1,\n \"Feb\": 2,\n \"Mar\": 3,\n \"Apr\": 4,\n \"May\": 5,\n \"Jun\": 6,\n \"Jul\": 7,\n \"Aug\": 8,\n \"Sep\": 9,\n \"Oct\": 10,\n \"Nov\": 11,\n \"Dec\": 12,\n}\n\n\ndef _is_ascii_digits(value, length=None):\n if length is not None and len(value) != length:\n return False\n\n if not value:\n return False\n\n for ch in value:\n if ord(ch) < ord(\"0\") or ord(ch) > ord(\"9\"):\n return False\n\n return True\n\n\ndef _parse_http_date(value):\n parts = value.split(\" \")\n if len(parts) != 6:\n return None\n\n weekday_part, day_part, month_part, year_part, time_part, tz_part = parts\n\n if len(weekday_part) != 4 or weekday_part[-1] != \",\":\n return None\n\n weekday = weekday_part[:-1]\n if weekday not in _WEEKDAY_SET:\n return None\n\n if not _is_ascii_digits(day_part, 2):\n return None\n\n month = _MONTHS.get(month_part)\n if month is None:\n return None\n\n if not _is_ascii_digits(year_part, 4):\n return None\n\n if tz_part != \"GMT\":\n return None\n\n time_parts = time_part.split(\":\")\n if len(time_parts) != 3:\n return None\n\n hour_part, minute_part, second_part = time_parts\n\n if (\n not _is_ascii_digits(hour_part, 2)\n or not _is_ascii_digits(minute_part, 2)\n or not _is_ascii_digits(second_part, 2)\n ):\n return None\n\n try:\n year = int(year_part)\n month = int(month)\n day = int(day_part)\n hour = int(hour_part)\n minute = int(minute_part)\n second = int(second_part)\n\n target = datetime(\n year,\n month,\n day,\n hour,\n minute,\n second,\n tzinfo=timezone.utc,\n )\n except (ValueError, OverflowError):\n return None\n\n if _WEEKDAYS[target.weekday()] != weekday:\n return None\n\n return target\n\n\ndef _ceil_delta_seconds(delta):\n numerator = (\n delta.days * 86_400_000_000\n + delta.seconds * 1_000_000\n + delta.microseconds\n )\n denominator = 1_000_000\n\n return -((-numerator) // denominator)\n\n\ndef retry_after(value, now):\n \"\"\"Parse an HTTP Retry-After header according to the task contract.\"\"\"\n if not isinstance(value, str):\n return None\n\n stripped = value.strip()\n\n if _is_ascii_digits(stripped):\n return int(stripped)\n\n target = _parse_http_date(stripped)\n if target is None:\n return None\n\n if not isinstance(now, datetime):\n return None\n\n if now.tzinfo is None:\n now = now.replace(tzinfo=timezone.utc)\n\n try:\n delta = target - now\n except TypeError:\n return None\n\n return max(0, _ceil_delta_seconds(delta))\n", "oracle": { "passed": 20, "total": 20, "cases": [ { "input": "0", "expected": 0, "actual": 0, "pass": true }, { "input": " 120\t", "expected": 120, "actual": 120, "pass": true }, { "input": "0007", "expected": 7, "actual": 7, "pass": true }, { "input": "-1", "expected": null, "actual": null, "pass": true }, { "input": "+1", "expected": null, "actual": null, "pass": true }, { "input": "1.5", "expected": null, "actual": null, "pass": true }, { "input": "１２", "expected": null, "actual": null, "pass": true }, { "input": "١٢", "expected": null, "actual": null, "pass": true }, { "input": "", "expected": null, "actual": null, "pass": true }, { "input": null, "expected": null, "actual": null, "pass": true }, { "input": 4, "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 GMT", "expected": 1, "actual": 1, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:01 GMT", "expected": 2, "actual": 2, "pass": true }, { "input": "Wed, 21 Oct 2015 07:27:00 GMT", "expected": 0, "actual": 0, "pass": true }, { "input": "Tue, 21 Oct 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 UTC", "expected": null, "actual": null, "pass": true }, { "input": "wed, 21 Oct 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 31 Feb 2015 07:28:00 GMT", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 GMT trailing", "expected": null, "actual": null, "pass": true }, { "input": "Wed, 21 Oct 2015 07:28:00 +0000", "expected": null, "actual": null, "pass": true } ] }, "providerCalls": [ { "requestEvidence": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683/daa05cea8f694c9da27dc77d097fb45a.request.json", "requestSha256": "dff63f4ce6de2d1b40003cfcbf795a85e7ef6f34ec0195c7b0649a09ccb5d3f0", "responseSha256": "13a8ae44c48e04b2b9aff4d07982d30720f88133f003fb589bd6a9be58deca4d", "httpStatus": 200, "requestedModel": "byok/qwen/qwen3.8-flash", "resolvedModel": "byok/qwen/qwen3.8-flash", "backendModel": "qwen/qwen3.8-flash", "resolutionSource": "explicit_byok_route", "generationControls": { "max_tokens": 8192, "temperature": 0, "reasoning_choice": { "mode": "default" }, "auto_failover": false, "stream": true }, "inputTokens": 934, "outputTokens": 10861, "totalTokens": 11795, "costUsd": 0.00524477, "energyWh": null, "finishReasons": [ "stop" ], "additiveReasoningTokens": null, "nestedReasoningTokens": 9997 } ], "gatewayBefore": { "status": "healthy", "service": "local_gateway", "mode": "full", "port": 6120, "timestamp": "2026-10-16T12:36:12.140738+00:00", "uptime_seconds": 181.4517719745636, "server_start_time": 1792153990.688971, "git_sha": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "build_time": "2026-10-16T08:32:43-04:00", "version": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "interop_min": 1, "interop_max": 1, "dependencies": { "event_stream": "http://localhost:6102" }, "lightrag": { "enabled": true, "status": "online", "files_indexed": 8020, "url": "http://127.0.0.1:6150" }, "cloud_relay": { "connected": true, "authenticated": true }, "context_status": { "summary": "ok", "target_context": 65536, "providers": [ { "provider": "ollama", "status": "ok", "actual_context": 131072, "target_context": 65536, "model": "deepseek-r1:7b" }, { "provider": "lmstudio", "status": "ok", "actual_context": 262144, "target_context": 65536, "model": "verdict-4-35b-toolterse-v5-mlx" }, { "provider": "llamacpp", "status": "context_unknown", "actual_context": null, "target_context": 65536, "model": null } ] } }, "gatewayAfter": { "status": "healthy", "service": "local_gateway", "mode": "full", "port": 6120, "timestamp": "2026-10-16T12:39:26.023034+00:00", "uptime_seconds": 375.3340699672699, "server_start_time": 1792153990.688971, "git_sha": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "build_time": "2026-10-16T08:32:43-04:00", "version": "a5d8f6a1c4fc2bd9144502167f3eaa8461dcfc25", "interop_min": 1, "interop_max": 1, "dependencies": { "event_stream": "http://localhost:6102" }, "lightrag": { "enabled": true, "status": "online", "files_indexed": 8020, "url": "http://127.0.0.1:6150" }, "cloud_relay": { "connected": true, "authenticated": true }, "context_status": { "summary": "ok", "target_context": 65536, "providers": [ { "provider": "ollama", "status": "ok", "actual_context": 131072, "target_context": 65536, "model": "deepseek-r1:7b" }, { "provider": "lmstudio", "status": "ok", "actual_context": 262144, "target_context": 65536, "model": "verdict-4-35b-toolterse-v5-mlx" }, { "provider": "llamacpp", "status": "context_unknown", "actual_context": null, "target_context": 65536, "model": null } ] } }, "rawArtifacts": "artifacts/agent_comparison_completion_repair_20261016/matrix/20261016T123611Z_aider_byok_qwen_qwen3.8-flash_5dd683", "rawResultSha256": "1daa94f5f316d9c03fdf0444d4b750a378c750b04f4c2496be57c98273509d9d", "rawEvidenceHashes": { "health_before.json": "690fd7da42123d0193b35f7d1c3b0ae62808f63cbd763ed4dfaa9ce5361b336d", "stdout.txt": "54c832ff163285f01ba43c52bf4a899834f8bb712263f4be5861205c85d632c5", "policy.json": "3163bbcc22b8d5d499225c93837b6be7a119a160855c41ab97209f56c997537d", "health_after.json": "d26c42476b712b225f253657577ece06f830d996893415fb29a9187694d5868e", "model_preflight.json": "c8e5d62d5b3eed8644996e6c1966e98f4cc831859adce57036f115dd006141b9", "sandbox_preflight.json": "8415d156075475eba2b82a04636f833e85484020de5e6424f6140cd0ac61390f", "budget_ledger_at_terminal.json": "37c475f1ceae66146860635e6a0a025c9f121e34609253857bdd3aca96818bf2", "prompt.txt": "6c229fd3a8dda211432b4ff70838fa0cb73b325bc778da52ab0cdd280619a5fe", "oracle.stdout.txt": "5b22a02f39f6be766ee28b0cdfca80c3244449283a14f0786cda512f6659a6fd", "sandbox.sb": "f7c4de2de4511c9fae802b8e235ccccd138ed2d2b3fcfb32875278097434e4fd", "stderr.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "launch.json": "252c7557a554c320428c65ad2938e50ed670c497d02c46bb0e9eadfcb85d6e85", "invocation.json": "ca269595ffd99d9c8dc18fe6c426e581e54b5be4465c842ddcdfff313dce160e", "oracle.stderr.txt": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "freeze.json": "a77bc6da8729af28b89978942b6aa89180637426da24ba2965ecd86c20e8521a", "product_source.json": "dbbdef3c8b1626c6e10404b9a184bf921e3b359c48f0e0fdd7c7dbaac0b9fbe4", "post-terminal-settlement.json": "86f1196886643528d63589853daff4056c40e5373c13de3622c2c864036154ed", "workspace/aider_model_metadata.json": "ca6e584ae0ec3f49c5fd105409e1df2658db69ea35dfa0e1f49ae965deca6dbb", "workspace/solution.py": "b5826b089e9c72d39d58665b6692d0dc219ed619b81506a03c3e0a9c8bd015e1", "workspace/chat_history.md": "310c15da99b80f9245a3f32a6d6522be3d1a672a4bbad41f99193675be19c38c", "workspace/aider_settings.yml": "34d42aeca49ec38ee9e07ea615a92560d2430d5c460d3dd433745b1b99ac1a4a", "terminal_evidence/daa05cea8f694c9da27dc77d097fb45a.http.json": "7b7d2ba7da38df349730b844f5a7ee404601546b1255c0860ee52f313dde4f73", "terminal_evidence/budget_ledger.json": "37c475f1ceae66146860635e6a0a025c9f121e34609253857bdd3aca96818bf2", "terminal_evidence/daa05cea8f694c9da27dc77d097fb45a.timing.json": "cf8cafc3db5a6ae36204ceb934abdc2d86f991bc03ee8438e536805c74f7735c", "terminal_evidence/daa05cea8f694c9da27dc77d097fb45a.request.json": "dff63f4ce6de2d1b40003cfcbf795a85e7ef6f34ec0195c7b0649a09ccb5d3f0", "terminal_evidence/daa05cea8f694c9da27dc77d097fb45a.response.txt": "13a8ae44c48e04b2b9aff4d07982d30720f88133f003fb589bd6a9be58deca4d", "source_snapshot/tools/verdict-code/VERSION": "98ddd8c64dd951f851d65ceaa4a8939727b829f146950a55439b56bd45e16208", "source_snapshot/tools/verdict-code/verdict_code/agent.py": "7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e", "source_snapshot/tools/verdict-code/verdict_code/main.py": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "source_snapshot/tools/verdict-code/verdict_code/tools/__init__.py": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "source_snapshot/scripts/benchmarks/agent_comparison_rerun_20261016.py": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "source_snapshot/scripts/benchmarks/agent_comparison_completion_repair_20261016.py": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa" }, "productSource": { "head": "409511b4c7a8ff3f29dff3ac2f029f5f1a469d41", "files": { "scripts/benchmarks/agent_comparison_rerun_20261016.py": "c625b691551051e1f0a3d8dea1fbb3ea6db797578005f86122099572c7acb0b0", "tools/verdict-code/verdict_code/agent.py": "7c0f6171d9cca143ca04f629570aceeb7d3c728e6c5cd3a30434b886c4007e0e", "tools/verdict-code/verdict_code/tools/__init__.py": "609350f15bbdec4099405afe9ddddddf7b2caa7d2d03cc53e0276d6ab8d3bd57", "tools/verdict-code/verdict_code/main.py": "af0a9ac892f38ce9c9fb422bf91facced4e72835ae63e0522b8cf75b1d1605ab", "tools/verdict-code/VERSION": "98ddd8c64dd951f851d65ceaa4a8939727b829f146950a55439b56bd45e16208", "scripts/benchmarks/agent_comparison_completion_repair_20261016.py": "827566af5635ef64554ef1201e9464a9544502d05813b0492020c77eb9ce07fa" }, "retainedSourceBytes": "source_snapshot/" }, "protocol": { "schema": "agent_comparison_rerun_policy.v1", "temperature": 0, "timeout_seconds": 240, "per_request_output_tokens": 8192, "proxy_shared_max_input_tokens": 160000, "proxy_shared_max_output_tokens": 32000, "proxy_reservation": "entire request UTF8 bytes +32/message+128; providerusage settled after eachcall; failclosed ifmissing", "maximum_input_tokens": 160000, "total_token_budget": 400000, "max_turns": 24, "proxy_read_timeout_seconds": 270, "verdict_stream_idle_timeout_seconds": 240, "aider_request_timeout_seconds": 240, "reasoning": { "mode": "default" }, "models": "exact pins, no failover", "task_file": "solution.py", "hidden_oracle_fed_to_agent": false, "producer_location": "outside candidate workspace; candidate read denied", "aider_context": "live exact catalog context metadata; history ceiling=min(inputbudget,context/2)", "appendix_model_changed": true } } }