{
  "schema": "opinion_bcb_evidence_v1",
  "date": "2026-10-05",
  "machine": "Trents-M4-Max-128G.local",
  "method": "M4 Max local LM Studio candidate calls using frozen BCB Complete prompts, temperature 0.2, 2048 output-token cap, reasoning off requested; code graded with BigCodeBenchAdapter. Five exact local variants, one answer per model and unique task. F15 Verdict Decision local replay on six prespecified 2/3/4-model panels, anchored by either Verdict model and adding Qwen, Gemma and GPT-OSS in that order with Analysis off and the product's 20-second Opinion Decision budget. Candidate generation reused across panels; account time is summed request wall, not end-to-end Opinion latency. This is a component replay, not a production or installed-client result.",
  "sourceSha256": {
    "candidates": "c2f4d2fda5be45d82658bb621d4bdc9084975dfd45b30657ce56a82f7aaab1fc",
    "decision": "48261a3c46a98d4eac637a4dd51bbfaf6e25e2db4386cd05084f219b6e4e9053"
  },
  "retriedCandidateFailures": 0,
  "retriedDecisionErrors": 0,
  "cohortManifestSha256": "90dbe2bb4e2cdc8a8070e478c5a3979be100272c509193d2f472464018c2ddc8",
  "sourceDatasetSha256": "eab8cc924ff6c80b53446210a951d13d0b5295aa6fa0f5602d47fbdeecf3de84",
  "models": [
    {
      "id": "verdict-4-35b-toolterse-v5-k-m",
      "name": "Verdict 4 35B Toolterse v5",
      "bytes": 21166757856,
      "format": "GGUF Q4_K_M"
    },
    {
      "id": "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
      "name": "Verdict 4 30B Terse",
      "bytes": 17190792774,
      "format": "MLX 4-bit"
    },
    {
      "id": "qwen3-coder",
      "name": "Qwen3 Coder 30B",
      "bytes": 18556689568,
      "format": "GGUF"
    },
    {
      "id": "google/gemma-4-26b-a4b",
      "name": "Gemma 4 26B A4B",
      "bytes": 17990934737,
      "format": "GGUF Q4_K_M"
    },
    {
      "id": "openai/gpt-oss-20b",
      "name": "GPT-OSS 20B",
      "bytes": 12104297682,
      "format": "Safetensors MXFP4"
    }
  ],
  "modelSummaries": [
    {
      "model": "verdict-4-35b-toolterse-v5-k-m",
      "cohort": "hard148",
      "n": 148,
      "solved": 41,
      "input": 56159,
      "output": 32341,
      "reasoning": 0,
      "reasoningResponses": 0,
      "seconds": 3072.372,
      "truncated": 0
    },
    {
      "model": "verdict-4-35b-toolterse-v5-k-m",
      "cohort": "complete100",
      "n": 100,
      "solved": 50,
      "input": 35182,
      "output": 16124,
      "reasoning": 0,
      "reasoningResponses": 0,
      "seconds": 1538.657,
      "truncated": 0
    },
    {
      "model": "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
      "cohort": "hard148",
      "n": 148,
      "solved": 38,
      "input": 52767,
      "output": 28401,
      "reasoning": 0,
      "reasoningResponses": 0,
      "seconds": 1356.121,
      "truncated": 0
    },
    {
      "model": "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
      "cohort": "complete100",
      "n": 100,
      "solved": 58,
      "input": 33063,
      "output": 13437,
      "reasoning": 0,
      "reasoningResponses": 0,
      "seconds": 729.6,
      "truncated": 0
    },
    {
      "model": "qwen3-coder",
      "cohort": "hard148",
      "n": 148,
      "solved": 50,
      "input": 52767,
      "output": 40215,
      "reasoning": 0,
      "reasoningResponses": 0,
      "seconds": 3369.77,
      "truncated": 0
    },
    {
      "model": "qwen3-coder",
      "cohort": "complete100",
      "n": 100,
      "solved": 59,
      "input": 33063,
      "output": 20677,
      "reasoning": 0,
      "reasoningResponses": 0,
      "seconds": 1632.575,
      "truncated": 0
    },
    {
      "model": "google/gemma-4-26b-a4b",
      "cohort": "hard148",
      "n": 148,
      "solved": 53,
      "input": 58745,
      "output": 72696,
      "reasoning": 0,
      "reasoningResponses": 0,
      "seconds": 6386.699,
      "truncated": 1
    },
    {
      "model": "google/gemma-4-26b-a4b",
      "cohort": "complete100",
      "n": 100,
      "solved": 55,
      "input": 36532,
      "output": 41451,
      "reasoning": 0,
      "reasoningResponses": 0,
      "seconds": 3432.425,
      "truncated": 1
    },
    {
      "model": "openai/gpt-oss-20b",
      "cohort": "hard148",
      "n": 148,
      "solved": 50,
      "input": 60302,
      "output": 51316,
      "reasoning": 7999,
      "reasoningResponses": 148,
      "seconds": 2632.803,
      "truncated": 0
    },
    {
      "model": "openai/gpt-oss-20b",
      "cohort": "complete100",
      "n": 100,
      "solved": 58,
      "input": 38104,
      "output": 27869,
      "reasoning": 5567,
      "reasoningResponses": 100,
      "seconds": 1622.567,
      "truncated": 0
    }
  ],
  "panels": [
    {
      "cohort": "hard148",
      "models": [
        "verdict-4-35b-toolterse-v5-k-m",
        "qwen3-coder"
      ],
      "n": 148,
      "bestSingleModel": "qwen3-coder",
      "bestSingleSolved": 50,
      "bestSingleWins": 5,
      "bestSingleLosses": 8,
      "bestSinglePairedP": 0.58105469,
      "firstSolved": 41,
      "anySolved": 64,
      "decisionSolved": 47,
      "decisionCompleted": 148,
      "decisionUnavailable": 0,
      "decisionAbstained": 0,
      "decisionWins": 15,
      "decisionLosses": 9,
      "pairedP": 0.30745625,
      "candidateInput": 108926,
      "candidateOutput": 72556,
      "decisionUsageMissing": 0,
      "decisionInput": 205177,
      "decisionOutput": 0,
      "candidateSeconds": 6442.142,
      "decisionSeconds": 1731.399
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict-4-35b-toolterse-v5-k-m",
        "qwen3-coder",
        "google/gemma-4-26b-a4b"
      ],
      "n": 148,
      "bestSingleModel": "google/gemma-4-26b-a4b",
      "bestSingleSolved": 53,
      "bestSingleWins": 9,
      "bestSingleLosses": 10,
      "bestSinglePairedP": 1.0,
      "firstSolved": 41,
      "anySolved": 74,
      "decisionSolved": 52,
      "decisionCompleted": 144,
      "decisionUnavailable": 4,
      "decisionAbstained": 0,
      "decisionWins": 23,
      "decisionLosses": 12,
      "pairedP": 0.08953108,
      "candidateInput": 167671,
      "candidateOutput": 145252,
      "decisionUsageMissing": 4,
      "decisionInput": 289720,
      "decisionOutput": 0,
      "candidateSeconds": 12828.841,
      "decisionSeconds": 2296.422
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict-4-35b-toolterse-v5-k-m",
        "qwen3-coder",
        "google/gemma-4-26b-a4b",
        "openai/gpt-oss-20b"
      ],
      "n": 148,
      "bestSingleModel": "google/gemma-4-26b-a4b",
      "bestSingleSolved": 53,
      "bestSingleWins": 6,
      "bestSingleLosses": 29,
      "bestSinglePairedP": 0.00011684,
      "firstSolved": 41,
      "anySolved": 81,
      "decisionSolved": 30,
      "decisionCompleted": 100,
      "decisionUnavailable": 48,
      "decisionAbstained": 0,
      "decisionWins": 8,
      "decisionLosses": 19,
      "pairedP": 0.05223899,
      "candidateInput": 227973,
      "candidateOutput": 196568,
      "decisionUsageMissing": 48,
      "decisionInput": 237293,
      "decisionOutput": 0,
      "candidateSeconds": 15461.644,
      "decisionSeconds": 2789.055
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
        "qwen3-coder"
      ],
      "n": 148,
      "bestSingleModel": "qwen3-coder",
      "bestSingleSolved": 50,
      "bestSingleWins": 4,
      "bestSingleLosses": 8,
      "bestSinglePairedP": 0.38769531,
      "firstSolved": 38,
      "anySolved": 59,
      "decisionSolved": 46,
      "decisionCompleted": 147,
      "decisionUnavailable": 1,
      "decisionAbstained": 0,
      "decisionWins": 13,
      "decisionLosses": 5,
      "pairedP": 0.09625244,
      "candidateInput": 105534,
      "candidateOutput": 68616,
      "decisionUsageMissing": 1,
      "decisionInput": 201686,
      "decisionOutput": 0,
      "candidateSeconds": 4725.891,
      "decisionSeconds": 1687.251
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
        "qwen3-coder",
        "google/gemma-4-26b-a4b"
      ],
      "n": 148,
      "bestSingleModel": "google/gemma-4-26b-a4b",
      "bestSingleSolved": 53,
      "bestSingleWins": 9,
      "bestSingleLosses": 15,
      "bestSinglePairedP": 0.30745625,
      "firstSolved": 38,
      "anySolved": 75,
      "decisionSolved": 47,
      "decisionCompleted": 143,
      "decisionUnavailable": 5,
      "decisionAbstained": 0,
      "decisionWins": 16,
      "decisionLosses": 7,
      "pairedP": 0.09313965,
      "candidateInput": 164279,
      "candidateOutput": 141312,
      "decisionUsageMissing": 5,
      "decisionInput": 285443,
      "decisionOutput": 0,
      "candidateSeconds": 11112.59,
      "decisionSeconds": 2278.33
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
        "qwen3-coder",
        "google/gemma-4-26b-a4b",
        "openai/gpt-oss-20b"
      ],
      "n": 148,
      "bestSingleModel": "google/gemma-4-26b-a4b",
      "bestSingleSolved": 53,
      "bestSingleWins": 7,
      "bestSingleLosses": 26,
      "bestSinglePairedP": 0.00131873,
      "firstSolved": 38,
      "anySolved": 79,
      "decisionSolved": 34,
      "decisionCompleted": 100,
      "decisionUnavailable": 48,
      "decisionAbstained": 0,
      "decisionWins": 13,
      "decisionLosses": 17,
      "pairedP": 0.58466471,
      "candidateInput": 224581,
      "candidateOutput": 192628,
      "decisionUsageMissing": 48,
      "decisionInput": 235703,
      "decisionOutput": 0,
      "candidateSeconds": 13745.393,
      "decisionSeconds": 2818.995
    },
    {
      "cohort": "complete100",
      "models": [
        "verdict-4-35b-toolterse-v5-k-m",
        "qwen3-coder"
      ],
      "n": 100,
      "bestSingleModel": "qwen3-coder",
      "bestSingleSolved": 59,
      "bestSingleWins": 1,
      "bestSingleLosses": 11,
      "bestSinglePairedP": 0.00634766,
      "firstSolved": 50,
      "anySolved": 65,
      "decisionSolved": 49,
      "decisionCompleted": 99,
      "decisionUnavailable": 1,
      "decisionAbstained": 0,
      "decisionWins": 5,
      "decisionLosses": 6,
      "pairedP": 1.0,
      "candidateInput": 68245,
      "candidateOutput": 36801,
      "decisionUsageMissing": 1,
      "decisionInput": 120342,
      "decisionOutput": 0,
      "candidateSeconds": 3171.232,
      "decisionSeconds": 1173.385
    },
    {
      "cohort": "complete100",
      "models": [
        "verdict-4-35b-toolterse-v5-k-m",
        "qwen3-coder",
        "google/gemma-4-26b-a4b"
      ],
      "n": 100,
      "bestSingleModel": "qwen3-coder",
      "bestSingleSolved": 59,
      "bestSingleWins": 4,
      "bestSingleLosses": 7,
      "bestSinglePairedP": 0.54882812,
      "firstSolved": 50,
      "anySolved": 67,
      "decisionSolved": 56,
      "decisionCompleted": 97,
      "decisionUnavailable": 3,
      "decisionAbstained": 0,
      "decisionWins": 11,
      "decisionLosses": 5,
      "pairedP": 0.21011353,
      "candidateInput": 104777,
      "candidateOutput": 78252,
      "decisionUsageMissing": 3,
      "decisionInput": 173062,
      "decisionOutput": 0,
      "candidateSeconds": 6603.657,
      "decisionSeconds": 1541.393
    },
    {
      "cohort": "complete100",
      "models": [
        "verdict-4-35b-toolterse-v5-k-m",
        "qwen3-coder",
        "google/gemma-4-26b-a4b",
        "openai/gpt-oss-20b"
      ],
      "n": 100,
      "bestSingleModel": "qwen3-coder",
      "bestSingleSolved": 59,
      "bestSingleWins": 1,
      "bestSingleLosses": 26,
      "bestSinglePairedP": 4.2e-07,
      "firstSolved": 50,
      "anySolved": 70,
      "decisionSolved": 34,
      "decisionCompleted": 59,
      "decisionUnavailable": 41,
      "decisionAbstained": 0,
      "decisionWins": 9,
      "decisionLosses": 25,
      "pairedP": 0.00904119,
      "candidateInput": 142881,
      "candidateOutput": 106121,
      "decisionUsageMissing": 41,
      "decisionInput": 122734,
      "decisionOutput": 0,
      "candidateSeconds": 8226.224,
      "decisionSeconds": 1874.935
    },
    {
      "cohort": "complete100",
      "models": [
        "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
        "qwen3-coder"
      ],
      "n": 100,
      "bestSingleModel": "qwen3-coder",
      "bestSingleSolved": 59,
      "bestSingleWins": 4,
      "bestSingleLosses": 6,
      "bestSinglePairedP": 0.75390625,
      "firstSolved": 58,
      "anySolved": 68,
      "decisionSolved": 57,
      "decisionCompleted": 100,
      "decisionUnavailable": 0,
      "decisionAbstained": 0,
      "decisionWins": 4,
      "decisionLosses": 5,
      "pairedP": 1.0,
      "candidateInput": 66126,
      "candidateOutput": 34114,
      "decisionUsageMissing": 0,
      "decisionInput": 120001,
      "decisionOutput": 0,
      "candidateSeconds": 2362.175,
      "decisionSeconds": 1138.292
    },
    {
      "cohort": "complete100",
      "models": [
        "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
        "qwen3-coder",
        "google/gemma-4-26b-a4b"
      ],
      "n": 100,
      "bestSingleModel": "qwen3-coder",
      "bestSingleSolved": 59,
      "bestSingleWins": 5,
      "bestSingleLosses": 9,
      "bestSinglePairedP": 0.4239502,
      "firstSolved": 58,
      "anySolved": 70,
      "decisionSolved": 55,
      "decisionCompleted": 94,
      "decisionUnavailable": 6,
      "decisionAbstained": 0,
      "decisionWins": 8,
      "decisionLosses": 11,
      "pairedP": 0.6476059,
      "candidateInput": 102658,
      "candidateOutput": 75565,
      "decisionUsageMissing": 6,
      "decisionInput": 164934,
      "decisionOutput": 0,
      "candidateSeconds": 5794.6,
      "decisionSeconds": 1527.088
    },
    {
      "cohort": "complete100",
      "models": [
        "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
        "qwen3-coder",
        "google/gemma-4-26b-a4b",
        "openai/gpt-oss-20b"
      ],
      "n": 100,
      "bestSingleModel": "qwen3-coder",
      "bestSingleSolved": 59,
      "bestSingleWins": 4,
      "bestSingleLosses": 27,
      "bestSinglePairedP": 3.395e-05,
      "firstSolved": 58,
      "anySolved": 72,
      "decisionSolved": 36,
      "decisionCompleted": 62,
      "decisionUnavailable": 38,
      "decisionAbstained": 0,
      "decisionWins": 4,
      "decisionLosses": 26,
      "pairedP": 5.948e-05,
      "candidateInput": 140762,
      "candidateOutput": 103434,
      "decisionUsageMissing": 38,
      "decisionInput": 130812,
      "decisionOutput": 0,
      "candidateSeconds": 7417.167,
      "decisionSeconds": 1872.903
    }
  ],
  "exampleCandidates": [
    {
      "cohort": "hard148",
      "models": [
        "verdict-4-35b-toolterse-v5-k-m",
        "qwen3-coder"
      ],
      "taskId": "BigCodeBench/1053",
      "selected": "qwen3-coder",
      "first": "verdict-4-35b-toolterse-v5-k-m",
      "kind": "decision_recovery",
      "detail": "The task plots word frequencies from CSV text. The Verdict answer lets vectorization errors escape on empty data; Qwen catches the error and returns None, matching the required error behavior and the benchmark test."
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict-4-35b-toolterse-v5-k-m",
        "qwen3-coder",
        "google/gemma-4-26b-a4b"
      ],
      "taskId": "BigCodeBench/1004",
      "selected": "qwen3-coder",
      "first": "verdict-4-35b-toolterse-v5-k-m",
      "kind": "decision_recovery",
      "detail": "The task counts words and plots a chart. The Verdict answer unpacks an empty list on empty input and raises; Qwen guards that case and returns an empty Counter and chart. Decision selected Qwen, which passed the tests."
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict-4-35b-toolterse-v5-k-m",
        "qwen3-coder",
        "google/gemma-4-26b-a4b"
      ],
      "taskId": "BigCodeBench/1053",
      "selected": "qwen3-coder",
      "first": "verdict-4-35b-toolterse-v5-k-m",
      "kind": "decision_recovery",
      "detail": "The task plots word frequencies from CSV text. The Verdict answer lets vectorization errors escape on empty data; Qwen catches the error and returns None, matching the required error behavior and the benchmark test."
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
        "qwen3-coder"
      ],
      "taskId": "BigCodeBench/100",
      "selected": "qwen3-coder",
      "first": "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
      "kind": "decision_loss",
      "detail": "Decision selected Qwen, whose date construction can produce an invalid day. The Verdict answer passed the tests but repeated the same date instead of covering 30 days. The test misses that requirement: neither selection nor a test pass establishes complete correctness."
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
        "qwen3-coder"
      ],
      "taskId": "BigCodeBench/1004",
      "selected": "qwen3-coder",
      "first": "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
      "kind": "decision_recovery",
      "detail": "The task counts words and plots a chart. The Verdict answer unpacks an empty list on empty input and raises; Qwen guards that case and returns an empty Counter and chart. Decision selected Qwen, which passed the tests."
    },
    {
      "cohort": "hard148",
      "models": [
        "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
        "qwen3-coder"
      ],
      "taskId": "BigCodeBench/1053",
      "selected": "qwen3-coder",
      "first": "verdict/verdict-4-30b-terse-mlx-4bit-ctx32k",
      "kind": "decision_recovery",
      "detail": "The task plots word frequencies from CSV text. The Verdict answer lets vectorization errors escape on empty data; Qwen catches the error and returns None, matching the required error behavior and the benchmark test."
    }
  ],
  "historicalDirect": [
    {
      "name": "Verdict 4 4B Quill",
      "runId": "0ef77808-d6ce-4b3a-8185-fd42e482c554",
      "solved": 45,
      "n": 100,
      "tokens": 56652,
      "sourceSha256": "62456be3447ef35e89fd3fca494925283972f1cc14930c7c1e9dbdc3abb0a201"
    },
    {
      "name": "Qwen3 4B Instruct 2507",
      "runId": "47881a5d-e44b-4293-b09f-d2bc63ac71e8",
      "solved": 51,
      "n": 100,
      "tokens": 78498,
      "sourceSha256": "f69c944ada35b8770dcd41080e4e916f86748bd9ad4c685afa44ea0ae11daebc"
    },
    {
      "name": "Verdict 4 30B Terse",
      "runId": "b69d897d-2a49-4b82-b205-f6982ed4d1f9",
      "solved": 61,
      "n": 100,
      "tokens": 49717,
      "sourceSha256": "76eb48d207d9f2022252a6457db48bc558c5dbac097941d183eb9302f398ce09"
    },
    {
      "name": "Qwen3 Coder 30B",
      "runId": "e30e4ba7-2b80-4a26-ab36-0ef90d8aa2aa",
      "solved": 62,
      "n": 100,
      "tokens": 70726,
      "sourceSha256": "1f64a82e10c23836255234c7a5e92548d3caae252ceccf7ad0ea7b9247464171"
    },
    {
      "name": "Verdict 4 35B Toolterse v5",
      "runId": "bigcodebench_20260909_025139_ddd352",
      "solved": 55,
      "n": 100,
      "tokens": 54239,
      "sourceSha256": "22cff12aacc4b8f9426c9d99b686a21e2e0611f0e524f6727038f274df064bf9"
    }
  ],
  "cohorts": {
    "hard148": 148,
    "complete100": 100
  },
  "panelDesign": "focused_six",
  "expectedUniqueDecisionEvaluations": 1422
}