{
  "schema": "s2s-bench-v1",
  "exhibit": "outside-judges",
  "published_utc": "2026-08-13",
  "status": "PUBLISHED — live at research.strata2signal.com.",
  "licence": "CC BY 4.0",
  "attribution": "strata→signal research, research.strata2signal.com",
  "hardware": "One 96G VRAM workstation, which also answered live requests for three of our apps throughout these trials — two public, one serving playtests for humans and agents alike. Contention is measured in both directions and published with the figures rather than assumed away.",
  "dataset": "outside-judges-agreement-matrix",
  "table": "inter-vendor agreement",
  "what_this_is": "Five vendors' judges scoring identical blinded comparisons, with the disagreements kept. For each round: every arm's rate under every vendor, the spread across the five, and all ten vendor pairs with their mean and maximum per-arm difference and their rank correlation.",
  "how_to_read": "A pair's mean_abs_diff is how far apart two vendors sit on a typical arm; its max_abs_diff is the worst single arm; its rank_spearman is whether they ORDER the arms the same way. The three answer different questions and none substitutes for another: two vendors can order the table identically and still disagree by a tenth on one arm.",
  "counting_rules": {
    "spread": "max rate minus min rate across the five vendors, on one arm, in one round — computed before rounding, so a spread cell can differ from subtracting the printed rates by 0.001.",
    "mean_abs_diff": "mean over arms of |rate(vendor A) − rate(vendor B)|.",
    "rank_spearman": "Spearman rank correlation over the six arms. Six arms is a very small basis for a rank statistic and it is a companion to the differences, never a substitute.",
    "divergence_leads": "divergence between vendors is a finding; it leads, it is not averaged away",
    "legB_panel_units": "the legB-panel figures are 0–10 dimension means, not rates, so its spreads and differences are in scale points and are NOT comparable with the rate rounds' figures."
  },
  "limits": [
    "Six arms and nine effective items. Every statistic here inherits that, and a rank correlation over six points moves a long way on one swapped pair.",
    "Each cloud judge read one original seat's shuffle, so a cross-vendor difference carries that seat's presentation with it.",
    "Agreement that is TOO good would be a fact about the replies being easy to separate, not about the panel being right. It is reported, not celebrated."
  ],
  "page_figures": {
    "mean_abs_diff": "0.051 over the six arms",
    "widest": "0.178, on muse-glimmer:30b-q8_0-dflash — mistral reads it at 0.450 against nvidia at 0.628"
  },
  "highlights": {
    "c4_pairwise": {
      "widest_arm_spread": {
        "arm": "muse-glimmer:30b-q8_0-dflash",
        "spread": 0.17777777777777776,
        "vendors": {
          "anthropic": 0.5888888888888889,
          "deepseek": 0.6111111111111112,
          "mistral": 0.45,
          "moonshot": 0.5722222222222222,
          "nvidia": 0.6277777777777778
        },
        "note": "the largest gap between any two vendors on any arm, and it is a real disagreement about one arm rather than about the table"
      },
      "weakest_rank_agreement": {
        "vendors": [
          "mistral",
          "nvidia"
        ],
        "rank_spearman": 0.6
      },
      "strongest_rank_agreement": {
        "vendors": [
          "anthropic",
          "deepseek"
        ],
        "rank_spearman": 0.9428571428571428
      },
      "anthropic_rank_agreement": {
        "with": {
          "anthropic/deepseek": 0.9428571428571428,
          "anthropic/mistral": 0.8285714285714286,
          "anthropic/moonshot": 0.9428571428571428,
          "anthropic/nvidia": 0.8285714285714286
        },
        "min": 0.8285714285714286,
        "max": 0.9428571428571428,
        "note": "what a reader discounting a single-vendor panel is entitled to price: our own judges' rank agreement with each other vendor, beside every other cross-vendor pair's"
      },
      "top_arm_by_vendor": {
        "anthropic": "qwen3.6:27b",
        "deepseek": "qwen3.6:27b",
        "mistral": "qwen3.6:27b",
        "moonshot": "qwen3.6:27b",
        "nvidia": "qwen3.6:27b"
      },
      "bottom_arm_by_vendor": {
        "anthropic": "muse-glimmer:30b-q8_0-dflash+think",
        "deepseek": "muse-glimmer:30b-q8_0-dflash+think",
        "mistral": "muse-glimmer:30b-q8_0-dflash+think",
        "moonshot": "muse-glimmer:30b-q8_0-dflash+think",
        "nvidia": "muse-glimmer:30b-q8_0-dflash+think"
      }
    }
  },
  "matrix": {
    "c4_pairwise": {
      "vendors": [
        "anthropic",
        "deepseek",
        "mistral",
        "moonshot",
        "nvidia"
      ],
      "per_arm_rate_by_vendor": {
        "gemma4:12b": {
          "anthropic": 0.6847222222222222,
          "deepseek": 0.6055555555555555,
          "mistral": 0.6555555555555556,
          "moonshot": 0.6333333333333333,
          "nvidia": 0.5444444444444444
        },
        "gemma4:26b": {
          "anthropic": 0.5388888888888889,
          "deepseek": 0.5166666666666667,
          "mistral": 0.55,
          "moonshot": 0.5777777777777777,
          "nvidia": 0.55
        },
        "muse-glimmer:30b-q8_0-dflash": {
          "anthropic": 0.5888888888888889,
          "deepseek": 0.6111111111111112,
          "mistral": 0.45,
          "moonshot": 0.5722222222222222,
          "nvidia": 0.6277777777777778
        },
        "muse-glimmer:30b-q8_0-dflash+think": {
          "anthropic": 0.11666666666666667,
          "deepseek": 0.08888888888888889,
          "mistral": 0.1,
          "moonshot": 0.11666666666666667,
          "nvidia": 0.10555555555555556
        },
        "nemotron-3.5-lightning:30b-a3b": {
          "anthropic": 0.3597222222222222,
          "deepseek": 0.4,
          "mistral": 0.49444444444444446,
          "moonshot": 0.4166666666666667,
          "nvidia": 0.4777777777777778
        },
        "qwen3.6:27b": {
          "anthropic": 0.7111111111111111,
          "deepseek": 0.7777777777777778,
          "mistral": 0.75,
          "moonshot": 0.6833333333333333,
          "nvidia": 0.6944444444444444
        }
      },
      "per_arm_spread": {
        "gemma4:12b": 0.14027777777777783,
        "gemma4:26b": 0.061111111111111005,
        "muse-glimmer:30b-q8_0-dflash": 0.17777777777777776,
        "muse-glimmer:30b-q8_0-dflash+think": 0.027777777777777776,
        "nemotron-3.5-lightning:30b-a3b": 0.13472222222222224,
        "qwen3.6:27b": 0.09444444444444444
      },
      "vendor_pairs": [
        {
          "vendors": [
            "anthropic",
            "deepseek"
          ],
          "arms": 6,
          "mean_abs_diff": 0.04305555555555556,
          "max_abs_diff": 0.07916666666666672,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.07916666666666672,
            "gemma4:26b": 0.022222222222222143,
            "muse-glimmer:30b-q8_0-dflash": 0.022222222222222254,
            "muse-glimmer:30b-q8_0-dflash+think": 0.027777777777777776,
            "nemotron-3.5-lightning:30b-a3b": 0.0402777777777778,
            "qwen3.6:27b": 0.06666666666666665
          },
          "rank_spearman": 0.9428571428571428
        },
        {
          "vendors": [
            "anthropic",
            "mistral"
          ],
          "arms": 6,
          "mean_abs_diff": 0.06157407407407409,
          "max_abs_diff": 0.1388888888888889,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.029166666666666674,
            "gemma4:26b": 0.011111111111111183,
            "muse-glimmer:30b-q8_0-dflash": 0.1388888888888889,
            "muse-glimmer:30b-q8_0-dflash+think": 0.016666666666666663,
            "nemotron-3.5-lightning:30b-a3b": 0.13472222222222224,
            "qwen3.6:27b": 0.03888888888888886
          },
          "rank_spearman": 0.8285714285714286
        },
        {
          "vendors": [
            "anthropic",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.03194444444444446,
          "max_abs_diff": 0.056944444444444464,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.05138888888888893,
            "gemma4:26b": 0.03888888888888886,
            "muse-glimmer:30b-q8_0-dflash": 0.01666666666666672,
            "muse-glimmer:30b-q8_0-dflash+think": 0.0,
            "nemotron-3.5-lightning:30b-a3b": 0.056944444444444464,
            "qwen3.6:27b": 0.02777777777777779
          },
          "rank_spearman": 0.9428571428571428
        },
        {
          "vendors": [
            "anthropic",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.05601851851851855,
          "max_abs_diff": 0.14027777777777783,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.14027777777777783,
            "gemma4:26b": 0.011111111111111183,
            "muse-glimmer:30b-q8_0-dflash": 0.03888888888888886,
            "muse-glimmer:30b-q8_0-dflash+think": 0.011111111111111113,
            "nemotron-3.5-lightning:30b-a3b": 0.11805555555555558,
            "qwen3.6:27b": 0.01666666666666672
          },
          "rank_spearman": 0.8285714285714286
        },
        {
          "vendors": [
            "deepseek",
            "mistral"
          ],
          "arms": 6,
          "mean_abs_diff": 0.06296296296296298,
          "max_abs_diff": 0.16111111111111115,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.050000000000000044,
            "gemma4:26b": 0.033333333333333326,
            "muse-glimmer:30b-q8_0-dflash": 0.16111111111111115,
            "muse-glimmer:30b-q8_0-dflash+think": 0.011111111111111113,
            "nemotron-3.5-lightning:30b-a3b": 0.09444444444444444,
            "qwen3.6:27b": 0.02777777777777779
          },
          "rank_spearman": 0.6571428571428571
        },
        {
          "vendors": [
            "deepseek",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.044444444444444446,
          "max_abs_diff": 0.09444444444444444,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.02777777777777779,
            "gemma4:26b": 0.061111111111111005,
            "muse-glimmer:30b-q8_0-dflash": 0.03888888888888897,
            "muse-glimmer:30b-q8_0-dflash+think": 0.027777777777777776,
            "nemotron-3.5-lightning:30b-a3b": 0.016666666666666663,
            "qwen3.6:27b": 0.09444444444444444
          },
          "rank_spearman": 0.8285714285714286
        },
        {
          "vendors": [
            "deepseek",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.04814814814814814,
          "max_abs_diff": 0.08333333333333337,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.061111111111111116,
            "gemma4:26b": 0.033333333333333326,
            "muse-glimmer:30b-q8_0-dflash": 0.016666666666666607,
            "muse-glimmer:30b-q8_0-dflash+think": 0.016666666666666663,
            "nemotron-3.5-lightning:30b-a3b": 0.07777777777777778,
            "qwen3.6:27b": 0.08333333333333337
          },
          "rank_spearman": 0.9428571428571428
        },
        {
          "vendors": [
            "mistral",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.05555555555555553,
          "max_abs_diff": 0.12222222222222218,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.022222222222222254,
            "gemma4:26b": 0.02777777777777768,
            "muse-glimmer:30b-q8_0-dflash": 0.12222222222222218,
            "muse-glimmer:30b-q8_0-dflash+think": 0.016666666666666663,
            "nemotron-3.5-lightning:30b-a3b": 0.07777777777777778,
            "qwen3.6:27b": 0.06666666666666665
          },
          "rank_spearman": 0.9428571428571428
        },
        {
          "vendors": [
            "mistral",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.061111111111111116,
          "max_abs_diff": 0.17777777777777776,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.11111111111111116,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.17777777777777776,
            "muse-glimmer:30b-q8_0-dflash+think": 0.00555555555555555,
            "nemotron-3.5-lightning:30b-a3b": 0.016666666666666663,
            "qwen3.6:27b": 0.05555555555555558
          },
          "rank_spearman": 0.6
        },
        {
          "vendors": [
            "moonshot",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.04259259259259258,
          "max_abs_diff": 0.0888888888888889,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.0888888888888889,
            "gemma4:26b": 0.02777777777777768,
            "muse-glimmer:30b-q8_0-dflash": 0.05555555555555558,
            "muse-glimmer:30b-q8_0-dflash+think": 0.011111111111111113,
            "nemotron-3.5-lightning:30b-a3b": 0.061111111111111116,
            "qwen3.6:27b": 0.011111111111111072
          },
          "rank_spearman": 0.7714285714285715
        }
      ],
      "mean_abs_diff_overall": 0.05074074074074074,
      "note": "divergence between vendors is a finding; it leads, it is not averaged away"
    },
    "c4_distinctness": {
      "vendors": [
        "anthropic",
        "deepseek",
        "mistral",
        "moonshot",
        "nvidia"
      ],
      "per_arm_rate_by_vendor": {
        "gemma4:12b": {
          "anthropic": 1.0,
          "deepseek": 1.0,
          "mistral": 1.0,
          "moonshot": 0.8888888888888888,
          "nvidia": 0.7777777777777778
        },
        "gemma4:26b": {
          "anthropic": 1.0,
          "deepseek": 1.0,
          "mistral": 0.8888888888888888,
          "moonshot": 1.0,
          "nvidia": 1.0
        },
        "muse-glimmer:30b-q8_0-dflash": {
          "anthropic": 1.0,
          "deepseek": 0.8888888888888888,
          "mistral": 1.0,
          "moonshot": 1.0,
          "nvidia": 1.0
        },
        "muse-glimmer:30b-q8_0-dflash+think": {
          "anthropic": 0.625,
          "deepseek": 0.5555555555555556,
          "mistral": 0.3888888888888889,
          "moonshot": 0.5555555555555556,
          "nvidia": 0.3333333333333333
        },
        "nemotron-3.5-lightning:30b-a3b": {
          "anthropic": 1.0,
          "deepseek": 1.0,
          "mistral": 0.6111111111111112,
          "moonshot": 1.0,
          "nvidia": 0.8888888888888888
        },
        "qwen3.6:27b": {
          "anthropic": 1.0,
          "deepseek": 1.0,
          "mistral": 0.7777777777777778,
          "moonshot": 1.0,
          "nvidia": 1.0
        }
      },
      "per_arm_spread": {
        "gemma4:12b": 0.2222222222222222,
        "gemma4:26b": 0.11111111111111116,
        "muse-glimmer:30b-q8_0-dflash": 0.11111111111111116,
        "muse-glimmer:30b-q8_0-dflash+think": 0.2916666666666667,
        "nemotron-3.5-lightning:30b-a3b": 0.38888888888888884,
        "qwen3.6:27b": 0.2222222222222222
      },
      "vendor_pairs": [
        {
          "vendors": [
            "anthropic",
            "deepseek"
          ],
          "arms": 6,
          "mean_abs_diff": 0.030092592592592598,
          "max_abs_diff": 0.11111111111111116,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.0,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.11111111111111116,
            "muse-glimmer:30b-q8_0-dflash+think": 0.06944444444444442,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.7745966692414834
        },
        {
          "vendors": [
            "anthropic",
            "mistral"
          ],
          "arms": 6,
          "mean_abs_diff": 0.1597222222222222,
          "max_abs_diff": 0.38888888888888884,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.0,
            "gemma4:26b": 0.11111111111111116,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.2361111111111111,
            "nemotron-3.5-lightning:30b-a3b": 0.38888888888888884,
            "qwen3.6:27b": 0.2222222222222222
          },
          "rank_spearman": 0.6642111641550715
        },
        {
          "vendors": [
            "anthropic",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.030092592592592598,
          "max_abs_diff": 0.11111111111111116,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.11111111111111116,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.06944444444444442,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.7745966692414834
        },
        {
          "vendors": [
            "anthropic",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.10416666666666667,
          "max_abs_diff": 0.2916666666666667,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.2222222222222222,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.2916666666666667,
            "nemotron-3.5-lightning:30b-a3b": 0.11111111111111116,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.6956083436402525
        },
        {
          "vendors": [
            "deepseek",
            "mistral"
          ],
          "arms": 6,
          "mean_abs_diff": 0.16666666666666666,
          "max_abs_diff": 0.38888888888888884,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.0,
            "gemma4:26b": 0.11111111111111116,
            "muse-glimmer:30b-q8_0-dflash": 0.11111111111111116,
            "muse-glimmer:30b-q8_0-dflash+think": 0.16666666666666669,
            "nemotron-3.5-lightning:30b-a3b": 0.38888888888888884,
            "qwen3.6:27b": 0.2222222222222222
          },
          "rank_spearman": 0.2572478777137633
        },
        {
          "vendors": [
            "deepseek",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.037037037037037056,
          "max_abs_diff": 0.11111111111111116,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.11111111111111116,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.11111111111111116,
            "muse-glimmer:30b-q8_0-dflash+think": 0.0,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.5
        },
        {
          "vendors": [
            "deepseek",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.11111111111111112,
          "max_abs_diff": 0.22222222222222227,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.2222222222222222,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.11111111111111116,
            "muse-glimmer:30b-q8_0-dflash+think": 0.22222222222222227,
            "nemotron-3.5-lightning:30b-a3b": 0.11111111111111116,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.3592106040535498
        },
        {
          "vendors": [
            "mistral",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.16666666666666666,
          "max_abs_diff": 0.38888888888888884,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.11111111111111116,
            "gemma4:26b": 0.11111111111111116,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.16666666666666669,
            "nemotron-3.5-lightning:30b-a3b": 0.38888888888888884,
            "qwen3.6:27b": 0.2222222222222222
          },
          "rank_spearman": 0.2572478777137633
        },
        {
          "vendors": [
            "mistral",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.14814814814814814,
          "max_abs_diff": 0.2777777777777777,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.2222222222222222,
            "gemma4:26b": 0.11111111111111116,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.05555555555555558,
            "nemotron-3.5-lightning:30b-a3b": 0.2777777777777777,
            "qwen3.6:27b": 0.2222222222222222
          },
          "rank_spearman": 0.4312287725435882
        },
        {
          "vendors": [
            "moonshot",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.07407407407407408,
          "max_abs_diff": 0.22222222222222227,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.11111111111111105,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.22222222222222227,
            "nemotron-3.5-lightning:30b-a3b": 0.11111111111111116,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.8980265101338746
        }
      ],
      "mean_abs_diff_overall": 0.10277777777777777,
      "note": "divergence between vendors is a finding; it leads, it is not averaged away"
    },
    "c4_canon": {
      "vendors": [
        "anthropic",
        "deepseek",
        "mistral",
        "moonshot",
        "nvidia"
      ],
      "per_arm_rate_by_vendor": {
        "gemma4:12b": {
          "anthropic": 0.2916666666666667,
          "deepseek": 0.3333333333333333,
          "mistral": 0.3333333333333333,
          "moonshot": 0.3333333333333333,
          "nvidia": 0.0
        },
        "gemma4:26b": {
          "anthropic": 0.2916666666666667,
          "deepseek": 0.0,
          "mistral": 0.0,
          "moonshot": 0.16666666666666666,
          "nvidia": 0.0
        },
        "muse-glimmer:30b-q8_0-dflash": {
          "anthropic": 0.2916666666666667,
          "deepseek": 0.5,
          "mistral": 0.5,
          "moonshot": 0.5,
          "nvidia": 0.5
        },
        "muse-glimmer:30b-q8_0-dflash+think": {
          "anthropic": 0.0,
          "deepseek": 0.0,
          "mistral": 0.3333333333333333,
          "moonshot": 0.0,
          "nvidia": 0.0
        },
        "nemotron-3.5-lightning:30b-a3b": {
          "anthropic": 0.5,
          "deepseek": 0.5,
          "mistral": 0.5,
          "moonshot": 0.5,
          "nvidia": 0.5
        },
        "qwen3.6:27b": {
          "anthropic": 0.20833333333333334,
          "deepseek": 0.0,
          "mistral": 0.0,
          "moonshot": 0.16666666666666666,
          "nvidia": 0.0
        }
      },
      "per_arm_spread": {
        "gemma4:12b": 0.3333333333333333,
        "gemma4:26b": 0.2916666666666667,
        "muse-glimmer:30b-q8_0-dflash": 0.20833333333333331,
        "muse-glimmer:30b-q8_0-dflash+think": 0.3333333333333333,
        "nemotron-3.5-lightning:30b-a3b": 0.0,
        "qwen3.6:27b": 0.20833333333333334
      },
      "vendor_pairs": [
        {
          "vendors": [
            "anthropic",
            "deepseek"
          ],
          "arms": 6,
          "mean_abs_diff": 0.125,
          "max_abs_diff": 0.2916666666666667,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.04166666666666663,
            "gemma4:26b": 0.2916666666666667,
            "muse-glimmer:30b-q8_0-dflash": 0.20833333333333331,
            "muse-glimmer:30b-q8_0-dflash+think": 0.0,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.20833333333333334
          },
          "rank_spearman": 0.7541997111515459
        },
        {
          "vendors": [
            "anthropic",
            "mistral"
          ],
          "arms": 6,
          "mean_abs_diff": 0.18055555555555555,
          "max_abs_diff": 0.3333333333333333,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.04166666666666663,
            "gemma4:26b": 0.2916666666666667,
            "muse-glimmer:30b-q8_0-dflash": 0.20833333333333331,
            "muse-glimmer:30b-q8_0-dflash+think": 0.3333333333333333,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.20833333333333334
          },
          "rank_spearman": 0.508000508000762
        },
        {
          "vendors": [
            "anthropic",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.06944444444444443,
          "max_abs_diff": 0.20833333333333331,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.04166666666666663,
            "gemma4:26b": 0.12500000000000003,
            "muse-glimmer:30b-q8_0-dflash": 0.20833333333333331,
            "muse-glimmer:30b-q8_0-dflash+think": 0.0,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.041666666666666685
          },
          "rank_spearman": 0.8441622892989752
        },
        {
          "vendors": [
            "anthropic",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.16666666666666666,
          "max_abs_diff": 0.2916666666666667,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.2916666666666667,
            "gemma4:26b": 0.2916666666666667,
            "muse-glimmer:30b-q8_0-dflash": 0.20833333333333331,
            "muse-glimmer:30b-q8_0-dflash+think": 0.0,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.20833333333333334
          },
          "rank_spearman": 0.6599120175960899
        },
        {
          "vendors": [
            "deepseek",
            "mistral"
          ],
          "arms": 6,
          "mean_abs_diff": 0.05555555555555555,
          "max_abs_diff": 0.3333333333333333,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.0,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.3333333333333333,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.9036961141150639
        },
        {
          "vendors": [
            "deepseek",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.05555555555555555,
          "max_abs_diff": 0.16666666666666666,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.0,
            "gemma4:26b": 0.16666666666666666,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.0,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.16666666666666666
          },
          "rank_spearman": 0.9534625892455922
        },
        {
          "vendors": [
            "deepseek",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.05555555555555555,
          "max_abs_diff": 0.3333333333333333,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.3333333333333333,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.0,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.8944271909999159
        },
        {
          "vendors": [
            "mistral",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.1111111111111111,
          "max_abs_diff": 0.3333333333333333,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.0,
            "gemma4:26b": 0.16666666666666666,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.3333333333333333,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.16666666666666666
          },
          "rank_spearman": 0.7385489458759964
        },
        {
          "vendors": [
            "mistral",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.1111111111111111,
          "max_abs_diff": 0.3333333333333333,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.3333333333333333,
            "gemma4:26b": 0.0,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.3333333333333333,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.8660254037844387
        },
        {
          "vendors": [
            "moonshot",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.1111111111111111,
          "max_abs_diff": 0.3333333333333333,
          "per_arm_abs_diff": {
            "gemma4:12b": 0.3333333333333333,
            "gemma4:26b": 0.16666666666666666,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.0,
            "nemotron-3.5-lightning:30b-a3b": 0.0,
            "qwen3.6:27b": 0.16666666666666666
          },
          "rank_spearman": 0.8528028654224418
        }
      ],
      "mean_abs_diff_overall": 0.10416666666666666,
      "note": "divergence between vendors is a finding; it leads, it is not averaged away"
    },
    "legB_panel": {
      "vendors": [
        "anthropic",
        "deepseek",
        "mistral",
        "moonshot",
        "nvidia"
      ],
      "per_arm_rate_by_vendor": {
        "gemma4:12b-it-q8_0": {
          "anthropic": 6.166666666666667,
          "deepseek": 7.0,
          "mistral": 7.666666666666667,
          "moonshot": 7.333333333333333,
          "nvidia": 5.666666666666667
        },
        "gemma4:26b": {
          "anthropic": 4.444444444444445,
          "deepseek": 4.333333333333333,
          "mistral": 6.333333333333333,
          "moonshot": 5.0,
          "nvidia": 4.666666666666667
        },
        "muse-glimmer:30b-q8_0-dflash": {
          "anthropic": 4.222222222222222,
          "deepseek": 5.0,
          "mistral": 7.666666666666667,
          "moonshot": 5.0,
          "nvidia": 4.666666666666667
        },
        "muse-glimmer:30b-q8_0-dflash+think": {
          "anthropic": 0.6666666666666666,
          "deepseek": 0.3333333333333333,
          "mistral": 1.3333333333333333,
          "moonshot": 1.0,
          "nvidia": 0.3333333333333333
        },
        "nemotron-3.5-lightning:30b-a3b": {
          "anthropic": 5.944444444444445,
          "deepseek": 8.333333333333334,
          "mistral": 9.333333333333334,
          "moonshot": 5.333333333333333,
          "nvidia": 8.0
        },
        "qwen3.6:27b": {
          "anthropic": 8.722222222222221,
          "deepseek": 8.0,
          "mistral": 9.0,
          "moonshot": 8.0,
          "nvidia": 7.333333333333333
        }
      },
      "per_arm_spread": {
        "gemma4:12b-it-q8_0": 2.0,
        "gemma4:26b": 2.0,
        "muse-glimmer:30b-q8_0-dflash": 3.4444444444444446,
        "muse-glimmer:30b-q8_0-dflash+think": 1.0,
        "nemotron-3.5-lightning:30b-a3b": 4.000000000000001,
        "qwen3.6:27b": 1.666666666666667
      },
      "vendor_pairs": [
        {
          "vendors": [
            "anthropic",
            "deepseek"
          ],
          "arms": 6,
          "mean_abs_diff": 0.861111111111111,
          "max_abs_diff": 2.3888888888888893,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 0.833333333333333,
            "gemma4:26b": 0.1111111111111116,
            "muse-glimmer:30b-q8_0-dflash": 0.7777777777777777,
            "muse-glimmer:30b-q8_0-dflash+think": 0.3333333333333333,
            "nemotron-3.5-lightning:30b-a3b": 2.3888888888888893,
            "qwen3.6:27b": 0.7222222222222214
          },
          "rank_spearman": 0.7714285714285715
        },
        {
          "vendors": [
            "anthropic",
            "mistral"
          ],
          "arms": 6,
          "mean_abs_diff": 1.8611111111111114,
          "max_abs_diff": 3.4444444444444446,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 1.5,
            "gemma4:26b": 1.8888888888888884,
            "muse-glimmer:30b-q8_0-dflash": 3.4444444444444446,
            "muse-glimmer:30b-q8_0-dflash+think": 0.6666666666666666,
            "nemotron-3.5-lightning:30b-a3b": 3.3888888888888893,
            "qwen3.6:27b": 0.27777777777777857
          },
          "rank_spearman": 0.6957252427829381
        },
        {
          "vendors": [
            "anthropic",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.6944444444444442,
          "max_abs_diff": 1.166666666666666,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 1.166666666666666,
            "gemma4:26b": 0.5555555555555554,
            "muse-glimmer:30b-q8_0-dflash": 0.7777777777777777,
            "muse-glimmer:30b-q8_0-dflash+think": 0.33333333333333337,
            "nemotron-3.5-lightning:30b-a3b": 0.6111111111111116,
            "qwen3.6:27b": 0.7222222222222214
          },
          "rank_spearman": 0.9856107606091623
        },
        {
          "vendors": [
            "anthropic",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.824074074074074,
          "max_abs_diff": 2.0555555555555554,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 0.5,
            "gemma4:26b": 0.22222222222222232,
            "muse-glimmer:30b-q8_0-dflash": 0.44444444444444464,
            "muse-glimmer:30b-q8_0-dflash+think": 0.3333333333333333,
            "nemotron-3.5-lightning:30b-a3b": 2.0555555555555554,
            "qwen3.6:27b": 1.3888888888888884
          },
          "rank_spearman": 0.8116794499134278
        },
        {
          "vendors": [
            "deepseek",
            "mistral"
          ],
          "arms": 6,
          "mean_abs_diff": 1.388888888888889,
          "max_abs_diff": 2.666666666666667,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 0.666666666666667,
            "gemma4:26b": 2.0,
            "muse-glimmer:30b-q8_0-dflash": 2.666666666666667,
            "muse-glimmer:30b-q8_0-dflash+think": 1.0,
            "nemotron-3.5-lightning:30b-a3b": 1.0,
            "qwen3.6:27b": 1.0
          },
          "rank_spearman": 0.9856107606091623
        },
        {
          "vendors": [
            "deepseek",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 0.777777777777778,
          "max_abs_diff": 3.000000000000001,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 0.33333333333333304,
            "gemma4:26b": 0.666666666666667,
            "muse-glimmer:30b-q8_0-dflash": 0.0,
            "muse-glimmer:30b-q8_0-dflash+think": 0.6666666666666667,
            "nemotron-3.5-lightning:30b-a3b": 3.000000000000001,
            "qwen3.6:27b": 0.0
          },
          "rank_spearman": 0.8116794499134278
        },
        {
          "vendors": [
            "deepseek",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 0.5000000000000001,
          "max_abs_diff": 1.333333333333333,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 1.333333333333333,
            "gemma4:26b": 0.3333333333333339,
            "muse-glimmer:30b-q8_0-dflash": 0.33333333333333304,
            "muse-glimmer:30b-q8_0-dflash+think": 0.0,
            "nemotron-3.5-lightning:30b-a3b": 0.3333333333333339,
            "qwen3.6:27b": 0.666666666666667
          },
          "rank_spearman": 0.9856107606091623
        },
        {
          "vendors": [
            "mistral",
            "moonshot"
          ],
          "arms": 6,
          "mean_abs_diff": 1.6111111111111114,
          "max_abs_diff": 4.000000000000001,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 0.3333333333333339,
            "gemma4:26b": 1.333333333333333,
            "muse-glimmer:30b-q8_0-dflash": 2.666666666666667,
            "muse-glimmer:30b-q8_0-dflash+think": 0.33333333333333326,
            "nemotron-3.5-lightning:30b-a3b": 4.000000000000001,
            "qwen3.6:27b": 1.0
          },
          "rank_spearman": 0.75
        },
        {
          "vendors": [
            "mistral",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 1.777777777777778,
          "max_abs_diff": 3.0,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 2.0,
            "gemma4:26b": 1.666666666666666,
            "muse-glimmer:30b-q8_0-dflash": 3.0,
            "muse-glimmer:30b-q8_0-dflash+think": 1.0,
            "nemotron-3.5-lightning:30b-a3b": 1.333333333333334,
            "qwen3.6:27b": 1.666666666666667
          },
          "rank_spearman": 0.9558823529411765
        },
        {
          "vendors": [
            "moonshot",
            "nvidia"
          ],
          "arms": 6,
          "mean_abs_diff": 1.0555555555555556,
          "max_abs_diff": 2.666666666666667,
          "per_arm_abs_diff": {
            "gemma4:12b-it-q8_0": 1.666666666666666,
            "gemma4:26b": 0.33333333333333304,
            "muse-glimmer:30b-q8_0-dflash": 0.33333333333333304,
            "muse-glimmer:30b-q8_0-dflash+think": 0.6666666666666667,
            "nemotron-3.5-lightning:30b-a3b": 2.666666666666667,
            "qwen3.6:27b": 0.666666666666667
          },
          "rank_spearman": 0.8235294117647058
        }
      ],
      "mean_abs_diff_overall": 1.1351851851851853,
      "note": "divergence between vendors is a finding; it leads, it is not averaged away"
    }
  },
  "provenance": [
    {
      "role": "the registered amendment",
      "artifact": "PREREG-PANEL-EXT.md",
      "sha256": "ca422ac6928bd01acfb6d501f0cd7aaea3d2d3b5e4cbfc386ed462cdd924721c"
    },
    {
      "role": "the agreement matrix as scored",
      "artifact": "panel-ext-scores.json",
      "sha256": "6a50dbd84371465cc97207af799a3f459da0884efbeecf2e46a5550c087874ba"
    }
  ],
  "data_boundary": "What left the box for judging was model outputs over public-domain text and published game canon — never user data, never a character's private canon, never telemetry. The cloud judges saw the sealed batches published in this kit's terms and nothing else: no letter map, no model name, no arm roster, no other judge's answer, no part of a key file.",
  "judge_dating": "Cloud judges are versionless hosted services. Every cloud verdict in this kit was produced 2026-08-12/13 and is dated rather than version-pinned; it may not reproduce against a later checkpoint of the same tag. The local instruments and scorers these judges audit are sha-frozen, and the asymmetry is stated rather than hidden.",
  "contamination_caveat": "Published 2026-08-13. Models with a later training cutoff may have seen these sets. We author fresh sets each cycle; this one is not a standard, it is our kit, yours to reuse.",
  "not_a_standard": "Our own trials, our own judges, our own instruments. Not first independent numbers, and not a benchmark.",
  "vendor_reads": {
    "note": "the matrix re-stated one vendor per row: their top pick, their single furthest departure from the other four vendors' mean on any arm (signed), and their rank agreement with the house column. Derived, not re-judged.",
    "rows": [
      {
        "vendor": "anthropic",
        "judge": "the four-seat house panel",
        "pairwise_judgments": 1080,
        "top_arm": "qwen3.6:27b",
        "top_rate": 0.7111111111111111,
        "top_rate_printed": "0.711",
        "furthest_from_field_arm": "nemotron-3.5-lightning:30b-a3b",
        "furthest_from_field_delta": -0.08750000000000002,
        "furthest_from_field_delta_printed": "0.088",
        "furthest_from_field_direction": "cooler",
        "rank_spearman_vs_house": null
      },
      {
        "vendor": "deepseek",
        "judge": "deepseek-v4-pro",
        "pairwise_judgments": 270,
        "top_arm": "qwen3.6:27b",
        "top_rate": 0.7777777777777778,
        "top_rate_printed": "0.778",
        "furthest_from_field_arm": "qwen3.6:27b",
        "furthest_from_field_delta": 0.06805555555555554,
        "furthest_from_field_delta_printed": "0.068",
        "furthest_from_field_direction": "warmer",
        "rank_spearman_vs_house": 0.9428571428571428
      },
      {
        "vendor": "mistral",
        "judge": "mistral-large-3:675b",
        "pairwise_judgments": 270,
        "top_arm": "qwen3.6:27b",
        "top_rate": 0.75,
        "top_rate_printed": "0.750",
        "furthest_from_field_arm": "muse-glimmer:30b-q8_0-dflash",
        "furthest_from_field_delta": -0.14999999999999997,
        "furthest_from_field_delta_printed": "0.150",
        "furthest_from_field_direction": "cooler",
        "rank_spearman_vs_house": 0.8285714285714286
      },
      {
        "vendor": "moonshot",
        "judge": "kimi-k3",
        "pairwise_judgments": 270,
        "top_arm": "qwen3.6:27b",
        "top_rate": 0.6833333333333333,
        "top_rate_printed": "0.683",
        "furthest_from_field_arm": "qwen3.6:27b",
        "furthest_from_field_delta": -0.050000000000000044,
        "furthest_from_field_delta_printed": "0.050",
        "furthest_from_field_direction": "cooler",
        "rank_spearman_vs_house": 0.9428571428571428
      },
      {
        "vendor": "nvidia",
        "judge": "nemotron-3-ultra",
        "pairwise_judgments": 270,
        "top_arm": "qwen3.6:27b",
        "top_rate": 0.6944444444444444,
        "top_rate_printed": "0.694",
        "furthest_from_field_arm": "gemma4:12b",
        "furthest_from_field_delta": -0.10034722222222225,
        "furthest_from_field_delta_printed": "0.100",
        "furthest_from_field_direction": "cooler",
        "rank_spearman_vs_house": 0.8285714285714286
      }
    ]
  }
}