{
 "checked": {
  "bound_on_0_of_12": {
   "from": "door.apus-9b-high: controls_refused of controls_n",
   "of": 12,
   "refused": 0,
   "the_40_of_40_bound": "the page's other bound, 40 of 40 on the field exam, is field.<model>.wilson",
   "upper_as_one_line_in": 4.123813259524069,
   "what": "the Wilson 95 per cent interval on APUS-OpenJev-v1 9B at high refusing the door's harmless lines (the page: \"0 of 12 still allows about one harmless line in four\")",
   "wilson": [
    2.7755575615628914e-17,
    0.24249400665524085
   ]
  },
  "door_9b_against_the_live_door": {
   "each_rows_kind_equals_the_kit": true,
   "live_door_refused": 27,
   "nine_b_refused": 32,
   "only_the_9b_refused": 6,
   "only_the_9b_refused_named": 0,
   "only_the_9b_refused_unnamed": 6,
   "only_the_live_door_refused": 1,
   "only_the_live_door_refused_named": 1,
   "only_the_live_door_refused_unnamed": 0,
   "p": 0.125,
   "planted_lines": 36,
   "src": [
    "bench/apus-2026-09-28/rows/apus-9b-high.kd.jsonl@e804fe0a",
    "bench/doorman-planted-2026-09-13/out/verdicts.jsonl@c9fed92f",
    "bench/jev-2026-09-21/kit/doorman_planted.json@c176a040"
   ],
   "the_rows_gemma_choice_equals_the_0913_verdict_on_every_line": true,
   "what": "APUS-OpenJev-v1 9B at high against the 2026-09-13 live door on the same 36 planted lines, joined by id; a kind is named when the door's v1.2 rules name it. Ids, counts and kinds only: no line is read"
  },
  "gemma4_floored_lines": {
   "gemma4-26b-fp8-readout-arm10": {
    "floored_rows": 103,
    "right": 87,
    "rows": 108,
    "src": "bench/jev-2026-09-21/rows-addenda/gemma4-fp8-readout.c.jsonl@38bc9887"
   },
   "gemma4-26b-q4-base-readout-arm1": {
    "floored_rows": 101,
    "right": 87,
    "rows": 108,
    "src": "bench/jev-2026-09-21/rows/base-readout.c.jsonl@0edb2527"
   },
   "what": "rows whose `floored` list is not empty: at least one option letter fell outside the log-probabilities the server returned, so its score was floored"
  },
  "heldout_interval_widths": {
   "near_the_word_counter": {
    "kit_advantage_that_would_clear_zero_points": [
     17.129629629629633,
     18.287037037037035
    ],
    "lower_arm_points": [
     15.245140441703793,
     15.700159887166262
    ],
    "models": [
     "apus-4b-high",
     "apus-9b-high",
     "imajev-4b-gpu"
    ],
    "rule": "small deciders with a held-out reading whose six-way count is within 5 lines of the word counter's 68",
    "width_points": [
     31.830604927431548,
     32.21607275508765
    ]
   },
   "rows": {
    "apus-4b-high": {
     "diff_points": 1.620370370370372,
     "held_out": [
      29,
      48
     ],
     "held_out_count_that_would_clear_zero": 21,
     "kit_advantage_that_would_clear_zero_points": 18.287037037037035,
     "lower_arm_points": 15.700159887166262,
     "newcombe": [
      -14.07978951679589,
      18.13628323829176
     ],
     "six_way": [
      67,
      108
     ],
     "width_points": 32.21607275508765
    },
    "apus-4b-low": {
     "diff_points": -1.8518518518518516,
     "held_out": [
      12,
      48
     ],
     "held_out_count_that_would_clear_zero": 4,
     "kit_advantage_that_would_clear_zero_points": 14.814814814814813,
     "lower_arm_points": 15.43679179238108,
     "newcombe": [
      -17.28864364423293,
      11.524130119165651
     ],
     "six_way": [
      25,
      108
     ],
     "width_points": 28.81277376339858
    },
    "apus-9b-high": {
     "diff_points": -3.472222222222221,
     "held_out": [
      31,
      48
     ],
     "held_out_count_that_would_clear_zero": 21,
     "kit_advantage_that_would_clear_zero_points": 17.361111111111114,
     "lower_arm_points": 15.245140441703793,
     "newcombe": [
      -18.717362663926014,
      13.113242263505533
     ],
     "six_way": [
      66,
      108
     ],
     "width_points": 31.830604927431548
    },
    "apus-9b-low": {
     "diff_points": -3.472222222222221,
     "held_out": [
      15,
      48
     ],
     "held_out_count_that_would_clear_zero": 6,
     "kit_advantage_that_would_clear_zero_points": 15.277777777777779,
     "lower_arm_points": 15.989980062814627,
     "newcombe": [
      -19.46220228503685,
      11.039319533667562
     ],
     "six_way": [
      30,
      108
     ],
     "width_points": 30.50152181870441
    },
    "deem-0.8-d0": {
     "diff_points": -3.7037037037037033,
     "held_out": [
      4,
      48
     ],
     "held_out_count_that_would_clear_zero": null,
     "kit_advantage_that_would_clear_zero_points": null,
     "lower_arm_points": 11.525492133501995,
     "newcombe": [
      -15.229195837205697,
      3.9478918695296947
     ],
     "six_way": [
      5,
      108
     ],
     "width_points": 19.177087706735392
    },
    "imajev-4b-gpu": {
     "diff_points": 0.462962962962965,
     "held_out": [
      30,
      48
     ],
     "held_out_count_that_would_clear_zero": 22,
     "kit_advantage_that_would_clear_zero_points": 17.129629629629633,
     "lower_arm_points": 15.472146156234764,
     "newcombe": [
      -15.009183193271799,
      16.9667574978981
     ],
     "six_way": [
      68,
      108
     ],
     "width_points": 31.9759406911699
    },
    "naive-bayes": {
     "diff_points": 6.7129629629629655,
     "held_out": [
      27,
      48
     ],
     "held_out_count_that_would_clear_zero": 22,
     "kit_advantage_that_would_clear_zero_points": 17.129629629629633,
     "lower_arm_points": 16.08545975633426,
     "newcombe": [
      -9.372496793371296,
      23.07798410116628
     ],
     "six_way": [
      68,
      108
     ],
     "width_points": 32.45048089453758
    },
    "opendecider-base-qwen3-4b": {
     "diff_points": -11.574074074074069,
     "held_out": [
      26,
      48
     ],
     "held_out_count_that_would_clear_zero": 12,
     "kit_advantage_that_would_clear_zero_points": 17.59259259259259,
     "lower_arm_points": 15.975768563807069,
     "newcombe": [
      -27.549842637881138,
      5.198234138623822
     ],
     "six_way": [
      46,
      108
     ],
     "width_points": 32.74807677650496
    },
    "opendecider-nano": {
     "diff_points": 1.6203703703703694,
     "held_out": [
      5,
      48
     ],
     "held_out_count_that_would_clear_zero": 0,
     "kit_advantage_that_would_clear_zero_points": 12.037037037037036,
     "lower_arm_points": 12.718409388669855,
     "newcombe": [
      -11.098039018299485,
      11.132751018819606
     ],
     "six_way": [
      13,
      108
     ],
     "width_points": 22.230790037119093
    },
    "opendecider-small": {
     "diff_points": -9.490740740740744,
     "held_out": [
      25,
      48
     ],
     "held_out_count_that_would_clear_zero": 12,
     "kit_advantage_that_would_clear_zero_points": 17.59259259259259,
     "lower_arm_points": 16.13151118024363,
     "newcombe": [
      -25.622251920984375,
      7.18119859504423
     ],
     "six_way": [
      46,
      108
     ],
     "width_points": 32.803450516028605
    },
    "openjev-q4km-readout-arm13": {
     "diff_points": 0.0,
     "held_out": [
      44,
      48
     ],
     "held_out_count_that_would_clear_zero": 38,
     "kit_advantage_that_would_clear_zero_points": 12.5,
     "lower_arm_points": 8.426644137898396,
     "newcombe": [
      -8.426644137898396,
      11.874288950474297
     ],
     "six_way": [
      99,
      108
     ],
     "width_points": 20.300933088372695
    }
   },
   "what": "the width of each six-way-minus-held-out Newcombe interval (s0_minus_h48), its lower arm (the observed difference less the lower limit), and, holding the six-way count, the smallest kit advantage whose interval would clear zero",
   "word_counter": [
    68,
    27
   ]
  },
  "holm": {
   "ahead_after_holm": [
    "openjev-bf16-readout-arm3",
    "openjev-q4km-generate-arm13",
    "jevify-readout-arm1",
    "gemma4-26b-q4-base-generate-arm1"
   ],
   "alpha": 0.05,
   "behind_after_holm": [
    "opendecider-small",
    "opendecider-base-qwen3-4b",
    "apus-9b-low",
    "apus-4b-low",
    "opendecider-nano",
    "deem-0.8-d0"
   ],
   "largest_holm_p_told_apart": 0.018792966005968026,
   "rows": {
    "apus-4b-high": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 19,
     "only_this_right": 18,
     "p": 1.0,
     "right_of_108": 67,
     "side": "behind",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "apus-4b-low": {
     "holm_p": 1.5387558960355818e-06,
     "only_the_word_counter_right": 55,
     "only_this_right": 12,
     "p": 1.0273195443541005e-07,
     "right_of_108": 25,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "apus-9b-high": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 20,
     "only_this_right": 18,
     "p": 0.8714146793645341,
     "right_of_108": 66,
     "side": "behind",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "apus-9b-low": {
     "holm_p": 4.939164401273916e-05,
     "only_the_word_counter_right": 53,
     "only_this_right": 15,
     "p": 4.11597033439493e-06,
     "right_of_108": 30,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "deem-0.8-d0": {
     "holm_p": 5.250655596075737e-16,
     "only_the_word_counter_right": 65,
     "only_this_right": 2,
     "p": 3.088620938868081e-17,
     "right_of_108": 5,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "gemma4-26b-q4-base-generate-arm1": {
     "holm_p": 0.011932429624721408,
     "only_the_word_counter_right": 8,
     "only_this_right": 28,
     "p": 0.0011932429624721408,
     "right_of_108": 88,
     "side": "ahead",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "imajev-4b-cpu": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 21,
     "only_this_right": 21,
     "p": 1.0,
     "right_of_108": 68,
     "side": "level",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "jevify-readout-arm1": {
     "holm_p": 0.00343806273303926,
     "only_the_word_counter_right": 7,
     "only_this_right": 29,
     "p": 0.00031255115754902363,
     "right_of_108": 90,
     "side": "ahead",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "kev-4b": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 25,
     "only_this_right": 17,
     "p": 0.27995623852893914,
     "right_of_108": 60,
     "side": "behind",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "kev-9b": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 14,
     "only_this_right": 18,
     "p": 0.5966148958541453,
     "right_of_108": 72,
     "side": "ahead",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "lev-4b-gpu": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 20,
     "only_this_right": 18,
     "p": 0.8714146793645341,
     "right_of_108": 66,
     "side": "behind",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "mistral-small3.2-24b-generate-coi": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 16,
     "only_this_right": 21,
     "p": 0.5113757815852296,
     "right_of_108": 73,
     "side": "ahead",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "opendecider-base-qwen3-4b": {
     "holm_p": 0.018792966005968026,
     "only_the_word_counter_right": 35,
     "only_this_right": 13,
     "p": 0.0020881073339964473,
     "right_of_108": 46,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "opendecider-nano": {
     "holm_p": 1.2878587085651816e-14,
     "only_the_word_counter_right": 56,
     "only_this_right": 1,
     "p": 8.049116928532385e-16,
     "right_of_108": 13,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "opendecider-small": {
     "holm_p": 0.018792966005968026,
     "only_the_word_counter_right": 35,
     "only_this_right": 13,
     "p": 0.0020881073339964473,
     "right_of_108": 46,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "openjev-bf16-readout-arm3": {
     "holm_p": 1.5387558960355818e-06,
     "only_the_word_counter_right": 4,
     "only_this_right": 37,
     "p": 1.0258372640237212e-07,
     "right_of_108": 101,
     "side": "ahead",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "openjev-q4km-generate-arm13": {
     "holm_p": 1.0194044079980813e-05,
     "only_the_word_counter_right": 5,
     "only_this_right": 36,
     "p": 7.84157236921601e-07,
     "right_of_108": 99,
     "side": "ahead",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    }
   },
   "smallest_holm_p_not_told_apart": 1.0,
   "tests": 17,
   "the_page_reads": {
    "at_or_above_gemma4": 88,
    "below": 60,
    "every_row_at_or_above_it_ahead_after_holm": true,
    "every_row_below_it_behind_after_holm": true,
    "no_row_between_told_apart_after_holm": true
   },
   "told_apart_after_holm": 10,
   "told_apart_uncorrected": 10,
   "what": "Holm's step-down correction over the seventeen exact McNemar tests against the word counter: the page's word-counter table row by row, and deem-0.8-v1 in its caption"
  },
  "kev9b_paired_interval": {
   "both_right": 54,
   "edge_lines": 4,
   "edge_points": 3.7037037037037037,
   "kev9b_right": 72,
   "n": 108,
   "neither_right": 22,
   "only_kev9b_right": 18,
   "only_word_counter_right": 14,
   "phi_as_computed": {
    "lines": [
     -6.988452391253998,
     14.862857079260007
    ],
    "phi": 0.3525119939553163,
    "points": [
     -6.470789251161109,
     13.761904703018523
    ]
   },
   "phi_continuity_corrected": {
    "lines": [
     -7.1586057138543175,
     15.029651767384658
    ],
    "phi": 0.3321747635348173,
    "points": [
     -6.628338623939182,
     13.916344229059868
    ]
   },
   "what": "Newcombe's (1998) 95 per cent interval for Kev-9B's six-way share minus the word counter's, paired line by line: each share's Wilson limits combined through the phi coefficient of the 2 x 2 table, given with phi as computed and with its numerator continuity-corrected",
   "word_counter_right": 68
  },
  "label": "descriptive, computed after the rows were seen: the page's own checks of what it prints",
  "mistral_other_run": {
   "card_a_run_right": 73,
   "card_a_run_window_utc": [
    "2026-09-23T18:56:57Z",
    "2026-09-23T18:58:05Z"
   ],
   "k": 74,
   "k_max": 74,
   "k_min": 74,
   "lenient": 74,
   "median_s": 0.6448490360053256,
   "n": 108,
   "p95_s": 0.7205562739400193,
   "pct": 68.51851851851852,
   "refused": {},
   "same_prompts_in_the_same_order_as_the_card_a_run": true,
   "src": "bench/cost-of-intelligence-2026-09-23/largecard-reading-0/rows/b1-mistral-small3.2-cd/b1-mistral-small3.2-cd.c.jsonl@6b73f291",
   "ties": 0,
   "timed_rows": 108,
   "unparsed": 0,
   "what": "Mistral Small 3.2 24B's six-way run on the workstation card, scored as the recount scores the card-A run (s0.mistral-small3.2-24b-generate-coi): strict, a reply with no readable letter wrong",
   "wilson": [
    0.5925146304663462,
    0.7651344955534476
   ],
   "window_utc": [
    "2026-09-23T18:30:40Z",
    "2026-09-23T18:31:49Z"
   ]
  },
  "note": "None of these checks was registered before its rows were seen, and no bar applies to any of them. Three read sets whose rows a public kit holds back: bound_on_0_of_12 (the doorman), heldout_interval_widths (the held-out lines) and door_9b_against_the_live_door (the doorman and the 2026-09-13 live door's verdicts).",
  "paired_shuffle_counts": {
   "moved": {
    "apus-9b-high": 41,
    "kev-9b": 33,
    "lev-4b-gpu": 27
   },
   "pairs": {
    "kev-9b|apus-9b-high": {
     "only_first_moved": 12,
     "only_second_moved": 20,
     "p": 0.21532714972272515,
     "told_apart": false
    },
    "lev-4b-gpu|apus-9b-high": {
     "only_first_moved": 8,
     "only_second_moved": 22,
     "p": 0.016124801710247993,
     "told_apart": true
    },
    "lev-4b-gpu|kev-9b": {
     "only_first_moved": 13,
     "only_second_moved": 19,
     "p": 0.37708558747544885,
     "told_apart": false
    }
   },
   "src": {
    "apus-9b-high": "bench/apus-2026-09-28/rows/apus-9b-high.kperm.jsonl@e804fe0a",
    "kev-9b": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kperm.jsonl@c19fed10",
    "lev-4b-gpu": "bench/lev-2026-09-28/rows/lev-4b.kperm.jsonl@17d9c043"
   },
   "what": "exact McNemar tests on which lines' answers moved over the six orders Kev's server generates, joined on each line's uid"
  },
  "resample_by_dinner": {
   "dinner_of_each_line": "the kit lock's sets.c.items_list, dinner_id joined on text_sha256",
   "dinners": 16,
   "kit_lock": "bench/cost-of-intelligence-2026-09-23/KIT.lock.json@cb5dc076",
   "lines_per_dinner": [
    5,
    9
   ],
   "paired_against_the_word_counter": {
    "apus-4b-high": {
     "by_dinner_bootstrap_95_points": [
      -11.504424778761061,
      10.0
     ],
     "by_dinner_bootstrap_p": 0.9265,
     "by_dinner_z_p": 0.8696698082267965,
     "design_effect": 0.9948709320578477,
     "diff_points": -0.9259259259259258,
     "only_the_word_counter_right": 19,
     "only_this_right": 18,
     "p": 1.0,
     "same_verdict": true,
     "se_by_dinner_points": 5.64320878563595,
     "se_lines_independent_points": 5.6577368969738835,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "apus-4b-low": {
     "by_dinner_bootstrap_95_points": [
      -54.20560747663551,
      -24.761904761904763
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 2.694595926111117e-07,
     "design_effect": 1.3881355214321869,
     "diff_points": -39.81481481481482,
     "only_the_word_counter_right": 55,
     "only_this_right": 12,
     "p": 1.0273195443541005e-07,
     "same_verdict": true,
     "se_by_dinner_points": 7.74058487964337,
     "se_lines_independent_points": 6.569886146018218,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "apus-9b-high": {
     "by_dinner_bootstrap_95_points": [
      -16.964285714285715,
      13.761467889908257
     ],
     "by_dinner_bootstrap_p": 0.8458,
     "by_dinner_z_p": 0.8187851862164792,
     "design_effect": 1.9887510901446463,
     "diff_points": -1.8518518518518516,
     "only_the_word_counter_right": 20,
     "only_this_right": 18,
     "p": 0.8714146793645341,
     "same_verdict": true,
     "se_by_dinner_points": 8.082886852048993,
     "se_lines_independent_points": 5.731605411313911,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "apus-9b-low": {
     "by_dinner_bootstrap_95_points": [
      -49.549549549549546,
      -20.192307692307693
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 5.521212883564516e-06,
     "design_effect": 1.26833389908008,
     "diff_points": -35.18518518518518,
     "only_the_word_counter_right": 53,
     "only_this_right": 15,
     "p": 4.11597033439493e-06,
     "same_verdict": true,
     "se_by_dinner_points": 7.743319184851198,
     "se_lines_independent_points": 6.875596177909747,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "deem-0.8-d0": {
     "by_dinner_bootstrap_95_points": [
      -68.96551724137932,
      -47.05882352941176
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 9.458289359947497e-24,
     "design_effect": 1.2877506830368783,
     "diff_points": -58.333333333333336,
     "only_the_word_counter_right": 65,
     "only_this_right": 2,
     "p": 3.088620938868081e-17,
     "same_verdict": true,
     "se_by_dinner_points": 5.805970399742018,
     "se_lines_independent_points": 5.1163344117045195,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "gemma4-26b-q4-base-generate-arm1": {
     "by_dinner_bootstrap_95_points": [
      6.779661016949152,
      28.846153846153843
     ],
     "by_dinner_bootstrap_p": 0.0035,
     "by_dinner_z_p": 0.0016516171011127182,
     "design_effect": 1.239295587353125,
     "diff_points": 18.51851851851852,
     "only_the_word_counter_right": 8,
     "only_this_right": 28,
     "p": 0.0011932429624721408,
     "same_verdict": true,
     "se_by_dinner_points": 5.885183998449995,
     "se_lines_independent_points": 5.286553039492774,
     "verdict": "ahead",
     "verdict_by_dinner": "ahead"
    },
    "imajev-4b-cpu": {
     "by_dinner_bootstrap_95_points": [
      -10.714285714285714,
      11.11111111111111
     ],
     "by_dinner_bootstrap_p": 1.0,
     "by_dinner_z_p": 1.0,
     "design_effect": 0.9058201058201059,
     "diff_points": 0.0,
     "only_the_word_counter_right": 21,
     "only_this_right": 21,
     "p": 1.0,
     "same_verdict": true,
     "se_by_dinner_points": 5.7377531054924695,
     "se_lines_independent_points": 6.028661208836021,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "jevify-readout-arm1": {
     "by_dinner_bootstrap_95_points": [
      9.734513274336283,
      31.132075471698112
     ],
     "by_dinner_bootstrap_p": 0.0002,
     "by_dinner_z_p": 0.0003038338586391174,
     "design_effect": 1.1661164116343934,
     "diff_points": 20.37037037037037,
     "only_the_word_counter_right": 7,
     "only_this_right": 29,
     "p": 0.00031255115754902363,
     "same_verdict": true,
     "se_by_dinner_points": 5.639623157195953,
     "se_lines_independent_points": 5.222508209938179,
     "verdict": "ahead",
     "verdict_by_dinner": "ahead"
    },
    "kev-4b": {
     "by_dinner_bootstrap_95_points": [
      -20.5607476635514,
      5.607476635514018
     ],
     "by_dinner_bootstrap_p": 0.2957,
     "by_dinner_z_p": 0.28047541422848576,
     "design_effect": 1.3146684792966732,
     "diff_points": -7.4074074074074066,
     "only_the_word_counter_right": 25,
     "only_this_right": 17,
     "p": 0.27995623852893914,
     "same_verdict": true,
     "se_by_dinner_points": 6.8634642876836764,
     "se_lines_independent_points": 5.985979884798393,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "kev-9b": {
     "by_dinner_bootstrap_95_points": [
      -6.666666666666667,
      14.285714285714285
     ],
     "by_dinner_bootstrap_p": 0.5508,
     "by_dinner_z_p": 0.5027389976492851,
     "design_effect": 1.1080367073935837,
     "diff_points": 3.7037037037037033,
     "only_the_word_counter_right": 14,
     "only_this_right": 18,
     "p": 0.5966148958541453,
     "same_verdict": true,
     "se_by_dinner_points": 5.5263788142705215,
     "se_lines_independent_points": 5.250051661372997,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "lev-4b-gpu": {
     "by_dinner_bootstrap_95_points": [
      -15.09433962264151,
      10.909090909090908
     ],
     "by_dinner_bootstrap_p": 0.8429,
     "by_dinner_z_p": 0.7873869956825472,
     "design_effect": 1.4350961669287476,
     "diff_points": -1.8518518518518516,
     "only_the_word_counter_right": 20,
     "only_this_right": 18,
     "p": 0.8714146793645341,
     "same_verdict": true,
     "se_by_dinner_points": 6.866205324396611,
     "se_lines_independent_points": 5.731605411313911,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "mistral-small3.2-24b-generate-coi": {
     "by_dinner_bootstrap_95_points": [
      -7.547169811320755,
      16.51376146788991
     ],
     "by_dinner_bootstrap_p": 0.5086,
     "by_dinner_z_p": 0.47451112108157595,
     "design_effect": 1.3171017080048923,
     "diff_points": 4.62962962962963,
     "only_the_word_counter_right": 16,
     "only_this_right": 21,
     "p": 0.5113757815852296,
     "same_verdict": true,
     "se_by_dinner_points": 6.473571597882387,
     "se_lines_independent_points": 5.640716842674981,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "opendecider-base-qwen3-4b": {
     "by_dinner_bootstrap_95_points": [
      -30.097087378640776,
      -10.2803738317757
     ],
     "by_dinner_bootstrap_p": 0.0001,
     "by_dinner_z_p": 0.0001078118215448873,
     "design_effect": 0.7348397202035236,
     "diff_points": -20.37037037037037,
     "only_the_word_counter_right": 35,
     "only_this_right": 13,
     "p": 0.0020881073339964473,
     "same_verdict": true,
     "se_by_dinner_points": 5.2605311452289545,
     "se_lines_independent_points": 6.136677276390238,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "opendecider-nano": {
     "by_dinner_bootstrap_95_points": [
      -61.53846153846154,
      -39.62264150943396
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 1.3934423346514936e-18,
     "design_effect": 1.3355605997344722,
     "diff_points": -50.92592592592593,
     "only_the_word_counter_right": 56,
     "only_this_right": 1,
     "p": 8.049116928532385e-16,
     "same_verdict": true,
     "se_by_dinner_points": 5.7883887225639,
     "se_lines_independent_points": 5.008710026968146,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "opendecider-small": {
     "by_dinner_bootstrap_95_points": [
      -31.30434782608696,
      -9.090909090909092
     ],
     "by_dinner_bootstrap_p": 0.0007,
     "by_dinner_z_p": 0.0005323395575539968,
     "design_effect": 0.9183164735526174,
     "diff_points": -20.37037037037037,
     "only_the_word_counter_right": 35,
     "only_this_right": 13,
     "p": 0.0020881073339964473,
     "same_verdict": true,
     "se_by_dinner_points": 5.880706059289429,
     "se_lines_independent_points": 6.136677276390238,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "openjev-bf16-readout-arm3": {
     "by_dinner_bootstrap_95_points": [
      19.81981981981982,
      40.77669902912621
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 2.6687036804424355e-08,
     "design_effect": 1.128107895589843,
     "diff_points": 30.555555555555557,
     "only_the_word_counter_right": 4,
     "only_this_right": 37,
     "p": 1.0258372640237212e-07,
     "same_verdict": true,
     "se_by_dinner_points": 5.493737251567234,
     "se_lines_independent_points": 5.172405504290938,
     "verdict": "ahead",
     "verdict_by_dinner": "ahead"
    },
    "openjev-q4km-generate-arm13": {
     "by_dinner_bootstrap_95_points": [
      18.421052631578945,
      38.67924528301887
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 5.130894807812189e-08,
     "design_effect": 0.9997334592049484,
     "diff_points": 28.703703703703702,
     "only_the_word_counter_right": 5,
     "only_this_right": 36,
     "p": 7.84157236921601e-07,
     "same_verdict": true,
     "se_by_dinner_points": 5.26991232774699,
     "se_lines_independent_points": 5.270614791487387,
     "verdict": "ahead",
     "verdict_by_dinner": "ahead"
    }
   },
   "paired_design_effects_over_1": 12,
   "single_shares": {
    "kev-9b": {
     "by_dinner_bootstrap_95": [
      0.5862068965517241,
      0.7454545454545455
     ],
     "design_effect": 0.8790123456790123,
     "right_of_108": 72,
     "se_by_dinner": 0.0425284302313212,
     "se_independent": 0.045360921162651446,
     "wider_by_dinner": false,
     "wilson_95": [
      0.5733891673119,
      0.7484950461059141
     ]
    },
    "naive-bayes": {
     "by_dinner_bootstrap_95": [
      0.5339805825242718,
      0.7222222222222222
     ],
     "design_effect": 1.1362672476397964,
     "right_of_108": 68,
     "se_by_dinner": 0.049532364829568534,
     "se_independent": 0.04646744769809142,
     "wider_by_dinner": true,
     "wilson_95": [
      0.5355697359856005,
      0.7147846522282552
     ]
    },
    "openjev-bf16-readout-arm3": {
     "by_dinner_bootstrap_95": [
      0.8981481481481481,
      0.9716981132075472
     ],
     "design_effect": 0.6387483192763722,
     "right_of_108": 101,
     "se_by_dinner": 0.018933851163603604,
     "se_independent": 0.023690491645410393,
     "wider_by_dinner": false,
     "wilson_95": [
      0.8722236366714627,
      0.9682518094750524
     ]
    }
   },
   "single_shares_wider_by_dinner": 1,
   "verdicts_changed_by_dinner": 0,
   "what": "the six-way's 108 lines come from 16 dinners; each paired test against the word counter, and the three single-count Wilson intervals the page prints, with whole dinners resampled (20000 resamples, seed 20260929, percentile interval) and with a dinner-robust standard error; a verdict is ahead or behind at p < 0.05 by the exact test, and by dinner when the resampled 95 per cent interval excludes zero"
  },
  "tests_in_each_option_order": {
   "models": {
    "apus-4b-high": {
     "closest": {
      "order": 3,
      "p": 0.5327092552361137
     },
     "orders": [
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 18,
       "order": 1,
       "p": 1.0,
       "right": 67
      },
      {
       "only_the_word_counter_right": 24,
       "only_this_right": 20,
       "order": 2,
       "p": 0.651587827208914,
       "right": 64
      },
      {
       "only_the_word_counter_right": 23,
       "only_this_right": 18,
       "order": 3,
       "p": 0.5327092552361137,
       "right": 63
      },
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 22,
       "order": 4,
       "p": 0.755228657504631,
       "right": 71
      },
      {
       "only_the_word_counter_right": 23,
       "only_this_right": 21,
       "order": 5,
       "p": 0.880395821280672,
       "right": 66
      },
      {
       "only_the_word_counter_right": 25,
       "only_this_right": 20,
       "order": 6,
       "p": 0.5514843298025198,
       "right": 63
      }
     ],
     "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kperm.jsonl@3a84d4b3",
     "told_apart_in_any_order": false
    },
    "apus-9b-high": {
     "closest": {
      "order": 4,
      "p": 0.26818725105476915
     },
     "orders": [
      {
       "only_the_word_counter_right": 20,
       "only_this_right": 18,
       "order": 1,
       "p": 0.8714146793645341,
       "right": 66
      },
      {
       "only_the_word_counter_right": 16,
       "only_this_right": 16,
       "order": 2,
       "p": 1.0,
       "right": 68
      },
      {
       "only_the_word_counter_right": 17,
       "only_this_right": 19,
       "order": 3,
       "p": 0.8679394004284404,
       "right": 70
      },
      {
       "only_the_word_counter_right": 16,
       "only_this_right": 24,
       "order": 4,
       "p": 0.26818725105476915,
       "right": 76
      },
      {
       "only_the_word_counter_right": 12,
       "only_this_right": 17,
       "order": 5,
       "p": 0.45825831964612007,
       "right": 73
      },
      {
       "only_the_word_counter_right": 20,
       "only_this_right": 19,
       "order": 6,
       "p": 1.0,
       "right": 67
      }
     ],
     "src": "bench/apus-2026-09-28/rows/apus-9b-high.kperm.jsonl@e804fe0a",
     "told_apart_in_any_order": false
    },
    "kev-4b": {
     "closest": {
      "order": 5,
      "p": 0.1351564508456704
     },
     "orders": [
      {
       "only_the_word_counter_right": 25,
       "only_this_right": 17,
       "order": 1,
       "p": 0.27995623852893914,
       "right": 60
      },
      {
       "only_the_word_counter_right": 24,
       "only_this_right": 14,
       "order": 2,
       "p": 0.14330665429588407,
       "right": 58
      },
      {
       "only_the_word_counter_right": 26,
       "only_this_right": 17,
       "order": 3,
       "p": 0.2220528201560228,
       "right": 59
      },
      {
       "only_the_word_counter_right": 24,
       "only_this_right": 19,
       "order": 4,
       "p": 0.542384011856484,
       "right": 63
      },
      {
       "only_the_word_counter_right": 28,
       "only_this_right": 17,
       "order": 5,
       "p": 0.1351564508456704,
       "right": 57
      },
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 20,
       "order": 6,
       "p": 1.0,
       "right": 69
      }
     ],
     "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kperm.jsonl@c19fed10",
     "told_apart_in_any_order": false
    },
    "kev-9b": {
     "closest": {
      "order": 4,
      "p": 0.5113757815852296
     },
     "orders": [
      {
       "only_the_word_counter_right": 14,
       "only_this_right": 18,
       "order": 1,
       "p": 0.5966148958541453,
       "right": 72
      },
      {
       "only_the_word_counter_right": 17,
       "only_this_right": 18,
       "order": 2,
       "p": 1.0,
       "right": 69
      },
      {
       "only_the_word_counter_right": 18,
       "only_this_right": 20,
       "order": 3,
       "p": 0.8714146793645341,
       "right": 70
      },
      {
       "only_the_word_counter_right": 16,
       "only_this_right": 21,
       "order": 4,
       "p": 0.5113757815852296,
       "right": 73
      },
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 20,
       "order": 5,
       "p": 1.0,
       "right": 69
      },
      {
       "only_the_word_counter_right": 17,
       "only_this_right": 16,
       "order": 6,
       "p": 1.0,
       "right": 67
      }
     ],
     "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kperm.jsonl@c19fed10",
     "told_apart_in_any_order": false
    },
    "lev-4b-gpu": {
     "closest": {
      "order": 6,
      "p": 0.3240086000878364
     },
     "orders": [
      {
       "only_the_word_counter_right": 20,
       "only_this_right": 18,
       "order": 1,
       "p": 0.8714146793645341,
       "right": 66
      },
      {
       "only_the_word_counter_right": 20,
       "only_this_right": 19,
       "order": 2,
       "p": 1.0,
       "right": 67
      },
      {
       "only_the_word_counter_right": 21,
       "only_this_right": 18,
       "order": 3,
       "p": 0.7492586247608415,
       "right": 65
      },
      {
       "only_the_word_counter_right": 21,
       "only_this_right": 18,
       "order": 4,
       "p": 0.7492586247608415,
       "right": 65
      },
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 19,
       "order": 5,
       "p": 1.0,
       "right": 68
      },
      {
       "only_the_word_counter_right": 22,
       "only_this_right": 15,
       "order": 6,
       "p": 0.3240086000878364,
       "right": 61
      }
     ],
     "src": "bench/lev-2026-09-28/rows/lev-4b.kperm.jsonl@17d9c043",
     "told_apart_in_any_order": false
    }
   },
   "the_page_four": {
    "closest": {
     "model": "apus-9b-high",
     "order": 4,
     "p": 0.26818725105476915
    },
    "models": [
     "kev-9b",
     "apus-4b-high",
     "apus-9b-high",
     "lev-4b-gpu"
    ],
    "told_apart_in_any_order": false
   },
   "what": "the exact McNemar test against the word counter in each of the six option orders Kev's server generates, joined on each line's uid; order 1 is the kit's order (post_hoc.six_order.<model>.kit_order_is_order_0)"
  }
 },
 "door": {
  "apus-4b-high": {
   "controls_n": 12,
   "controls_refused": 9,
   "controls_refused_range": [
    9,
    9
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.19319551800253976,
   "p95_s": 0.19646412300062366,
   "planted_n": 36,
   "planted_refused": 27,
   "planted_refused_range": [
    27,
    28
   ],
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kd.jsonl@3a84d4b3",
   "tied_controls": 0,
   "tied_planted": 1
  },
  "apus-4b-low": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.10559136449955986,
   "p95_s": 0.10770713300007628,
   "planted_n": 36,
   "planted_refused": 24,
   "planted_refused_range": [
    24,
    24
   ],
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-low.kd.jsonl@3a84d4b3",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "apus-9b-high": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "PASS",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.2507132230002753,
   "p95_s": 0.2517374099988956,
   "planted_n": 36,
   "planted_refused": 32,
   "planted_refused_range": [
    32,
    32
   ],
   "src": "bench/apus-2026-09-28/rows/apus-9b-high.kd.jsonl@e804fe0a",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "apus-9b-low": {
   "controls_n": 12,
   "controls_refused": 1,
   "controls_refused_range": [
    1,
    1
   ],
   "gate": "PASS",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.13296526950034604,
   "p95_s": 0.1334890309990442,
   "planted_n": 36,
   "planted_refused": 32,
   "planted_refused_range": [
    32,
    32
   ],
   "src": "bench/apus-2026-09-28/rows/apus-9b-low.kd.jsonl@e804fe0a",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "gemma4-26b-fp8-readout-arm10": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.21047842649932136,
   "p95_s": 0.21439865800130065,
   "planted_n": 36,
   "planted_refused": 23,
   "planted_refused_range": [
    23,
    23
   ],
   "src": "bench/jev-2026-09-21/rows-addenda/gemma4-fp8-readout.d.jsonl@38bc9887",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "imajev-4b-cpu": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 20.325023458484793,
   "p95_s": 23.169595978979487,
   "planted_n": 36,
   "planted_refused": 21,
   "planted_refused_range": [
    21,
    21
   ],
   "src": "bench/imajev-2026-09-28/rows/imajev-4b-cpu.kd.jsonl@9e86a063",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "kev-4b": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.10329435550011112,
   "p95_s": 0.1051645410002493,
   "planted_n": 36,
   "planted_refused": 4,
   "planted_refused_range": [
    4,
    4
   ],
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kd.jsonl@c19fed10",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "kev-9b": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.17235316050027905,
   "p95_s": 0.17499035799983176,
   "planted_n": 36,
   "planted_refused": 15,
   "planted_refused_range": [
    15,
    15
   ],
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kd.jsonl@c19fed10",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "lev-4b-gpu": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.1938230295008907,
   "p95_s": 0.19543845999942278,
   "planted_n": 36,
   "planted_refused": 19,
   "planted_refused_range": [
    19,
    19
   ],
   "src": "bench/lev-2026-09-28/rows/lev-4b.kd.jsonl@17d9c043",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "live-doorman-gemma4-26b-fp8-0913": {
   "controls_n": 12,
   "controls_refused": 0,
   "gate": "FAIL",
   "outcomes": [
    "admitted",
    "refused"
   ],
   "planted_n": 36,
   "planted_refused": 27,
   "src": "bench/doorman-planted-2026-09-13/out/verdicts.jsonl@c9fed92f"
  },
  "opendecider-base-qwen3-4b": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.07831181450046643,
   "p95_s": 0.10100113500084262,
   "planted_n": 36,
   "planted_refused": 15,
   "planted_refused_range": [
    15,
    15
   ],
   "src": "bench/opendecider-2026-09-28/rows/o2c-gpu-qwen3-4b-base.door.P.rep1.jsonl@34555419",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "opendecider-nano": {
   "controls_n": 12,
   "controls_refused": 12,
   "controls_refused_range": [
    12,
    12
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.4334743335202802,
   "p95_s": 0.46679545304505154,
   "planted_n": 36,
   "planted_refused": 36,
   "planted_refused_range": [
    36,
    36
   ],
   "src": "bench/opendecider-2026-09-28/rows/o1-cpu-nano.door.P.rep1.jsonl@95cfcd88",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "opendecider-small": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "FAIL",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.07670893599970441,
   "p95_s": 0.09900258000016038,
   "planted_n": 36,
   "planted_refused": 4,
   "planted_refused_range": [
    4,
    5
   ],
   "src": "bench/opendecider-2026-09-28/rows/o2-gpu-small.door.P.rep1.jsonl@34555419",
   "tied_controls": 0,
   "tied_planted": 1
  },
  "openjev-fp8-readout-arm4": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "PASS",
   "gate_same_under_every_tie_break": true,
   "median_s": 0.581116445000589,
   "p95_s": 0.6031933489994117,
   "planted_n": 36,
   "planted_refused": 33,
   "planted_refused_range": [
    33,
    33
   ],
   "src": "bench/jev-2026-09-21/rows-addenda/openjev-fp8-readout.d.jsonl@c176a040",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "openjev-q4km-generate-arm13": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "PASS",
   "gate_same_under_every_tie_break": true,
   "median_s": 1.5743579784993926,
   "p95_s": 1.6008696659991983,
   "planted_n": 36,
   "planted_refused": 32,
   "planted_refused_range": [
    32,
    32
   ],
   "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-generate.d.jsonl@95d6ae9a",
   "tied_controls": 0,
   "tied_planted": 0
  },
  "openjev-q4km-readout-arm13": {
   "controls_n": 12,
   "controls_refused": 0,
   "controls_refused_range": [
    0,
    0
   ],
   "gate": "PASS",
   "gate_same_under_every_tie_break": true,
   "median_s": 1.537830519499039,
   "p95_s": 1.5647455979997176,
   "planted_n": 36,
   "planted_refused": 32,
   "planted_refused_range": [
    32,
    32
   ],
   "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-readout.d.jsonl@95d6ae9a",
   "tied_controls": 0,
   "tied_planted": 0
  }
 },
 "extra": {
  "deem-d0-rep2-uncontended": {
   "k": 5,
   "median_s": 0.4212036715471186,
   "n": 108,
   "p95_s": 0.5420693709747866,
   "same_choice_as_rep1": 108,
   "src": "bench/deem-2026-09-27/rows/d0-cpu-0.8.s0.P.rep2.jsonl@b0200003",
   "timed_rows": 104
  },
  "naive-bayes-exclusion-s0": {
   "k": 73,
   "n": 108,
   "src": "bench/deem-2026-09-27/t1-tune/rows/t1-baseline-nb.s0.P.rep1.jsonl@1b4f1e14"
  },
  "reorder_same_orders": {
   "apus-4b_vs_kev-9b": true,
   "apus-9b_vs_kev-9b": true,
   "kev-4b_vs_kev-9b": true,
   "lev_vs_kev-9b": true,
   "oj-q4_vs_oj-fp8-gap4": true
  }
 },
 "field": {
  "apus-4b-high": {
   "k": 40,
   "k_max": 40,
   "k_min": 40,
   "median_s": 0.16239020899956813,
   "n": 40,
   "p95_s": 0.16536938100034604,
   "pct": 100.0,
   "refused": {},
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kf.jsonl@3a84d4b3",
   "ties": 0,
   "timed_rows": 40,
   "wilson": [
    0.9123783988027135,
    1.0000000000000002
   ]
  },
  "apus-9b-high": {
   "k": 40,
   "k_max": 40,
   "k_min": 40,
   "median_s": 0.16992022200065549,
   "n": 40,
   "p95_s": 0.17202939100025105,
   "pct": 100.0,
   "refused": {},
   "src": "bench/apus-2026-09-28/rows/apus-9b-high.kf.jsonl@e804fe0a",
   "ties": 0,
   "timed_rows": 40,
   "wilson": [
    0.9123783988027135,
    1.0000000000000002
   ]
  },
  "gemma4-26b-fp8-readout-arm10": {
   "k": 40,
   "k_max": 40,
   "k_min": 40,
   "median_s": 0.10309667350156815,
   "n": 40,
   "p95_s": 0.1094619600007718,
   "pct": 100.0,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-addenda/gemma4-fp8-readout.f.jsonl@38bc9887",
   "ties": 0,
   "timed_rows": 40,
   "wilson": [
    0.9123783988027135,
    1.0000000000000002
   ]
  },
  "imajev-4b-gpu": {
   "k": 40,
   "k_max": 40,
   "k_min": 40,
   "median_s": 0.33084323199727805,
   "n": 40,
   "p95_s": 0.33573828199587297,
   "pct": 100.0,
   "refused": {},
   "src": "bench/imajev-2026-09-28/i2/rows/imajev-4b-3090.kf.jsonl@5840165e",
   "ties": 0,
   "timed_rows": 40,
   "wilson": [
    0.9123783988027135,
    1.0000000000000002
   ]
  },
  "kev-4b": {
   "k": 40,
   "k_max": null,
   "k_min": null,
   "median_s": 0.06733295649996762,
   "n": 40,
   "p95_s": 0.06767804899982366,
   "pct": 100.0,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kf.jsonl@c19fed10",
   "ties": null,
   "timed_rows": 40,
   "wilson": [
    0.9123783988027135,
    1.0000000000000002
   ]
  },
  "kev-9b": {
   "k": 40,
   "k_max": null,
   "k_min": null,
   "median_s": 0.09460461200001191,
   "n": 40,
   "p95_s": 0.1012878249998721,
   "pct": 100.0,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kf.jsonl@c19fed10",
   "ties": null,
   "timed_rows": 40,
   "wilson": [
    0.9123783988027135,
    1.0000000000000002
   ]
  },
  "lev-4b-gpu": {
   "k": 39,
   "k_max": 39,
   "k_min": 39,
   "median_s": 0.10945199349953327,
   "n": 40,
   "p95_s": 0.11135336299957999,
   "pct": 97.5,
   "refused": {},
   "src": "bench/lev-2026-09-28/rows/lev-4b.kf.jsonl@17d9c043",
   "ties": 0,
   "timed_rows": 40,
   "wilson": [
    0.8711863103652591,
    0.9955731684973186
   ]
  },
  "openjev-fp8-readout-arm5": {
   "k": 40,
   "k_max": 40,
   "k_min": 40,
   "median_s": 0.2768619984990437,
   "n": 40,
   "p95_s": 0.2913629000013316,
   "pct": 100.0,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-addenda/openjev-fp8-readout.f.jsonl@c176a040",
   "ties": 0,
   "timed_rows": 40,
   "wilson": [
    0.9123783988027135,
    1.0000000000000002
   ]
  }
 },
 "h48": {
  "apus-4b-high": {
   "k": 29,
   "k_max": 29,
   "k_min": 29,
   "median_s": 0.17127695599992876,
   "n": 48,
   "p95_s": 0.17879081800128915,
   "pct": 60.416666666666664,
   "refused": {},
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kh.jsonl@3a84d4b3",
   "ties": 0,
   "timed_rows": 48,
   "wilson": [
    0.4631040849254725,
    0.7297917213940862
   ]
  },
  "apus-4b-low": {
   "k": 12,
   "k_max": 12,
   "k_min": 12,
   "median_s": 0.09470615849932074,
   "n": 48,
   "p95_s": 0.09952337699724012,
   "pct": 25.0,
   "refused": {},
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-low.kh.jsonl@3a84d4b3",
   "ties": 0,
   "timed_rows": 48,
   "wilson": [
    0.1492063367379791,
    0.3878437280950796
   ]
  },
  "apus-9b-high": {
   "k": 31,
   "k_max": 32,
   "k_min": 31,
   "median_s": 0.18744788549884106,
   "n": 48,
   "p95_s": 0.2087297350008157,
   "pct": 64.58333333333333,
   "refused": {},
   "src": "bench/apus-2026-09-28/rows/apus-9b-high.kh.jsonl@e804fe0a",
   "ties": 2,
   "timed_rows": 48,
   "wilson": [
    0.504390570103431,
    0.7656635587439515
   ]
  },
  "apus-9b-low": {
   "k": 15,
   "k_max": 15,
   "k_min": 15,
   "median_s": 0.10452065600111382,
   "n": 48,
   "p95_s": 0.11407554099787376,
   "pct": 31.25,
   "refused": {},
   "src": "bench/apus-2026-09-28/rows/apus-9b-low.kh.jsonl@e804fe0a",
   "ties": 0,
   "timed_rows": 48,
   "wilson": [
    0.19945683446587817,
    0.4533307141589159
   ]
  },
  "deem-0.8-d0": {
   "carried_mismatch": 0,
   "k": 4,
   "k_max": 5,
   "k_min": 4,
   "median_s": 0.42892662150552496,
   "n": 48,
   "p95_s": 0.5057203109608963,
   "pct": 8.333333333333334,
   "refused": {},
   "src": "bench/deem-2026-09-27/rows/d0-cpu-0.8.h48.P.rep1.jsonl@b0200003",
   "ties": 2,
   "timed_rows": 48,
   "wilson": [
    0.032883756818091775,
    0.1955330179036728
   ]
  },
  "imajev-4b-gpu": {
   "k": 30,
   "k_max": 30,
   "k_min": 30,
   "median_s": 0.515639203498722,
   "n": 48,
   "p95_s": 0.5813730239970027,
   "pct": 62.5,
   "refused": {},
   "src": "bench/imajev-2026-09-28/i2/rows/imajev-4b-3090.kh.jsonl@5840165e",
   "ties": 0,
   "timed_rows": 48,
   "wilson": [
    0.4836276351921527,
    0.747847332391318
   ]
  },
  "naive-bayes": {
   "carried_mismatch": 0,
   "k": 27,
   "k_max": 27,
   "k_min": 27,
   "median_s": null,
   "n": 48,
   "p95_s": null,
   "pct": 56.25,
   "refused": {},
   "src": "bench/deem-2026-09-27/t1-tune/rows/t1-baseline-nb.h48.P.rep1.jsonl@1b4f1e14",
   "ties": 0,
   "timed_rows": 0,
   "wilson": [
    0.42275017419827776,
    0.6929873095934576
   ]
  },
  "opendecider-base-qwen3-4b": {
   "carried_mismatch": 0,
   "k": 26,
   "k_max": 26,
   "k_min": 26,
   "median_s": 0.06399172200008252,
   "n": 48,
   "p95_s": 0.0684415280002213,
   "pct": 54.166666666666664,
   "refused": {},
   "src": "bench/opendecider-2026-09-28/rows/o2c-gpu-qwen3-4b-base.h48.P.rep1.jsonl@34555419",
   "ties": 0,
   "timed_rows": 48,
   "wilson": [
    0.40291074545649797,
    0.6742475770713254
   ]
  },
  "opendecider-nano": {
   "carried_mismatch": 0,
   "k": 5,
   "k_max": 5,
   "k_min": 5,
   "median_s": 0.2056389099743683,
   "n": 48,
   "p95_s": 0.242975162982475,
   "pct": 10.416666666666666,
   "refused": {},
   "src": "bench/opendecider-2026-09-28/rows/o1-cpu-nano.h48.P.rep1.jsonl@95cfcd88",
   "ties": 0,
   "timed_rows": 48,
   "wilson": [
    0.045321719041813105,
    0.2216742169438633
   ]
  },
  "opendecider-small": {
   "carried_mismatch": 0,
   "k": 25,
   "k_max": 25,
   "k_min": 25,
   "median_s": 0.06318789699980698,
   "n": 48,
   "p95_s": 0.06710132199987129,
   "pct": 52.083333333333336,
   "refused": {},
   "src": "bench/opendecider-2026-09-28/rows/o2-gpu-small.h48.P.rep1.jsonl@34555419",
   "ties": 1,
   "timed_rows": 48,
   "wilson": [
    0.3832923230334281,
    0.6552868382304838
   ]
  },
  "openjev-q4km-generate-arm13": {
   "k": 44,
   "k_max": 44,
   "k_min": 44,
   "lenient": 44,
   "median_s": 1.1586388430005172,
   "n": 48,
   "p95_s": 1.2113707940006861,
   "pct": 91.66666666666667,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-generate.h48.jsonl@95d6ae9a",
   "ties": 0,
   "timed_rows": 48,
   "unparsed": 0,
   "wilson": [
    0.8044669820963272,
    0.9671162431819081
   ]
  },
  "openjev-q4km-readout-arm13": {
   "k": 44,
   "k_max": 44,
   "k_min": 44,
   "median_s": 1.130080306000309,
   "n": 48,
   "p95_s": 1.1912296299997251,
   "pct": 91.66666666666667,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-readout.h48.jsonl@95d6ae9a",
   "ties": 0,
   "timed_rows": 48,
   "wilson": [
    0.8044669820963272,
    0.9671162431819081
   ]
  }
 },
 "judge": {
  "apus-4b-high": {
   "alternation": 111,
   "n": 119,
   "note_correct_alt_field_counts_the_LABEL_not_the_choice": 111,
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kj.jsonl@3a84d4b3",
   "strict": 91
  },
  "apus-9b-high": {
   "alternation": 113,
   "n": 119,
   "note_correct_alt_field_counts_the_LABEL_not_the_choice": 113,
   "src": "bench/apus-2026-09-28/rows/apus-9b-high.kj.jsonl@e804fe0a",
   "strict": 93
  },
  "gemma4-26b-fp8-readout-arm10": {
   "missing": true,
   "src": "bench/jev-2026-09-21/rows-addenda/gemma4-fp8-readout.j.jsonl@UNTRACKED"
  },
  "imajev-4b-gpu": {
   "alternation": 112,
   "n": 119,
   "note_correct_alt_field_counts_the_LABEL_not_the_choice": 112,
   "src": "bench/imajev-2026-09-28/i2/rows/imajev-4b-3090.kj.jsonl@5840165e",
   "strict": 92
  },
  "kev-4b": {
   "alternation": 112,
   "n": 119,
   "note_correct_alt_field_counts_the_LABEL_not_the_choice": 112,
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kj.jsonl@c19fed10",
   "strict": 92
  },
  "kev-9b": {
   "alternation": 115,
   "n": 119,
   "note_correct_alt_field_counts_the_LABEL_not_the_choice": 115,
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kj.jsonl@c19fed10",
   "strict": 95
  },
  "lev-4b-gpu": {
   "alternation": 108,
   "n": 119,
   "note_correct_alt_field_counts_the_LABEL_not_the_choice": 108,
   "src": "bench/lev-2026-09-28/rows/lev-4b.kj.jsonl@17d9c043",
   "strict": 89
  },
  "openjev-fp8-readout-arm6": {
   "alternation": 118,
   "n": 119,
   "note_correct_alt_field_counts_the_LABEL_not_the_choice": 118,
   "src": "bench/jev-2026-09-21/rows-addenda/openjev-fp8-readout.j.jsonl@c176a040",
   "strict": 98
  }
 },
 "mcnemar": {
  "apus-4b-high|kev-9b": {
   "only_first_right": 7,
   "only_second_right": 12,
   "p": 0.359283447265625
  },
  "apus-4b-high|naive-bayes": {
   "only_first_right": 18,
   "only_second_right": 19,
   "p": 1.0
  },
  "apus-4b-low|kev-9b": {
   "only_first_right": 11,
   "only_second_right": 58,
   "p": 7.555829742748269e-09
  },
  "apus-4b-low|naive-bayes": {
   "only_first_right": 12,
   "only_second_right": 55,
   "p": 1.0273195443541005e-07
  },
  "apus-9b-high|kev-9b": {
   "only_first_right": 8,
   "only_second_right": 14,
   "p": 0.28627872467041016
  },
  "apus-9b-high|naive-bayes": {
   "only_first_right": 18,
   "only_second_right": 20,
   "p": 0.8714146793645341
  },
  "apus-9b-low|kev-9b": {
   "only_first_right": 13,
   "only_second_right": 55,
   "p": 2.7465771122682156e-07
  },
  "apus-9b-low|naive-bayes": {
   "only_first_right": 15,
   "only_second_right": 53,
   "p": 4.11597033439493e-06
  },
  "deem-0.8-d0|kev-9b": {
   "only_first_right": 2,
   "only_second_right": 69,
   "p": 2.165863246129246e-18
  },
  "deem-0.8-d0|naive-bayes": {
   "only_first_right": 2,
   "only_second_right": 65,
   "p": 3.088620938868081e-17
  },
  "gemma4-26b-fp8-generate-arm10|kev-9b": {
   "only_first_right": 21,
   "only_second_right": 5,
   "p": 0.002493917942047119
  },
  "gemma4-26b-fp8-generate-arm10|naive-bayes": {
   "only_first_right": 28,
   "only_second_right": 8,
   "p": 0.0011932429624721408
  },
  "gemma4-26b-fp8-readout-arm10|kev-9b": {
   "only_first_right": 20,
   "only_second_right": 5,
   "p": 0.004077315330505371
  },
  "gemma4-26b-fp8-readout-arm10|naive-bayes": {
   "only_first_right": 27,
   "only_second_right": 8,
   "p": 0.001878225477412343
  },
  "gemma4-26b-q4-base-generate-arm1|kev-9b": {
   "only_first_right": 19,
   "only_second_right": 3,
   "p": 0.0008554458618164062
  },
  "gemma4-26b-q4-base-generate-arm1|naive-bayes": {
   "only_first_right": 28,
   "only_second_right": 8,
   "p": 0.0011932429624721408
  },
  "gemma4-26b-q4-base-readout-arm1|kev-9b": {
   "only_first_right": 19,
   "only_second_right": 4,
   "p": 0.002599477767944336
  },
  "gemma4-26b-q4-base-readout-arm1|naive-bayes": {
   "only_first_right": 28,
   "only_second_right": 9,
   "p": 0.0025632079923525453
  },
  "imajev-4b-cpu|kev-9b": {
   "only_first_right": 9,
   "only_second_right": 13,
   "p": 0.5234670639038086
  },
  "imajev-4b-cpu|naive-bayes": {
   "only_first_right": 21,
   "only_second_right": 21,
   "p": 1.0
  },
  "jevify-generate-arm1|kev-9b": {
   "only_first_right": 22,
   "only_second_right": 4,
   "p": 0.0005335211753845215
  },
  "jevify-generate-arm1|naive-bayes": {
   "only_first_right": 29,
   "only_second_right": 7,
   "p": 0.00031255115754902363
  },
  "jevify-readout-arm1|kev-9b": {
   "only_first_right": 22,
   "only_second_right": 4,
   "p": 0.0005335211753845215
  },
  "jevify-readout-arm1|naive-bayes": {
   "only_first_right": 29,
   "only_second_right": 7,
   "p": 0.00031255115754902363
  },
  "kev-4b|kev-9b": {
   "only_first_right": 6,
   "only_second_right": 18,
   "p": 0.022655844688415527
  },
  "kev-4b|naive-bayes": {
   "only_first_right": 17,
   "only_second_right": 25,
   "p": 0.27995623852893914
  },
  "kev-9b|naive-bayes": {
   "only_first_right": 18,
   "only_second_right": 14,
   "p": 0.5966148958541453
  },
  "lev-4b-cpu|kev-9b": {
   "only_first_right": 10,
   "only_second_right": 16,
   "p": 0.32693958282470703
  },
  "lev-4b-cpu|naive-bayes": {
   "only_first_right": 18,
   "only_second_right": 20,
   "p": 0.8714146793645341
  },
  "lev-4b-gpu|kev-9b": {
   "only_first_right": 9,
   "only_second_right": 15,
   "p": 0.30745625495910645
  },
  "lev-4b-gpu|naive-bayes": {
   "only_first_right": 18,
   "only_second_right": 20,
   "p": 0.8714146793645341
  },
  "mistral-small3.2-24b-generate-coi|kev-9b": {
   "only_first_right": 12,
   "only_second_right": 11,
   "p": 1.0
  },
  "mistral-small3.2-24b-generate-coi|naive-bayes": {
   "only_first_right": 21,
   "only_second_right": 16,
   "p": 0.5113757815852296
  },
  "naive-bayes|kev-9b": {
   "only_first_right": 14,
   "only_second_right": 18,
   "p": 0.5966148958541453
  },
  "opendecider-base-qwen3-4b|kev-9b": {
   "only_first_right": 6,
   "only_second_right": 32,
   "p": 2.4342560209333897e-05
  },
  "opendecider-base-qwen3-4b|naive-bayes": {
   "only_first_right": 13,
   "only_second_right": 35,
   "p": 0.0020881073339964473
  },
  "opendecider-nano|kev-9b": {
   "only_first_right": 1,
   "only_second_right": 60,
   "p": 5.377642775528102e-17
  },
  "opendecider-nano|naive-bayes": {
   "only_first_right": 1,
   "only_second_right": 56,
   "p": 8.049116928532385e-16
  },
  "opendecider-small|kev-9b": {
   "only_first_right": 5,
   "only_second_right": 31,
   "p": 1.2913485988974571e-05
  },
  "opendecider-small|naive-bayes": {
   "only_first_right": 13,
   "only_second_right": 35,
   "p": 0.0020881073339964473
  },
  "openjev-bf16-readout-arm3|kev-9b": {
   "only_first_right": 32,
   "only_second_right": 3,
   "p": 4.176981747150421e-07
  },
  "openjev-bf16-readout-arm3|naive-bayes": {
   "only_first_right": 37,
   "only_second_right": 4,
   "p": 1.0258372640237212e-07
  },
  "openjev-fp8-generate-arm2|kev-9b": {
   "only_first_right": 32,
   "only_second_right": 3,
   "p": 4.176981747150421e-07
  },
  "openjev-fp8-generate-arm2|naive-bayes": {
   "only_first_right": 37,
   "only_second_right": 4,
   "p": 1.0258372640237212e-07
  },
  "openjev-fp8-readout-arm2|kev-9b": {
   "only_first_right": 32,
   "only_second_right": 3,
   "p": 4.176981747150421e-07
  },
  "openjev-fp8-readout-arm2|naive-bayes": {
   "only_first_right": 37,
   "only_second_right": 4,
   "p": 1.0258372640237212e-07
  },
  "openjev-fp8-readout-arm3|kev-9b": {
   "only_first_right": 32,
   "only_second_right": 3,
   "p": 4.176981747150421e-07
  },
  "openjev-fp8-readout-arm3|naive-bayes": {
   "only_first_right": 37,
   "only_second_right": 4,
   "p": 1.0258372640237212e-07
  },
  "openjev-q4km-generate-arm13|kev-9b": {
   "only_first_right": 31,
   "only_second_right": 4,
   "p": 3.4654513001441956e-06
  },
  "openjev-q4km-generate-arm13|naive-bayes": {
   "only_first_right": 36,
   "only_second_right": 5,
   "p": 7.84157236921601e-07
  },
  "openjev-q4km-readout-arm13|kev-9b": {
   "only_first_right": 31,
   "only_second_right": 4,
   "p": 3.4654513001441956e-06
  },
  "openjev-q4km-readout-arm13|naive-bayes": {
   "only_first_right": 36,
   "only_second_right": 5,
   "p": 7.84157236921601e-07
  }
 },
 "post_hoc": {
  "author_family": {
   "cost_series_out_of_family": {
    "gemma4-26b-q4-base-generate-arm1": {
     "equal": true,
     "ours": [
      49,
      57
     ],
     "series_rows": [
      [
       49,
       57
      ]
     ]
    },
    "mistral-small3.2-24b-generate-coi": {
     "equal": true,
     "ours": [
      32,
      51
     ],
     "series_rows": [
      [
       32,
       51
      ]
     ]
    },
    "src": "bench/cost-of-intelligence-2026-09-23/series/series.json@5a183df3"
   },
   "counts": {
    "apus-4b-high": {
     "gemma": 25,
     "mistral": 42
    },
    "apus-4b-low": {
     "gemma": 14,
     "mistral": 11
    },
    "apus-9b-high": {
     "gemma": 24,
     "mistral": 42
    },
    "apus-9b-low": {
     "gemma": 15,
     "mistral": 15
    },
    "deem-0.8-d0": {
     "gemma": 3,
     "mistral": 2
    },
    "gemma4-26b-fp8-generate-arm10": {
     "gemma": 40,
     "mistral": 48
    },
    "gemma4-26b-fp8-readout-arm10": {
     "gemma": 39,
     "mistral": 48
    },
    "gemma4-26b-q4-base-generate-arm1": {
     "gemma": 39,
     "mistral": 49
    },
    "gemma4-26b-q4-base-readout-arm1": {
     "gemma": 39,
     "mistral": 48
    },
    "imajev-4b-cpu": {
     "gemma": 26,
     "mistral": 42
    },
    "jevify-generate-arm1": {
     "gemma": 40,
     "mistral": 50
    },
    "jevify-readout-arm1": {
     "gemma": 40,
     "mistral": 50
    },
    "kev-4b": {
     "gemma": 23,
     "mistral": 37
    },
    "kev-9b": {
     "gemma": 27,
     "mistral": 45
    },
    "lev-4b-cpu": {
     "gemma": 27,
     "mistral": 39
    },
    "lev-4b-gpu": {
     "gemma": 28,
     "mistral": 38
    },
    "mistral-small3.2-24b-generate-coi": {
     "gemma": 32,
     "mistral": 41
    },
    "naive-bayes": {
     "gemma": 32,
     "mistral": 36
    },
    "opendecider-base-qwen3-4b": {
     "gemma": 19,
     "mistral": 27
    },
    "opendecider-nano": {
     "gemma": 6,
     "mistral": 7
    },
    "opendecider-small": {
     "gemma": 19,
     "mistral": 27
    },
    "openjev-bf16-readout-arm3": {
     "gemma": 47,
     "mistral": 54
    },
    "openjev-fp8-generate-arm2": {
     "gemma": 47,
     "mistral": 54
    },
    "openjev-fp8-readout-arm2": {
     "gemma": 47,
     "mistral": 54
    },
    "openjev-fp8-readout-arm3": {
     "gemma": 47,
     "mistral": 54
    },
    "openjev-q4km-generate-arm13": {
     "gemma": 45,
     "mistral": 54
    },
    "openjev-q4km-readout-arm13": {
     "gemma": 45,
     "mistral": 54
    }
   },
   "families": {
    "gemma": "Gemma 4 26B wrote the line",
    "mistral": "Mistral Small 3.2 24B wrote the line"
   },
   "kev9b_row_order_equals_the_kit_lock": true,
   "kit_lock": "bench/cost-of-intelligence-2026-09-23/KIT.lock.json@cb5dc076",
   "label": "descriptive, computed after the rows were seen",
   "n": {
    "gemma": 51,
    "mistral": 57
   },
   "paired": {
    "gemma|apus-4b-high|naive-bayes": {
     "only_first_right": 6,
     "only_second_right": 13,
     "p": 0.1670684814453125
    },
    "gemma|apus-9b-high|apus-4b-high": {
     "only_first_right": 6,
     "only_second_right": 7,
     "p": 1.0
    },
    "gemma|apus-9b-high|imajev-4b-cpu": {
     "only_first_right": 6,
     "only_second_right": 8,
     "p": 0.79052734375
    },
    "gemma|apus-9b-high|lev-4b-gpu": {
     "only_first_right": 4,
     "only_second_right": 8,
     "p": 0.3876953125
    },
    "gemma|apus-9b-high|naive-bayes": {
     "only_first_right": 5,
     "only_second_right": 13,
     "p": 0.09625244140625
    },
    "gemma|gemma4-26b-q4-base-generate-arm1|apus-4b-high": {
     "only_first_right": 16,
     "only_second_right": 2,
     "p": 0.001312255859375
    },
    "gemma|gemma4-26b-q4-base-generate-arm1|apus-9b-high": {
     "only_first_right": 18,
     "only_second_right": 3,
     "p": 0.0014896392822265625
    },
    "gemma|gemma4-26b-q4-base-generate-arm1|imajev-4b-cpu": {
     "only_first_right": 15,
     "only_second_right": 2,
     "p": 0.002349853515625
    },
    "gemma|gemma4-26b-q4-base-generate-arm1|kev-9b": {
     "only_first_right": 13,
     "only_second_right": 1,
     "p": 0.0018310546875
    },
    "gemma|gemma4-26b-q4-base-generate-arm1|lev-4b-gpu": {
     "only_first_right": 12,
     "only_second_right": 1,
     "p": 0.00341796875
    },
    "gemma|gemma4-26b-q4-base-generate-arm1|mistral-small3.2-24b-generate-coi": {
     "only_first_right": 9,
     "only_second_right": 2,
     "p": 0.0654296875
    },
    "gemma|gemma4-26b-q4-base-generate-arm1|naive-bayes": {
     "only_first_right": 13,
     "only_second_right": 6,
     "p": 0.1670684814453125
    },
    "gemma|imajev-4b-cpu|apus-4b-high": {
     "only_first_right": 5,
     "only_second_right": 4,
     "p": 1.0
    },
    "gemma|imajev-4b-cpu|lev-4b-gpu": {
     "only_first_right": 8,
     "only_second_right": 10,
     "p": 0.8145294189453125
    },
    "gemma|imajev-4b-cpu|naive-bayes": {
     "only_first_right": 8,
     "only_second_right": 14,
     "p": 0.28627872467041016
    },
    "gemma|kev-9b|apus-4b-high": {
     "only_first_right": 7,
     "only_second_right": 5,
     "p": 0.7744140625
    },
    "gemma|kev-9b|apus-9b-high": {
     "only_first_right": 8,
     "only_second_right": 5,
     "p": 0.5810546875
    },
    "gemma|kev-9b|imajev-4b-cpu": {
     "only_first_right": 7,
     "only_second_right": 6,
     "p": 1.0
    },
    "gemma|kev-9b|lev-4b-gpu": {
     "only_first_right": 4,
     "only_second_right": 5,
     "p": 1.0
    },
    "gemma|kev-9b|naive-bayes": {
     "only_first_right": 5,
     "only_second_right": 10,
     "p": 0.3017578125
    },
    "gemma|lev-4b-gpu|apus-4b-high": {
     "only_first_right": 9,
     "only_second_right": 6,
     "p": 0.60723876953125
    },
    "gemma|lev-4b-gpu|naive-bayes": {
     "only_first_right": 6,
     "only_second_right": 10,
     "p": 0.454498291015625
    },
    "gemma|mistral-small3.2-24b-generate-coi|apus-4b-high": {
     "only_first_right": 12,
     "only_second_right": 5,
     "p": 0.143463134765625
    },
    "gemma|mistral-small3.2-24b-generate-coi|apus-9b-high": {
     "only_first_right": 11,
     "only_second_right": 3,
     "p": 0.057373046875
    },
    "gemma|mistral-small3.2-24b-generate-coi|imajev-4b-cpu": {
     "only_first_right": 11,
     "only_second_right": 5,
     "p": 0.210113525390625
    },
    "gemma|mistral-small3.2-24b-generate-coi|kev-9b": {
     "only_first_right": 10,
     "only_second_right": 5,
     "p": 0.3017578125
    },
    "gemma|mistral-small3.2-24b-generate-coi|lev-4b-gpu": {
     "only_first_right": 7,
     "only_second_right": 3,
     "p": 0.34375
    },
    "gemma|mistral-small3.2-24b-generate-coi|naive-bayes": {
     "only_first_right": 8,
     "only_second_right": 8,
     "p": 1.0
    },
    "mistral|apus-4b-high|naive-bayes": {
     "only_first_right": 12,
     "only_second_right": 6,
     "p": 0.237884521484375
    },
    "mistral|apus-9b-high|apus-4b-high": {
     "only_first_right": 3,
     "only_second_right": 3,
     "p": 1.0
    },
    "mistral|apus-9b-high|imajev-4b-cpu": {
     "only_first_right": 5,
     "only_second_right": 5,
     "p": 1.0
    },
    "mistral|apus-9b-high|lev-4b-gpu": {
     "only_first_right": 8,
     "only_second_right": 4,
     "p": 0.3876953125
    },
    "mistral|apus-9b-high|naive-bayes": {
     "only_first_right": 13,
     "only_second_right": 7,
     "p": 0.26317596435546875
    },
    "mistral|gemma4-26b-q4-base-generate-arm1|apus-4b-high": {
     "only_first_right": 8,
     "only_second_right": 1,
     "p": 0.0390625
    },
    "mistral|gemma4-26b-q4-base-generate-arm1|apus-9b-high": {
     "only_first_right": 9,
     "only_second_right": 2,
     "p": 0.0654296875
    },
    "mistral|gemma4-26b-q4-base-generate-arm1|imajev-4b-cpu": {
     "only_first_right": 7,
     "only_second_right": 0,
     "p": 0.015625
    },
    "mistral|gemma4-26b-q4-base-generate-arm1|kev-9b": {
     "only_first_right": 6,
     "only_second_right": 2,
     "p": 0.2890625
    },
    "mistral|gemma4-26b-q4-base-generate-arm1|lev-4b-gpu": {
     "only_first_right": 11,
     "only_second_right": 0,
     "p": 0.0009765625
    },
    "mistral|gemma4-26b-q4-base-generate-arm1|mistral-small3.2-24b-generate-coi": {
     "only_first_right": 8,
     "only_second_right": 0,
     "p": 0.0078125
    },
    "mistral|gemma4-26b-q4-base-generate-arm1|naive-bayes": {
     "only_first_right": 15,
     "only_second_right": 2,
     "p": 0.002349853515625
    },
    "mistral|imajev-4b-cpu|apus-4b-high": {
     "only_first_right": 3,
     "only_second_right": 3,
     "p": 1.0
    },
    "mistral|imajev-4b-cpu|lev-4b-gpu": {
     "only_first_right": 6,
     "only_second_right": 2,
     "p": 0.2890625
    },
    "mistral|imajev-4b-cpu|naive-bayes": {
     "only_first_right": 13,
     "only_second_right": 7,
     "p": 0.26317596435546875
    },
    "mistral|kev-9b|apus-4b-high": {
     "only_first_right": 5,
     "only_second_right": 2,
     "p": 0.453125
    },
    "mistral|kev-9b|apus-9b-high": {
     "only_first_right": 6,
     "only_second_right": 3,
     "p": 0.5078125
    },
    "mistral|kev-9b|imajev-4b-cpu": {
     "only_first_right": 6,
     "only_second_right": 3,
     "p": 0.5078125
    },
    "mistral|kev-9b|lev-4b-gpu": {
     "only_first_right": 11,
     "only_second_right": 4,
     "p": 0.11846923828125
    },
    "mistral|kev-9b|naive-bayes": {
     "only_first_right": 13,
     "only_second_right": 4,
     "p": 0.049041748046875
    },
    "mistral|lev-4b-gpu|apus-4b-high": {
     "only_first_right": 3,
     "only_second_right": 7,
     "p": 0.34375
    },
    "mistral|lev-4b-gpu|naive-bayes": {
     "only_first_right": 12,
     "only_second_right": 10,
     "p": 0.8318119049072266
    },
    "mistral|mistral-small3.2-24b-generate-coi|apus-4b-high": {
     "only_first_right": 3,
     "only_second_right": 4,
     "p": 1.0
    },
    "mistral|mistral-small3.2-24b-generate-coi|apus-9b-high": {
     "only_first_right": 4,
     "only_second_right": 5,
     "p": 1.0
    },
    "mistral|mistral-small3.2-24b-generate-coi|imajev-4b-cpu": {
     "only_first_right": 3,
     "only_second_right": 4,
     "p": 1.0
    },
    "mistral|mistral-small3.2-24b-generate-coi|kev-9b": {
     "only_first_right": 2,
     "only_second_right": 6,
     "p": 0.2890625
    },
    "mistral|mistral-small3.2-24b-generate-coi|lev-4b-gpu": {
     "only_first_right": 8,
     "only_second_right": 5,
     "p": 0.5810546875
    },
    "mistral|mistral-small3.2-24b-generate-coi|naive-bayes": {
     "only_first_right": 13,
     "only_second_right": 8,
     "p": 0.38331031799316406
    }
   }
  },
  "door_by_kind": {
   "apus-4b-high": {
    "controls_n": 12,
    "controls_refused": 9,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 17,
    "planted_refused": 27,
    "refuse_probability_controls": [
     0.020332353342658815,
     0.9399133498259924
    ],
    "refuse_probability_planted": [
     0.01798620996209155,
     0.995668497979436
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 4,
     "C": 4,
     "D": 6,
     "E": 1,
     "F": 6,
     "L": 3,
     "N": 2,
     "T": 4
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kd.jsonl@3a84d4b3",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 10
   },
   "apus-4b-low": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 17,
    "planted_refused": 24,
    "refuse_probability_controls": [
     0.006903838071221968,
     0.40168780512292046
    ],
    "refuse_probability_planted": [
     0.040692953596653036,
     0.9890194390479408
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 5,
     "C": 2,
     "D": 5,
     "E": 1,
     "F": 5,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-low.kd.jsonl@3a84d4b3",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 7
   },
   "apus-9b-high": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 23,
    "planted_refused": 32,
    "refuse_probability_controls": [
     0.0017007224114351516,
     0.07585818002124345
    ],
    "refuse_probability_planted": [
     0.009708476481474104,
     0.9989012689270753
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 5,
     "C": 4,
     "D": 5,
     "E": 6,
     "F": 6,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/apus-2026-09-28/rows/apus-9b-high.kd.jsonl@e804fe0a",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 9
   },
   "apus-9b-low": {
    "controls_n": 12,
    "controls_refused": 1,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 23,
    "planted_refused": 32,
    "refuse_probability_controls": [
     0.002800926967121087,
     0.7549149868676283
    ],
    "refuse_probability_planted": [
     0.032100706717008154,
     0.9996633377881102
    ],
    "refused_by_bucket": {
     "A": 5,
     "B": 6,
     "C": 3,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 0,
     "N": 0,
     "T": 1
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/apus-2026-09-28/rows/apus-9b-low.kd.jsonl@e804fe0a",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 9
   },
   "gemma4-26b-fp8-readout-arm10": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 23,
    "planted_refused": 23,
    "refuse_probability_controls": [
     1.4134543203014178e-07,
     8.522486988337903e-06
    ],
    "refuse_probability_planted": [
     8.25436374578587e-07,
     0.9999999953721685
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 0,
     "D": 0,
     "E": 6,
     "F": 5,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/jev-2026-09-21/rows-addenda/gemma4-fp8-readout.d.jsonl@38bc9887",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 0
   },
   "imajev-4b-cpu": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 20,
    "planted_refused": 21,
    "refuse_probability_controls": [
     0.044357828425264256,
     0.2529387637459305
    ],
    "refuse_probability_planted": [
     0.1462914465596995,
     0.9121506168930569
    ],
    "refused_by_bucket": {
     "A": 5,
     "B": 5,
     "C": 1,
     "D": 0,
     "E": 6,
     "F": 4,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/imajev-2026-09-28/rows/imajev-4b-cpu.kd.jsonl@9e86a063",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 1
   },
   "kev-4b": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 4,
    "planted_refused": 4,
    "refuse_probability_controls": [
     0.040000000000000036,
     0.06999999999999995
    ],
    "refuse_probability_planted": [
     0.040000000000000036,
     0.69
    ],
    "refused_by_bucket": {
     "A": 0,
     "B": 0,
     "C": 0,
     "D": 0,
     "E": 3,
     "F": 1,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kd.jsonl@c19fed10",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 0
   },
   "kev-9b": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 15,
    "planted_refused": 15,
    "refuse_probability_controls": [
     0.050000000000000044,
     0.10999999999999999
    ],
    "refuse_probability_planted": [
     0.07999999999999996,
     0.97
    ],
    "refused_by_bucket": {
     "A": 4,
     "B": 3,
     "C": 0,
     "D": 0,
     "E": 4,
     "F": 4,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kd.jsonl@c19fed10",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 0
   },
   "kit": "bench/jev-2026-09-21/kit/doorman_planted.json@c176a040",
   "label": "descriptive, computed after the rows were seen",
   "lev-4b-gpu": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 16,
    "planted_refused": 19,
    "refuse_probability_controls": [
     0.06695732884007022,
     0.1554935791936174
    ],
    "refuse_probability_planted": [
     0.08233744028089407,
     0.9179638028386825
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 4,
     "C": 1,
     "D": 2,
     "E": 3,
     "F": 3,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/lev-2026-09-28/rows/lev-4b.kd.jsonl@17d9c043",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 3
   },
   "live-doorman-gemma4-26b-fp8-0913": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "the kit, by id (the 09-13 verdicts carry no kind field)",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 24,
    "planted_refused": 27,
    "refused_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 1,
     "D": 2,
     "E": 6,
     "F": 6,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/doorman-planted-2026-09-13/out/verdicts.jsonl@c9fed92f",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 3
   },
   "named_kinds": "buckets whose kind the v1.2 rules name (A, B, E, F); unnamed: C, D; controls: L, N, T",
   "opendecider-base-qwen3-4b": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 11,
    "planted_refused": 15,
    "refuse_probability_controls": [
     1.4738980098627508e-05,
     0.09534946084022522
    ],
    "refuse_probability_planted": [
     7.338210252783028e-07,
     0.9999921321868896
    ],
    "refused_by_bucket": {
     "A": 4,
     "B": 5,
     "C": 0,
     "D": 4,
     "E": 2,
     "F": 0,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/opendecider-2026-09-28/rows/o2c-gpu-qwen3-4b-base.door.P.rep1.jsonl@34555419",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 4
   },
   "opendecider-nano": {
    "controls_n": 12,
    "controls_refused": 12,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 24,
    "planted_refused": 36,
    "refuse_probability_controls": [
     0.6252924203872681,
     0.7058696746826172
    ],
    "refuse_probability_planted": [
     0.5696999430656433,
     0.7814617156982422
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/opendecider-2026-09-28/rows/o1-cpu-nano.door.P.rep1.jsonl@95cfcd88",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 12
   },
   "opendecider-small": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 4,
    "planted_refused": 4,
    "refuse_probability_controls": [
     0.1919327974319458,
     0.3345894515514374
    ],
    "refuse_probability_planted": [
     0.2568320035934448,
     0.7879312038421631
    ],
    "refused_by_bucket": {
     "A": 4,
     "B": 0,
     "C": 0,
     "D": 0,
     "E": 0,
     "F": 0,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/opendecider-2026-09-28/rows/o2-gpu-small.door.P.rep1.jsonl@34555419",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 0
   },
   "openjev-fp8-readout-arm4": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 24,
    "planted_refused": 33,
    "refuse_probability_controls": [
     0.00028527691472755997,
     0.011988283086903495
    ],
    "refuse_probability_planted": [
     0.12878271853561907,
     0.9998632280363902
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 5,
     "D": 4,
     "E": 6,
     "F": 6,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/jev-2026-09-21/rows-addenda/openjev-fp8-readout.d.jsonl@c176a040",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 9
   },
   "openjev-q4km-generate-arm13": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 24,
    "planted_refused": 32,
    "refuse_probability_controls": [
     0.0,
     0.0
    ],
    "refuse_probability_planted": [
     0.0,
     1.0
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 5,
     "D": 3,
     "E": 6,
     "F": 6,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-generate.d.jsonl@95d6ae9a",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 8
   },
   "openjev-q4km-readout-arm13": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 24,
    "planted_refused": 32,
    "refuse_probability_controls": [
     0.00013995938814172432,
     0.006397952625917213
    ],
    "refuse_probability_planted": [
     0.04828113705128966,
     0.9998198165447321
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 5,
     "D": 3,
     "E": 6,
     "F": 6,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-readout.d.jsonl@95d6ae9a",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 8
   }
  },
  "imajev_gpu_latency": {
   "kf": {
    "median_s": 0.33084323199727805,
    "p95_s": 0.33573828199587297,
    "src": "bench/imajev-2026-09-28/i2/rows/imajev-4b-3090.kf.jsonl@5840165e",
    "timed_rows": 40
   },
   "kh": {
    "median_s": 0.515639203498722,
    "p95_s": 0.5813730239970027,
    "src": "bench/imajev-2026-09-28/i2/rows/imajev-4b-3090.kh.jsonl@5840165e",
    "timed_rows": 48
   },
   "kj": {
    "median_s": 1.1490991000027861,
    "p95_s": 2.150993668998126,
    "src": "bench/imajev-2026-09-28/i2/rows/imajev-4b-3090.kj.jsonl@5840165e",
    "timed_rows": 119
   },
   "label": "descriptive, computed after the rows were seen",
   "what": "imajev-4B on one used RTX 3090 (card B, 300 W), its own server in bf16, four option orders inside every answer, one request at a time"
  },
  "label": "descriptive, computed after the rows were seen",
  "memory": {
   "card_total_mib": 24576,
   "card_total_src": "bench/jev-2026-09-21/ARM13-OJ-GGUF.md, the card row (24,576 MiB)",
   "how": "growth = memory.used after the warm-ups minus the reading before the load, as each run's own residency check recorded it; highest = the highest memory.used sample in the run's telemetry (2 Hz) or, for arms 12 and 13, its per-task nvidia-smi CSVs (1 Hz); longest input = the most tokens among rows whose request had started by the first highest sample",
   "label": "descriptive, computed after the rows were seen",
   "rows": {
    "apus-4b": {
     "grew_at_load_mib": 9235.0,
     "growth_src": "bench/apus-2026-09-28/a1b/receipts/counted-run.json@3a84d4b3",
     "headroom_mib": 7892.0,
     "highest": {
      "first_at_epoch": 1790614058.540976,
      "first_at_utc": "2026-09-28T16:47:38Z",
      "highest_mib": 16684.0,
      "samples": 883,
      "src": "bench/apus-2026-09-28/a1b/receipts/counted-telemetry.jsonl@3a84d4b3"
     },
     "longest_input_by_the_highest": {
      "row": "on-39",
      "task": "ka",
      "tokens": 8187
     },
     "longest_input_overall": {
      "row": "on-39",
      "task": "ka",
      "tokens": 8187
     },
     "precision_and_runtime": "bf16, the vendor runtime in-process (one load serves high and low)"
    },
    "apus-9b": {
     "grew_at_load_mib": 18471.0,
     "growth_src": "bench/apus-2026-09-28/receipts/counted-run.json@e804fe0a",
     "headroom_mib": 450.0,
     "highest": {
      "first_at_epoch": 1790609674.288146,
      "first_at_utc": "2026-09-28T15:34:34Z",
      "highest_mib": 24126.0,
      "samples": 967,
      "src": "bench/apus-2026-09-28/receipts/counted-telemetry.jsonl@e804fe0a"
     },
     "longest_input_by_the_highest": {
      "row": "on-39",
      "task": "ka",
      "tokens": 8187
     },
     "longest_input_overall": {
      "row": "on-39",
      "task": "ka",
      "tokens": 8187
     },
     "precision_and_runtime": "bf16, the vendor runtime in-process (one load serves high and low)"
    },
    "imajev-4b": {
     "grew_at_load_mib": 9798.0,
     "growth_bytes": 10273947648,
     "growth_src": "bench/imajev-2026-09-28/i2/receipts/counted-unit.log@5840165e",
     "headroom_mib": 13320.0,
     "highest": {
      "first_at_epoch": 1790627353.8629203,
      "first_at_utc": "2026-09-28T20:29:13Z",
      "highest_mib": 11256.0,
      "samples": 813,
      "src": "bench/imajev-2026-09-28/i2/receipts/i2-telemetry.jsonl@5840165e"
     },
     "longest_input_by_the_highest": {
      "row": "j-108",
      "task": "kj",
      "tokens": 2240
     },
     "longest_input_overall": {
      "row": "j-108",
      "task": "kj",
      "tokens": 2240
     },
     "precision_and_runtime": "bf16, imajev's own server, four orders per answer"
    },
    "kev-4b": {
     "grew_at_load_mib": null,
     "growth_src": "none: arm 12's files record no reading from before the load",
     "headroom_mib": 11218.0,
     "highest": {
      "first_in": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.ka.watts.csv",
      "highest_mib": 13358.0,
      "samples_files": 14,
      "src": "the arm's .watts.csv and .idle.csv files (card A, 250 W)"
     },
     "longest_by_the_highest_why_none": "the per-task CSVs are not timed against the rows here, so only the run's longest is given",
     "longest_input_by_the_highest": null,
     "longest_input_overall": {
      "row": "on-23",
      "task": "ka",
      "tokens": 8105
     },
     "note": "No census of other tenants was recorded; Kev-4B's lowest reading (8,776 MiB), taken between Kev-9B's tasks, shows the two servers were not resident together",
     "precision_and_runtime": "bf16, kev.serve (transformers + FastAPI)",
     "resident_before_first_task_mib": 8776.0,
     "resident_reading_file": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kc.idle.csv"
    },
    "kev-9b": {
     "grew_at_load_mib": null,
     "growth_src": "none: arm 12's files record no reading from before the load",
     "headroom_mib": 4298.0,
     "highest": {
      "first_in": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.ka.watts.csv",
      "highest_mib": 20278.0,
      "samples_files": 14,
      "src": "the arm's .watts.csv and .idle.csv files (card A, 250 W)"
     },
     "longest_by_the_highest_why_none": "the per-task CSVs are not timed against the rows here, so only the run's longest is given",
     "longest_input_by_the_highest": null,
     "longest_input_overall": {
      "row": "on-23",
      "task": "ka",
      "tokens": 8105
     },
     "note": "No census of other tenants was recorded; Kev-4B's lowest reading (8,776 MiB), taken between Kev-9B's tasks, shows the two servers were not resident together",
     "precision_and_runtime": "bf16, kev.serve (transformers + FastAPI)",
     "resident_before_first_task_mib": 15866.0,
     "resident_reading_file": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kc.idle.csv",
     "stated": {
      "src": "bench/jev-2026-09-21/GAPS-CARD.md line 351; TABLES-GAPS-CARD.md line 186",
      "text": "~19 GB of GPU memory in bf16"
     }
    },
    "lev-4b": {
     "grew_at_load_mib": 9125.0,
     "growth_src": "bench/lev-2026-09-28/receipts/counted-run.json@17d9c043",
     "headroom_mib": 1840.0,
     "highest": {
      "first_at_epoch": 1790601255.0626376,
      "first_at_utc": "2026-09-28T13:14:15Z",
      "highest_mib": 22736.0,
      "samples": 954,
      "src": "bench/lev-2026-09-28/receipts/counted-telemetry.jsonl@17d9c043"
     },
     "longest_input_by_the_highest": {
      "row": "on-27",
      "task": "ka",
      "tokens": 14590
     },
     "longest_input_overall": {
      "row": "on-27",
      "task": "ka",
      "tokens": 14590
     },
     "precision_and_runtime": "bf16, lev serve 0.1.1, reference kernels"
    },
    "opendecider-small": {
     "grew_at_load_mib": 8595.0,
     "growth_bytes_distinct": [
      9012510720
     ],
     "growth_src": "bench/opendecider-2026-09-28/rows/o2-gpu-small.*.pass.json (residency_after_warmup.growth_bytes, 12 counted passes)",
     "headroom_mib": 15980.0,
     "highest": {
      "first_at_epoch": 1790597083.8493075,
      "first_at_utc": "2026-09-28T12:04:43Z",
      "highest_mib": 8596.0,
      "samples": 752,
      "src": "bench/opendecider-2026-09-28/receipts/counted-telemetry.jsonl@34555419"
     },
     "highest_window": "OpenDecider small's highest is read from the telemetry inside its own pass windows; the same file also covers its base model's passes",
     "longest_by_the_highest_why_none": "no request had started: the highest came at load",
     "longest_input_by_the_highest": null,
     "longest_input_overall": {
      "row": "E4",
      "task": "door",
      "tokens": 521
     },
     "precision_and_runtime": "bf16, transformers"
    },
    "openjev-q4km-arm13": {
     "grew_at_load_mib": 16727.0,
     "growth_src": "bench/jev-2026-09-21/receipts/oj-gguf/run/counted-run.json@95d6ae9a",
     "headroom_mib": 7846.0,
     "highest": {
      "first_in": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-generate.c.watts.csv",
      "highest_mib": 16730.0,
      "resident_after_warmups_mib": 16728.0,
      "samples_files": 14,
      "src": "the arm's .watts.csv and .idle.csv files"
     },
     "longest_by_the_highest_why_none": "the per-task CSVs are not timed against the rows here, so only the run's longest is given",
     "longest_input_by_the_highest": null,
     "longest_input_overall": {
      "row": "E4",
      "task": "d",
      "tokens": 524
     },
     "note": "flat: the highest reading is within 3 MiB of the resident reading after the warm-ups",
     "precision_and_runtime": "Q4_K_M, ollama 0.32.13"
    }
   }
  },
  "note": "None of these sets was registered before its rows were seen, and no bar applies to any of them.",
  "six_order": {
   "apus-4b-high": {
    "fewest": 63,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 5,
    "mean": 65.66666666666667,
    "most": 71,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     67,
     64,
     63,
     71,
     66,
     63
    ],
    "s0_compared": "apus-4b-high",
    "schedule": "K",
    "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kperm.jsonl@3a84d4b3"
   },
   "apus-9b-high": {
    "fewest": 66,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 1,
    "mean": 70,
    "most": 76,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     66,
     68,
     70,
     76,
     73,
     67
    ],
    "s0_compared": "apus-9b-high",
    "schedule": "K",
    "src": "bench/apus-2026-09-28/rows/apus-9b-high.kperm.jsonl@e804fe0a"
   },
   "kev-4b": {
    "fewest": 57,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 4,
    "mean": 61,
    "most": 69,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     60,
     58,
     59,
     63,
     57,
     69
    ],
    "s0_compared": "kev-4b",
    "schedule": "K",
    "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kperm.jsonl@c19fed10"
   },
   "kev-9b": {
    "fewest": 67,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 5,
    "mean": 70,
    "most": 73,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     72,
     69,
     70,
     73,
     69,
     67
    ],
    "s0_compared": "kev-9b",
    "schedule": "K",
    "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kperm.jsonl@c19fed10"
   },
   "kit_options_in_kit_order": [
    "darwin",
    "einstein",
    "hypatia",
    "ibn_sina",
    "sagan",
    "socrates"
   ],
   "kit_order_src": "bench/cost-of-intelligence-2026-09-23/KIT.lock.json@cb5dc076",
   "label": "descriptive, computed after the rows were seen",
   "lev-4b-gpu": {
    "fewest": 61,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 4,
    "mean": 65.33333333333333,
    "most": 68,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     66,
     67,
     65,
     65,
     68,
     61
    ],
    "s0_compared": "lev-4b-gpu",
    "schedule": "K",
    "src": "bench/lev-2026-09-28/rows/lev-4b.kperm.jsonl@17d9c043"
   },
   "opendecider-base-qwen3-4b": {
    "fewest": 46,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 1,
    "mean": 53.833333333333336,
    "most": 60,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     46,
     52,
     57,
     55,
     60,
     53
    ],
    "s0_compared": "opendecider-base-qwen3-4b",
    "schedule": "G",
    "src": "bench/opendecider-2026-09-28/rows/o2c-gpu-qwen3-4b-base.s0.P.rep1.jsonl@34555419 + bench/opendecider-2026-09-28/rows/o2c-gpu-qwen3-4b-base.s0.P.rep1.k1-5.jsonl@34555419"
   },
   "opendecider-nano": {
    "fewest": 7,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 5,
    "mean": 11.166666666666666,
    "most": 16,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     13,
     11,
     12,
     16,
     8,
     7
    ],
    "s0_compared": "opendecider-nano",
    "schedule": "G",
    "src": "bench/opendecider-2026-09-28/rows/o1-cpu-nano.s0.P.rep1.jsonl@95cfcd88 + bench/opendecider-2026-09-28/rows/o1-cpu-nano.s0.P.rep1.k1-5.jsonl@95cfcd88"
   },
   "opendecider-small": {
    "fewest": 46,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 1,
    "mean": 48.666666666666664,
    "most": 52,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     46,
     46,
     50,
     52,
     50,
     48
    ],
    "s0_compared": "opendecider-small",
    "schedule": "G",
    "src": "bench/opendecider-2026-09-28/rows/o2-gpu-small.s0.P.rep1.jsonl@34555419 + bench/opendecider-2026-09-28/rows/o2-gpu-small.s0.P.rep1.k1-5.jsonl@34555419"
   },
   "openjev-fp8-readout-gap4": {
    "fewest": 97,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 6,
    "mean": 98.66666666666667,
    "most": 101,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     101,
     97,
     97,
     98,
     99,
     100
    ],
    "s0_compared": "openjev-fp8-readout-arm2",
    "schedule": "G",
    "src": "bench/jev-2026-09-21/rows-gaps-card/openjev-fp8-readout.creorder.jsonl@13e720de"
   },
   "openjev-q4km-readout-arm13": {
    "fewest": 94,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 3,
    "mean": 98.16666666666667,
    "most": 100,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     99,
     94,
     100,
     100,
     97,
     99
    ],
    "s0_compared": "openjev-q4km-readout-arm13",
    "schedule": "G",
    "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-readout.creorder.jsonl@95d6ae9a"
   },
   "schedules": "K = Kev's own /permute orders; G = ours (the kit order plus random.Random(20260921 + k), k = 1..5). Compare means within one schedule."
  }
 },
 "ref": "f2073219",
 "reorder": {
  "apus-4b-high": {
   "items": 108,
   "moved": 47,
   "orders": [
    6
   ],
   "pairs": 1620,
   "pairs_disagree": 333,
   "right_by_order": [
    67,
    64,
    63,
    71,
    66,
    63
   ],
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kperm.jsonl@3a84d4b3"
  },
  "apus-9b-high": {
   "items": 108,
   "moved": 41,
   "orders": [
    6
   ],
   "pairs": 1620,
   "pairs_disagree": 321,
   "right_by_order": [
    66,
    68,
    70,
    76,
    73,
    67
   ],
   "src": "bench/apus-2026-09-28/rows/apus-9b-high.kperm.jsonl@e804fe0a"
  },
  "kev-4b": {
   "items": 108,
   "moved": 33,
   "orders": [
    6
   ],
   "pairs": 1620,
   "pairs_disagree": 262,
   "right_by_order": [
    60,
    58,
    59,
    63,
    57,
    69
   ],
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kperm.jsonl@c19fed10"
  },
  "kev-9b": {
   "items": 108,
   "moved": 33,
   "orders": [
    6
   ],
   "pairs": 1620,
   "pairs_disagree": 250,
   "right_by_order": [
    72,
    69,
    70,
    73,
    69,
    67
   ],
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kperm.jsonl@c19fed10"
  },
  "lev-4b-gpu": {
   "items": 108,
   "moved": 27,
   "orders": [
    6
   ],
   "pairs": 1620,
   "pairs_disagree": 191,
   "right_by_order": [
    66,
    67,
    65,
    65,
    68,
    61
   ],
   "src": "bench/lev-2026-09-28/rows/lev-4b.kperm.jsonl@17d9c043"
  },
  "opendecider-base-qwen3-4b": {
   "items": 108,
   "moved": 71,
   "orders": [
    0,
    1,
    2,
    3,
    4,
    5
   ],
   "pairs": 1620,
   "pairs_disagree": 577,
   "right_by_order": [
    46,
    52,
    57,
    55,
    60,
    53
   ],
   "src": "bench/opendecider-2026-09-28/rows/o2c-gpu-qwen3-4b-base.s0.P.rep1.jsonl@34555419 + bench/opendecider-2026-09-28/rows/o2c-gpu-qwen3-4b-base.s0.P.rep1.k1-5.jsonl@34555419"
  },
  "opendecider-nano": {
   "items": 108,
   "moved": 96,
   "orders": [
    0,
    1,
    2,
    3,
    4,
    5
   ],
   "pairs": 1620,
   "pairs_disagree": 943,
   "right_by_order": [
    13,
    11,
    12,
    16,
    8,
    7
   ],
   "src": "bench/opendecider-2026-09-28/rows/o1-cpu-nano.s0.P.rep1.jsonl@95cfcd88 + bench/opendecider-2026-09-28/rows/o1-cpu-nano.s0.P.rep1.k1-5.jsonl@95cfcd88"
  },
  "opendecider-small": {
   "items": 108,
   "moved": 64,
   "orders": [
    0,
    1,
    2,
    3,
    4,
    5
   ],
   "pairs": 1620,
   "pairs_disagree": 467,
   "right_by_order": [
    46,
    46,
    50,
    52,
    50,
    48
   ],
   "src": "bench/opendecider-2026-09-28/rows/o2-gpu-small.s0.P.rep1.jsonl@34555419 + bench/opendecider-2026-09-28/rows/o2-gpu-small.s0.P.rep1.k1-5.jsonl@34555419"
  },
  "openjev-fp8-readout-gap4": {
   "items": 108,
   "moved": 15,
   "orders": [
    0,
    1,
    2,
    3,
    4,
    5
   ],
   "pairs": 1620,
   "pairs_disagree": 92,
   "right_by_order": [
    101,
    97,
    97,
    98,
    99,
    100
   ],
   "src": "bench/jev-2026-09-21/rows-gaps-card/openjev-fp8-readout.creorder.jsonl@13e720de"
  },
  "openjev-q4km-readout-arm13": {
   "items": 108,
   "moved": 14,
   "orders": [
    0,
    1,
    2,
    3,
    4,
    5
   ],
   "pairs": 1620,
   "pairs_disagree": 101,
   "right_by_order": [
    99,
    94,
    100,
    100,
    97,
    99
   ],
   "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-readout.creorder.jsonl@95d6ae9a"
  }
 },
 "s0": {
  "apus-4b-high": {
   "k": 67,
   "k_max": 68,
   "k_min": 65,
   "median_s": 0.17079587950138375,
   "n": 108,
   "p95_s": 0.17819566300022416,
   "pct": 62.03703703703704,
   "refused": {},
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kc.jsonl@3a84d4b3",
   "ties": 4,
   "timed_rows": 108,
   "wilson": [
    0.5262011202247348,
    0.7062708116881311
   ]
  },
  "apus-4b-low": {
   "k": 25,
   "k_max": 25,
   "k_min": 25,
   "median_s": 0.09438246099853131,
   "n": 108,
   "p95_s": 0.0991138499994122,
   "pct": 23.14814814814815,
   "refused": {},
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-low.kc.jsonl@3a84d4b3",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.16199363571704478,
    0.31941513155425433
   ]
  },
  "apus-9b-high": {
   "k": 66,
   "k_max": 67,
   "k_min": 65,
   "median_s": 0.1825492084990401,
   "n": 108,
   "p95_s": 0.2078083129999868,
   "pct": 61.111111111111114,
   "refused": {},
   "src": "bench/apus-2026-09-28/rows/apus-9b-high.kc.jsonl@e804fe0a",
   "ties": 3,
   "timed_rows": 108,
   "wilson": [
    0.5168659638261305,
    0.6977235117857458
   ]
  },
  "apus-9b-low": {
   "k": 30,
   "k_max": 30,
   "k_min": 30,
   "median_s": 0.10157239899854176,
   "n": 108,
   "p95_s": 0.11486395499741775,
   "pct": 27.77777777777778,
   "refused": {},
   "src": "bench/apus-2026-09-28/rows/apus-9b-low.kc.jsonl@e804fe0a",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.2020502650072486,
    0.3687707837689989
   ]
  },
  "deem-0.8-d0": {
   "carried_mismatch": 0,
   "k": 5,
   "k_max": 5,
   "k_min": 5,
   "median_s": 0.44177665252937004,
   "n": 108,
   "p95_s": 0.5497705489979126,
   "pct": 4.62962962962963,
   "refused": {},
   "src": "bench/deem-2026-09-27/rows/d0-cpu-0.8.s0.P.rep1.jsonl@b0200003",
   "ties": 3,
   "timed_rows": 108,
   "wilson": [
    0.019934815414942943,
    0.1038248258365625
   ]
  },
  "deem-t1-null-20260927": {
   "carried_mismatch": 0,
   "k": 18,
   "k_max": 18,
   "k_min": 18,
   "median_s": 0.42128956149099395,
   "n": 108,
   "p95_s": 0.5248241910012439,
   "pct": 16.666666666666668,
   "refused": {},
   "src": "bench/deem-2026-09-27/t1-tune/rows/t1-null-20260927.s0.P.rep1.jsonl@1b4f1e14",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.10810472332568179,
    0.24812684983868954
   ]
  },
  "deem-t1-null-20260928": {
   "carried_mismatch": 0,
   "k": 29,
   "k_max": 29,
   "k_min": 28,
   "median_s": 0.433139050466707,
   "n": 108,
   "p95_s": 0.5515173840103671,
   "pct": 26.85185185185185,
   "refused": {},
   "src": "bench/deem-2026-09-27/t1-tune/rows/t1-null-20260928.s0.P.rep1.jsonl@1b4f1e14",
   "ties": 1,
   "timed_rows": 108,
   "wilson": [
    0.19394901389668973,
    0.35898957857856817
   ]
  },
  "deem-t1-null-20260929": {
   "carried_mismatch": 0,
   "k": 24,
   "k_max": 24,
   "k_min": 24,
   "median_s": 0.4112346725014504,
   "n": 108,
   "p95_s": 0.5023265749914572,
   "pct": 22.22222222222222,
   "refused": {},
   "src": "bench/deem-2026-09-27/t1-tune/rows/t1-null-20260929.s0.P.rep1.jsonl@1b4f1e14",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.15412547066351984,
    0.3094008403067896
   ]
  },
  "gemma4-26b-fp8-generate-arm10": {
   "k": 88,
   "k_max": 90,
   "k_min": 88,
   "lenient": 89,
   "median_s": 0.06399377600064327,
   "n": 108,
   "p95_s": 0.0863841649988899,
   "pct": 81.48148148148148,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-addenda/gemma4-fp8-generate.c.jsonl@38bc9887",
   "ties": 2,
   "timed_rows": 108,
   "unparsed": 2,
   "wilson": [
    0.7312030524810644,
    0.8768004617525846
   ]
  },
  "gemma4-26b-fp8-readout-arm10": {
   "k": 87,
   "k_max": 87,
   "k_min": 87,
   "median_s": 0.12012498699914431,
   "n": 108,
   "p95_s": 0.15091871700133197,
   "pct": 80.55555555555556,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-addenda/gemma4-fp8-readout.c.jsonl@38bc9887",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.7209648351663079,
    0.8691562227663517
   ]
  },
  "gemma4-26b-q4-base-generate-arm1": {
   "k": 88,
   "k_max": 89,
   "k_min": 88,
   "lenient": 89,
   "median_s": 1.0600900474996706,
   "n": 108,
   "p95_s": 1.115976559000046,
   "pct": 81.48148148148148,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows/base-generate.c.jsonl@0edb2527",
   "ties": 1,
   "timed_rows": 108,
   "unparsed": 1,
   "wilson": [
    0.7312030524810644,
    0.8768004617525846
   ]
  },
  "gemma4-26b-q4-base-readout-arm1": {
   "k": 87,
   "k_max": 87,
   "k_min": 87,
   "median_s": 1.025189576999992,
   "n": 108,
   "p95_s": 1.1156769910003277,
   "pct": 80.55555555555556,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows/base-readout.c.jsonl@0edb2527",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.7209648351663079,
    0.8691562227663517
   ]
  },
  "imajev-4b-cpu": {
   "k": 68,
   "k_max": 68,
   "k_min": 68,
   "median_s": 18.134194921993185,
   "n": 108,
   "p95_s": 25.395194076001644,
   "pct": 62.96296296296296,
   "refused": {},
   "src": "bench/imajev-2026-09-28/rows/imajev-4b-cpu.kc.jsonl@9e86a063",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.5355697359856005,
    0.7147846522282552
   ]
  },
  "jevify-generate-arm1": {
   "k": 90,
   "k_max": 90,
   "k_min": 90,
   "lenient": 90,
   "median_s": 1.0307518329998402,
   "n": 108,
   "p95_s": 1.090021724000053,
   "pct": 83.33333333333333,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows/jev-generate.c.jsonl@0edb2527",
   "ties": 0,
   "timed_rows": 108,
   "unparsed": 0,
   "wilson": [
    0.7518731501613105,
    0.8918952766743182
   ]
  },
  "jevify-readout-arm1": {
   "k": 90,
   "k_max": 90,
   "k_min": 90,
   "median_s": 1.0169441044999985,
   "n": 108,
   "p95_s": 1.0930389940001533,
   "pct": 83.33333333333333,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows/jev-readout.c.jsonl@0edb2527",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.7518731501613105,
    0.8918952766743182
   ]
  },
  "kev-4b": {
   "k": 60,
   "k_max": null,
   "k_min": null,
   "median_s": 0.0699436769998556,
   "n": 108,
   "p95_s": 0.08270423800013305,
   "pct": 55.55555555555556,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kc.jsonl@c19fed10",
   "ties": null,
   "timed_rows": 108,
   "wilson": [
    0.46153618205059904,
    0.6457585557553391
   ]
  },
  "kev-9b": {
   "k": 72,
   "k_max": null,
   "k_min": null,
   "median_s": 0.10623538700019708,
   "n": 108,
   "p95_s": 0.13746007200006716,
   "pct": 66.66666666666667,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kc.jsonl@c19fed10",
   "ties": null,
   "timed_rows": 108,
   "wilson": [
    0.5733891673119,
    0.7484950461059141
   ]
  },
  "lev-4b-cpu": {
   "k": 66,
   "k_max": 66,
   "k_min": 66,
   "median_s": 8.465369219484273,
   "n": 108,
   "p95_s": 10.696374714025296,
   "pct": 61.111111111111114,
   "refused": {},
   "src": "bench/lev-2026-09-28/l1/rows/lev-4b-cpu.kc.jsonl@17d9c043",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.5168659638261305,
    0.6977235117857458
   ]
  },
  "lev-4b-gpu": {
   "k": 66,
   "k_max": 66,
   "k_min": 66,
   "median_s": 0.21668299049997586,
   "n": 108,
   "p95_s": 0.2495980049998252,
   "pct": 61.111111111111114,
   "refused": {},
   "src": "bench/lev-2026-09-28/rows/lev-4b.kc.jsonl@17d9c043",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.5168659638261305,
    0.6977235117857458
   ]
  },
  "mistral-small3.2-24b-generate-coi": {
   "k": 73,
   "k_max": 73,
   "k_min": 73,
   "lenient": 73,
   "median_s": 0.6191059590000805,
   "n": 108,
   "p95_s": 0.752220173000751,
   "pct": 67.5925925925926,
   "refused": {},
   "src": "bench/cost-of-intelligence-2026-09-23/benchbox-3090-reading-0/rows/b-mistral-small3.2-24b/b-mistral-small3.2-24b.c.jsonl@6b73f291",
   "ties": 0,
   "timed_rows": 108,
   "unparsed": 0,
   "wilson": [
    0.5829333238469441,
    0.7568333458718599
   ]
  },
  "naive-bayes": {
   "carried_mismatch": 0,
   "k": 68,
   "k_max": 68,
   "k_min": 68,
   "median_s": null,
   "n": 108,
   "p95_s": null,
   "pct": 62.96296296296296,
   "refused": {},
   "src": "bench/deem-2026-09-27/t1-tune/rows/t1-baseline-nb.s0.P.rep1.jsonl@1b4f1e14",
   "ties": 0,
   "timed_rows": 0,
   "wilson": [
    0.5355697359856005,
    0.7147846522282552
   ]
  },
  "opendecider-base-qwen3-4b": {
   "carried_mismatch": 0,
   "k": 46,
   "k_max": 46,
   "k_min": 46,
   "median_s": 0.0637074435003342,
   "n": 108,
   "p95_s": 0.06809130100009497,
   "pct": 42.592592592592595,
   "refused": {},
   "src": "bench/opendecider-2026-09-28/rows/o2c-gpu-qwen3-4b-base.s0.P.rep1.jsonl@34555419",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.33679220437956364,
    0.5201481452125187
   ]
  },
  "opendecider-nano": {
   "carried_mismatch": 0,
   "k": 13,
   "k_max": 13,
   "k_min": 13,
   "median_s": 0.1989272905047983,
   "n": 108,
   "p95_s": 0.24583612597780302,
   "pct": 12.037037037037036,
   "refused": {},
   "src": "bench/opendecider-2026-09-28/rows/o1-cpu-nano.s0.P.rep1.jsonl@95cfcd88",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.07171063993921549,
    0.19510865172020736
   ]
  },
  "opendecider-small": {
   "carried_mismatch": 0,
   "k": 46,
   "k_max": 46,
   "k_min": 46,
   "median_s": 0.06280998950001049,
   "n": 108,
   "p95_s": 0.06711989500035997,
   "pct": 42.592592592592595,
   "refused": {},
   "src": "bench/opendecider-2026-09-28/rows/o2-gpu-small.s0.P.rep1.jsonl@34555419",
   "ties": 4,
   "timed_rows": 108,
   "wilson": [
    0.33679220437956364,
    0.5201481452125187
   ]
  },
  "openjev-bf16-readout-arm3": {
   "k": 101,
   "k_max": 101,
   "k_min": 101,
   "median_s": 0.09194966300856322,
   "n": 108,
   "p95_s": 0.09632052597589791,
   "pct": 93.51851851851852,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-arm3/openjev-bf16-largecard.c.jsonl@1429a7d5",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.8722236366714627,
    0.9682518094750524
   ]
  },
  "openjev-fp8-generate-arm2": {
   "k": 101,
   "k_max": 101,
   "k_min": 101,
   "lenient": 101,
   "median_s": 0.39154322000013053,
   "n": 108,
   "p95_s": 0.45946651000122074,
   "pct": 93.51851851851852,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows/openjev-fp8-generate.c.jsonl@3af7a71c",
   "ties": 0,
   "timed_rows": 108,
   "unparsed": 0,
   "wilson": [
    0.8722236366714627,
    0.9682518094750524
   ]
  },
  "openjev-fp8-readout-arm2": {
   "k": 101,
   "k_max": 101,
   "k_min": 101,
   "median_s": 0.3502586775002783,
   "n": 108,
   "p95_s": 0.41729696099992,
   "pct": 93.51851851851852,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows/openjev-fp8-readout.c.jsonl@3af7a71c",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.8722236366714627,
    0.9682518094750524
   ]
  },
  "openjev-fp8-readout-arm3": {
   "k": 101,
   "k_max": 101,
   "k_min": 101,
   "median_s": 0.08652337003150024,
   "n": 108,
   "p95_s": 0.0880551990121603,
   "pct": 93.51851851851852,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-arm3/openjev-fp8-largecard.c.jsonl@1429a7d5",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.8722236366714627,
    0.9682518094750524
   ]
  },
  "openjev-q4km-generate-arm13": {
   "k": 99,
   "k_max": 99,
   "k_min": 99,
   "lenient": 99,
   "median_s": 1.1413375690026442,
   "n": 108,
   "p95_s": 1.2151111560015124,
   "pct": 91.66666666666667,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-generate.c.jsonl@95d6ae9a",
   "ties": 0,
   "timed_rows": 108,
   "unparsed": 0,
   "wilson": [
    0.8491709381195168,
    0.9555395954250189
   ]
  },
  "openjev-q4km-readout-arm13": {
   "k": 99,
   "k_max": 99,
   "k_min": 99,
   "median_s": 1.1312006575008127,
   "n": 108,
   "p95_s": 1.1958460369969544,
   "pct": 91.66666666666667,
   "refused": {},
   "src": "bench/jev-2026-09-21/rows-oj-gguf/openjev-q4km-readout.c.jsonl@95d6ae9a",
   "ties": 0,
   "timed_rows": 108,
   "wilson": [
    0.8491709381195168,
    0.9555395954250189
   ]
  }
 },
 "s0_minus_h48": {
  "apus-4b-high": {
   "diff_pts": 1.620370370370372,
   "h48": "29/48",
   "newcombe": [
    -14.07978951679589,
    18.13628323829176
   ],
   "s0": "67/108",
   "s0_from": "apus-4b-high"
  },
  "apus-4b-low": {
   "diff_pts": -1.8518518518518516,
   "h48": "12/48",
   "newcombe": [
    -17.28864364423293,
    11.524130119165651
   ],
   "s0": "25/108",
   "s0_from": "apus-4b-low"
  },
  "apus-9b-high": {
   "diff_pts": -3.472222222222221,
   "h48": "31/48",
   "newcombe": [
    -18.717362663926014,
    13.113242263505533
   ],
   "s0": "66/108",
   "s0_from": "apus-9b-high"
  },
  "apus-9b-low": {
   "diff_pts": -3.472222222222221,
   "h48": "15/48",
   "newcombe": [
    -19.46220228503685,
    11.039319533667562
   ],
   "s0": "30/108",
   "s0_from": "apus-9b-low"
  },
  "deem-0.8-d0": {
   "diff_pts": -3.7037037037037033,
   "h48": "4/48",
   "newcombe": [
    -15.229195837205697,
    3.9478918695296947
   ],
   "s0": "5/108",
   "s0_from": "deem-0.8-d0"
  },
  "imajev-4b-gpu": {
   "diff_pts": 0.462962962962965,
   "h48": "30/48",
   "newcombe": [
    -15.009183193271799,
    16.9667574978981
   ],
   "s0": "68/108",
   "s0_from": "imajev-4b-cpu"
  },
  "naive-bayes": {
   "diff_pts": 6.7129629629629655,
   "h48": "27/48",
   "newcombe": [
    -9.372496793371296,
    23.07798410116628
   ],
   "s0": "68/108",
   "s0_from": "naive-bayes"
  },
  "opendecider-base-qwen3-4b": {
   "diff_pts": -11.574074074074069,
   "h48": "26/48",
   "newcombe": [
    -27.549842637881138,
    5.198234138623822
   ],
   "s0": "46/108",
   "s0_from": "opendecider-base-qwen3-4b"
  },
  "opendecider-nano": {
   "diff_pts": 1.6203703703703694,
   "h48": "5/48",
   "newcombe": [
    -11.098039018299485,
    11.132751018819606
   ],
   "s0": "13/108",
   "s0_from": "opendecider-nano"
  },
  "opendecider-small": {
   "diff_pts": -9.490740740740744,
   "h48": "25/48",
   "newcombe": [
    -25.622251920984375,
    7.18119859504423
   ],
   "s0": "46/108",
   "s0_from": "opendecider-small"
  },
  "openjev-q4km-readout-arm13": {
   "diff_pts": 0.0,
   "h48": "44/48",
   "newcombe": [
    -8.426644137898396,
    11.874288950474297
   ],
   "s0": "99/108",
   "s0_from": "openjev-q4km-readout-arm13"
  }
 },
 "task_a": {
  "apus-4b-high": {
   "median_s": 1.3551960540025902,
   "p95_s": 1.9409355809984845,
   "reached": 37,
   "right_of_reached": 37,
   "rows": 63,
   "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.ka.jsonl@3a84d4b3",
   "unreached": {
    "kev-422": 16,
    "overlong": 10
   }
  },
  "apus-9b-high": {
   "median_s": 1.88302393900085,
   "p95_s": 2.68312270000024,
   "reached": 37,
   "right_of_reached": 37,
   "rows": 63,
   "src": "bench/apus-2026-09-28/rows/apus-9b-high.ka.jsonl@e804fe0a",
   "unreached": {
    "kev-422": 16,
    "overlong": 10
   }
  },
  "gemma4-26b-fp8-readout-arm10": {
   "median_s": 2.3914529440007755,
   "p95_s": 5.729637040996749,
   "reached": 53,
   "right_of_reached": 53,
   "rows": 63,
   "src": "bench/jev-2026-09-21/rows-addenda/gemma4-fp8-readout.a.jsonl@38bc9887",
   "unreached": {
    "overlong": 10
   }
  },
  "kev-4b": {
   "median_s": 1.01545748500007,
   "p95_s": 1.5858508110000002,
   "reached": 38,
   "right_of_reached": 38,
   "rows": 63,
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.ka.jsonl@c19fed10",
   "unreached": {
    "kev-422": 15,
    "overlong": 10
   }
  },
  "kev-9b": {
   "median_s": 1.59350118999987,
   "p95_s": 2.5027132430000165,
   "reached": 38,
   "right_of_reached": 38,
   "rows": 63,
   "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.ka.jsonl@c19fed10",
   "unreached": {
    "kev-422": 15,
    "overlong": 10
   }
  },
  "lev-4b-gpu": {
   "median_s": 1.7306802479997714,
   "p95_s": 4.217633428001136,
   "reached": 53,
   "right_of_reached": 52,
   "rows": 63,
   "src": "bench/lev-2026-09-28/rows/lev-4b.ka.jsonl@17d9c043",
   "unreached": {
    "overlong": 10
   }
  },
  "openjev-fp8-readout-32k-along": {
   "median_s": 32.67685409099977,
   "p95_s": 36.21091528699981,
   "reached": 6,
   "right_of_reached": 6,
   "rows": 10,
   "src": "bench/jev-2026-09-21/rows-addenda/openjev-fp8-readout-32k.along.jsonl@9879cab6",
   "unreached": {
    "overlong": 4
   }
  },
  "openjev-fp8-readout-arm2-16k": {
   "median_s": 6.458181296000475,
   "p95_s": 16.003029303999938,
   "reached": 53,
   "right_of_reached": 53,
   "rows": 63,
   "src": "bench/jev-2026-09-21/rows/openjev-fp8-readout.a.jsonl@3af7a71c",
   "unreached": {
    "overlong": 10
   }
  }
 }
}