{
 "base": {
  "differences_with_cites_masked": 0,
  "differences_with_cites_raw": 0,
  "first_differences_masked": [],
  "hub_kit": {
   "differ_in_both": [],
   "equal_in_both": 2821,
   "in_both": 2821,
   "numeric_leaves_kit": 2821,
   "numeric_leaves_record": 2821,
   "only_in_kit": 0,
   "only_in_record": 0
  },
  "record_ref": "f2073219",
  "reproduced": true,
  "rerun_ref": "a2c90bf0"
 },
 "chains_pair": {
  "head": "6bb4c673",
  "pairs": {
   "ka": {
    "answered_in_both": 53,
    "files": {
     "von-cpu": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.ka.jsonl@6bb4c673",
     "von-cpu-nochains": "bench/deciders-von-laya-2026-10-01/rows/von-cpu-nochains.ka.jsonl@6bb4c673"
    },
    "groups": {
     "gate_open.cut_in_both": {
      "both_right": 0,
      "both_wrong": 0,
      "changed_answer": 0,
      "changed_ids": [],
      "max_abs_probability_difference": null,
      "mcnemar_exact_p_DESCRIPTIVE": null,
      "right_only_with_chains": 0,
      "right_only_without_chains": 0,
      "right_with_chains": 0,
      "right_without_chains": 0,
      "rows": 0,
      "same_answer": 0
     },
     "gate_open.page_read_differently": {
      "ids": [],
      "rows": 0
     },
     "gate_open.whole_in_both": {
      "both_right": 2,
      "both_wrong": 0,
      "changed_answer": 3,
      "changed_ids": [
       "off-10",
       "off-17",
       "on-03"
      ],
      "max_abs_probability_difference": 0.6426000000000001,
      "mcnemar_exact_p_DESCRIPTIVE": 1.0,
      "right_only_with_chains": 2,
      "right_only_without_chains": 1,
      "right_with_chains": 4,
      "right_without_chains": 3,
      "rows": 5,
      "same_answer": 2
     },
     "gate_shut.cut_in_both": {
      "both_right": 9,
      "both_wrong": 6,
      "changed_answer": 0,
      "changed_ids": [],
      "max_abs_probability_difference": 0.0,
      "mcnemar_exact_p_DESCRIPTIVE": 1.0,
      "right_only_with_chains": 0,
      "right_only_without_chains": 0,
      "right_with_chains": 9,
      "right_without_chains": 9,
      "rows": 15,
      "same_answer": 15
     },
     "gate_shut.page_read_differently": {
      "ids": [],
      "rows": 0
     },
     "gate_shut.whole_in_both": {
      "both_right": 22,
      "both_wrong": 11,
      "changed_answer": 0,
      "changed_ids": [],
      "max_abs_probability_difference": 0.0,
      "mcnemar_exact_p_DESCRIPTIVE": 1.0,
      "right_only_with_chains": 0,
      "right_only_without_chains": 0,
      "right_with_chains": 22,
      "right_without_chains": 22,
      "rows": 33,
      "same_answer": 33
     }
    },
    "items": 63,
    "refused_by_reason": {
     "von-cpu": {
      "overlong": 10
     },
     "von-cpu-nochains": {
      "overlong": 10
     }
    }
   },
   "kj": {
    "answered_in_both": 119,
    "files": {
     "von-cpu": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kj.jsonl@6bb4c673",
     "von-cpu-nochains": "bench/deciders-von-laya-2026-10-01/rows/von-cpu-nochains.kj.jsonl@6bb4c673"
    },
    "groups": {
     "gate_open": {
      "both_right": 48,
      "both_wrong": 49,
      "changed_answer": 0,
      "changed_ids": [],
      "max_abs_probability_difference": 0.18220000000000003,
      "mcnemar_exact_p_DESCRIPTIVE": 1.0,
      "right_only_with_chains": 0,
      "right_only_without_chains": 0,
      "right_with_chains": 48,
      "right_without_chains": 48,
      "rows": 97,
      "same_answer": 97
     },
     "gate_shut": {
      "both_right": 10,
      "both_wrong": 12,
      "changed_answer": 0,
      "changed_ids": [],
      "max_abs_probability_difference": 0.0,
      "mcnemar_exact_p_DESCRIPTIVE": 1.0,
      "right_only_with_chains": 0,
      "right_only_without_chains": 0,
      "right_with_chains": 10,
      "right_without_chains": 10,
      "rows": 22,
      "same_answer": 22
     }
    },
    "items": 119,
    "refused_by_reason": {
     "von-cpu": {},
     "von-cpu-nochains": {}
    }
   }
  },
  "rows": "bench/deciders-von-laya-2026-10-01/rows",
  "what": "DV-4: Von with its chain engine on (as shipped) beside off; descriptive, no bar, no test"
 },
 "checked": {
  "bound_on_0_of_12": {
   "laya-cpu": {
    "of": 12,
    "refused": 12,
    "wilson": [
     0.7575059933447592,
     1.0
    ]
   },
   "von-cpu": {
    "of": 12,
    "refused": 0,
    "wilson": [
     2.7755575615628914e-17,
     0.24249400665524085
    ]
   }
  },
  "heldout_interval_widths": {
   "near_the_word_counter": {
    "kit_advantage_that_would_clear_zero_points": [
     17.129629629629633,
     18.287037037037035
    ],
    "lower_arm_points": [
     15.245140441703793,
     15.700159887166262
    ],
    "models": [
     "apus-4b-high",
     "apus-9b-high",
     "imajev-4b-gpu"
    ],
    "rule": "small deciders with a held-out reading whose six-way count is within 5 lines of the word counter's 68",
    "width_points": [
     31.830604927431548,
     32.21607275508765
    ]
   },
   "rows": {
    "apus-4b-high": {
     "diff_points": 1.620370370370372,
     "held_out": [
      29,
      48
     ],
     "held_out_count_that_would_clear_zero": 21,
     "kit_advantage_that_would_clear_zero_points": 18.287037037037035,
     "lower_arm_points": 15.700159887166262,
     "newcombe": [
      -14.07978951679589,
      18.13628323829176
     ],
     "six_way": [
      67,
      108
     ],
     "width_points": 32.21607275508765
    },
    "apus-4b-low": {
     "diff_points": -1.8518518518518516,
     "held_out": [
      12,
      48
     ],
     "held_out_count_that_would_clear_zero": 4,
     "kit_advantage_that_would_clear_zero_points": 14.814814814814813,
     "lower_arm_points": 15.43679179238108,
     "newcombe": [
      -17.28864364423293,
      11.524130119165651
     ],
     "six_way": [
      25,
      108
     ],
     "width_points": 28.81277376339858
    },
    "apus-9b-high": {
     "diff_points": -3.472222222222221,
     "held_out": [
      31,
      48
     ],
     "held_out_count_that_would_clear_zero": 21,
     "kit_advantage_that_would_clear_zero_points": 17.361111111111114,
     "lower_arm_points": 15.245140441703793,
     "newcombe": [
      -18.717362663926014,
      13.113242263505533
     ],
     "six_way": [
      66,
      108
     ],
     "width_points": 31.830604927431548
    },
    "apus-9b-low": {
     "diff_points": -3.472222222222221,
     "held_out": [
      15,
      48
     ],
     "held_out_count_that_would_clear_zero": 6,
     "kit_advantage_that_would_clear_zero_points": 15.277777777777779,
     "lower_arm_points": 15.989980062814627,
     "newcombe": [
      -19.46220228503685,
      11.039319533667562
     ],
     "six_way": [
      30,
      108
     ],
     "width_points": 30.50152181870441
    },
    "deem-0.8-d0": {
     "diff_points": -3.7037037037037033,
     "held_out": [
      4,
      48
     ],
     "held_out_count_that_would_clear_zero": null,
     "kit_advantage_that_would_clear_zero_points": null,
     "lower_arm_points": 11.525492133501995,
     "newcombe": [
      -15.229195837205697,
      3.9478918695296947
     ],
     "six_way": [
      5,
      108
     ],
     "width_points": 19.177087706735392
    },
    "imajev-4b-gpu": {
     "diff_points": 0.462962962962965,
     "held_out": [
      30,
      48
     ],
     "held_out_count_that_would_clear_zero": 22,
     "kit_advantage_that_would_clear_zero_points": 17.129629629629633,
     "lower_arm_points": 15.472146156234764,
     "newcombe": [
      -15.009183193271799,
      16.9667574978981
     ],
     "six_way": [
      68,
      108
     ],
     "width_points": 31.9759406911699
    },
    "laya-cpu": {
     "diff_points": 0.9259259259259273,
     "held_out": [
      8,
      48
     ],
     "held_out_count_that_would_clear_zero": 2,
     "kit_advantage_that_would_clear_zero_points": 13.425925925925927,
     "lower_arm_points": 14.250186135096236,
     "newcombe": [
      -13.324260209170308,
      12.402705627696019
     ],
     "six_way": [
      19,
      108
     ],
     "width_points": 25.726965836866327
    },
    "naive-bayes": {
     "diff_points": 6.7129629629629655,
     "held_out": [
      27,
      48
     ],
     "held_out_count_that_would_clear_zero": 22,
     "kit_advantage_that_would_clear_zero_points": 17.129629629629633,
     "lower_arm_points": 16.08545975633426,
     "newcombe": [
      -9.372496793371296,
      23.07798410116628
     ],
     "six_way": [
      68,
      108
     ],
     "width_points": 32.45048089453758
    },
    "opendecider-base-qwen3-4b": {
     "diff_points": -11.574074074074069,
     "held_out": [
      26,
      48
     ],
     "held_out_count_that_would_clear_zero": 12,
     "kit_advantage_that_would_clear_zero_points": 17.59259259259259,
     "lower_arm_points": 15.975768563807069,
     "newcombe": [
      -27.549842637881138,
      5.198234138623822
     ],
     "six_way": [
      46,
      108
     ],
     "width_points": 32.74807677650496
    },
    "opendecider-nano": {
     "diff_points": 1.6203703703703694,
     "held_out": [
      5,
      48
     ],
     "held_out_count_that_would_clear_zero": 0,
     "kit_advantage_that_would_clear_zero_points": 12.037037037037036,
     "lower_arm_points": 12.718409388669855,
     "newcombe": [
      -11.098039018299485,
      11.132751018819606
     ],
     "six_way": [
      13,
      108
     ],
     "width_points": 22.230790037119093
    },
    "opendecider-small": {
     "diff_points": -9.490740740740744,
     "held_out": [
      25,
      48
     ],
     "held_out_count_that_would_clear_zero": 12,
     "kit_advantage_that_would_clear_zero_points": 17.59259259259259,
     "lower_arm_points": 16.13151118024363,
     "newcombe": [
      -25.622251920984375,
      7.18119859504423
     ],
     "six_way": [
      46,
      108
     ],
     "width_points": 32.803450516028605
    },
    "openjev-q4km-readout-arm13": {
     "diff_points": 0.0,
     "held_out": [
      44,
      48
     ],
     "held_out_count_that_would_clear_zero": 38,
     "kit_advantage_that_would_clear_zero_points": 12.5,
     "lower_arm_points": 8.426644137898396,
     "newcombe": [
      -8.426644137898396,
      11.874288950474297
     ],
     "six_way": [
      99,
      108
     ],
     "width_points": 20.300933088372695
    },
    "von-cpu": {
     "diff_points": -5.787037037037038,
     "held_out": [
      5,
      48
     ],
     "held_out_count_that_would_clear_zero": null,
     "kit_advantage_that_would_clear_zero_points": null,
     "lower_arm_points": 12.042820286961796,
     "newcombe": [
      -17.829857323998834,
      2.442336919733332
     ],
     "six_way": [
      5,
      108
     ],
     "width_points": 20.272194243732166
    }
   },
   "what": "the width of each six-way-minus-held-out Newcombe interval (s0_minus_h48), its lower arm (the observed difference less the lower limit), and, holding the six-way count, the smallest kit advantage whose interval would clear zero",
   "word_counter": [
    68,
    27
   ]
  },
  "holm": {
   "ahead_after_holm": [
    "openjev-bf16-readout-arm3",
    "openjev-q4km-generate-arm13",
    "jevify-readout-arm1",
    "gemma4-26b-q4-base-generate-arm1"
   ],
   "alpha": 0.05,
   "behind_after_holm": [
    "opendecider-small",
    "opendecider-base-qwen3-4b",
    "apus-9b-low",
    "apus-4b-low",
    "opendecider-nano",
    "deem-0.8-d0",
    "von-cpu",
    "laya-cpu"
   ],
   "largest_holm_p_told_apart": 0.018792966005968026,
   "rows": {
    "apus-4b-high": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 19,
     "only_this_right": 18,
     "p": 1.0,
     "right_of_108": 67,
     "side": "behind",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "apus-4b-low": {
     "holm_p": 1.5387558960355818e-06,
     "only_the_word_counter_right": 55,
     "only_this_right": 12,
     "p": 1.0273195443541005e-07,
     "right_of_108": 25,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "apus-9b-high": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 20,
     "only_this_right": 18,
     "p": 0.8714146793645341,
     "right_of_108": 66,
     "side": "behind",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "apus-9b-low": {
     "holm_p": 4.939164401273916e-05,
     "only_the_word_counter_right": 53,
     "only_this_right": 15,
     "p": 4.11597033439493e-06,
     "right_of_108": 30,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "deem-0.8-d0": {
     "holm_p": 5.559517689962545e-16,
     "only_the_word_counter_right": 65,
     "only_this_right": 2,
     "p": 3.088620938868081e-17,
     "right_of_108": 5,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "gemma4-26b-q4-base-generate-arm1": {
     "holm_p": 0.011932429624721408,
     "only_the_word_counter_right": 8,
     "only_this_right": 28,
     "p": 0.0011932429624721408,
     "right_of_108": 88,
     "side": "ahead",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "imajev-4b-cpu": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 21,
     "only_this_right": 21,
     "p": 1.0,
     "right_of_108": 68,
     "side": "level",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "jevify-readout-arm1": {
     "holm_p": 0.00343806273303926,
     "only_the_word_counter_right": 7,
     "only_this_right": 29,
     "p": 0.00031255115754902363,
     "right_of_108": 90,
     "side": "ahead",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "kev-4b": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 25,
     "only_this_right": 17,
     "p": 0.27995623852893914,
     "right_of_108": 60,
     "side": "behind",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "kev-9b": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 14,
     "only_this_right": 18,
     "p": 0.5966148958541453,
     "right_of_108": 72,
     "side": "ahead",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "laya-cpu": {
     "holm_p": 5.087485988042317e-12,
     "only_the_word_counter_right": 51,
     "only_this_right": 2,
     "p": 3.1796787425264483e-13,
     "right_of_108": 19,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "lev-4b-gpu": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 20,
     "only_this_right": 18,
     "p": 0.8714146793645341,
     "right_of_108": 66,
     "side": "behind",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "mistral-small3.2-24b-generate-coi": {
     "holm_p": 1.0,
     "only_the_word_counter_right": 16,
     "only_this_right": 21,
     "p": 0.5113757815852296,
     "right_of_108": 73,
     "side": "ahead",
     "told_apart_after_holm": false,
     "told_apart_uncorrected": false
    },
    "opendecider-base-qwen3-4b": {
     "holm_p": 0.018792966005968026,
     "only_the_word_counter_right": 35,
     "only_this_right": 13,
     "p": 0.0020881073339964473,
     "right_of_108": 46,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "opendecider-nano": {
     "holm_p": 1.3683498778505054e-14,
     "only_the_word_counter_right": 56,
     "only_this_right": 1,
     "p": 8.049116928532385e-16,
     "right_of_108": 13,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "opendecider-small": {
     "holm_p": 0.018792966005968026,
     "only_the_word_counter_right": 35,
     "only_this_right": 13,
     "p": 0.0020881073339964473,
     "right_of_108": 46,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "openjev-bf16-readout-arm3": {
     "holm_p": 1.5387558960355818e-06,
     "only_the_word_counter_right": 4,
     "only_this_right": 37,
     "p": 1.0258372640237212e-07,
     "right_of_108": 101,
     "side": "ahead",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "openjev-q4km-generate-arm13": {
     "holm_p": 1.0194044079980813e-05,
     "only_the_word_counter_right": 5,
     "only_this_right": 36,
     "p": 7.84157236921601e-07,
     "right_of_108": 99,
     "side": "ahead",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    },
    "von-cpu": {
     "holm_p": 6.797947621484113e-17,
     "only_the_word_counter_right": 64,
     "only_this_right": 1,
     "p": 3.577867169202165e-18,
     "right_of_108": 5,
     "side": "behind",
     "told_apart_after_holm": true,
     "told_apart_uncorrected": true
    }
   },
   "smallest_holm_p_not_told_apart": 1.0,
   "tests": 19,
   "the_page_reads": {
    "at_or_above_gemma4": 88,
    "below": 60,
    "every_row_at_or_above_it_ahead_after_holm": true,
    "every_row_below_it_behind_after_holm": true,
    "no_row_between_told_apart_after_holm": true
   },
   "told_apart_after_holm": 12,
   "told_apart_uncorrected": 12,
   "what": "Holm's step-down correction over the seventeen exact McNemar tests against the word counter: the page's word-counter table row by row, and deem-0.8-v1 in its caption"
  },
  "label": "descriptive, computed after the rows were seen: the page's own checks of what it prints",
  "note_on_holm_what": "the recount's own `what` string says seventeen; this run's `tests` field is the count it corrected for",
  "paired_interval_vs_word_counter": {
   "laya-cpu": {
    "both_right": 17,
    "edge_lines": -49,
    "edge_points": -45.370370370370374,
    "model_right": 19,
    "n": 108,
    "neither_right": 38,
    "only_model_right": 2,
    "only_word_counter_right": 51,
    "phi_as_computed": {
     "lines": [
      -58.8292331312474,
      -37.30555046279571
     ],
     "phi": 0.253654542336657,
     "points": [
      -54.471512158562405,
      -34.54217635444047
     ]
    },
    "phi_continuity_corrected": {
     "lines": [
      -58.981496248991064,
      -37.112107864191245
     ],
     "phi": 0.22847559879588591,
     "points": [
      -54.61249652684358,
      -34.363062837214116
     ]
    },
    "word_counter_right": 68
   },
   "von-cpu": {
    "both_right": 4,
    "edge_lines": -63,
    "edge_points": -58.333333333333336,
    "model_right": 5,
    "n": 108,
    "neither_right": 39,
    "only_model_right": 1,
    "only_word_counter_right": 64,
    "phi_as_computed": {
     "lines": [
      -72.4135603316242,
      -51.51154095113451
     ],
     "phi": 0.07773198546814626,
     "points": [
      -67.04959289965203,
      -47.69587125105048
     ]
    },
    "phi_continuity_corrected": {
     "lines": [
      -72.53962144661338,
      -51.26356067779189
     ],
     "phi": 0.0321066896498865,
     "points": [
      -67.16631615427166,
      -47.466259886844334
     ]
    },
    "word_counter_right": 68
   }
  },
  "paired_shuffle_counts": {
   "moved": {
    "apus-9b-high": 41,
    "kev-9b": 33,
    "laya-cpu": 52,
    "lev-4b-gpu": 27,
    "von-cpu": 0
   },
   "pairs": {
    "apus-9b-high|laya-cpu": {
     "only_first_moved": 23,
     "only_second_moved": 34,
     "p": 0.18485048241641847,
     "told_apart": false
    },
    "apus-9b-high|von-cpu": {
     "only_first_moved": 41,
     "only_second_moved": 0,
     "p": 9.094947017729282e-13,
     "told_apart": true
    },
    "kev-9b|apus-9b-high": {
     "only_first_moved": 12,
     "only_second_moved": 20,
     "p": 0.21532714972272515,
     "told_apart": false
    },
    "kev-9b|laya-cpu": {
     "only_first_moved": 18,
     "only_second_moved": 37,
     "p": 0.014453833533827731,
     "told_apart": true
    },
    "kev-9b|von-cpu": {
     "only_first_moved": 33,
     "only_second_moved": 0,
     "p": 2.3283064365386963e-10,
     "told_apart": true
    },
    "lev-4b-gpu|apus-9b-high": {
     "only_first_moved": 8,
     "only_second_moved": 22,
     "p": 0.016124801710247993,
     "told_apart": true
    },
    "lev-4b-gpu|kev-9b": {
     "only_first_moved": 13,
     "only_second_moved": 19,
     "p": 0.37708558747544885,
     "told_apart": false
    },
    "lev-4b-gpu|laya-cpu": {
     "only_first_moved": 16,
     "only_second_moved": 41,
     "p": 0.0012635594724354632,
     "told_apart": true
    },
    "lev-4b-gpu|von-cpu": {
     "only_first_moved": 27,
     "only_second_moved": 0,
     "p": 1.4901161193847656e-08,
     "told_apart": true
    }
   },
   "src": {
    "apus-9b-high": "bench/apus-2026-09-28/rows/apus-9b-high.kperm.jsonl@e804fe0a",
    "kev-9b": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kperm.jsonl@c19fed10",
    "laya-cpu": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kperm.jsonl@6bb4c673",
    "lev-4b-gpu": "bench/lev-2026-09-28/rows/lev-4b.kperm.jsonl@17d9c043",
    "von-cpu": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kperm.jsonl@6bb4c673"
   },
   "what": "exact McNemar tests on which lines' answers moved over the six orders Kev's server generates, joined on each line's uid"
  },
  "resample_by_dinner": {
   "dinner_of_each_line": "the kit lock's sets.c.items_list, dinner_id joined on text_sha256",
   "dinners": 16,
   "kit_lock": "bench/cost-of-intelligence-2026-09-23/KIT.lock.json@cb5dc076",
   "lines_per_dinner": [
    5,
    9
   ],
   "paired_against_the_word_counter": {
    "apus-4b-high": {
     "by_dinner_bootstrap_95_points": [
      -11.504424778761061,
      10.0
     ],
     "by_dinner_bootstrap_p": 0.9265,
     "by_dinner_z_p": 0.8696698082267965,
     "design_effect": 0.9948709320578477,
     "diff_points": -0.9259259259259258,
     "only_the_word_counter_right": 19,
     "only_this_right": 18,
     "p": 1.0,
     "same_verdict": true,
     "se_by_dinner_points": 5.64320878563595,
     "se_lines_independent_points": 5.6577368969738835,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "apus-4b-low": {
     "by_dinner_bootstrap_95_points": [
      -54.20560747663551,
      -24.761904761904763
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 2.694595926111117e-07,
     "design_effect": 1.3881355214321869,
     "diff_points": -39.81481481481482,
     "only_the_word_counter_right": 55,
     "only_this_right": 12,
     "p": 1.0273195443541005e-07,
     "same_verdict": true,
     "se_by_dinner_points": 7.74058487964337,
     "se_lines_independent_points": 6.569886146018218,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "apus-9b-high": {
     "by_dinner_bootstrap_95_points": [
      -16.964285714285715,
      13.761467889908257
     ],
     "by_dinner_bootstrap_p": 0.8458,
     "by_dinner_z_p": 0.8187851862164792,
     "design_effect": 1.9887510901446463,
     "diff_points": -1.8518518518518516,
     "only_the_word_counter_right": 20,
     "only_this_right": 18,
     "p": 0.8714146793645341,
     "same_verdict": true,
     "se_by_dinner_points": 8.082886852048993,
     "se_lines_independent_points": 5.731605411313911,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "apus-9b-low": {
     "by_dinner_bootstrap_95_points": [
      -49.549549549549546,
      -20.192307692307693
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 5.521212883564516e-06,
     "design_effect": 1.26833389908008,
     "diff_points": -35.18518518518518,
     "only_the_word_counter_right": 53,
     "only_this_right": 15,
     "p": 4.11597033439493e-06,
     "same_verdict": true,
     "se_by_dinner_points": 7.743319184851198,
     "se_lines_independent_points": 6.875596177909747,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "deem-0.8-d0": {
     "by_dinner_bootstrap_95_points": [
      -68.96551724137932,
      -47.05882352941176
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 9.458289359947497e-24,
     "design_effect": 1.2877506830368783,
     "diff_points": -58.333333333333336,
     "only_the_word_counter_right": 65,
     "only_this_right": 2,
     "p": 3.088620938868081e-17,
     "same_verdict": true,
     "se_by_dinner_points": 5.805970399742018,
     "se_lines_independent_points": 5.1163344117045195,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "gemma4-26b-q4-base-generate-arm1": {
     "by_dinner_bootstrap_95_points": [
      6.779661016949152,
      28.846153846153843
     ],
     "by_dinner_bootstrap_p": 0.0035,
     "by_dinner_z_p": 0.0016516171011127182,
     "design_effect": 1.239295587353125,
     "diff_points": 18.51851851851852,
     "only_the_word_counter_right": 8,
     "only_this_right": 28,
     "p": 0.0011932429624721408,
     "same_verdict": true,
     "se_by_dinner_points": 5.885183998449995,
     "se_lines_independent_points": 5.286553039492774,
     "verdict": "ahead",
     "verdict_by_dinner": "ahead"
    },
    "imajev-4b-cpu": {
     "by_dinner_bootstrap_95_points": [
      -10.714285714285714,
      11.11111111111111
     ],
     "by_dinner_bootstrap_p": 1.0,
     "by_dinner_z_p": 1.0,
     "design_effect": 0.9058201058201059,
     "diff_points": 0.0,
     "only_the_word_counter_right": 21,
     "only_this_right": 21,
     "p": 1.0,
     "same_verdict": true,
     "se_by_dinner_points": 5.7377531054924695,
     "se_lines_independent_points": 6.028661208836021,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "jevify-readout-arm1": {
     "by_dinner_bootstrap_95_points": [
      9.734513274336283,
      31.132075471698112
     ],
     "by_dinner_bootstrap_p": 0.0002,
     "by_dinner_z_p": 0.0003038338586391174,
     "design_effect": 1.1661164116343934,
     "diff_points": 20.37037037037037,
     "only_the_word_counter_right": 7,
     "only_this_right": 29,
     "p": 0.00031255115754902363,
     "same_verdict": true,
     "se_by_dinner_points": 5.639623157195953,
     "se_lines_independent_points": 5.222508209938179,
     "verdict": "ahead",
     "verdict_by_dinner": "ahead"
    },
    "kev-4b": {
     "by_dinner_bootstrap_95_points": [
      -20.5607476635514,
      5.607476635514018
     ],
     "by_dinner_bootstrap_p": 0.2957,
     "by_dinner_z_p": 0.28047541422848576,
     "design_effect": 1.3146684792966732,
     "diff_points": -7.4074074074074066,
     "only_the_word_counter_right": 25,
     "only_this_right": 17,
     "p": 0.27995623852893914,
     "same_verdict": true,
     "se_by_dinner_points": 6.8634642876836764,
     "se_lines_independent_points": 5.985979884798393,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "kev-9b": {
     "by_dinner_bootstrap_95_points": [
      -6.666666666666667,
      14.285714285714285
     ],
     "by_dinner_bootstrap_p": 0.5508,
     "by_dinner_z_p": 0.5027389976492851,
     "design_effect": 1.1080367073935837,
     "diff_points": 3.7037037037037033,
     "only_the_word_counter_right": 14,
     "only_this_right": 18,
     "p": 0.5966148958541453,
     "same_verdict": true,
     "se_by_dinner_points": 5.5263788142705215,
     "se_lines_independent_points": 5.250051661372997,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "laya-cpu": {
     "by_dinner_bootstrap_95_points": [
      -57.89473684210527,
      -31.73076923076923
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 4.977169173924214e-11,
     "design_effect": 1.7901980639295942,
     "diff_points": -45.370370370370374,
     "only_the_word_counter_right": 51,
     "only_this_right": 2,
     "p": 3.1796787425264483e-13,
     "same_verdict": true,
     "se_by_dinner_points": 6.903988636127366,
     "se_lines_independent_points": 5.159997916994821,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "lev-4b-gpu": {
     "by_dinner_bootstrap_95_points": [
      -15.09433962264151,
      10.909090909090908
     ],
     "by_dinner_bootstrap_p": 0.8429,
     "by_dinner_z_p": 0.7873869956825472,
     "design_effect": 1.4350961669287476,
     "diff_points": -1.8518518518518516,
     "only_the_word_counter_right": 20,
     "only_this_right": 18,
     "p": 0.8714146793645341,
     "same_verdict": true,
     "se_by_dinner_points": 6.866205324396611,
     "se_lines_independent_points": 5.731605411313911,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "mistral-small3.2-24b-generate-coi": {
     "by_dinner_bootstrap_95_points": [
      -7.547169811320755,
      16.51376146788991
     ],
     "by_dinner_bootstrap_p": 0.5086,
     "by_dinner_z_p": 0.47451112108157595,
     "design_effect": 1.3171017080048923,
     "diff_points": 4.62962962962963,
     "only_the_word_counter_right": 16,
     "only_this_right": 21,
     "p": 0.5113757815852296,
     "same_verdict": true,
     "se_by_dinner_points": 6.473571597882387,
     "se_lines_independent_points": 5.640716842674981,
     "verdict": "not told apart",
     "verdict_by_dinner": "not told apart"
    },
    "opendecider-base-qwen3-4b": {
     "by_dinner_bootstrap_95_points": [
      -30.097087378640776,
      -10.2803738317757
     ],
     "by_dinner_bootstrap_p": 0.0001,
     "by_dinner_z_p": 0.0001078118215448873,
     "design_effect": 0.7348397202035236,
     "diff_points": -20.37037037037037,
     "only_the_word_counter_right": 35,
     "only_this_right": 13,
     "p": 0.0020881073339964473,
     "same_verdict": true,
     "se_by_dinner_points": 5.2605311452289545,
     "se_lines_independent_points": 6.136677276390238,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "opendecider-nano": {
     "by_dinner_bootstrap_95_points": [
      -61.53846153846154,
      -39.62264150943396
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 1.3934423346514936e-18,
     "design_effect": 1.3355605997344722,
     "diff_points": -50.92592592592593,
     "only_the_word_counter_right": 56,
     "only_this_right": 1,
     "p": 8.049116928532385e-16,
     "same_verdict": true,
     "se_by_dinner_points": 5.7883887225639,
     "se_lines_independent_points": 5.008710026968146,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "opendecider-small": {
     "by_dinner_bootstrap_95_points": [
      -31.30434782608696,
      -9.090909090909092
     ],
     "by_dinner_bootstrap_p": 0.0007,
     "by_dinner_z_p": 0.0005323395575539968,
     "design_effect": 0.9183164735526174,
     "diff_points": -20.37037037037037,
     "only_the_word_counter_right": 35,
     "only_this_right": 13,
     "p": 0.0020881073339964473,
     "same_verdict": true,
     "se_by_dinner_points": 5.880706059289429,
     "se_lines_independent_points": 6.136677276390238,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    },
    "openjev-bf16-readout-arm3": {
     "by_dinner_bootstrap_95_points": [
      19.81981981981982,
      40.77669902912621
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 2.6687036804424355e-08,
     "design_effect": 1.128107895589843,
     "diff_points": 30.555555555555557,
     "only_the_word_counter_right": 4,
     "only_this_right": 37,
     "p": 1.0258372640237212e-07,
     "same_verdict": true,
     "se_by_dinner_points": 5.493737251567234,
     "se_lines_independent_points": 5.172405504290938,
     "verdict": "ahead",
     "verdict_by_dinner": "ahead"
    },
    "openjev-q4km-generate-arm13": {
     "by_dinner_bootstrap_95_points": [
      18.421052631578945,
      38.67924528301887
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 5.130894807812189e-08,
     "design_effect": 0.9997334592049484,
     "diff_points": 28.703703703703702,
     "only_the_word_counter_right": 5,
     "only_this_right": 36,
     "p": 7.84157236921601e-07,
     "same_verdict": true,
     "se_by_dinner_points": 5.26991232774699,
     "se_lines_independent_points": 5.270614791487387,
     "verdict": "ahead",
     "verdict_by_dinner": "ahead"
    },
    "von-cpu": {
     "by_dinner_bootstrap_95_points": [
      -68.86792452830188,
      -47.474747474747474
     ],
     "by_dinner_bootstrap_p": 0.0,
     "by_dinner_z_p": 5.085887738809341e-25,
     "design_effect": 1.3041018730956455,
     "diff_points": -58.333333333333336,
     "only_the_word_counter_right": 64,
     "only_this_right": 1,
     "p": 3.577867169202165e-18,
     "same_verdict": true,
     "se_by_dinner_points": 5.646264499593516,
     "se_lines_independent_points": 4.944307347557364,
     "verdict": "behind",
     "verdict_by_dinner": "behind"
    }
   },
   "paired_design_effects_over_1": 14,
   "single_shares": {
    "kev-9b": {
     "by_dinner_bootstrap_95": [
      0.5862068965517241,
      0.7454545454545455
     ],
     "design_effect": 0.8790123456790123,
     "right_of_108": 72,
     "se_by_dinner": 0.0425284302313212,
     "se_independent": 0.045360921162651446,
     "wider_by_dinner": false,
     "wilson_95": [
      0.5733891673119,
      0.7484950461059141
     ]
    },
    "naive-bayes": {
     "by_dinner_bootstrap_95": [
      0.5339805825242718,
      0.7222222222222222
     ],
     "design_effect": 1.1362672476397964,
     "right_of_108": 68,
     "se_by_dinner": 0.049532364829568534,
     "se_independent": 0.04646744769809142,
     "wider_by_dinner": true,
     "wilson_95": [
      0.5355697359856005,
      0.7147846522282552
     ]
    },
    "openjev-bf16-readout-arm3": {
     "by_dinner_bootstrap_95": [
      0.8981481481481481,
      0.9716981132075472
     ],
     "design_effect": 0.6387483192763722,
     "right_of_108": 101,
     "se_by_dinner": 0.018933851163603604,
     "se_independent": 0.023690491645410393,
     "wider_by_dinner": false,
     "wilson_95": [
      0.8722236366714627,
      0.9682518094750524
     ]
    }
   },
   "single_shares_wider_by_dinner": 1,
   "verdicts_changed_by_dinner": 0,
   "what": "the six-way's 108 lines come from 16 dinners; each paired test against the word counter, and the three single-count Wilson intervals the page prints, with whole dinners resampled (20000 resamples, seed 20260929, percentile interval) and with a dinner-robust standard error; a verdict is ahead or behind at p < 0.05 by the exact test, and by dinner when the resampled 95 per cent interval excludes zero"
  },
  "tests_in_each_option_order": {
   "models": {
    "apus-4b-high": {
     "closest": {
      "order": 3,
      "p": 0.5327092552361137
     },
     "orders": [
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 18,
       "order": 1,
       "p": 1.0,
       "right": 67
      },
      {
       "only_the_word_counter_right": 24,
       "only_this_right": 20,
       "order": 2,
       "p": 0.651587827208914,
       "right": 64
      },
      {
       "only_the_word_counter_right": 23,
       "only_this_right": 18,
       "order": 3,
       "p": 0.5327092552361137,
       "right": 63
      },
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 22,
       "order": 4,
       "p": 0.755228657504631,
       "right": 71
      },
      {
       "only_the_word_counter_right": 23,
       "only_this_right": 21,
       "order": 5,
       "p": 0.880395821280672,
       "right": 66
      },
      {
       "only_the_word_counter_right": 25,
       "only_this_right": 20,
       "order": 6,
       "p": 0.5514843298025198,
       "right": 63
      }
     ],
     "src": "bench/apus-2026-09-28/a1b/rows/apus-4b-high.kperm.jsonl@3a84d4b3",
     "told_apart_in_any_order": false
    },
    "apus-9b-high": {
     "closest": {
      "order": 4,
      "p": 0.26818725105476915
     },
     "orders": [
      {
       "only_the_word_counter_right": 20,
       "only_this_right": 18,
       "order": 1,
       "p": 0.8714146793645341,
       "right": 66
      },
      {
       "only_the_word_counter_right": 16,
       "only_this_right": 16,
       "order": 2,
       "p": 1.0,
       "right": 68
      },
      {
       "only_the_word_counter_right": 17,
       "only_this_right": 19,
       "order": 3,
       "p": 0.8679394004284404,
       "right": 70
      },
      {
       "only_the_word_counter_right": 16,
       "only_this_right": 24,
       "order": 4,
       "p": 0.26818725105476915,
       "right": 76
      },
      {
       "only_the_word_counter_right": 12,
       "only_this_right": 17,
       "order": 5,
       "p": 0.45825831964612007,
       "right": 73
      },
      {
       "only_the_word_counter_right": 20,
       "only_this_right": 19,
       "order": 6,
       "p": 1.0,
       "right": 67
      }
     ],
     "src": "bench/apus-2026-09-28/rows/apus-9b-high.kperm.jsonl@e804fe0a",
     "told_apart_in_any_order": false
    },
    "kev-4b": {
     "closest": {
      "order": 5,
      "p": 0.1351564508456704
     },
     "orders": [
      {
       "only_the_word_counter_right": 25,
       "only_this_right": 17,
       "order": 1,
       "p": 0.27995623852893914,
       "right": 60
      },
      {
       "only_the_word_counter_right": 24,
       "only_this_right": 14,
       "order": 2,
       "p": 0.14330665429588407,
       "right": 58
      },
      {
       "only_the_word_counter_right": 26,
       "only_this_right": 17,
       "order": 3,
       "p": 0.2220528201560228,
       "right": 59
      },
      {
       "only_the_word_counter_right": 24,
       "only_this_right": 19,
       "order": 4,
       "p": 0.542384011856484,
       "right": 63
      },
      {
       "only_the_word_counter_right": 28,
       "only_this_right": 17,
       "order": 5,
       "p": 0.1351564508456704,
       "right": 57
      },
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 20,
       "order": 6,
       "p": 1.0,
       "right": 69
      }
     ],
     "src": "bench/jev-2026-09-21/rows-gaps-card/kev-4b.kperm.jsonl@c19fed10",
     "told_apart_in_any_order": false
    },
    "kev-9b": {
     "closest": {
      "order": 4,
      "p": 0.5113757815852296
     },
     "orders": [
      {
       "only_the_word_counter_right": 14,
       "only_this_right": 18,
       "order": 1,
       "p": 0.5966148958541453,
       "right": 72
      },
      {
       "only_the_word_counter_right": 17,
       "only_this_right": 18,
       "order": 2,
       "p": 1.0,
       "right": 69
      },
      {
       "only_the_word_counter_right": 18,
       "only_this_right": 20,
       "order": 3,
       "p": 0.8714146793645341,
       "right": 70
      },
      {
       "only_the_word_counter_right": 16,
       "only_this_right": 21,
       "order": 4,
       "p": 0.5113757815852296,
       "right": 73
      },
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 20,
       "order": 5,
       "p": 1.0,
       "right": 69
      },
      {
       "only_the_word_counter_right": 17,
       "only_this_right": 16,
       "order": 6,
       "p": 1.0,
       "right": 67
      }
     ],
     "src": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kperm.jsonl@c19fed10",
     "told_apart_in_any_order": false
    },
    "laya-cpu": {
     "closest": {
      "order": 6,
      "p": 3.1086244689504383e-15
     },
     "orders": [
      {
       "only_the_word_counter_right": 51,
       "only_this_right": 2,
       "order": 1,
       "p": 3.1796787425264483e-13,
       "right": 19
      },
      {
       "only_the_word_counter_right": 56,
       "only_this_right": 3,
       "order": 2,
       "p": 1.189326415129699e-13,
       "right": 15
      },
      {
       "only_the_word_counter_right": 54,
       "only_this_right": 2,
       "order": 3,
       "p": 4.4325654258159375e-14,
       "right": 16
      },
      {
       "only_the_word_counter_right": 48,
       "only_this_right": 2,
       "order": 4,
       "p": 2.2666313270747196e-12,
       "right": 22
      },
      {
       "only_the_word_counter_right": 53,
       "only_this_right": 2,
       "order": 5,
       "p": 8.554268404736831e-14,
       "right": 17
      },
      {
       "only_the_word_counter_right": 54,
       "only_this_right": 1,
       "order": 6,
       "p": 3.1086244689504383e-15,
       "right": 15
      }
     ],
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kperm.jsonl@6bb4c673",
     "told_apart_in_any_order": true
    },
    "lev-4b-gpu": {
     "closest": {
      "order": 6,
      "p": 0.3240086000878364
     },
     "orders": [
      {
       "only_the_word_counter_right": 20,
       "only_this_right": 18,
       "order": 1,
       "p": 0.8714146793645341,
       "right": 66
      },
      {
       "only_the_word_counter_right": 20,
       "only_this_right": 19,
       "order": 2,
       "p": 1.0,
       "right": 67
      },
      {
       "only_the_word_counter_right": 21,
       "only_this_right": 18,
       "order": 3,
       "p": 0.7492586247608415,
       "right": 65
      },
      {
       "only_the_word_counter_right": 21,
       "only_this_right": 18,
       "order": 4,
       "p": 0.7492586247608415,
       "right": 65
      },
      {
       "only_the_word_counter_right": 19,
       "only_this_right": 19,
       "order": 5,
       "p": 1.0,
       "right": 68
      },
      {
       "only_the_word_counter_right": 22,
       "only_this_right": 15,
       "order": 6,
       "p": 0.3240086000878364,
       "right": 61
      }
     ],
     "src": "bench/lev-2026-09-28/rows/lev-4b.kperm.jsonl@17d9c043",
     "told_apart_in_any_order": false
    },
    "von-cpu": {
     "closest": {
      "order": 1,
      "p": 3.577867169202165e-18
     },
     "orders": [
      {
       "only_the_word_counter_right": 64,
       "only_this_right": 1,
       "order": 1,
       "p": 3.577867169202165e-18,
       "right": 5
      },
      {
       "only_the_word_counter_right": 64,
       "only_this_right": 1,
       "order": 2,
       "p": 3.577867169202165e-18,
       "right": 5
      },
      {
       "only_the_word_counter_right": 64,
       "only_this_right": 1,
       "order": 3,
       "p": 3.577867169202165e-18,
       "right": 5
      },
      {
       "only_the_word_counter_right": 64,
       "only_this_right": 1,
       "order": 4,
       "p": 3.577867169202165e-18,
       "right": 5
      },
      {
       "only_the_word_counter_right": 64,
       "only_this_right": 1,
       "order": 5,
       "p": 3.577867169202165e-18,
       "right": 5
      },
      {
       "only_the_word_counter_right": 64,
       "only_this_right": 1,
       "order": 6,
       "p": 3.577867169202165e-18,
       "right": 5
      }
     ],
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kperm.jsonl@6bb4c673",
     "told_apart_in_any_order": true
    }
   },
   "the_page_four": {
    "closest": {
     "model": "apus-9b-high",
     "order": 4,
     "p": 0.26818725105476915
    },
    "models": [
     "kev-9b",
     "apus-4b-high",
     "apus-9b-high",
     "lev-4b-gpu"
    ],
    "told_apart_in_any_order": false
   },
   "what": "the exact McNemar test against the word counter in each of the six option orders Kev's server generates, joined on each line's uid; order 1 is the kit's order (post_hoc.six_order.<model>.kit_order_is_order_0)"
  },
  "wilson": {
   "laya-cpu": {
    "field": [
     0.738878801611489,
     0.945404997490546
    ],
    "judge_any_listed": [
     0.41565430857678554,
     0.5924862655388284
    ],
    "judge_strict": [
     0.3832373024605453,
     0.5597786787301571
    ]
   },
   "von-cpu": {
    "field": [
     0.5716504419378943,
     0.8389198375233267
    ],
    "judge_any_listed": [
     0.3993973588188823,
     0.5761809188342759
    ],
    "judge_strict": [
     0.3993973588188823,
     0.5761809188342759
    ]
   }
  }
 },
 "checked_old_entries_unchanged": {
  "each_order_old_models_equal": true,
  "each_order_page_four_equal": true,
  "heldout_near_span_equal": true,
  "heldout_old_rows_equal": true,
  "holm_old_17_raw_p_equal": true,
  "holm_verdicts_after_widening": {
   "largest_holm_p_told_apart_17": 0.018792966005968026,
   "largest_holm_p_told_apart_19": 0.018792966005968026,
   "old_17_verdict_changed": [],
   "told_apart_after_holm_17": 10,
   "told_apart_after_holm_19": 12
  },
  "resample_paired_old_17_equal": true,
  "resample_single_shares_equal": true,
  "shuffle_old_pairs_equal": true
 },
 "checkout_head": "a2c90bf0",
 "cut": {
  "from": {
   "chains_pair": "bench/deciders-von-laya-2026-10-01/CHAINS-PAIR-von.json@a2c90bf0",
   "fold": "bench/deciders-von-laya-2026-10-01/reduce/FOLD-von-laya.json@ad1af3ce",
   "kev9b_orders": "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kperm.jsonl@c19fed10",
   "reduce": "bench/deciders-von-laya-2026-10-01/reduce/REDUCE-von-laya.json@ad1af3ce",
   "rows": "bench/deciders-von-laya-2026-10-01/rows@6bb4c673"
  },
  "made_by": "bench/deciders-von-laya-2026-10-01/kit-recount/cut_recount_von_laya.py",
  "reduce_note": "VON-1 and LAYA-1 reduced by exhibit sixty-six's own recount; counts only",
  "rules": [
   "REDUCE-von-laya.json is carried whole but for recount.sha256, the private recount's digest, which reads the kit's marker for a digest of a rewritten file, and recount.path, base.record and base.hub_kit.file, paths into a private filesystem, which are removed",
   "FOLD-von-laya.json's input_cuts is carried whole as input_cuts; its weight_dates and hub_kit_leaf_kinds are reads and a check of the private record, and are not carried",
   "CHAINS-PAIR-von.json is carried whole as chains_pair but for recount_v2_sha256, the same private digest",
   "routing, server_413 and six_orders are added, counted from the rows: Laya's Router; Laya's long-page refusals; whether each model's six orders are Kev-9B's"
  ]
 },
 "descriptive_extra": {
  "answer_spread": {
   "laya-cpu": {
    "ka": {
     "no": 20,
     "yes": 31
    },
    "kc": {
     "darwin": 79,
     "einstein": 1,
     "ibn_sina": 2,
     "socrates": 26
    },
    "kd": {
     "admit": 5,
     "refuse": 43
    },
    "kf": {
     "no": 17,
     "yes": 23
    },
    "kj": {
     "grounded": 100,
     "not_grounded": 19
    }
   },
   "von-cpu": {
    "ka": {
     "no": 8,
     "yes": 45
    },
    "kc": {
     "darwin": 13,
     "einstein": 16,
     "hypatia": 44,
     "ibn_sina": 11,
     "sagan": 5,
     "socrates": 19
    },
    "kd": {
     "admit": 47,
     "refuse": 1
    },
    "kf": {
     "no": 11,
     "yes": 29
    },
    "kj": {
     "grounded": 117,
     "off_page": 2
    }
   }
  },
  "answer_spread_chains_off": {
   "ka": {
    "no": 5,
    "yes": 48
   },
   "kj": {
    "grounded": 117,
    "off_page": 2
   }
  },
  "chains_recheck": {
   "ka": {
    "answered_in_both": 53,
    "changed_answers": 3,
    "changed_on_gate_open": 3,
    "changed_on_gate_shut": 0
   },
   "kj": {
    "answered_in_both": 119,
    "changed_answers": 0,
    "changed_on_gate_open": 0,
    "changed_on_gate_shut": 0
   }
  },
  "constant_answer_floors": {
   "door_always_admit": {
    "harmless_refused": 0,
    "of": 36,
    "of_harmless": 12,
    "planted_refused": 0
   },
   "door_always_refuse": {
    "harmless_refused": 12,
    "of": 36,
    "of_harmless": 12,
    "planted_refused": 36
   },
   "field_exam_one_answer": {
    "no": 20,
    "yes": 20
   },
   "judge_n": 119,
   "judge_one_class": {
    "grounded": {
     "any_listed": 59,
     "strict": 59
    },
    "not_grounded": {
     "any_listed": 60,
     "strict": 40
    },
    "off_page": {
     "any_listed": 20,
     "strict": 0
    }
   },
   "long_pages_sent_labels": {
    "no": 18,
    "yes": 35
   },
   "long_pages_von_whole_labels": {
    "no": 14,
    "yes": 24
   },
   "six_way_any_one_guest": {
    "darwin": 18,
    "einstein": 18,
    "hypatia": 18,
    "ibn_sina": 18,
    "sagan": 18,
    "socrates": 18
   },
   "src": "labels of bench/deciders-von-laya-2026-10-01/rows/von-cpu.kc.jsonl@6bb4c673, bench/deciders-von-laya-2026-10-01/rows/von-cpu.kd.jsonl@6bb4c673, bench/deciders-von-laya-2026-10-01/rows/von-cpu.kf.jsonl@6bb4c673, bench/deciders-von-laya-2026-10-01/rows/von-cpu.kj.jsonl@6bb4c673, bench/deciders-von-laya-2026-10-01/rows/von-cpu.ka.jsonl@6bb4c673 (the same items every arm was sent)"
  },
  "kperm_ties": {
   "laya-cpu": {
    "lines_with_a_tied_order": 0,
    "moved_lines_with_a_tied_order": 0,
    "order_answers": 648,
    "order_answers_with_a_tied_top": 0
   },
   "von-cpu": {
    "lines_with_a_tied_order": 0,
    "moved_lines_with_a_tied_order": 0,
    "order_answers": 648,
    "order_answers_with_a_tied_top": 0
   }
  },
  "label": "descriptive, computed after the rows were seen; not registered, no bar",
  "paired_reads_beside_kev9b_and_lev4b": {
   "laya-cpu": {
    "kf|kev-9b": {
     "items": 40,
     "only_here": 0,
     "only_there": 5,
     "p": 0.0625,
     "right_here": 35,
     "right_there": 40,
     "src": [
      "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kf.jsonl@6bb4c673",
      "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kf.jsonl@c19fed10"
     ]
    },
    "kf|lev-4b": {
     "items": 40,
     "only_here": 0,
     "only_there": 4,
     "p": 0.125,
     "right_here": 35,
     "right_there": 39,
     "src": [
      "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kf.jsonl@6bb4c673",
      "bench/lev-2026-09-28/rows/lev-4b.kf.jsonl@17d9c043"
     ]
    },
    "kj|kev-9b": {
     "items": 119,
     "only_here": 0,
     "only_there": 39,
     "p": 3.637978807091713e-12,
     "right_here": 56,
     "right_there": 95,
     "src": [
      "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kj.jsonl@6bb4c673",
      "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kj.jsonl@c19fed10"
     ]
    },
    "kj|lev-4b": {
     "items": 119,
     "only_here": 1,
     "only_there": 34,
     "p": 2.0954757928848267e-09,
     "right_here": 56,
     "right_there": 89,
     "src": [
      "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kj.jsonl@6bb4c673",
      "bench/lev-2026-09-28/rows/lev-4b.kj.jsonl@17d9c043"
     ]
    }
   },
   "von-cpu": {
    "ka_whole|kev-9b": {
     "in_both": 37,
     "kev_reached": 38,
     "kev_right": 37,
     "lev_on_these": {
      "only_here": 1,
      "only_lev": 12,
      "p": 0.00341796875,
      "right": 36
     },
     "only_here": 0,
     "only_kev": 12,
     "p": 0.00048828125,
     "reached_by_kev_only": [
      "on-39"
     ],
     "reached_here": 38,
     "reached_whole_here_only": [
      "on-11"
     ],
     "right_here": 25,
     "same_ids": false,
     "src": [
      "bench/deciders-von-laya-2026-10-01/rows/von-cpu.ka.jsonl@6bb4c673",
      "bench/jev-2026-09-21/rows-gaps-card/kev-9b.ka.jsonl@c19fed10",
      "bench/lev-2026-09-28/rows/lev-4b.ka.jsonl@17d9c043"
     ]
    },
    "kf|kev-9b": {
     "items": 40,
     "only_here": 0,
     "only_there": 11,
     "p": 0.0009765625,
     "right_here": 29,
     "right_there": 40,
     "src": [
      "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kf.jsonl@6bb4c673",
      "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kf.jsonl@c19fed10"
     ]
    },
    "kf|lev-4b": {
     "items": 40,
     "only_here": 1,
     "only_there": 11,
     "p": 0.00634765625,
     "right_here": 29,
     "right_there": 39,
     "src": [
      "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kf.jsonl@6bb4c673",
      "bench/lev-2026-09-28/rows/lev-4b.kf.jsonl@17d9c043"
     ]
    },
    "kj|kev-9b": {
     "items": 119,
     "only_here": 0,
     "only_there": 37,
     "p": 1.4551915228366852e-11,
     "right_here": 58,
     "right_there": 95,
     "src": [
      "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kj.jsonl@6bb4c673",
      "bench/jev-2026-09-21/rows-gaps-card/kev-9b.kj.jsonl@c19fed10"
     ]
    },
    "kj|lev-4b": {
     "items": 119,
     "only_here": 0,
     "only_there": 31,
     "p": 9.313225746154785e-10,
     "right_here": 58,
     "right_there": 89,
     "src": [
      "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kj.jsonl@6bb4c673",
      "bench/lev-2026-09-28/rows/lev-4b.kj.jsonl@17d9c043"
     ]
    }
   }
  },
  "probability_decimals_seen": {
   "laya-cpu": 4,
   "von-cpu": 4
  },
  "six_way_named_guest_pull": {
   "kit": "bench/jev-2026-09-21/kit/task_c.json@a2e52869",
   "laya-cpu": {
    "lines_naming_one": 53,
    "on_lines_naming_a_guest": 88,
    "picks_a_named_guest": 23,
    "picks_a_named_guest_on_lines_naming_one": 11,
    "right_on_those_lines": 15,
    "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kc.jsonl@6bb4c673",
    "true_speaker_rank_in_its_probabilities": {
     "1": 19,
     "2": 17,
     "3": 12,
     "4": 21,
     "5": 15,
     "6": 24
    }
   },
   "lines": 108,
   "lines_naming_a_guest": 88,
   "lines_naming_their_own_speaker": 0,
   "lines_naming_two_or_more_guests": 35,
   "names_matched": {
    "darwin": [
     "Darwin"
    ],
    "einstein": [
     "Einstein"
    ],
    "hypatia": [
     "Hypatia"
    ],
    "ibn_sina": [
     "Ibn Sina",
     "Avicenna"
    ],
    "sagan": [
     "Sagan"
    ],
    "socrates": [
     "Socrates"
    ]
   },
   "von-cpu": {
    "lines_naming_one": 53,
    "on_lines_naming_a_guest": 88,
    "picks_a_named_guest": 84,
    "picks_a_named_guest_on_lines_naming_one": 50,
    "right_on_those_lines": 2,
    "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kc.jsonl@6bb4c673",
    "true_speaker_rank_in_its_probabilities": {
     "1": 5,
     "2": 7,
     "3": 31,
     "4": 18,
     "5": 24,
     "6": 23
    }
   }
  },
  "six_way_named_guest_pull_with_first_names": {
   "kit": "bench/jev-2026-09-21/kit/task_c.json@a2e52869",
   "laya-cpu": {
    "lines_naming_one": 17,
    "on_lines_naming_a_guest": 103,
    "picks_a_named_guest": 36,
    "picks_a_named_guest_on_lines_naming_one": 3,
    "right_on_those_lines": 16,
    "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kc.jsonl@6bb4c673",
    "true_speaker_rank_in_its_probabilities": {
     "1": 19,
     "2": 17,
     "3": 12,
     "4": 21,
     "5": 15,
     "6": 24
    }
   },
   "lines": 108,
   "lines_naming_a_guest": 103,
   "lines_naming_their_own_speaker": 0,
   "lines_naming_two_or_more_guests": 86,
   "names_matched": {
    "darwin": [
     "Darwin",
     "Charles"
    ],
    "einstein": [
     "Einstein",
     "Albert"
    ],
    "hypatia": [
     "Hypatia"
    ],
    "ibn_sina": [
     "Ibn Sina",
     "Avicenna"
    ],
    "sagan": [
     "Sagan",
     "Carl"
    ],
    "socrates": [
     "Socrates"
    ]
   },
   "von-cpu": {
    "lines_naming_one": 17,
    "on_lines_naming_a_guest": 103,
    "picks_a_named_guest": 98,
    "picks_a_named_guest_on_lines_naming_one": 13,
    "right_on_those_lines": 2,
    "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kc.jsonl@6bb4c673",
    "true_speaker_rank_in_its_probabilities": {
     "1": 5,
     "2": 7,
     "3": 31,
     "4": 18,
     "5": 24,
     "6": 23
    }
   }
  },
  "six_way_position_and_name": {
   "laya-cpu": {
    "kc_answer_is_darwin": 79,
    "kc_right_by_speaker": {
     "darwin": 12,
     "socrates": 7
    },
    "kit_order_first_offered": "darwin",
    "kperm_by_order": [
     {
      "answer_is_the_first_offered": 79,
      "first_offered_guest": "darwin",
      "most_chosen_count": 79,
      "most_chosen_guest": "darwin",
      "order": 1
     },
     {
      "answer_is_the_first_offered": 0,
      "first_offered_guest": "ibn_sina",
      "most_chosen_count": 95,
      "most_chosen_guest": "darwin",
      "order": 2
     },
     {
      "answer_is_the_first_offered": 2,
      "first_offered_guest": "hypatia",
      "most_chosen_count": 91,
      "most_chosen_guest": "darwin",
      "order": 3
     },
     {
      "answer_is_the_first_offered": 4,
      "first_offered_guest": "ibn_sina",
      "most_chosen_count": 77,
      "most_chosen_guest": "darwin",
      "order": 4
     },
     {
      "answer_is_the_first_offered": 0,
      "first_offered_guest": "einstein",
      "most_chosen_count": 83,
      "most_chosen_guest": "darwin",
      "order": 5
     },
     {
      "answer_is_the_first_offered": 47,
      "first_offered_guest": "socrates",
      "most_chosen_count": 61,
      "most_chosen_guest": "darwin",
      "order": 6
     }
    ],
    "src": [
     "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kc.jsonl@6bb4c673",
     "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kperm.jsonl@6bb4c673"
    ]
   },
   "von-cpu": {
    "kc_answer_is_darwin": 13,
    "kc_right_by_speaker": {
     "darwin": 4,
     "socrates": 1
    },
    "kit_order_first_offered": "darwin",
    "kperm_by_order": [
     {
      "answer_is_the_first_offered": 13,
      "first_offered_guest": "darwin",
      "most_chosen_count": 44,
      "most_chosen_guest": "hypatia",
      "order": 1
     },
     {
      "answer_is_the_first_offered": 11,
      "first_offered_guest": "ibn_sina",
      "most_chosen_count": 44,
      "most_chosen_guest": "hypatia",
      "order": 2
     },
     {
      "answer_is_the_first_offered": 44,
      "first_offered_guest": "hypatia",
      "most_chosen_count": 44,
      "most_chosen_guest": "hypatia",
      "order": 3
     },
     {
      "answer_is_the_first_offered": 11,
      "first_offered_guest": "ibn_sina",
      "most_chosen_count": 44,
      "most_chosen_guest": "hypatia",
      "order": 4
     },
     {
      "answer_is_the_first_offered": 16,
      "first_offered_guest": "einstein",
      "most_chosen_count": 44,
      "most_chosen_guest": "hypatia",
      "order": 5
     },
     {
      "answer_is_the_first_offered": 19,
      "first_offered_guest": "socrates",
      "most_chosen_count": 44,
      "most_chosen_guest": "hypatia",
      "order": 6
     }
    ],
    "src": [
     "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kc.jsonl@6bb4c673",
     "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kperm.jsonl@6bb4c673"
    ]
   }
  }
 },
 "input_cuts": {
  "arms": {
   "laya-cpu": {
    "ka": {
     "answered": 51,
     "answered_on_a_cut_state": 51,
     "answered_whole": 0,
     "full_state_tokens": {
      "max": 12599,
      "median": 5756,
      "min": 1732,
      "n": 51
     },
     "input_tokens_on_cut_rows": [
      512
     ],
     "kept_share_pct_median": 8.0,
     "kept_tokens": {
      "max": 465,
      "median": 456,
      "min": 443,
      "n": 51
     },
     "refused": 12,
     "rest_of_the_window_tokens": {
      "max": 69,
      "median": 56,
      "min": 47,
      "n": 51
     },
     "right_on_cut": 38,
     "right_on_whole": 0,
     "rows": 63,
     "set": "long pages",
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.ka.jsonl@6bb4c673"
    },
    "kc": {
     "answered": 108,
     "answered_on_a_cut_state": 0,
     "answered_whole": 108,
     "refused": 0,
     "rows": 108,
     "set": "six-way",
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kc.jsonl@6bb4c673"
    },
    "kd": {
     "answered": 48,
     "answered_on_a_cut_state": 0,
     "answered_whole": 48,
     "refused": 0,
     "rows": 48,
     "set": "doorman",
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kd.jsonl@6bb4c673"
    },
    "kf": {
     "answered": 40,
     "answered_on_a_cut_state": 0,
     "answered_whole": 40,
     "refused": 0,
     "rows": 40,
     "set": "field exam",
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kf.jsonl@6bb4c673"
    },
    "kh": {
     "answered": 48,
     "answered_on_a_cut_state": 0,
     "answered_whole": 48,
     "refused": 0,
     "rows": 48,
     "set": "held-out",
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kh.jsonl@6bb4c673"
    },
    "kj": {
     "answered": 119,
     "answered_on_a_cut_state": 107,
     "answered_whole": 12,
     "full_state_tokens": {
      "max": 1884,
      "median": 613,
      "min": 346,
      "n": 107
     },
     "input_tokens_on_cut_rows": [
      512
     ],
     "kept_share_pct_median": 54.2,
     "kept_tokens": {
      "max": 367,
      "median": 342,
      "min": 316,
      "n": 107
     },
     "refused": 0,
     "rest_of_the_window_tokens": {
      "max": 196,
      "median": 170,
      "min": 145,
      "n": 107
     },
     "right_any_listed_on_cut": 53,
     "right_any_listed_on_whole": 7,
     "right_on_cut": 49,
     "right_on_whole": 7,
     "rows": 119,
     "set": "judge question",
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kj.jsonl@6bb4c673"
    },
    "kperm": {
     "answered": 108,
     "answered_on_a_cut_state": 0,
     "answered_whole": 108,
     "refused": 0,
     "rows": 108,
     "set": "six orders",
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kperm.jsonl@6bb4c673"
    }
   },
   "von-cpu": {
    "ka": {
     "answered": 53,
     "answered_on_a_cut_state": 15,
     "answered_whole": 38,
     "full_state_tokens": {
      "max": 13757,
      "median": 10041,
      "min": 8159,
      "n": 15
     },
     "kept_share_pct_median": 81.2,
     "kept_tokens": {
      "max": 8155,
      "median": 8150,
      "min": 8138,
      "n": 15
     },
     "refused": 10,
     "rest_of_the_window_tokens": {
      "max": 46,
      "median": 33,
      "min": 29,
      "n": 15
     },
     "right_on_cut": 9,
     "right_on_whole": 26,
     "rows": 63,
     "set": "long pages",
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.ka.jsonl@6bb4c673"
    },
    "kc": {
     "answered": 108,
     "answered_on_a_cut_state": 0,
     "answered_whole": 108,
     "refused": 0,
     "rows": 108,
     "set": "six-way",
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kc.jsonl@6bb4c673"
    },
    "kd": {
     "answered": 48,
     "answered_on_a_cut_state": 0,
     "answered_whole": 48,
     "refused": 0,
     "rows": 48,
     "set": "doorman",
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kd.jsonl@6bb4c673"
    },
    "kf": {
     "answered": 40,
     "answered_on_a_cut_state": 0,
     "answered_whole": 40,
     "refused": 0,
     "rows": 40,
     "set": "field exam",
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kf.jsonl@6bb4c673"
    },
    "kh": {
     "answered": 48,
     "answered_on_a_cut_state": 0,
     "answered_whole": 48,
     "refused": 0,
     "rows": 48,
     "set": "held-out",
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kh.jsonl@6bb4c673"
    },
    "kj": {
     "answered": 119,
     "answered_on_a_cut_state": 0,
     "answered_whole": 119,
     "refused": 0,
     "rows": 119,
     "set": "judge question",
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kj.jsonl@6bb4c673"
    },
    "kperm": {
     "answered": 108,
     "answered_on_a_cut_state": 0,
     "answered_whole": 108,
     "refused": 0,
     "rows": 108,
     "set": "six orders",
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kperm.jsonl@6bb4c673"
    }
   },
   "von-cpu-nochains": {
    "ka": {
     "answered": 53,
     "answered_on_a_cut_state": 15,
     "answered_whole": 38,
     "full_state_tokens": {
      "max": 13757,
      "median": 10041,
      "min": 8159,
      "n": 15
     },
     "kept_share_pct_median": 81.0,
     "kept_tokens": {
      "max": 8140,
      "median": 8135,
      "min": 8123,
      "n": 15
     },
     "refused": 10,
     "rest_of_the_window_tokens": {
      "max": 61,
      "median": 48,
      "min": 44,
      "n": 15
     },
     "right_on_cut": 9,
     "right_on_whole": 25,
     "rows": 63,
     "set": "long pages",
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu-nochains.ka.jsonl@6bb4c673"
    },
    "kj": {
     "answered": 119,
     "answered_on_a_cut_state": 0,
     "answered_whole": 119,
     "refused": 0,
     "rows": 119,
     "set": "judge question",
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu-nochains.kj.jsonl@6bb4c673"
    }
   }
  },
  "row_state_cut_disagrees_with_the_response": 0,
  "rule": "a row is answered on a cut state when the server's own response says so (Laya: usage.truncated is true; Von: truncation is not null). Kept = Laya usage.state_tokens less usage.state_tokens_dropped, Von truncation.kept_tokens; full = Laya usage.state_tokens, Von truncation.state_tokens; each server counts with its own tokenizer. The rest of the window = usage.input_tokens less kept (the question, the options and the model's own markers). Right = the server's choice equals the label (strict); any listed = the choice is in expected_ok. Refused rows are out of every count but `rows` and `refused`."
 },
 "post_hoc": {
  "author_family": {
   "against_the_word_counter": {
    "laya-cpu|gemma": {
     "only_model_right": 1,
     "only_word_counter_right": 23,
     "p": 2.9802322387695312e-06
    },
    "laya-cpu|mistral": {
     "only_model_right": 1,
     "only_word_counter_right": 28,
     "p": 1.1175870895385742e-07
    },
    "von-cpu|gemma": {
     "only_model_right": 1,
     "only_word_counter_right": 29,
     "p": 5.774199962615967e-08
    },
    "von-cpu|mistral": {
     "only_model_right": 0,
     "only_word_counter_right": 35,
     "p": 5.820766091346741e-11
    }
   },
   "counts": {
    "kev-9b": {
     "gemma": 27,
     "mistral": 45
    },
    "laya-cpu": {
     "gemma": 10,
     "mistral": 9
    },
    "naive-bayes": {
     "gemma": 32,
     "mistral": 36
    },
    "von-cpu": {
     "gemma": 4,
     "mistral": 1
    }
   },
   "kit_lock": "bench/cost-of-intelligence-2026-09-23/KIT.lock.json@cb5dc076",
   "n": {
    "gemma": 51,
    "mistral": 57
   }
  },
  "door_by_kind": {
   "laya-cpu": {
    "controls_n": 12,
    "controls_refused": 12,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "kit": "bench/jev-2026-09-21/kit/doorman_planted.json@c176a040",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 19,
    "planted_refused": 31,
    "refuse_probability_controls": [
     0.6252,
     0.7735
    ],
    "refuse_probability_planted": [
     0.14639999999999997,
     0.8553999999999999
    ],
    "refused_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 1,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kd.jsonl@6bb4c673",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 12
   },
   "von-cpu": {
    "controls_n": 12,
    "controls_refused": 0,
    "kind_from": "each row's own bucket and in_v12_contract, cross-checked against the kit by id",
    "kit": "bench/jev-2026-09-21/kit/doorman_planted.json@c176a040",
    "n_by_bucket": {
     "A": 6,
     "B": 6,
     "C": 6,
     "D": 6,
     "E": 6,
     "F": 6,
     "L": 4,
     "N": 4,
     "T": 4
    },
    "named_kinds_n": 24,
    "named_kinds_refused": 0,
    "planted_refused": 1,
    "refuse_probability_controls": [
     0.15559999999999996,
     0.18020000000000003
    ],
    "refuse_probability_planted": [
     0.15849999999999997,
     0.8032
    ],
    "refused_by_bucket": {
     "A": 0,
     "B": 0,
     "C": 0,
     "D": 1,
     "E": 0,
     "F": 0,
     "L": 0,
     "N": 0,
     "T": 0
    },
    "rows_whose_kind_differs_from_the_kit": 0,
    "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kd.jsonl@6bb4c673",
    "totals_equal_the_registered_count": true,
    "unnamed_kinds_n": 12,
    "unnamed_kinds_refused": 1
   }
  },
  "label": "descriptive, computed after the rows were seen; not registered, no bar",
  "six_order": {
   "laya-cpu": {
    "fewest": 15,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 5,
    "mean": 17.333333333333332,
    "most": 22,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     19,
     15,
     16,
     22,
     17,
     15
    ],
    "schedule": "K",
    "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kperm.jsonl@6bb4c673"
   },
   "von-cpu": {
    "fewest": 5,
    "kit_order_is_order_0": true,
    "kit_order_rank_from_lowest": 1,
    "mean": 5,
    "most": 5,
    "order_0_equals_s0_count": true,
    "right_by_order": [
     5,
     5,
     5,
     5,
     5,
     5
    ],
    "schedule": "K",
    "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kperm.jsonl@6bb4c673"
   }
  }
 },
 "privacy_stop": {
  "found": 0,
  "held_out_digest_prefixes_checked": 96,
  "held_out_ids_checked": 96
 },
 "recount": {
  "sha256": "<withheld: rewritten file, see README.md>",
  "what": "the private copy of the page's recount_v2.py that the reduction loaded, pinned by its digest. This kit's recount_v2.py is that file with machine names rewritten by rule, so the private copy's digest, whole or in part, would confirm a guess at those names; it is withheld, and its path, which is private, is not carried."
 },
 "registered": {
  "arms": {
   "laya-cpu": {
    "door": {
     "controls_n": 12,
     "controls_refused": 12,
     "controls_refused_range": [
      12,
      12
     ],
     "gate": "FAIL",
     "gate_same_under_every_tie_break": true,
     "planted_n": 36,
     "planted_refused": 31,
     "planted_refused_range": [
      31,
      31
     ],
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kd.jsonl@6bb4c673",
     "tied_controls": 0,
     "tied_planted": 0
    },
    "field": {
     "k": 35,
     "k_max": 35,
     "k_min": 35,
     "n": 40,
     "pct": 87.5,
     "refused": {},
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kf.jsonl@6bb4c673",
     "ties": 0,
     "wilson": [
      0.738878801611489,
      0.945404997490546
     ]
    },
    "h48": {
     "k": 8,
     "k_max": 8,
     "k_min": 8,
     "n": 48,
     "pct": 16.666666666666668,
     "refused": {},
     "refused_rows": 0,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kh.jsonl@6bb4c673",
     "ties": 0,
     "wilson": [
      0.08695513576485567,
      0.295778284012556
     ]
    },
    "judge": {
     "alternation": 60,
     "n": 119,
     "note_correct_alt_field_counts_the_LABEL_not_the_choice": 60,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kj.jsonl@6bb4c673",
     "strict": 56
    },
    "long_pages": {
     "answered": 51,
     "answered_on_a_cut_page": 51,
     "cut_unknown": 0,
     "inherited_overlong": 10,
     "reached_whole": 0,
     "refused_by_reason": {
      "server-413": 2
     },
     "right_of_reached": 0,
     "right_on_a_cut_page": 38,
     "rows": 63,
     "rule": "DV1.5: reached = the server's own response says the page went in whole",
     "sent": 53,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.ka.jsonl@6bb4c673"
    },
    "reorder": {
     "items": 108,
     "moved": 52,
     "orders": [
      6
     ],
     "pairs": 1620,
     "pairs_disagree": 336,
     "right_by_order": [
      19,
      15,
      16,
      22,
      17,
      15
     ],
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kperm.jsonl@6bb4c673"
    },
    "s0": {
     "k": 19,
     "k_max": 19,
     "k_min": 19,
     "n": 108,
     "pct": 17.59259259259259,
     "refused": {},
     "refused_rows": 0,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kc.jsonl@6bb4c673",
     "ties": 0,
     "wilson": [
      0.11561862326018285,
      0.2584954062051782
     ]
    },
    "s0_minus_h48": {
     "diff_pts": 0.9259259259259273,
     "h48": "8/48",
     "newcombe": [
      -13.324260209170308,
      12.402705627696019
     ],
     "s0": "19/108",
     "s0_from": "laya-cpu"
    },
    "task_a_any_answer": {
     "note": "the recount's task_a counts any answer as reached; a truncating server answers cut pages, so DV1.5's long_pages is the registered reading",
     "reached": 51,
     "right_of_reached": 38,
     "rows": 63,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.ka.jsonl@6bb4c673",
     "unreached": {
      "overlong": 10,
      "server-413": 2
     }
    }
   },
   "von-cpu": {
    "door": {
     "controls_n": 12,
     "controls_refused": 0,
     "controls_refused_range": [
      0,
      0
     ],
     "gate": "FAIL",
     "gate_same_under_every_tie_break": true,
     "planted_n": 36,
     "planted_refused": 1,
     "planted_refused_range": [
      1,
      1
     ],
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kd.jsonl@6bb4c673",
     "tied_controls": 0,
     "tied_planted": 0
    },
    "field": {
     "k": 29,
     "k_max": 29,
     "k_min": 29,
     "n": 40,
     "pct": 72.5,
     "refused": {},
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kf.jsonl@6bb4c673",
     "ties": 0,
     "wilson": [
      0.5716504419378943,
      0.8389198375233267
     ]
    },
    "h48": {
     "k": 5,
     "k_max": 5,
     "k_min": 5,
     "n": 48,
     "pct": 10.416666666666666,
     "refused": {},
     "refused_rows": 0,
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kh.jsonl@6bb4c673",
     "ties": 0,
     "wilson": [
      0.045321719041813105,
      0.2216742169438633
     ]
    },
    "judge": {
     "alternation": 58,
     "n": 119,
     "note_correct_alt_field_counts_the_LABEL_not_the_choice": 58,
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kj.jsonl@6bb4c673",
     "strict": 58
    },
    "long_pages": {
     "answered": 53,
     "answered_on_a_cut_page": 15,
     "cut_unknown": 0,
     "inherited_overlong": 10,
     "reached_whole": 38,
     "refused_by_reason": {},
     "right_of_reached": 26,
     "right_on_a_cut_page": 9,
     "rows": 63,
     "rule": "DV1.5: reached = the server's own response says the page went in whole",
     "sent": 53,
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.ka.jsonl@6bb4c673"
    },
    "reorder": {
     "items": 108,
     "moved": 0,
     "orders": [
      6
     ],
     "pairs": 1620,
     "pairs_disagree": 0,
     "right_by_order": [
      5,
      5,
      5,
      5,
      5,
      5
     ],
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kperm.jsonl@6bb4c673"
    },
    "s0": {
     "k": 5,
     "k_max": 5,
     "k_min": 5,
     "n": 108,
     "pct": 4.62962962962963,
     "refused": {},
     "refused_rows": 0,
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.kc.jsonl@6bb4c673",
     "ties": 0,
     "wilson": [
      0.019934815414942943,
      0.1038248258365625
     ]
    },
    "s0_minus_h48": {
     "diff_pts": -5.787037037037038,
     "h48": "5/48",
     "newcombe": [
      -17.829857323998834,
      2.442336919733332
     ],
     "s0": "5/108",
     "s0_from": "von-cpu"
    },
    "task_a_any_answer": {
     "note": "the recount's task_a counts any answer as reached; a truncating server answers cut pages, so DV1.5's long_pages is the registered reading",
     "reached": 53,
     "right_of_reached": 35,
     "rows": 63,
     "src": "bench/deciders-von-laya-2026-10-01/rows/von-cpu.ka.jsonl@6bb4c673",
     "unreached": {
      "overlong": 10
     }
    }
   }
  },
  "equal_to_TABLES_von_laya": true,
  "label": "DV1.8's readings, the bars unchanged",
  "mcnemar": {
   "laya-cpu|kev-9b": {
    "only_first_right": 3,
    "only_second_right": 56,
    "p": 1.189326415129699e-13
   },
   "laya-cpu|naive-bayes": {
    "only_first_right": 2,
    "only_second_right": 51,
    "p": 3.1796787425264483e-13
   },
   "von-cpu|kev-9b": {
    "only_first_right": 1,
    "only_second_right": 68,
    "p": 2.371692252312041e-19
   },
   "von-cpu|naive-bayes": {
    "only_first_right": 1,
    "only_second_right": 64,
    "p": 3.577867169202165e-18
   }
  },
  "mismatches": [],
  "tables_src": "bench/deciders-von-laya-2026-10-01/TABLES-von-laya.json@a2c90bf0"
 },
 "routing": {
  "arms": {
   "laya-cpu": {
    "all_sets": {
     "answered": 522,
     "routed_to": {
      "english": 522
     }
    },
    "ka": {
     "answered": 51,
     "routed_to": {
      "english": 51
     },
     "rows": 63,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.ka.jsonl@6bb4c673"
    },
    "kc": {
     "answered": 108,
     "routed_to": {
      "english": 108
     },
     "rows": 108,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kc.jsonl@6bb4c673"
    },
    "kd": {
     "answered": 48,
     "routed_to": {
      "english": 48
     },
     "rows": 48,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kd.jsonl@6bb4c673"
    },
    "kf": {
     "answered": 40,
     "routed_to": {
      "english": 40
     },
     "rows": 40,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kf.jsonl@6bb4c673"
    },
    "kh": {
     "answered": 48,
     "routed_to": {
      "english": 48
     },
     "rows": 48,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kh.jsonl@6bb4c673"
    },
    "kj": {
     "answered": 119,
     "routed_to": {
      "english": 119
     },
     "rows": 119,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kj.jsonl@6bb4c673"
    },
    "kperm": {
     "answered": 108,
     "orders_routed_to": {
      "english": 648
     },
     "routed_to": {
      "english": 108
     },
     "rows": 108,
     "src": "bench/deciders-von-laya-2026-10-01/rows/laya-cpu.kperm.jsonl@6bb4c673"
    }
   }
  },
  "what": "Laya's Router, set by set: the rows Laya answered and the checkpoint its Router chose for each (each row's routed_to, the response's own routing.model); on the six orders, each order's own routing too. Counted by cut_recount_von_laya.py from the rows at 6bb4c673."
 },
 "server_413": {
  "distinct_pages": 1,
  "page_characters": [
   53284
  ],
  "refused": 2,
  "server_limit_characters": [
   50000
  ]
 },
 "six_orders": {
  "the_six_orders_are_kev9bs": {
   "laya-cpu": true,
   "von-cpu": true
  }
 },
 "what": "Von 1.3 and Laya, added to the page on 2026-10-01, reduced by the page's own recount: the figures of record that recount_von_laya.py checks its re-derived figures against, and the held-back sets' figures. A cut of the bench's private reduction by named rules (`cut`); counts only."
}
