{
 "n": 500,
 "n_answerable": 470,
 "arms": {
  "S1": {
   "counts": {
    "every@5": 416,
    "any@5_500": 487,
    "any@5_470": 458,
    "any@1": 413,
    "all@10": 441,
    "all@5_500": 437,
    "any@1_500": 436,
    "all@10_500": 466
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 72,
     "any@5_all": 78,
     "any@1": 70,
     "all@10": 72,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 97,
     "any@5_all": 129,
     "any@1": 106,
     "all@10": 110,
     "any@5_470": 117
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 28,
     "any@5_all": 28,
     "any@1": 16,
     "all@10": 29,
     "any@5_470": 28
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 62,
     "any@5_all": 67,
     "any@1": 55,
     "all@10": 62,
     "any@5_470": 62
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 101,
     "any@5_all": 129,
     "any@1": 110,
     "all@10": 112,
     "any@5_470": 123
    }
   }
  },
  "F1": {
   "counts": {
    "every@5": 417,
    "any@5_500": 490,
    "any@5_470": 460,
    "any@1": 421,
    "all@10": 447,
    "all@5_500": 439,
    "any@1_500": 446,
    "all@10_500": 470
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 71,
     "any@5_all": 78,
     "any@1": 70,
     "all@10": 72,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 96,
     "any@5_all": 131,
     "any@1": 110,
     "all@10": 112,
     "any@5_470": 119
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 27,
     "any@5_all": 27,
     "any@1": 17,
     "all@10": 29,
     "any@5_470": 27
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 62,
     "any@5_all": 68,
     "any@1": 58,
     "all@10": 63,
     "any@5_470": 62
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 105,
     "any@5_all": 130,
     "any@1": 110,
     "all@10": 115,
     "any@5_470": 124
    }
   }
  },
  "F2": {
   "counts": {
    "every@5": 422,
    "any@5_500": 491,
    "any@5_470": 462,
    "any@1": 415,
    "all@10": 449,
    "all@5_500": 444,
    "any@1_500": 438,
    "all@10_500": 478
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 70,
     "any@5_all": 78,
     "any@1": 69,
     "all@10": 72,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 103,
     "any@5_all": 132,
     "any@1": 106,
     "all@10": 113,
     "any@5_470": 120
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 28,
     "any@5_all": 28,
     "any@1": 20,
     "all@10": 29,
     "any@5_470": 28
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 62,
     "any@5_all": 67,
     "any@1": 57,
     "all@10": 63,
     "any@5_470": 62
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 103,
     "any@5_all": 130,
     "any@1": 107,
     "all@10": 116,
     "any@5_470": 124
    }
   }
  },
  "F3": {
   "counts": {
    "every@5": 425,
    "any@5_500": 492,
    "any@5_470": 462,
    "any@1": 420,
    "all@10": 455,
    "all@5_500": 447,
    "any@1_500": 447,
    "all@10_500": 484
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 70,
     "any@5_all": 78,
     "any@1": 70,
     "all@10": 72,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 104,
     "any@5_all": 132,
     "any@1": 108,
     "all@10": 115,
     "any@5_470": 120
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 27,
     "any@5_all": 27,
     "any@1": 22,
     "all@10": 29,
     "any@5_470": 27
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 63,
     "any@5_all": 69,
     "any@1": 58,
     "all@10": 64,
     "any@5_470": 63
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 105,
     "any@5_all": 130,
     "any@1": 106,
     "all@10": 119,
     "any@5_470": 124
    }
   }
  },
  "F1h": {
   "counts": {
    "every@5": 415,
    "any@5_500": 492,
    "any@5_470": 462,
    "any@1": 424,
    "all@10": 448,
    "all@5_500": 438,
    "any@1_500": 450,
    "all@10_500": 473
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 71,
     "any@5_all": 78,
     "any@1": 71,
     "all@10": 71,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 94,
     "any@5_all": 132,
     "any@1": 110,
     "all@10": 113,
     "any@5_470": 120
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 27,
     "any@5_all": 27,
     "any@1": 19,
     "all@10": 29,
     "any@5_470": 27
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 63,
     "any@5_all": 69,
     "any@1": 59,
     "all@10": 64,
     "any@5_470": 63
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 104,
     "any@5_all": 130,
     "any@1": 109,
     "all@10": 115,
     "any@5_470": 124
    }
   }
  },
  "F2h": {
   "counts": {
    "every@5": 418,
    "any@5_500": 487,
    "any@5_470": 458,
    "any@1": 411,
    "all@10": 448,
    "all@5_500": 441,
    "any@1_500": 434,
    "all@10_500": 476
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 68,
     "any@5_all": 78,
     "any@1": 69,
     "all@10": 72,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 102,
     "any@5_all": 133,
     "any@1": 104,
     "all@10": 113,
     "any@5_470": 121
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 27,
     "any@5_all": 27,
     "any@1": 24,
     "all@10": 29,
     "any@5_470": 27
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 62,
     "any@5_all": 67,
     "any@1": 52,
     "all@10": 63,
     "any@5_470": 62
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 103,
     "any@5_all": 126,
     "any@1": 106,
     "all@10": 115,
     "any@5_470": 120
    }
   }
  },
  "F3h": {
   "counts": {
    "every@5": 421,
    "any@5_500": 490,
    "any@5_470": 460,
    "any@1": 416,
    "all@10": 454,
    "all@5_500": 442,
    "any@1_500": 440,
    "all@10_500": 482
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 70,
     "any@5_all": 78,
     "any@1": 69,
     "all@10": 71,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 101,
     "any@5_all": 132,
     "any@1": 107,
     "all@10": 116,
     "any@5_470": 120
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 27,
     "any@5_all": 27,
     "any@1": 22,
     "all@10": 29,
     "any@5_470": 27
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 63,
     "any@5_all": 69,
     "any@1": 55,
     "all@10": 64,
     "any@5_470": 63
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 104,
     "any@5_all": 128,
     "any@1": 107,
     "all@10": 118,
     "any@5_470": 122
    }
   }
  },
  "A0": {
   "counts": {
    "every@5": 400,
    "any@5_500": 485,
    "any@5_470": 455,
    "any@1": 412,
    "all@10": 430,
    "all@5_500": 421,
    "any@1_500": 437,
    "all@10_500": 454
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 71,
     "any@5_all": 78,
     "any@1": 70,
     "all@10": 71,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 85,
     "any@5_all": 131,
     "any@1": 109,
     "all@10": 101,
     "any@5_470": 119
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 25,
     "any@5_all": 25,
     "any@1": 15,
     "all@10": 28,
     "any@5_470": 25
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 64,
     "any@5_all": 70,
     "any@1": 60,
     "all@10": 64,
     "any@5_470": 64
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 99,
     "any@5_all": 125,
     "any@1": 102,
     "all@10": 110,
     "any@5_470": 119
    }
   }
  },
  "A1": {
   "counts": {
    "every@5": 403,
    "any@5_500": 481,
    "any@5_470": 453,
    "any@1": 398,
    "all@10": 437,
    "all@5_500": 427,
    "any@1_500": 421,
    "all@10_500": 466
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 66,
     "any@5_all": 78,
     "any@1": 69,
     "all@10": 68,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 95,
     "any@5_all": 131,
     "any@1": 99,
     "all@10": 110,
     "any@5_470": 120
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 27,
     "any@5_all": 27,
     "any@1": 21,
     "all@10": 28,
     "any@5_470": 27
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 60,
     "any@5_all": 65,
     "any@1": 50,
     "all@10": 63,
     "any@5_470": 60
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 99,
     "any@5_all": 124,
     "any@1": 103,
     "all@10": 112,
     "any@5_470": 118
    }
   }
  },
  "A2": {
   "counts": {
    "every@5": 409,
    "any@5_500": 489,
    "any@5_470": 460,
    "any@1": 415,
    "all@10": 450,
    "all@5_500": 432,
    "any@1_500": 438,
    "all@10_500": 478
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 66,
     "any@5_all": 78,
     "any@1": 69,
     "all@10": 71,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 96,
     "any@5_all": 133,
     "any@1": 106,
     "all@10": 114,
     "any@5_470": 121
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 27,
     "any@5_all": 27,
     "any@1": 23,
     "all@10": 29,
     "any@5_470": 27
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 63,
     "any@5_all": 68,
     "any@1": 54,
     "all@10": 64,
     "any@5_470": 63
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 101,
     "any@5_all": 127,
     "any@1": 107,
     "all@10": 116,
     "any@5_470": 121
    }
   }
  },
  "A5": {
   "counts": {
    "every@5": 417,
    "any@5_500": 487,
    "any@5_470": 457,
    "any@1": 421,
    "all@10": 450,
    "all@5_500": 442,
    "any@1_500": 446,
    "all@10_500": 479
   },
   "by_type": {
    "knowledge-update": {
     "n": 72,
     "n_all": 78,
     "every@5": 70,
     "any@5_all": 78,
     "any@1": 69,
     "all@10": 72,
     "any@5_470": 72
    },
    "multi-session": {
     "n": 121,
     "n_all": 133,
     "every@5": 99,
     "any@5_all": 132,
     "any@1": 110,
     "all@10": 115,
     "any@5_470": 120
    },
    "single-session-assistant": {
     "n": 56,
     "n_all": 56,
     "every@5": 56,
     "any@5_all": 56,
     "any@1": 56,
     "all@10": 56,
     "any@5_470": 56
    },
    "single-session-preference": {
     "n": 30,
     "n_all": 30,
     "every@5": 27,
     "any@5_all": 27,
     "any@1": 23,
     "all@10": 28,
     "any@5_470": 27
    },
    "single-session-user": {
     "n": 64,
     "n_all": 70,
     "every@5": 62,
     "any@5_all": 68,
     "any@1": 55,
     "all@10": 63,
     "any@5_470": 62
    },
    "temporal-reasoning": {
     "n": 127,
     "n_all": 133,
     "every@5": 103,
     "any@5_all": 126,
     "any@1": 108,
     "all@10": 116,
     "any@5_470": 120
    }
   }
  }
 },
 "family_crosscheck": {
  "primary": [
   {
    "contrast": "S1 vs A0",
    "harness": {
     "b_arm_only": 27,
     "c_ref_only": 11,
     "p_two_sided": 0.013852965261321515,
     "delta_hits": 16
    },
    "family": {
     "b_arm_only": 27,
     "c_ref_only": 11,
     "p": 0.013852965261321515,
     "delta": 16,
     "p_holm": 0.06926482630660757,
     "words": "NO MATERIAL DIFFERENCE between S1 and BM25"
    },
    "holm_recomputed": 0.06926482630660757,
    "equal": true
   },
   {
    "contrast": "S1 vs A1",
    "harness": {
     "b_arm_only": 26,
     "c_ref_only": 13,
     "p_two_sided": 0.05325191409792751,
     "delta_hits": 13
    },
    "family": {
     "b_arm_only": 26,
     "c_ref_only": 13,
     "p": 0.05325191409792751,
     "delta": 13,
     "p_holm": 0.21300765639171004,
     "words": "NO MATERIAL DIFFERENCE between S1 and the dense embedder"
    },
    "holm_recomputed": 0.21300765639171004,
    "equal": true
   },
   {
    "contrast": "F2 vs S1",
    "harness": {
     "b_arm_only": 16,
     "c_ref_only": 10,
     "p_two_sided": 0.32693958282470703,
     "delta_hits": 6
    },
    "family": {
     "b_arm_only": 16,
     "c_ref_only": 10,
     "p": 0.32693958282470703,
     "delta": 6,
     "p_holm": 0.4014015197753906,
     "words": "NO MATERIAL DIFFERENCE: this set cannot say the hybrid is worse (or better) than sparse alone"
    },
    "holm_recomputed": 0.4014015197753906,
    "equal": true
   },
   {
    "contrast": "F1 vs A2",
    "harness": {
     "b_arm_only": 16,
     "c_ref_only": 8,
     "p_two_sided": 0.15158963203430176,
     "delta_hits": 8
    },
    "family": {
     "b_arm_only": 16,
     "c_ref_only": 8,
     "p": 0.15158963203430176,
     "delta": 8,
     "p_holm": 0.4014015197753906,
     "words": "NO MATERIAL DIFFERENCE between the sparse swap and the house hybrid"
    },
    "holm_recomputed": 0.4014015197753906,
    "equal": true
   },
   {
    "contrast": "F3 vs A5",
    "harness": {
     "b_arm_only": 15,
     "c_ref_only": 7,
     "p_two_sided": 0.13380050659179688,
     "delta_hits": 8
    },
    "family": {
     "b_arm_only": 15,
     "c_ref_only": 7,
     "p": 0.13380050659179688,
     "delta": 8,
     "p_holm": 0.4014015197753906,
     "words": "NO MATERIAL DIFFERENCE between the three-leg blend and A5"
    },
    "holm_recomputed": 0.4014015197753906,
    "equal": true
   }
  ],
  "secondary": [
   {
    "contrast": "S1 vs A0",
    "equal": true
   },
   {
    "contrast": "S1 vs A1",
    "equal": true
   },
   {
    "contrast": "F2 vs S1",
    "equal": true
   },
   {
    "contrast": "F1 vs A2",
    "equal": true
   },
   {
    "contrast": "F3 vs A5",
    "equal": true
   }
  ],
  "all_equal": true
 },
 "family_ids": {
  "S1 vs A0": {
   "only_arm": [
    "gpt4_59c863d7",
    "gpt4_d84a3211",
    "28dc39ac",
    "2e6d26dc",
    "d682f1a2",
    "gpt4_5501fe77",
    "2318644b",
    "gpt4_d12ceb0e",
    "129d1232",
    "gpt4_194be4b3",
    "gpt4_e05b82a6",
    "e3038f8c",
    "2b8f3739",
    "32260d93",
    "195a1a1b",
    "06f04340",
    "d905b33f",
    "3c1045c8",
    "a1cc6108",
    "gpt4_f420262c",
    "gpt4_1e4a8aec",
    "gpt4_f420262d",
    "6e984302",
    "gpt4_93159ced",
    "gpt4_93f6379c",
    "gpt4_2f56ae70",
    "0977f2af"
   ],
   "only_ref": [
    "5d3d2817",
    "577d4d32",
    "00ca467f",
    "60472f9c",
    "a9f6b44c",
    "10d9b85a",
    "0bc8ad92",
    "gpt4_b5700ca0",
    "gpt4_5dcc0aab",
    "d01c6aa8",
    "gpt4_5438fa52"
   ]
  },
  "S1 vs A1": {
   "only_arm": [
    "e47becba",
    "726462e0",
    "aae3761f",
    "gpt4_2ba83207",
    "2788b940",
    "129d1232",
    "gpt4_e05b82a6",
    "c2ac3c61",
    "1c0ddc50",
    "d905b33f",
    "1192316e",
    "67e0d0f2",
    "af082822",
    "gpt4_468eb063",
    "gpt4_21adecb5",
    "0bc8ad93",
    "6e984302",
    "08f4fc43",
    "bbf86515",
    "gpt4_0a05b494",
    "830ce83f",
    "184da446",
    "45dc21b6",
    "7e974930",
    "5831f84d",
    "0977f2af"
   ],
   "only_ref": [
    "gpt4_15e38248",
    "gpt4_7fce9456",
    "2ce6a0f2",
    "a9f6b44c",
    "bc149d6b",
    "157a136e",
    "8e91e7d9",
    "gpt4_45189cb4",
    "2ebe6c92",
    "gpt4_b5700ca0",
    "gpt4_5dcc0aab",
    "d01c6aa8",
    "6613b389"
   ]
  },
  "F2 vs S1": {
   "only_arm": [
    "gpt4_15e38248",
    "gpt4_7fce9456",
    "2ce6a0f2",
    "00ca467f",
    "gpt4_31ff4165",
    "a9f6b44c",
    "gpt4_372c3eed",
    "92a0aa75",
    "bc149d6b",
    "157a136e",
    "c18a7dc8",
    "gpt4_d6585ce8",
    "gpt4_59149c78",
    "gpt4_b5700ca0",
    "gpt4_5dcc0aab",
    "d01c6aa8"
   ],
   "only_ref": [
    "aae3761f",
    "2788b940",
    "129d1232",
    "d905b33f",
    "1192316e",
    "gpt4_21adecb5",
    "0bc8ad93",
    "6e984302",
    "830ce83f",
    "45dc21b6"
   ]
  },
  "F1 vs A2": {
   "only_arm": [
    "e47becba",
    "aae3761f",
    "gpt4_5501fe77",
    "129d1232",
    "gpt4_e05b82a6",
    "d905b33f",
    "gpt4_21adecb5",
    "gpt4_59149c78",
    "0bc8ad93",
    "6e984302",
    "bbf86515",
    "830ce83f",
    "184da446",
    "45dc21b6",
    "7e974930",
    "5831f84d"
   ],
   "only_ref": [
    "5d3d2817",
    "577d4d32",
    "gpt4_7fce9456",
    "2ce6a0f2",
    "gpt4_731e37d7",
    "92a0aa75",
    "bc149d6b",
    "gpt4_7f6b06db"
   ]
  },
  "F3 vs A5": {
   "only_arm": [
    "5d3d2817",
    "726462e0",
    "gpt4_5501fe77",
    "gpt4_31ff4165",
    "2b8f3739",
    "gpt4_372c3eed",
    "3c1045c8",
    "a1cc6108",
    "1192316e",
    "078150f1",
    "af082822",
    "gpt4_7f6b06db",
    "gpt4_468eb063",
    "gpt4_d6585ce8",
    "184da446"
   ],
   "only_ref": [
    "577d4d32",
    "gpt4_f2262a51",
    "dd2973ad",
    "c18a7dc8",
    "gpt4_45189cb4",
    "gpt4_5438fa52",
    "45dc21b6"
   ]
  }
 },
 "exploratory": [
  {
   "contrast": "F1h vs A2",
   "b_arm_only": 15,
   "c_ref_only": 9,
   "p_two_sided": 0.30745625495910645,
   "delta_hits": 6
  },
  {
   "contrast": "F2h vs S1",
   "b_arm_only": 17,
   "c_ref_only": 15,
   "p_two_sided": 0.860050065908581,
   "delta_hits": 2
  },
  {
   "contrast": "F3h vs A5",
   "b_arm_only": 13,
   "c_ref_only": 9,
   "p_two_sided": 0.5234670639038086,
   "delta_hits": 4
  }
 ],
 "ctrl_a2": [
  500,
  0
 ],
 "ties": {
  "F1": [
   0,
   0
  ],
  "F2": [
   10,
   8
  ],
  "F3": [
   0,
   0
  ],
  "F1h": [
   8,
   6
  ],
  "F2h": [
   0,
   0
  ],
  "F3h": [
   0,
   0
  ]
 },
 "f2_ties_reversed_exploratory": {
  "every@5": 422,
  "n": 470,
  "gain_vs_F2": [],
  "loss_vs_F2": [],
  "F2_tie_at_5_ids": [
   "77eafa52",
   "ef9cf60a",
   "gpt4_fa19884c",
   "gpt4_7f6b06db",
   "gpt4_85da3956",
   "08f4fc43",
   "982b5123_abs",
   "0e4e4c46",
   "9bbe84a2",
   "e8a79c70"
  ],
  "vs_S1": {
   "b_arm_only": 16,
   "c_ref_only": 10,
   "p_two_sided": 0.32693958282470703,
   "delta_hits": 6
  }
 },
 "g0_extras": {
  "model_sha_pinned": true,
  "device": "cpu",
  "torch": "2.8.0+cpu",
  "torch_cuda_build": null,
  "env": {
   "TZ": "UTC",
   "PYTHONDONTWRITEBYTECODE": "1",
   "HF_HUB_OFFLINE": "1",
   "CUDA_VISIBLE_DEVICES": "",
   "DO_NOT_TRACK": "1",
   "TRANSFORMERS_OFFLINE": "1"
  },
  "ctrl_card": 11.1105,
  "activation": "card-v3",
  "window": 510,
  "started_utc": "2026-11-07T03:57:08Z",
  "finished_utc": "2026-11-07T06:25:43Z",
  "secs_model_load": 1.69,
  "threads": 16
 },
 "s1_run": {
  "timing": {
   "per_question_secs_p50": 17.8697,
   "per_question_secs_p95": 18.2361,
   "per_question_secs_max": 18.5796,
   "total_secs": 8913.5,
   "encode_secs": 8847.11,
   "windows": 180813,
   "tokens": 59091860,
   "windows_per_sec": 20.438,
   "tokens_per_sec": 6679.2,
   "index_bytes": 468768587,
   "nnz_texts": 38872666,
   "maxrss_mb": 3291,
   "secs_model_load": 1.69
  },
  "texts": 125667,
  "texts_windowed": 54802,
  "windows": 180813,
  "tokens": 59091860,
  "load1_p50": 15.07,
  "load1_p95": 15.9,
  "load1_max": 16.15,
  "load1_min": 0.95,
  "secs_mean": 17.827,
  "scored_files_min": 34,
  "lists_short_of_50": 429,
  "query_terms_min_max": [
   7,
   56
  ]
 },
 "s1_misses": {
  "worst_evidence_rank": {
   "6-10": 25,
   "11-20": 22,
   "21-50": 7,
   "absent": 0
  },
  "partial": 42,
  "none_in_top5": 12,
  "by_evidence_count_misses_of_n": {
   "1": [
    7,
    170
   ],
   "2": [
    17,
    229
   ],
   "3": [
    11,
    39
   ],
   "4": [
    9,
    18
   ],
   "5": [
    7,
    11
   ],
   "6": [
    3,
    3
   ]
  }
 },
 "sources": {
  "rescore-S1": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-s1.json",
   "withheld-2026-09-29"
  ],
  "rescore-F1": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-F1.json",
   "withheld-2026-09-29"
  ],
  "rescore-F2": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-F2.json",
   "withheld-2026-09-29"
  ],
  "rescore-F3": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-F3.json",
   "withheld-2026-09-29"
  ],
  "rescore-F1h": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-F1h.json",
   "withheld-2026-09-29"
  ],
  "rescore-F2h": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-F2h.json",
   "withheld-2026-09-29"
  ],
  "rescore-F3h": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-F3h.json",
   "withheld-2026-09-29"
  ],
  "tsv-A0": [
   "longmemeval-2026-10/a0/a0-per-question.tsv",
   "83584b3f7520a5cce1e03359e370f20d32705d71fafa759db353253bc3f3de9e"
  ],
  "tsv-A1": [
   "longmemeval-2026-10/w7/w7-per-question.tsv",
   "612af3d11aa62e71b92fd01a2c6d25141306bcd750c7c76732c2884cfdea8c20"
  ],
  "tsv-A2": [
   "longmemeval-2026-10/w7/w7-per-question.tsv",
   "612af3d11aa62e71b92fd01a2c6d25141306bcd750c7c76732c2884cfdea8c20"
  ],
  "tsv-A5": [
   "longmemeval-2026-10/a5/a5-per-question.tsv",
   "34dc22ad11181b4bba43f2c6d5e8875dccde9b13d35a347578b1f0d01725c8cc"
  ],
  "family": [
   "learned-sparse-2026-11-07/full-s1/score/family.json",
   "withheld-2026-09-29"
  ],
  "fuse": [
   "learned-sparse-2026-11-07/full-s1/fuse/fuse.json",
   "withheld-2026-09-29"
  ],
  "s1-rows": [
   "learned-sparse-2026-11-07/full-s1/run/rows.ndjson",
   "724a352b8417129bb059fbfdbc28eaba64c061da25d9a1edbf7ab08819887432"
  ],
  "a0-rows": [
   "longmemeval-2026-10/a0/run/rows.ndjson",
   "4d545fdcd364ce86f688250b8b46f8f36db782aba683fde49497425c6f421e55"
  ],
  "a1-legs": [
   "longmemeval-2026-10/w7/rows/a1/legs.ndjson",
   "6a4299d76c89e38818fe6d69062c3fb6b9b9bbc2836915793374bdc889bf7994"
  ],
  "F2-rows": [
   "learned-sparse-2026-11-07/full-s1/fuse/F2.rows.ndjson",
   "65595bad63ee4c831950539e264af93d7de2f9580cfff3782617bbc24159434e"
  ],
  "s1-run.json": [
   "learned-sparse-2026-11-07/full-s1/run/run.json",
   "02a152f7f182d79ef43970c2922baf06a1737cb2b6e67330f5154330aa982961"
  ],
  "s1-summary": [
   "learned-sparse-2026-11-07/full-s1/run/summary.json",
   "8d1559929ec7185d583edb8f6b37f253af36c2b12ce337add36976ff7a449b79"
  ],
  "control-card": [
   "learned-sparse-2026-11-07/full-s1/control-card.json",
   "b5356f0e5ad05e7fa216e91db09b9dab93f3d4762ea05ea6291f520d009ef0d0"
  ]
 }
}