{
 "n": 500,
 "n_answerable": 470,
 "sources": {
  "S1": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-s1.json",
   "withheld-2026-09-29"
  ],
  "F1": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-F1.json",
   "withheld-2026-09-29"
  ],
  "F2": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-F2.json",
   "withheld-2026-09-29"
  ],
  "F3": [
   "learned-sparse-2026-11-07/full-s1/score/rescore-F3.json",
   "withheld-2026-09-29"
  ],
  "A0": [
   "longmemeval-2026-10/a0/a0-per-question.tsv",
   "83584b3f7520a5cce1e03359e370f20d32705d71fafa759db353253bc3f3de9e"
  ],
  "A1": [
   "longmemeval-2026-10/w7/w7-per-question.tsv",
   "612af3d11aa62e71b92fd01a2c6d25141306bcd750c7c76732c2884cfdea8c20"
  ],
  "A2": [
   "longmemeval-2026-10/w7/w7-per-question.tsv",
   "612af3d11aa62e71b92fd01a2c6d25141306bcd750c7c76732c2884cfdea8c20"
  ],
  "A5": [
   "longmemeval-2026-10/a5/a5-per-question.tsv",
   "34dc22ad11181b4bba43f2c6d5e8875dccde9b13d35a347578b1f0d01725c8cc"
  ]
 },
 "primary": [
  {
   "contrast": "S1 vs A0",
   "metric": "all@5",
   "n": 470,
   "margin": 10,
   "arm_hits": 416,
   "ref_hits": 400,
   "b_arm_only": 27,
   "c_ref_only": 11,
   "p": 0.013852965261321515,
   "delta": 16,
   "p_holm": 0.06926482630660757,
   "words": "NO MATERIAL DIFFERENCE between S1 and BM25"
  },
  {
   "contrast": "S1 vs A1",
   "metric": "all@5",
   "n": 470,
   "margin": 10,
   "arm_hits": 416,
   "ref_hits": 403,
   "b_arm_only": 26,
   "c_ref_only": 13,
   "p": 0.05325191409792751,
   "delta": 13,
   "p_holm": 0.21300765639171004,
   "words": "NO MATERIAL DIFFERENCE between S1 and the dense embedder"
  },
  {
   "contrast": "F2 vs S1",
   "metric": "all@5",
   "n": 470,
   "margin": 10,
   "arm_hits": 422,
   "ref_hits": 416,
   "b_arm_only": 16,
   "c_ref_only": 10,
   "p": 0.32693958282470703,
   "delta": 6,
   "p_holm": 0.4014015197753906,
   "words": "NO MATERIAL DIFFERENCE: this set cannot say the hybrid is worse (or better) than sparse alone"
  },
  {
   "contrast": "F1 vs A2",
   "metric": "all@5",
   "n": 470,
   "margin": 10,
   "arm_hits": 417,
   "ref_hits": 409,
   "b_arm_only": 16,
   "c_ref_only": 8,
   "p": 0.15158963203430176,
   "delta": 8,
   "p_holm": 0.4014015197753906,
   "words": "NO MATERIAL DIFFERENCE between the sparse swap and the house hybrid"
  },
  {
   "contrast": "F3 vs A5",
   "metric": "all@5",
   "n": 470,
   "margin": 10,
   "arm_hits": 425,
   "ref_hits": 417,
   "b_arm_only": 15,
   "c_ref_only": 7,
   "p": 0.13380050659179688,
   "delta": 8,
   "p_holm": 0.4014015197753906,
   "words": "NO MATERIAL DIFFERENCE between the three-leg blend and A5"
  }
 ],
 "secondary": [
  {
   "contrast": "S1 vs A0",
   "metric": "any@5",
   "n": 500,
   "margin": 5,
   "arm_hits": 487,
   "ref_hits": 485,
   "b_arm_only": 9,
   "c_ref_only": 7,
   "p": 0.803619384765625,
   "delta": 2,
   "p_unadjusted_rule": 0.803619384765625,
   "words": "(secondary) NO MATERIAL DIFFERENCE between S1 and BM25"
  },
  {
   "contrast": "S1 vs A1",
   "metric": "any@5",
   "n": 500,
   "margin": 5,
   "arm_hits": 487,
   "ref_hits": 481,
   "b_arm_only": 12,
   "c_ref_only": 6,
   "p": 0.237884521484375,
   "delta": 6,
   "p_unadjusted_rule": 0.237884521484375,
   "words": "(secondary) NO MATERIAL DIFFERENCE between S1 and the dense embedder"
  },
  {
   "contrast": "F2 vs S1",
   "metric": "any@5",
   "n": 500,
   "margin": 5,
   "arm_hits": 491,
   "ref_hits": 487,
   "b_arm_only": 6,
   "c_ref_only": 2,
   "p": 0.2890625,
   "delta": 4,
   "p_unadjusted_rule": 0.2890625,
   "words": "(secondary) NO MATERIAL DIFFERENCE: this set cannot say the hybrid is worse (or better) than sparse alone"
  },
  {
   "contrast": "F1 vs A2",
   "metric": "any@5",
   "n": 500,
   "margin": 5,
   "arm_hits": 490,
   "ref_hits": 489,
   "b_arm_only": 5,
   "c_ref_only": 4,
   "p": 1.0,
   "delta": 1,
   "p_unadjusted_rule": 1.0,
   "words": "(secondary) NO MATERIAL DIFFERENCE between the sparse swap and the house hybrid"
  },
  {
   "contrast": "F3 vs A5",
   "metric": "any@5",
   "n": 500,
   "margin": 5,
   "arm_hits": 492,
   "ref_hits": 487,
   "b_arm_only": 8,
   "c_ref_only": 3,
   "p": 0.2265625,
   "delta": 5,
   "p_unadjusted_rule": 0.2265625,
   "words": "(secondary) NO MATERIAL DIFFERENCE between the three-leg blend and A5"
  }
 ]
}