{
 "script": "score_registered_w7.py",
 "script_sha256": "975fca6c1bf753d8fb9c6fa2b4f2aeac1481bd7973ecc618eccdb8d51ad0360a",
 "harness_pins": {
  "metrics.py": "614f0ed8bae5ca6855b701f0ad5057619737167b1695ffefe6ff6f4a2e03cf21",
  "data.py": "b72b19b63db76c8af8bdf3058e4af7df1fe15c17eea8f1837ec082fec722d05d"
 },
 "data_sha256": "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442",
 "a0_rows_sha256": "4d545fdcd364ce86f688250b8b46f8f36db782aba683fde49497425c6f421e55",
 "inputs": {
  "legs.ndjson": "6a4299d76c89e38818fe6d69062c3fb6b9b9bbc2836915793374bdc889bf7994",
  "a2.ndjson": "eb489cdba861b7795d07bd2ec187eb585ed0e9d5a90ea97d939e41fa3bcdd8a6"
 },
 "n_rows": {
  "A1": 500,
  "A2": 500,
  "A0": 500
 },
 "A1": {
  "n_all": 500,
  "n_answerable": 470,
  "n_abstention": 30,
  "headline": {
   "every@5 (all@5, answerable)": {
    "hits": 403,
    "n": 470,
    "misses": 67
   },
   "any@5 (all questions)": {
    "hits": 481,
    "n": 500,
    "misses": 19
   }
  },
  "answerable": {
   "any@1": {
    "hits": 398,
    "n": 470,
    "misses": 72
   },
   "any@5": {
    "hits": 453,
    "n": 470,
    "misses": 17
   },
   "all@5": {
    "hits": 403,
    "n": 470,
    "misses": 67
   },
   "all@10": {
    "hits": 437,
    "n": 470,
    "misses": 33
   }
  },
  "all_questions": {
   "any@1": {
    "hits": 421,
    "n": 500,
    "misses": 79
   },
   "any@5": {
    "hits": 481,
    "n": 500,
    "misses": 19
   },
   "all@5": {
    "hits": 427,
    "n": 500,
    "misses": 73
   },
   "all@10": {
    "hits": 466,
    "n": 500,
    "misses": 34
   }
  },
  "by_type_answerable": {
   "knowledge-update": {
    "any@1": {
     "hits": 69,
     "n": 72,
     "misses": 3
    },
    "any@5": {
     "hits": 72,
     "n": 72,
     "misses": 0
    },
    "all@5": {
     "hits": 66,
     "n": 72,
     "misses": 6
    },
    "all@10": {
     "hits": 68,
     "n": 72,
     "misses": 4
    }
   },
   "multi-session": {
    "any@1": {
     "hits": 99,
     "n": 121,
     "misses": 22
    },
    "any@5": {
     "hits": 120,
     "n": 121,
     "misses": 1
    },
    "all@5": {
     "hits": 95,
     "n": 121,
     "misses": 26
    },
    "all@10": {
     "hits": 110,
     "n": 121,
     "misses": 11
    }
   },
   "single-session-assistant": {
    "any@1": {
     "hits": 56,
     "n": 56,
     "misses": 0
    },
    "any@5": {
     "hits": 56,
     "n": 56,
     "misses": 0
    },
    "all@5": {
     "hits": 56,
     "n": 56,
     "misses": 0
    },
    "all@10": {
     "hits": 56,
     "n": 56,
     "misses": 0
    }
   },
   "single-session-preference": {
    "any@1": {
     "hits": 21,
     "n": 30,
     "misses": 9
    },
    "any@5": {
     "hits": 27,
     "n": 30,
     "misses": 3
    },
    "all@5": {
     "hits": 27,
     "n": 30,
     "misses": 3
    },
    "all@10": {
     "hits": 28,
     "n": 30,
     "misses": 2
    }
   },
   "single-session-user": {
    "any@1": {
     "hits": 50,
     "n": 64,
     "misses": 14
    },
    "any@5": {
     "hits": 60,
     "n": 64,
     "misses": 4
    },
    "all@5": {
     "hits": 60,
     "n": 64,
     "misses": 4
    },
    "all@10": {
     "hits": 63,
     "n": 64,
     "misses": 1
    }
   },
   "temporal-reasoning": {
    "any@1": {
     "hits": 103,
     "n": 127,
     "misses": 24
    },
    "any@5": {
     "hits": 118,
     "n": 127,
     "misses": 9
    },
    "all@5": {
     "hits": 99,
     "n": 127,
     "misses": 28
    },
    "all@10": {
     "hits": 112,
     "n": 127,
     "misses": 15
    }
   }
  },
  "by_type_all_questions": {
   "knowledge-update": {
    "any@5": {
     "hits": 78,
     "n": 78,
     "misses": 0
    }
   },
   "multi-session": {
    "any@5": {
     "hits": 131,
     "n": 133,
     "misses": 2
    }
   },
   "single-session-assistant": {
    "any@5": {
     "hits": 56,
     "n": 56,
     "misses": 0
    }
   },
   "single-session-preference": {
    "any@5": {
     "hits": 27,
     "n": 30,
     "misses": 3
    }
   },
   "single-session-user": {
    "any@5": {
     "hits": 65,
     "n": 70,
     "misses": 5
    }
   },
   "temporal-reasoning": {
    "any@5": {
     "hits": 124,
     "n": 133,
     "misses": 9
    }
   }
  }
 },
 "A2": {
  "n_all": 500,
  "n_answerable": 470,
  "n_abstention": 30,
  "headline": {
   "every@5 (all@5, answerable)": {
    "hits": 409,
    "n": 470,
    "misses": 61
   },
   "any@5 (all questions)": {
    "hits": 489,
    "n": 500,
    "misses": 11
   }
  },
  "answerable": {
   "any@1": {
    "hits": 415,
    "n": 470,
    "misses": 55
   },
   "any@5": {
    "hits": 460,
    "n": 470,
    "misses": 10
   },
   "all@5": {
    "hits": 409,
    "n": 470,
    "misses": 61
   },
   "all@10": {
    "hits": 450,
    "n": 470,
    "misses": 20
   }
  },
  "all_questions": {
   "any@1": {
    "hits": 438,
    "n": 500,
    "misses": 62
   },
   "any@5": {
    "hits": 489,
    "n": 500,
    "misses": 11
   },
   "all@5": {
    "hits": 432,
    "n": 500,
    "misses": 68
   },
   "all@10": {
    "hits": 478,
    "n": 500,
    "misses": 22
   }
  },
  "by_type_answerable": {
   "knowledge-update": {
    "any@1": {
     "hits": 69,
     "n": 72,
     "misses": 3
    },
    "any@5": {
     "hits": 72,
     "n": 72,
     "misses": 0
    },
    "all@5": {
     "hits": 66,
     "n": 72,
     "misses": 6
    },
    "all@10": {
     "hits": 71,
     "n": 72,
     "misses": 1
    }
   },
   "multi-session": {
    "any@1": {
     "hits": 106,
     "n": 121,
     "misses": 15
    },
    "any@5": {
     "hits": 121,
     "n": 121,
     "misses": 0
    },
    "all@5": {
     "hits": 96,
     "n": 121,
     "misses": 25
    },
    "all@10": {
     "hits": 114,
     "n": 121,
     "misses": 7
    }
   },
   "single-session-assistant": {
    "any@1": {
     "hits": 56,
     "n": 56,
     "misses": 0
    },
    "any@5": {
     "hits": 56,
     "n": 56,
     "misses": 0
    },
    "all@5": {
     "hits": 56,
     "n": 56,
     "misses": 0
    },
    "all@10": {
     "hits": 56,
     "n": 56,
     "misses": 0
    }
   },
   "single-session-preference": {
    "any@1": {
     "hits": 23,
     "n": 30,
     "misses": 7
    },
    "any@5": {
     "hits": 27,
     "n": 30,
     "misses": 3
    },
    "all@5": {
     "hits": 27,
     "n": 30,
     "misses": 3
    },
    "all@10": {
     "hits": 29,
     "n": 30,
     "misses": 1
    }
   },
   "single-session-user": {
    "any@1": {
     "hits": 54,
     "n": 64,
     "misses": 10
    },
    "any@5": {
     "hits": 63,
     "n": 64,
     "misses": 1
    },
    "all@5": {
     "hits": 63,
     "n": 64,
     "misses": 1
    },
    "all@10": {
     "hits": 64,
     "n": 64,
     "misses": 0
    }
   },
   "temporal-reasoning": {
    "any@1": {
     "hits": 107,
     "n": 127,
     "misses": 20
    },
    "any@5": {
     "hits": 121,
     "n": 127,
     "misses": 6
    },
    "all@5": {
     "hits": 101,
     "n": 127,
     "misses": 26
    },
    "all@10": {
     "hits": 116,
     "n": 127,
     "misses": 11
    }
   }
  },
  "by_type_all_questions": {
   "knowledge-update": {
    "any@5": {
     "hits": 78,
     "n": 78,
     "misses": 0
    }
   },
   "multi-session": {
    "any@5": {
     "hits": 133,
     "n": 133,
     "misses": 0
    }
   },
   "single-session-assistant": {
    "any@5": {
     "hits": 56,
     "n": 56,
     "misses": 0
    }
   },
   "single-session-preference": {
    "any@5": {
     "hits": 27,
     "n": 30,
     "misses": 3
    }
   },
   "single-session-user": {
    "any@5": {
     "hits": 68,
     "n": 70,
     "misses": 2
    }
   },
   "temporal-reasoning": {
    "any@5": {
     "hits": 127,
     "n": 133,
     "misses": 6
    }
   }
  }
 },
 "A0_recomputed": {
  "every@5": {
   "hits": 400,
   "n": 470,
   "misses": 70
  },
  "any@5": {
   "hits": 485,
   "n": 500,
   "misses": 15
  }
 },
 "a2_stored_rows_differing": [],
 "G-WIN A2 vs A0 (every@5, 470)": {
  "b_arm_only": 23,
  "c_ref_only": 14,
  "p_two_sided": 0.18774155899882317,
  "delta_hits": 9,
  "margin": 10,
  "alpha": 0.05,
  "verdict": "NO MATERIAL DIFFERENCE",
  "holm_bound": {
   "lower": 0.18774155899882317,
   "upper": 0.7509662359952927,
   "decided": true,
   "note": "Holm over the four G-WIN contrasts; the other three (A3 vs A2, A4 vs A2, A4 vs A3) have no p-value yet. For the contrast ranked i-th, adjusted = max_{j<=i} min(1,(5-j)p(j)), which lies in [p, min(1,4p)] whatever the other three read."
  }
 },
 "G-WIN-2 A2 vs A0 (any@5, 500)": {
  "b_arm_only": 8,
  "c_ref_only": 4,
  "p_two_sided": 0.3876953125,
  "delta_hits": 4,
  "margin": 5,
  "alpha": 0.05,
  "verdict": "NO MATERIAL DIFFERENCE"
 },
 "wall_secs": 1.72,
 "utc": "2026-10-17T10:15:42Z"
}