[ { "ended": "2026-09-21T22:18:05Z", "floor": { "counts": { "darwin": 5, "einstein": 5, "hypatia": 5, "ibn_sina": 5, "sagan": 5, "socrates": 5 }, "kind": "majority class on the held-out set", "value": 0.16666666666666666 }, "gemma4": { "accuracy": 0.7, "correct": 21, "n": 30, "source": "rows-addenda/gemma4-fp8-readout.c.jsonl, filtered to the held-out uids" }, "heldout_n": 30, "leak_check": "passed: train/dev share no uid with test", "model_bytes": 598479299, "model_mb": 598.479299, "openjev": { "accuracy": 0.8666666666666667, "correct": 26, "n": 30, "source": "rows/openjev-fp8-readout.c.jsonl, filtered to the held-out uids" }, "question": "speaker", "seed": 20260921, "split": { "dev": 12, "test": 30, "train": 66 }, "started": "2026-09-21T22:17:44Z", "task": "longtable_voices", "train_rc": 0, "train_seconds": 14.279769847002171, "train_tail": " acc 95% CI macroF1 ECE Brier\nspeaker 12 0.1667 [0.000,0.417] 0.0476 0.045 0.8360\n\n-- dev after epoch 1\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.1667 [0.000,0.417] 0.0476 0.048 0.8345\n\n-- dev after epoch 2\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.1667 [0.000,0.417] 0.0476 0.031 0.8357\n\n-- dev after epoch 3\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.2500 [0.000,0.500] 0.1624 0.063 0.8337\n\n-- dev after epoch 4\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.0833 [0.000,0.250] 0.0278 0.110 0.8358\n ep5 step 50/54 loss 0.1800 7s\n\n-- dev after epoch 5\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.0833 [0.000,0.250] 0.0278 0.112 0.8360\n\nfitted temperatures: {'speaker': 6.787}\n\n== TEST (raw)\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 30 0.2333 [0.100,0.400] 0.1263 0.039 0.8343\n\n== TEST (temperature-scaled)\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 30 0.2333 [0.100,0.400] 0.1263 0.063 0.8330\n\nsaved to checkpoints/longtable_voices/model.pt (598 MB, 9s total)\n", "trained": { "accuracy": 0.23333333333333334, "brier": 0.8342921905341106, "correct": 7, "n": 30 } }, { "ended": "2026-09-21T22:18:21Z", "floor": { "counts": { "false": 4, "true": 11 }, "kind": "majority class on the held-out set", "value": 0.7333333333333333 }, "gemma4": { "accuracy": 0.9333333333333333, "correct": 14, "n": 15, "source": "rows-addenda/gemma4-fp8-readout.d.jsonl, filtered to the held-out uids" }, "heldout_n": 15, "leak_check": "passed: train/dev share no uid with test", "model_bytes": 598479299, "model_mb": 598.479299, "openjev": { "accuracy": 1.0, "correct": 15, "n": 15, "source": "rows-addenda/openjev-fp8-readout.d.jsonl, filtered to the held-out uids" }, "question": "refuse", "seed": 20260921, "split": { "dev": 6, "test": 15, "train": 27 }, "started": "2026-09-21T22:18:05Z", "task": "doorman_gate", "train_rc": 0, "train_seconds": 8.943027540000912, "train_tail": " epoch 0\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.166 0.3331\n\n-- dev after epoch 1\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.477 0.4636\n\n-- dev after epoch 2\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.144 0.3301\n\n-- dev after epoch 3\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.180 0.3128\n\n-- dev after epoch 4\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.077 0.3122\n\n-- dev after epoch 5\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.080 0.3150\n\nfitted temperatures: {'refuse': 0.799}\n\n== TEST (raw)\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 15 0.7333 [0.533,0.933] 0.4231 0.149 0.3622\n\n== TEST (temperature-scaled)\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 15 0.7333 [0.533,0.933] 0.4231 0.137 0.3688\n\nsaved to checkpoints/doorman_gate/model.pt (598 MB, 3s total)\n", "trained": { "accuracy": 0.7333333333333333, "brier": 0.36219782782332743, "correct": 11, "n": 15 } }, { "ended": "2026-09-21T22:18:36Z", "floor": { "counts": { "false": 6, "true": 6 }, "kind": "majority class on the held-out set", "value": 0.5 }, "gemma4": { "accuracy": 1.0, "correct": 12, "n": 12, "source": "rows-addenda/gemma4-fp8-readout.f.jsonl, filtered to the held-out uids" }, "heldout_n": 12, "leak_check": "passed: train/dev share no uid with test", "model_bytes": 598479299, "model_mb": 598.479299, "openjev": { "accuracy": 1.0, "correct": 12, "n": 12, "source": "rows-addenda/openjev-fp8-readout.f.jsonl, filtered to the held-out uids" }, "question": "answerable", "seed": 20260921, "split": { "dev": 4, "test": 12, "train": 24 }, "started": "2026-09-21T22:18:21Z", "task": "fieldexam_grounded", "train_rc": 0, "train_seconds": 8.860026867998386, "train_tail": "stion n acc 95% CI macroF1 ECE Brier\nanswerable 4 0.5000 [0.000,1.000] 0.3333 0.321 0.5370\n\n-- dev after epoch 1\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 0.5000 [0.000,1.000] 0.3333 0.297 0.5459\n\n-- dev after epoch 2\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 0.5000 [0.000,1.000] 0.3333 0.301 0.5996\n\n-- dev after epoch 3\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 0.2500 [0.000,0.750] 0.2000 0.576 0.6850\n\n-- dev after epoch 4\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 0.5000 [0.000,1.000] 0.5000 0.588 0.7248\n\n-- dev after epoch 5\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 0.5000 [0.000,1.000] 0.5000 0.586 0.7256\n\nfitted temperatures: {'answerable': 175.449}\n\n== TEST (raw)\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 12 0.5000 [0.250,0.750] 0.4857 0.193 0.4860\n\n== TEST (temperature-scaled)\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 12 0.5000 [0.250,0.750] 0.4857 0.001 0.4995\n\nsaved to checkpoints/fieldexam_grounded/model.pt (598 MB, 3s total)\n", "trained": { "accuracy": 0.5, "brier": 0.4860280410160171, "correct": 6, "n": 12 } } ]