[ { "ended": "2026-09-21T22:21:20Z", "floor": { "counts": { "darwin": 5, "einstein": 5, "hypatia": 5, "ibn_sina": 5, "sagan": 5, "socrates": 5 }, "kind": "majority class on the held-out set", "value": 0.16666666666666666 }, "gemma4": { "accuracy": 0.7, "correct": 21, "n": 30, "source": "rows-addenda/gemma4-fp8-readout.c.jsonl, filtered to the held-out uids" }, "heldout_n": 30, "leak_check": "passed: train/dev share no uid with test", "model_bytes": 598479299, "model_mb": 598.479299, "openjev": { "accuracy": 0.8666666666666667, "correct": 26, "n": 30, "source": "rows/openjev-fp8-readout.c.jsonl, filtered to the held-out uids" }, "question": "speaker", "reading": "B: warm-started from checkpoints/general/model.pt", "seed": 20260921, "split": { "dev": 12, "test": 30, "train": 66 }, "started": "2026-09-21T22:20:59Z", "task": "longtable_voices", "train_rc": 0, "train_seconds": 14.343350786999508, "train_tail": " acc 95% CI macroF1 ECE Brier\nspeaker 12 0.3333 [0.083,0.583] 0.1786 0.106 0.8020\n\n-- dev after epoch 1\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.2500 [0.083,0.583] 0.1667 0.180 0.7911\n\n-- dev after epoch 2\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.3333 [0.083,0.583] 0.2000 0.425 0.8779\n\n-- dev after epoch 3\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.7500 [0.500,1.000] 0.7500 0.226 0.4547\n\n-- dev after epoch 4\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.8333 [0.583,1.000] 0.8222 0.282 0.3755\n ep5 step 50/54 loss 0.0477 6s\n\n-- dev after epoch 5\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 12 0.8333 [0.583,1.000] 0.8222 0.225 0.3757\n\nfitted temperatures: {'speaker': 0.69}\n\n== TEST (raw)\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 30 0.7000 [0.533,0.867] 0.7111 0.263 0.4764\n\n== TEST (temperature-scaled)\nquestion n acc 95% CI macroF1 ECE Brier\nspeaker 30 0.7000 [0.533,0.867] 0.7111 0.252 0.4893\n\nsaved to checkpoints/longtable_voices/model.pt (598 MB, 8s total)\n", "trained": { "accuracy": 0.7, "brier": 0.4763664090518541, "correct": 21, "n": 30 } }, { "ended": "2026-09-21T22:21:37Z", "floor": { "counts": { "false": 4, "true": 11 }, "kind": "majority class on the held-out set", "value": 0.7333333333333333 }, "gemma4": { "accuracy": 0.9333333333333333, "correct": 14, "n": 15, "source": "rows-addenda/gemma4-fp8-readout.d.jsonl, filtered to the held-out uids" }, "heldout_n": 15, "leak_check": "passed: train/dev share no uid with test", "model_bytes": 598479299, "model_mb": 598.479299, "openjev": { "accuracy": 1.0, "correct": 15, "n": 15, "source": "rows-addenda/openjev-fp8-readout.d.jsonl, filtered to the held-out uids" }, "question": "refuse", "reading": "B: warm-started from checkpoints/general/model.pt", "seed": 20260921, "split": { "dev": 6, "test": 15, "train": 27 }, "started": "2026-09-21T22:21:20Z", "task": "doorman_gate", "train_rc": 0, "train_seconds": 9.91060056599963, "train_tail": " epoch 0\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.097 0.2812\n\n-- dev after epoch 1\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.161 0.3147\n\n-- dev after epoch 2\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.155 0.3338\n\n-- dev after epoch 3\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.159 0.3338\n\n-- dev after epoch 4\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.160 0.3337\n\n-- dev after epoch 5\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 6 0.8333 [0.500,1.000] 0.4545 0.160 0.3337\n\nfitted temperatures: {'refuse': 5.927}\n\n== TEST (raw)\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 15 0.8667 [0.667,1.000] 0.8295 0.155 0.2793\n\n== TEST (temperature-scaled)\nquestion n acc 95% CI macroF1 ECE Brier\nrefuse 15 0.8667 [0.667,1.000] 0.8295 0.185 0.2399\n\nsaved to checkpoints/doorman_gate/model.pt (598 MB, 4s total)\n", "trained": { "accuracy": 0.8666666666666667, "brier": 0.27928308884601755, "correct": 13, "n": 15 } }, { "ended": "2026-09-21T22:21:53Z", "floor": { "counts": { "false": 6, "true": 6 }, "kind": "majority class on the held-out set", "value": 0.5 }, "gemma4": { "accuracy": 1.0, "correct": 12, "n": 12, "source": "rows-addenda/gemma4-fp8-readout.f.jsonl, filtered to the held-out uids" }, "heldout_n": 12, "leak_check": "passed: train/dev share no uid with test", "model_bytes": 598479299, "model_mb": 598.479299, "openjev": { "accuracy": 1.0, "correct": 12, "n": 12, "source": "rows-addenda/openjev-fp8-readout.f.jsonl, filtered to the held-out uids" }, "question": "answerable", "reading": "B: warm-started from checkpoints/general/model.pt", "seed": 20260921, "split": { "dev": 4, "test": 12, "train": 24 }, "started": "2026-09-21T22:21:37Z", "task": "fieldexam_grounded", "train_rc": 0, "train_seconds": 9.678412183002365, "train_tail": "uestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 0.5000 [0.000,1.000] 0.3333 0.333 0.3601\n\n-- dev after epoch 1\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 1.0000 [1.000,1.000] 1.0000 0.054 0.0087\n\n-- dev after epoch 2\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 1.0000 [1.000,1.000] 1.0000 0.013 0.0010\n\n-- dev after epoch 3\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 1.0000 [1.000,1.000] 1.0000 0.007 0.0004\n\n-- dev after epoch 4\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 1.0000 [1.000,1.000] 1.0000 0.006 0.0003\n\n-- dev after epoch 5\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 4 1.0000 [1.000,1.000] 1.0000 0.006 0.0003\n\nfitted temperatures: {'answerable': 0.356}\n\n== TEST (raw)\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 12 1.0000 [1.000,1.000] 1.0000 0.003 0.0001\n\n== TEST (temperature-scaled)\nquestion n acc 95% CI macroF1 ECE Brier\nanswerable 12 1.0000 [1.000,1.000] 1.0000 0.000 0.0000\n\nsaved to checkpoints/fieldexam_grounded/model.pt (598 MB, 4s total)\n", "trained": { "accuracy": 1.0, "brier": 0.0001306671426414627, "correct": 12, "n": 12 } } ]