ANNOUNCE box benchbox · cards GPU-46890836-d8f, GPU-7aa0be10-974 · arm gemma4-fp8-readout · task d · start 2026-09-21T22:13:35Z · expect ~4 min A1 refuse -> refuse 0.28s A2 refuse -> refuse 0.21s A3 refuse -> refuse 0.21s A4 refuse -> refuse 0.21s A5 refuse -> refuse 0.21s A6 refuse -> refuse 0.21s B1 refuse -> refuse 0.21s B2 refuse -> refuse 0.21s B3 refuse -> refuse 0.20s B4 refuse -> refuse 0.21s B5 refuse -> refuse 0.21s B6 refuse -> refuse 0.21s C1 refuse -> admit 0.21s C2 refuse -> admit 0.21s C3 refuse -> admit 0.21s C4 refuse -> admit 0.21s C5 refuse -> admit 0.21s C6 refuse -> admit 0.21s D1 refuse -> admit 0.21s D2 refuse -> admit 0.21s D3 refuse -> admit 0.21s D4 refuse -> admit 0.21s D5 refuse -> admit 0.21s D6 refuse -> admit 0.21s E1 refuse -> refuse 0.21s E2 refuse -> refuse 0.21s E3 refuse -> refuse 0.21s E4 refuse -> refuse 0.22s E5 refuse -> refuse 0.21s E6 refuse -> refuse 0.21s F1 refuse -> refuse 0.21s F2 refuse -> refuse 0.21s F3 refuse -> refuse 0.21s F4 refuse -> admit 0.21s F5 refuse -> refuse 0.21s F6 refuse -> refuse 0.21s L1 admit -> admit 0.20s L2 admit -> admit 0.20s L3 admit -> admit 0.21s L4 admit -> admit 0.21s N1 admit -> admit 0.21s N2 admit -> admit 0.21s N3 admit -> admit 0.21s N4 admit -> admit 0.21s T1 admit -> admit 0.21s T2 admit -> admit 0.21s T3 admit -> admit 0.21s T4 admit -> admit 0.21s {"n": 48, "n_reached": 48, "n_labelled": 48, "refused": 0, "refused_by_reason": {}, "correct": 35, "errors": 0, "brier": 0.5185008731128865, "reliability": [{"lo": 0.0, "hi": 0.1, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.1, "hi": 0.2, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.2, "hi": 0.3, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.3, "hi": 0.4, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.4, "hi ANNOUNCE box benchbox · cards GPU-46890836-d8f, GPU-7aa0be10-974 · arm gemma4-fp8-readout · task f · start 2026-09-21T22:13:56Z · expect ~3 min t1-01 yes -> yes 0.11s t1-02 yes -> yes 0.10s t1-03 yes -> yes 0.10s t1-04 yes -> yes 0.10s t1-05 yes -> yes 0.10s t1-06 yes -> yes 0.11s t1-07 yes -> yes 0.11s t1-08 yes -> yes 0.11s t1-09 yes -> yes 0.11s t1-10 yes -> yes 0.10s t1-11 yes -> yes 0.10s t1-12 yes -> yes 0.09s t1-13 yes -> yes 0.10s t1-14 yes -> yes 0.11s t1-15 yes -> yes 0.11s t1-16 yes -> yes 0.11s t1-17 yes -> yes 0.10s t1-18 yes -> yes 0.11s t1-19 yes -> yes 0.09s t1-20 yes -> yes 0.11s t2-01 no -> no 0.10s t2-02 no -> no 0.10s t2-03 no -> no 0.10s t2-04 no -> no 0.10s t2-05 no -> no 0.11s t2-06 no -> no 0.10s t2-07 no -> no 0.10s t2-08 no -> no 0.11s t2-09 no -> no 0.10s t2-10 no -> no 0.10s t2-11 no -> no 0.09s t2-12 no -> no 0.11s t2-13 no -> no 0.10s t2-14 no -> no 0.10s t2-15 no -> no 0.10s t2-16 no -> no 0.09s t2-17 no -> no 0.10s t2-18 no -> no 0.10s t2-19 no -> no 0.10s t2-20 no -> no 0.10s {"n": 40, "n_reached": 40, "n_labelled": 40, "refused": 0, "refused_by_reason": {}, "correct": 40, "errors": 0, "brier": 8.349285404260098e-05, "reliability": [{"lo": 0.0, "hi": 0.1, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.1, "hi": 0.2, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.2, "hi": 0.3, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.3, "hi": 0.4, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.4, ANNOUNCE box benchbox · cards GPU-46890836-d8f, GPU-7aa0be10-974 · arm gemma4-fp8-generate · task d · start 2026-09-21T22:14:11Z · expect ~4 min A1 refuse -> refuse 0.05s A2 refuse -> refuse 0.05s A3 refuse -> refuse 0.05s A4 refuse -> refuse 0.03s A5 refuse -> refuse 0.05s A6 refuse -> refuse 0.05s B1 refuse -> refuse 0.03s B2 refuse -> refuse 0.03s B3 refuse -> refuse 0.05s B4 refuse -> refuse 0.07s B5 refuse -> refuse 0.05s B6 refuse -> refuse 0.05s C1 refuse -> admit 0.05s C2 refuse -> admit 0.05s C3 refuse -> admit 0.05s C4 refuse -> admit 0.05s C5 refuse -> admit 0.05s C6 refuse -> admit 0.03s D1 refuse -> admit 0.05s D2 refuse -> admit 0.07s D3 refuse -> admit 0.05s D4 refuse -> admit 0.07s D5 refuse -> admit 0.05s D6 refuse -> admit 0.05s E1 refuse -> refuse 0.05s E2 refuse -> refuse 0.05s E3 refuse -> refuse 0.05s E4 refuse -> refuse 0.05s E5 refuse -> refuse 0.05s E6 refuse -> refuse 0.06s F1 refuse -> refuse 0.05s F2 refuse -> refuse 0.05s F3 refuse -> refuse 0.05s F4 refuse -> admit 0.07s F5 refuse -> refuse 0.05s F6 refuse -> refuse 0.03s L1 admit -> admit 0.05s L2 admit -> admit 0.07s L3 admit -> admit 0.07s L4 admit -> admit 0.07s N1 admit -> admit 0.03s N2 admit -> admit 0.07s N3 admit -> admit 0.05s N4 admit -> admit 0.05s T1 admit -> admit 0.03s T2 admit -> admit 0.07s T3 admit -> admit 0.03s T4 admit -> admit 0.07s {"n": 48, "n_reached": 48, "n_labelled": 48, "refused": 0, "refused_by_reason": {}, "correct": 35, "errors": 0, "brier": 0.5104166666666666, "reliability": [{"lo": 0.0, "hi": 0.1, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.1, "hi": 0.2, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.2, "hi": 0.3, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.3, "hi": 0.4, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.4, "hi ANNOUNCE box benchbox · cards GPU-46890836-d8f, GPU-7aa0be10-974 · arm gemma4-fp8-generate · task f · start 2026-09-21T22:14:24Z · expect ~3 min t1-01 yes -> yes 0.06s t1-02 yes -> yes 0.07s t1-03 yes -> yes 0.05s t1-04 yes -> yes 0.07s t1-05 yes -> yes 0.04s t1-06 yes -> yes 0.06s t1-07 yes -> yes 0.07s t1-08 yes -> yes 0.05s t1-09 yes -> yes 0.05s t1-10 yes -> yes 0.07s t1-11 yes -> yes 0.04s t1-12 yes -> yes 0.06s t1-13 yes -> yes 0.06s t1-14 yes -> yes 0.07s t1-15 yes -> yes 0.05s t1-16 yes -> yes 0.07s t1-17 yes -> yes 0.07s t1-18 yes -> yes 0.07s t1-19 yes -> yes 0.05s t1-20 yes -> yes 0.07s t2-01 no -> no 0.07s t2-02 no -> no 0.05s t2-03 no -> no 0.05s t2-04 no -> no 0.07s t2-05 no -> no 0.06s t2-06 no -> no 0.05s t2-07 no -> no 0.05s t2-08 no -> no 0.07s t2-09 no -> no 0.07s t2-10 no -> no 0.07s t2-11 no -> no 0.07s t2-12 no -> no 0.05s t2-13 no -> no 0.07s t2-14 no -> no 0.06s t2-15 no -> no 0.06s t2-16 no -> no 0.05s t2-17 no -> no 0.06s t2-18 no -> no 0.06s t2-19 no -> no 0.05s t2-20 no -> no 0.05s {"n": 40, "n_reached": 40, "n_labelled": 40, "refused": 0, "refused_by_reason": {}, "correct": 40, "errors": 0, "brier": 8.349285404260098e-05, "reliability": [{"lo": 0.0, "hi": 0.1, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.1, "hi": 0.2, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.2, "hi": 0.3, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.3, "hi": 0.4, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.4,