ANNOUNCE box benchbox · cards GPU-46890836-d8f, GPU-7aa0be10-974 · arm gemma4-fp8-readout · task c · start 2026-09-21T22:01:06Z · expect ~5 min c-darwin-01-dessert-01 darwin -> darwin 0.83s c-darwin-01-dessert-01 darwin -> darwin 0.15s c-darwin-01-dessert-01 darwin -> darwin 0.10s c-darwin-01-dessert-02 darwin -> darwin 0.16s c-darwin-01-dessert-03 darwin -> hypatia 0.16s c-darwin-01-dessert-03 darwin -> darwin 0.14s c-darwin-01-main-01 darwin -> darwin 0.12s c-darwin-01-main-02 darwin -> ibn_sina 0.12s c-darwin-01-main-02 darwin -> darwin 0.14s c-darwin-01-main-02 darwin -> darwin 0.16s c-darwin-01-main-02 darwin -> darwin 0.14s c-darwin-01-main-03 darwin -> darwin 0.14s c-darwin-01-starter-01 darwin -> sagan 0.09s c-darwin-01-starter-01 darwin -> darwin 0.09s c-darwin-01-starter-01 darwin -> sagan 0.11s c-darwin-01-starter-02 darwin -> darwin 0.09s c-darwin-01-starter-03 darwin -> darwin 0.15s c-darwin-01-starter-03 darwin -> darwin 0.14s c-einstein-01-dessert-01 einstein -> sagan 0.14s c-einstein-01-dessert-01 einstein -> sagan 0.14s c-einstein-01-dessert-02 einstein -> einstein 0.12s c-einstein-01-dessert-02 einstein -> einstein 0.09s c-einstein-01-dessert-03 einstein -> einstein 0.15s c-einstein-01-main-01 einstein -> socrates 0.11s c-einstein-01-main-01 einstein -> einstein 0.11s c-einstein-01-main-02 einstein -> einstein 0.13s c-einstein-01-main-03 einstein -> sagan 0.13s c-einstein-01-main-03 einstein -> einstein 0.12s c-einstein-01-main-03 einstein -> einstein 0.12s c-einstein-01-main-03 einstein -> einstein 0.14s c-einstein-01-starter-01 einstein -> einstein 0.11s c-einstein-01-starter-02 einstein -> einstein 0.09s c-einstein-01-starter-02 einstein -> einstein 0.11s c-einstein-01-starter-03 einstein -> einstein 0.13s c-einstein-01-starter-03 einstein -> einstein 0.12s c-einstein-01-starter-03 einstein -> einstein 0.09s c-hypatia-01-dessert-01 hypatia -> hypatia 0.15s c-hypatia-01-dessert-02 hypatia -> hypatia 0.12s c-hypatia-01-dessert-02 hypatia -> sagan 0.09s c-hypatia-01-dessert-03 hypatia -> ibn_sina 0.11s c-hypatia-01-main-01 hypatia -> hypatia 0.14s c-hypatia-01-main-01 hypatia -> einstein 0.10s c-hypatia-01-main-01 hypatia -> hypatia 0.13s c-hypatia-01-main-01 hypatia -> hypatia 0.11s c-hypatia-01-main-01 hypatia -> sagan 0.10s c-hypatia-01-main-02 hypatia -> einstein 0.12s c-hypatia-01-main-03 hypatia -> ibn_sina 0.12s c-hypatia-01-main-03 hypatia -> hypatia 0.11s c-hypatia-01-main-03 hypatia -> hypatia 0.15s c-hypatia-01-starter-01 hypatia -> hypatia 0.09s c-hypatia-01-starter-01 hypatia -> socrates 0.14s c-hypatia-01-starter-02 hypatia -> hypatia 0.10s c-hypatia-01-starter-02 hypatia -> hypatia 0.09s c-hypatia-01-starter-02 hypatia -> hypatia 0.11s c-ibn_sina-01-dessert-01 ibn_sina -> ibn_sina 0.10s c-ibn_sina-01-dessert-02 ibn_sina -> ibn_sina 0.12s c-ibn_sina-01-dessert-02 ibn_sina -> ibn_sina 0.11s c-ibn_sina-01-dessert-02 ibn_sina -> ibn_sina 0.11s c-ibn_sina-01-dessert-03 ibn_sina -> ibn_sina 0.12s c-ibn_sina-01-dessert-03 ibn_sina -> ibn_sina 0.13s c-ibn_sina-01-main-02 ibn_sina -> hypatia 0.11s c-ibn_sina-01-main-02 ibn_sina -> ibn_sina 0.13s c-ibn_sina-01-main-02 ibn_sina -> ibn_sina 0.12s c-ibn_sina-01-main-03 ibn_sina -> ibn_sina 0.14s c-ibn_sina-01-main-03 ibn_sina -> ibn_sina 0.13s c-ibn_sina-01-main-03 ibn_sina -> ibn_sina 0.13s c-ibn_sina-01-starter-01 ibn_sina -> hypatia 0.11s c-ibn_sina-01-starter-02 ibn_sina -> socrates 0.10s c-ibn_sina-01-starter-03 ibn_sina -> ibn_sina 0.13s c-ibn_sina-01-starter-03 ibn_sina -> ibn_sina 0.12s c-ibn_sina-01-starter-03 ibn_sina -> ibn_sina 0.12s c-ibn_sina-01-starter-03 ibn_sina -> ibn_sina 0.13s c-sagan-01-dessert-01 sagan -> sagan 0.14s c-sagan-01-dessert-01 sagan -> sagan 0.10s c-sagan-01-dessert-01 sagan -> sagan 0.09s c-sagan-01-dessert-03 sagan -> sagan 0.12s c-sagan-01-dessert-03 sagan -> sagan 0.14s c-sagan-01-dessert-03 sagan -> sagan 0.11s c-sagan-01-dessert-03 sagan -> sagan 0.14s c-sagan-01-dessert-03 sagan -> sagan 0.13s c-sagan-01-main-01 sagan -> sagan 0.12s c-sagan-01-main-02 sagan -> sagan 0.13s c-sagan-01-main-02 sagan -> sagan 0.11s c-sagan-01-main-03 sagan -> sagan 0.14s c-sagan-01-main-03 sagan -> sagan 0.13s c-sagan-01-starter-01 sagan -> sagan 0.10s c-sagan-01-starter-02 sagan -> sagan 0.15s c-sagan-01-starter-02 sagan -> sagan 0.09s c-sagan-01-starter-03 sagan -> darwin 0.10s c-sagan-01-starter-03 sagan -> sagan 0.13s c-socrates-01-dessert-01 socrates -> socrates 0.15s c-socrates-01-dessert-01 socrates -> socrates 0.10s c-socrates-01-dessert-01 socrates -> socrates 0.12s c-socrates-01-dessert-02 socrates -> sagan 0.11s c-socrates-01-dessert-02 socrates -> socrates 0.12s c-socrates-01-dessert-02 socrates -> socrates 0.11s c-socrates-01-main-01 socrates -> socrates 0.09s c-socrates-01-main-02 socrates -> socrates 0.15s c-socrates-01-main-02 socrates -> socrates 0.09s c-socrates-01-main-02 socrates -> socrates 0.12s c-socrates-01-main-02 socrates -> ibn_sina 0.12s c-socrates-01-main-03 socrates -> socrates 0.12s c-socrates-01-main-03 socrates -> socrates 0.14s c-socrates-01-starter-01 socrates -> socrates 0.09s c-socrates-01-starter-02 socrates -> socrates 0.10s c-socrates-01-starter-03 socrates -> socrates 0.11s c-socrates-01-starter-03 socrates -> socrates 0.12s c-socrates-01-starter-03 socrates -> socrates 0.12s {"n": 108, "n_reached": 108, "n_labelled": 108, "refused": 0, "refused_by_reason": {}, "correct": 87, "errors": 0, "brier": 0.33806863300148976, "reliability": [{"lo": 0.0, "hi": 0.1, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.1, "hi": 0.2, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.2, "hi": 0.3, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.3, "hi": 0.4, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.4, ANNOUNCE box benchbox · cards GPU-46890836-d8f, GPU-7aa0be10-974 · arm gemma4-fp8-generate · task c · start 2026-09-21T22:01:31Z · expect ~5 min c-darwin-01-dessert-01 darwin -> darwin 0.05s c-darwin-01-dessert-01 darwin -> darwin 0.10s c-darwin-01-dessert-01 darwin -> darwin 0.07s c-darwin-01-dessert-02 darwin -> darwin 0.08s c-darwin-01-dessert-03 darwin -> darwin 0.09s c-darwin-01-dessert-03 darwin -> darwin 0.06s c-darwin-01-main-01 darwin -> darwin 0.06s c-darwin-01-main-02 darwin -> ibn_sina 0.07s c-darwin-01-main-02 darwin -> darwin 0.06s c-darwin-01-main-02 darwin -> darwin 0.07s c-darwin-01-main-02 darwin -> darwin 0.06s c-darwin-01-main-03 darwin -> darwin 0.08s c-darwin-01-starter-01 darwin -> sagan 0.06s c-darwin-01-starter-01 darwin -> darwin 0.06s c-darwin-01-starter-01 darwin -> darwin 0.07s c-darwin-01-starter-02 darwin -> darwin 0.07s c-darwin-01-starter-03 darwin -> darwin 0.07s c-darwin-01-starter-03 darwin -> darwin 0.06s c-einstein-01-dessert-01 einstein -> sagan 0.07s c-einstein-01-dessert-01 einstein -> sagan 0.07s c-einstein-01-dessert-02 einstein -> einstein 0.07s c-einstein-01-dessert-02 einstein -> einstein 0.07s c-einstein-01-dessert-03 einstein -> einstein 0.05s c-einstein-01-main-01 einstein -> socrates 0.07s c-einstein-01-main-01 einstein -> einstein 0.06s c-einstein-01-main-02 einstein -> einstein 0.06s c-einstein-01-main-03 einstein -> sagan 0.06s c-einstein-01-main-03 einstein -> einstein 0.07s c-einstein-01-main-03 einstein -> einstein 0.06s c-einstein-01-main-03 einstein -> einstein 0.07s c-einstein-01-starter-01 einstein -> einstein 0.07s c-einstein-01-starter-02 einstein -> einstein 0.07s c-einstein-01-starter-02 einstein -> einstein 0.06s c-einstein-01-starter-03 einstein -> einstein 0.08s c-einstein-01-starter-03 einstein -> einstein 0.06s c-einstein-01-starter-03 einstein -> einstein 0.06s c-hypatia-01-dessert-01 hypatia -> hypatia 0.09s c-hypatia-01-dessert-02 hypatia -> hypatia 0.09s c-hypatia-01-dessert-02 hypatia -> sagan 0.07s c-hypatia-01-dessert-03 hypatia -> ibn_sina 0.06s c-hypatia-01-main-01 hypatia -> hypatia 0.06s c-hypatia-01-main-01 hypatia -> einstein 0.06s c-hypatia-01-main-01 hypatia -> hypatia 0.06s c-hypatia-01-main-01 hypatia -> hypatia 0.07s c-hypatia-01-main-01 hypatia -> sagan 0.07s c-hypatia-01-main-02 hypatia -> einstein 0.06s c-hypatia-01-main-03 hypatia -> einstein 0.06s c-hypatia-01-main-03 hypatia -> hypatia 0.06s c-hypatia-01-main-03 hypatia -> hypatia 0.08s c-hypatia-01-starter-01 hypatia -> hypatia 0.07s c-hypatia-01-starter-01 hypatia -> darwin 0.09s c-hypatia-01-starter-02 hypatia -> hypatia 0.05s c-hypatia-01-starter-02 hypatia -> hypatia 0.06s c-hypatia-01-starter-02 hypatia -> hypatia 0.06s c-ibn_sina-01-dessert-01 ibn_sina -> ibn_sina 0.04s c-ibn_sina-01-dessert-02 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-dessert-02 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-dessert-02 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-dessert-03 ibn_sina -> ibn_sina 0.07s c-ibn_sina-01-dessert-03 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-main-02 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-main-02 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-main-02 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-main-03 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-main-03 ibn_sina -> ibn_sina 0.07s c-ibn_sina-01-main-03 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-starter-01 ibn_sina -> hypatia 0.08s c-ibn_sina-01-starter-02 ibn_sina -> socrates 0.06s c-ibn_sina-01-starter-03 ibn_sina -> ibn_sina 0.08s c-ibn_sina-01-starter-03 ibn_sina -> ibn_sina 0.08s c-ibn_sina-01-starter-03 ibn_sina -> ibn_sina 0.06s c-ibn_sina-01-starter-03 ibn_sina -> ibn_sina 0.06s c-sagan-01-dessert-01 sagan -> sagan 0.06s c-sagan-01-dessert-01 sagan -> sagan 0.06s c-sagan-01-dessert-01 sagan -> sagan 0.07s c-sagan-01-dessert-03 sagan -> sagan 0.06s c-sagan-01-dessert-03 sagan -> sagan 0.06s c-sagan-01-dessert-03 sagan -> sagan 0.06s c-sagan-01-dessert-03 sagan -> sagan 0.06s c-sagan-01-dessert-03 sagan -> sagan 0.07s c-sagan-01-main-01 sagan -> sagan 0.06s c-sagan-01-main-02 sagan -> sagan 0.06s c-sagan-01-main-02 sagan -> sagan 0.06s c-sagan-01-main-03 sagan -> sagan 0.07s c-sagan-01-main-03 sagan -> sagan 0.06s c-sagan-01-starter-01 sagan -> sagan 0.06s c-sagan-01-starter-02 sagan -> sagan 0.06s c-sagan-01-starter-02 sagan -> sagan 0.06s c-sagan-01-starter-03 sagan -> darwin 0.06s c-sagan-01-starter-03 sagan -> einstein 0.06s c-socrates-01-dessert-01 socrates -> socrates 0.05s c-socrates-01-dessert-01 socrates -> socrates 0.07s c-socrates-01-dessert-01 socrates -> socrates 0.06s c-socrates-01-dessert-02 socrates -> sagan 0.05s c-socrates-01-dessert-02 socrates -> socrates 0.06s c-socrates-01-dessert-02 socrates -> socrates 0.06s c-socrates-01-main-01 socrates -> socrates 0.07s c-socrates-01-main-02 socrates -> socrates 0.06s c-socrates-01-main-02 socrates -> socrates 0.06s c-socrates-01-main-02 socrates -> socrates 0.06s c-socrates-01-main-02 socrates -> hypatia 0.09s c-socrates-01-main-03 socrates -> socrates 0.06s c-socrates-01-main-03 socrates -> socrates 0.06s c-socrates-01-starter-01 socrates -> socrates 0.09s c-socrates-01-starter-02 socrates -> socrates 0.06s c-socrates-01-starter-03 socrates -> socrates 0.07s c-socrates-01-starter-03 socrates -> socrates 0.06s c-socrates-01-starter-03 socrates -> socrates 0.07s {"n": 108, "n_reached": 108, "n_labelled": 108, "refused": 0, "refused_by_reason": {}, "correct": 89, "errors": 0, "brier": 0.3487654320987654, "reliability": [{"lo": 0.0, "hi": 0.1, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.1, "hi": 0.2, "n": 2, "sum_p": 0.3333333333333333, "correct": 1, "mean_p": 0.16666666666666666, "accuracy": 0.5, "gap": -0.33333333333333337}, {"lo": 0.2, "hi": 0.3, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null, "accuracy": null, "gap": null}, {"lo": 0.3, "hi": 0.4, "n": 0, "sum_p": 0.0, "correct": 0, "mean_p": null,