{ "builder_check": { "of": 108, "s0_rebuilt_equal_kev9b": 108 }, "card": "RTX 3090 (fp 33a0b4acb8e8)", "depths": { "high": 32, "low": 16 }, "driver": [ "595.84", "NVIDIA GeForce RTX 3090", "24576", "16" ], "dry": 0, "ended": "2026-09-28T16:50:39Z", "env": { "CUDA_VISIBLE_DEVICES": "", "DO_NOT_TRACK": null, "GRADIO_ANALYTICS_ENABLED": "False", "HF_HOME": "/workshop/bench-scratch-apus-a1b/sandbox-home/hf", "HF_HUB_DISABLE_TELEMETRY": "1", "HF_HUB_OFFLINE": "1", "ORT_DISABLE_TELEMETRY": "1", "TORCH_DISABLE_NATIVE_JIT": "1", "TRANSFORMERS_OFFLINE": "1", "VLLM_NO_USAGE_STATS": "1" }, "files": { "all_equal_pins": true, "bridge": "7cb3e56d42651087e35b7a620b2407dc96801a09ae942bbe803cb202ee187e65", "files": 36, "seconds": 4.8 }, "guard_heartbeat_age_s": 0.8, "h48_sha256": "", "hostos_sha256": "a6a74bba57250be1936825466c6614876d757cf81373fcf6fa0b48587cf0e3ad", "instrument_sha256": { "run.py": "", "run_addenda.py": "a34aca61833ed0c5860c91c7e2e227429e8ea9e2e50fec68d7b7fec15d89acab", "run_kev.py": "1fbce7a41746ed3d612a8a488e35dbebba789bc55086daa7d71ffce2cd010b05" }, "load_s": 3.1, "mem_available_bytes": 30868168704, "os": { "arch": "x86_64", "id": "ubuntu", "kernel": "7.0.0-31-generic", "pretty_name": "Ubuntu 26.04 LTS", "source": "/etc/os-release", "version_id": "26.04" }, "peak_memory": { "max_allocated_mib": 10701.40966796875, "max_reserved_mib": 16358.0 }, "pins_sha256": "afe301e75b5c58e9a6305ddfba85bdf5122c489366903007b978fedd716b0e8c", "posture_after_release": { "compute_apps": 1, "empty": false, "memory_used_mib": 346.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P2", "temp_c": 60.0, "utc": "2026-09-28T16:50:39Z", "when": "after the model was released (this process still holds a context)" }, "posture_after_tasks": { "compute_apps": 1, "empty": false, "memory_used_mib": 16684.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P2", "temp_c": 61.0, "utc": "2026-09-28T16:50:38Z", "when": "after the tasks, model still resident" }, "posture_before": { "compute_apps": 0, "empty": true, "memory_used_mib": 1.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P8", "temp_c": 36.0, "utc": "2026-09-28T16:44:18Z", "when": "before the load" }, "posture_loaded": { "compute_apps": 1, "empty": false, "memory_used_mib": 9058.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P2", "temp_c": 38.0, "utc": "2026-09-28T16:44:29Z", "when": "after the load" }, "posture_resident": { "compute_apps": 1, "empty": false, "memory_used_mib": 9236.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P2", "temp_c": 42.0, "utc": "2026-09-28T16:44:30Z", "when": "after the warm-ups" }, "preflight": false, "residency": { "bar_mib": 8225.1, "compute_apps": 1, "growth_mib": 9235.0, "ok": true }, "runner_sha256": "4eb415638ae570e5f27c26dd59a92de0172de0a23d104cfdfe277cdc1717eafd", "started": "2026-09-28T16:44:18Z", "summaries": { "apus-4b-high.ka": { "accuracy": 1.0, "brier": 9.131412137662564e-06, "correct": 37, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 37, "joules": 15110.268151744604, "joules_per_decision": 408.3856257228271, "mean_watts": 290.5150374406601, "net_joules_per_decision": 224.88875125548395, "peak_watts": 299.53, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 15110.268151744604, "mean_watts": 290.5150374406601, "min_watts": 115.16, "peak_watts": 299.53, "samples": 53, "seconds": 52.01200008392334 } }, "samples": 53, "seconds": 52.01200008392334 }, "errors": 0, "eval_tokens_total": 37, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1175.98943256855, "mean_watts": 130.53496009260877, "peak_watts": 221.01, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1175.98943256855, "mean_watts": 130.53496009260877, "min_watts": 115.13, "peak_watts": 221.01, "samples": 10, "seconds": 9.008999824523926 } }, "samples": 10, "seconds": 9.008999824523926, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 1.3551960540025902, "n": 63, "n_labelled": 37, "n_reached": 37, "p95_seconds": 1.9210954629998014, "prompt_tokens_total": 198260, "refused": 26, "refused_by_reason": { "kev-422": 16, "overlong": 10 }, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.6, "lo": 0.5, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.7, "lo": 0.6, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.8, "lo": 0.7, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.9, "lo": 0.8, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 1.0, "correct": 37, "gap": -0.001761425767536573, "hi": 1.0, "lo": 0.9, "mean_p": 0.9982385742324634, "n": 37, "sum_p": 36.934827246601145 } ], "unparsed": 0, "wall_seconds": 52.32673817500108 }, "apus-4b-high.kc": { "accuracy": 0.6203703703703703, "brier": 0.5655715046465185, "correct": 67, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 108, "joules": 4707.487003295421, "joules_per_decision": 43.58784262310575, "mean_watts": 261.4253907521237, "net_joules_per_decision": 22.962639319718946, "peak_watts": 276.07, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 4707.487003295421, "mean_watts": 261.4253907521237, "min_watts": 109.24, "peak_watts": 276.07, "samples": 19, "seconds": 18.006999969482422 } }, "samples": 19, "seconds": 18.006999969482422 }, "errors": 0, "eval_tokens_total": 108, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1114.3176315391063, "mean_watts": 123.70311326378045, "peak_watts": 165.33, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1114.3176315391063, "mean_watts": 123.70311326378045, "min_watts": 109.24, "peak_watts": 165.33, "samples": 10, "seconds": 9.007999897003174 } }, "samples": 10, "seconds": 9.007999897003174, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.17079587950138375, "n": 108, "n_labelled": 108, "n_reached": 108, "p95_seconds": 0.17819566300022416, "prompt_tokens_total": 36905, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.7142857142857143, "correct": 5, "gap": -0.3499396064900012, "hi": 0.4, "lo": 0.3, "mean_p": 0.3643461077957131, "n": 7, "sum_p": 2.5504227545699916 }, { "accuracy": 0.2857142857142857, "correct": 2, "gap": 0.15944601864033742, "hi": 0.5, "lo": 0.4, "mean_p": 0.4451603043546231, "n": 7, "sum_p": 3.1161221304823616 }, { "accuracy": 0.25, "correct": 3, "gap": 0.3087051935852353, "hi": 0.6, "lo": 0.5, "mean_p": 0.5587051935852353, "n": 12, "sum_p": 6.704462323022824 }, { "accuracy": 0.36363636363636365, "correct": 4, "gap": 0.29526054070474084, "hi": 0.7, "lo": 0.6, "mean_p": 0.6588969043411045, "n": 11, "sum_p": 7.24786594775215 }, { "accuracy": 0.6153846153846154, "correct": 8, "gap": 0.13533251282148384, "hi": 0.8, "lo": 0.7, "mean_p": 0.7507171282060993, "n": 13, "sum_p": 9.75932266667929 }, { "accuracy": 0.5, "correct": 6, "gap": 0.3424368303729496, "hi": 0.9, "lo": 0.8, "mean_p": 0.8424368303729496, "n": 12, "sum_p": 10.109241964475395 }, { "accuracy": 0.8478260869565217, "correct": 39, "gap": 0.11665105455948488, "hi": 1.0, "lo": 0.9, "mean_p": 0.9644771415160066, "n": 46, "sum_p": 44.365948509736306 } ], "unparsed": 0, "wall_seconds": 18.504091474002053 }, "apus-4b-high.kd": { "accuracy": 0.625, "brier": 0.5497210125079637, "correct": 30, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 48, "joules": 2470.565664833784, "joules_per_decision": 51.4701180173705, "mean_watts": 274.32441187238214, "net_joules_per_decision": 27.250702377390656, "peak_watts": 297.33, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 2470.565664833784, "mean_watts": 274.32441187238214, "min_watts": 111.73, "peak_watts": 297.33, "samples": 10, "seconds": 9.00600004196167 } }, "samples": 10, "seconds": 9.00600004196167 }, "errors": 0, "eval_tokens_total": 48, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1162.6610562992096, "mean_watts": 129.08416003802415, "peak_watts": 249.49, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1162.6610562992096, "mean_watts": 129.08416003802415, "min_watts": 111.72, "peak_watts": 249.49, "samples": 10, "seconds": 9.007000207901001 } }, "samples": 10, "seconds": 9.007000207901001, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.19319551800253976, "n": 48, "n_labelled": 48, "n_reached": 48, "p95_seconds": 0.19646412300062366, "prompt_tokens_total": 28918, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.4, "correct": 2, "gap": 0.16193724033594725, "hi": 0.6, "lo": 0.5, "mean_p": 0.5619372403359473, "n": 5, "sum_p": 2.809686201679736 }, { "accuracy": 0.75, "correct": 3, "gap": -0.09539913655247112, "hi": 0.7, "lo": 0.6, "mean_p": 0.6546008634475289, "n": 4, "sum_p": 2.6184034537901155 }, { "accuracy": 0.625, "correct": 5, "gap": 0.13087736765726843, "hi": 0.8, "lo": 0.7, "mean_p": 0.7558773676572684, "n": 8, "sum_p": 6.047018941258147 }, { "accuracy": 0.45454545454545453, "correct": 5, "gap": 0.40127996999809973, "hi": 0.9, "lo": 0.8, "mean_p": 0.8558254245435543, "n": 11, "sum_p": 9.414079669979097 }, { "accuracy": 0.75, "correct": 15, "gap": 0.20500702328364184, "hi": 1.0, "lo": 0.9, "mean_p": 0.9550070232836418, "n": 20, "sum_p": 19.100140465672837 } ], "unparsed": 0, "wall_seconds": 9.340095682000538 }, "apus-4b-high.kf": { "accuracy": 1.0, "brier": 2.481534288537669e-06, "correct": 40, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 40, "joules": 1525.2506810140608, "joules_per_decision": 38.13126702535152, "mean_watts": 253.9967880939198, "net_joules_per_decision": 17.269764894795365, "peak_watts": 287.05, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1525.2506810140608, "mean_watts": 253.9967880939198, "min_watts": 122.49, "peak_watts": 287.05, "samples": 7, "seconds": 6.004999876022339 } }, "samples": 7, "seconds": 6.004999876022339 }, "errors": 0, "eval_tokens_total": 40, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1251.7596529352666, "mean_watts": 138.960883005877, "peak_watts": 225.23, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1251.7596529352666, "mean_watts": 138.960883005877, "min_watts": 122.95, "peak_watts": 225.23, "samples": 10, "seconds": 9.008000135421753 } }, "samples": 10, "seconds": 9.008000135421753, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.16239020899956813, "n": 40, "n_labelled": 40, "n_reached": 40, "p95_seconds": 0.16536938100034604, "prompt_tokens_total": 11156, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.6, "lo": 0.5, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.7, "lo": 0.6, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.8, "lo": 0.7, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.9, "lo": 0.8, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 1.0, "correct": 40, "gap": -0.0009444067393589162, "hi": 1.0, "lo": 0.9, "mean_p": 0.9990555932606411, "n": 40, "sum_p": 39.96222373042564 } ], "unparsed": 0, "wall_seconds": 6.526520731000346 }, "apus-4b-high.kh": { "accuracy": 0.6041666666666666, "brier": 0.5220315713928496, "correct": 29, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 48, "joules": 2036.766190340519, "joules_per_decision": 42.432628965427476, "mean_watts": 254.404968731609, "net_joules_per_decision": 21.07141844072605, "peak_watts": 283.29, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 2036.766190340519, "mean_watts": 254.404968731609, "min_watts": 110.2, "peak_watts": 283.29, "samples": 9, "seconds": 8.00600004196167 } }, "samples": 9, "seconds": 8.00600004196167 }, "errors": 0, "eval_tokens_total": 48, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1153.6654381084443, "mean_watts": 128.07120906964613, "peak_watts": 259.65, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1153.6654381084443, "mean_watts": 128.07120906964613, "min_watts": 110.19, "peak_watts": 259.65, "samples": 10, "seconds": 9.007999897003174 } }, "samples": 10, "seconds": 9.007999897003174, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.17127695599992876, "n": 48, "n_labelled": 48, "n_reached": 48, "p95_seconds": 0.17879081800128915, "prompt_tokens_total": 16837, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.0, "correct": 0, "gap": 0.32859482569736986, "hi": 0.4, "lo": 0.3, "mean_p": 0.32859482569736986, "n": 2, "sum_p": 0.6571896513947397 }, { "accuracy": 0.0, "correct": 0, "gap": 0.45169131936719803, "hi": 0.5, "lo": 0.4, "mean_p": 0.45169131936719803, "n": 4, "sum_p": 1.8067652774687921 }, { "accuracy": 0.5, "correct": 2, "gap": 0.049844899647008334, "hi": 0.6, "lo": 0.5, "mean_p": 0.5498448996470083, "n": 4, "sum_p": 2.1993795985880333 }, { "accuracy": 0.2, "correct": 1, "gap": 0.4641718740215381, "hi": 0.7, "lo": 0.6, "mean_p": 0.6641718740215381, "n": 5, "sum_p": 3.3208593701076907 }, { "accuracy": 0.4, "correct": 4, "gap": 0.35157413272409854, "hi": 0.8, "lo": 0.7, "mean_p": 0.7515741327240986, "n": 10, "sum_p": 7.515741327240986 }, { "accuracy": 1.0, "correct": 2, "gap": -0.1508106723388194, "hi": 0.9, "lo": 0.8, "mean_p": 0.8491893276611806, "n": 2, "sum_p": 1.6983786553223612 }, { "accuracy": 0.9523809523809523, "correct": 20, "gap": 0.027357205740614066, "hi": 1.0, "lo": 0.9, "mean_p": 0.9797381581215664, "n": 21, "sum_p": 20.574501320552894 } ], "unparsed": 0, "wall_seconds": 8.276120432998141 }, "apus-4b-high.kj": { "accuracy": 0.7647058823529411, "brier": 0.27524192532194686, "correct": 91, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 119, "joules": 9922.953669581413, "joules_per_decision": 83.3861652906001, "mean_watts": 291.77434359188464, "net_joules_per_decision": 43.90285517403331, "peak_watts": 298.9, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 9922.953669581413, "mean_watts": 291.77434359188464, "min_watts": 120.15, "peak_watts": 298.9, "samples": 35, "seconds": 34.009000062942505 } }, "samples": 35, "seconds": 34.009000062942505 }, "errors": 0, "eval_tokens_total": 119, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1244.5003911912443, "mean_watts": 138.15501470715475, "peak_watts": 260.13, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1244.5003911912443, "mean_watts": 138.15501470715475, "min_watts": 120.33, "peak_watts": 260.13, "samples": 10, "seconds": 9.008000135421753 } }, "samples": 10, "seconds": 9.008000135421753, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.2687984679978399, "n": 119, "n_labelled": 119, "n_reached": 119, "p95_seconds": 0.49118050199831487, "prompt_tokens_total": 111435, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.6666666666666666, "correct": 2, "gap": -0.0964173588672772, "hi": 0.6, "lo": 0.5, "mean_p": 0.5702493077993894, "n": 3, "sum_p": 1.7107479233981682 }, { "accuracy": 0.0, "correct": 0, "gap": 0.6766117282278755, "hi": 0.7, "lo": 0.6, "mean_p": 0.6766117282278755, "n": 1, "sum_p": 0.6766117282278755 }, { "accuracy": 1.0, "correct": 1, "gap": -0.24713480048555792, "hi": 0.8, "lo": 0.7, "mean_p": 0.7528651995144421, "n": 1, "sum_p": 0.7528651995144421 }, { "accuracy": 0.6, "correct": 3, "gap": 0.25285333871986426, "hi": 0.9, "lo": 0.8, "mean_p": 0.8528533387198642, "n": 5, "sum_p": 4.264266693599321 }, { "accuracy": 0.7798165137614679, "correct": 85, "gap": 0.2096700776450673, "hi": 1.0, "lo": 0.9, "mean_p": 0.9894865914065352, "n": 109, "sum_p": 107.85403846331234 } ], "unparsed": 0, "wall_seconds": 34.83520143699934 }, "apus-4b-high.kperm": { "accuracy": 0.6203703703703703, "brier": 0.5655715046465185, "correct": 67, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 108, "joules": 32175.309275598527, "joules_per_decision": 297.919530329616, "mean_watts": 289.7971594375859, "net_joules_per_decision": 143.2622569138462, "peak_watts": 296.91, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 32175.309275598527, "mean_watts": 289.7971594375859, "min_watts": 121.88, "peak_watts": 296.91, "samples": 112, "seconds": 111.02700018882751 } }, "samples": 112, "seconds": 111.02700018882751 }, "errors": 0, "eval_tokens_total": 0, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1355.3207546389103, "mean_watts": 150.44075315460012, "peak_watts": 297.55, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1355.3207546389103, "mean_watts": 150.44075315460012, "min_watts": 122.83, "peak_watts": 297.55, "samples": 10, "seconds": 9.009000062942505 } }, "samples": 10, "seconds": 9.009000062942505, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.17167393491702873, "n": 108, "n_labelled": 108, "n_reached": 108, "p95_seconds": 0.1804556140001902, "prompt_tokens_total": 36905, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.7142857142857143, "correct": 5, "gap": -0.3499396064900012, "hi": 0.4, "lo": 0.3, "mean_p": 0.3643461077957131, "n": 7, "sum_p": 2.5504227545699916 }, { "accuracy": 0.2857142857142857, "correct": 2, "gap": 0.15944601864033742, "hi": 0.5, "lo": 0.4, "mean_p": 0.4451603043546231, "n": 7, "sum_p": 3.1161221304823616 }, { "accuracy": 0.25, "correct": 3, "gap": 0.3087051935852353, "hi": 0.6, "lo": 0.5, "mean_p": 0.5587051935852353, "n": 12, "sum_p": 6.704462323022824 }, { "accuracy": 0.36363636363636365, "correct": 4, "gap": 0.29526054070474084, "hi": 0.7, "lo": 0.6, "mean_p": 0.6588969043411045, "n": 11, "sum_p": 7.24786594775215 }, { "accuracy": 0.6153846153846154, "correct": 8, "gap": 0.13533251282148384, "hi": 0.8, "lo": 0.7, "mean_p": 0.7507171282060993, "n": 13, "sum_p": 9.75932266667929 }, { "accuracy": 0.5, "correct": 6, "gap": 0.3424368303729496, "hi": 0.9, "lo": 0.8, "mean_p": 0.8424368303729496, "n": 12, "sum_p": 10.109241964475395 }, { "accuracy": 0.8478260869565217, "correct": 39, "gap": 0.11665105455948488, "hi": 1.0, "lo": 0.9, "mean_p": 0.9644771415160066, "n": 46, "sum_p": 44.365948509736306 } ], "unparsed": 0, "wall_seconds": 111.45537506700202 }, "apus-4b-low.kc": { "accuracy": 0.23148148148148148, "brier": 0.9086866775378496, "correct": 25, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 108, "joules": 2587.473707690239, "joules_per_decision": 23.95808988602073, "mean_watts": 258.5922143553146, "net_joules_per_decision": 10.79557559222759, "peak_watts": 277.82, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 2587.473707690239, "mean_watts": 258.5922143553146, "min_watts": 124.98, "peak_watts": 277.82, "samples": 11, "seconds": 10.00600004196167 } }, "samples": 11, "seconds": 10.00600004196167 }, "errors": 0, "eval_tokens_total": 108, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1279.9078446161748, "mean_watts": 142.06991182971902, "peak_watts": 238.03, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1279.9078446161748, "mean_watts": 142.06991182971902, "min_watts": 125.49, "peak_watts": 238.03, "samples": 10, "seconds": 9.009000062942505 } }, "samples": 10, "seconds": 9.009000062942505, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.09438246099853131, "n": 108, "n_labelled": 108, "n_reached": 108, "p95_seconds": 0.0991138499994122, "prompt_tokens_total": 36905, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.1935483870967742, "correct": 6, "gap": 0.07477417052407426, "hi": 0.3, "lo": 0.2, "mean_p": 0.26832255762084845, "n": 31, "sum_p": 8.317999286246302 }, { "accuracy": 0.2857142857142857, "correct": 8, "gap": 0.06003492579511688, "hi": 0.4, "lo": 0.3, "mean_p": 0.3457492115094026, "n": 28, "sum_p": 9.680977922263272 }, { "accuracy": 0.2727272727272727, "correct": 3, "gap": 0.1842377504149263, "hi": 0.5, "lo": 0.4, "mean_p": 0.456965023142199, "n": 11, "sum_p": 5.026615254564189 }, { "accuracy": 0.08333333333333333, "correct": 1, "gap": 0.4640215365319214, "hi": 0.6, "lo": 0.5, "mean_p": 0.5473548698652547, "n": 12, "sum_p": 6.568258438383056 }, { "accuracy": 0.0, "correct": 0, "gap": 0.665845380443368, "hi": 0.7, "lo": 0.6, "mean_p": 0.665845380443368, "n": 12, "sum_p": 7.990144565320417 }, { "accuracy": 0.42857142857142855, "correct": 3, "gap": 0.3254555239645555, "hi": 0.8, "lo": 0.7, "mean_p": 0.7540269525359841, "n": 7, "sum_p": 5.2781886677518886 }, { "accuracy": 0.0, "correct": 0, "gap": 0.8712117405082737, "hi": 0.9, "lo": 0.8, "mean_p": 0.8712117405082737, "n": 1, "sum_p": 0.8712117405082737 }, { "accuracy": 0.6666666666666666, "correct": 4, "gap": 0.2807683593231086, "hi": 1.0, "lo": 0.9, "mean_p": 0.9474350259897752, "n": 6, "sum_p": 5.6846101559386515 } ], "unparsed": 0, "wall_seconds": 10.233292810000421 }, "apus-4b-low.kd": { "accuracy": 0.75, "brier": 0.3392385981344586, "correct": 36, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 48, "joules": 1255.1760025238991, "joules_per_decision": 26.14950005258123, "mean_watts": 250.78441035442577, "net_joules_per_decision": 11.8046232912388, "peak_watts": 289.91, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1255.1760025238991, "mean_watts": 250.78441035442577, "min_watts": 119.53, "peak_watts": 289.91, "samples": 6, "seconds": 5.005000114440918 } }, "samples": 6, "seconds": 5.005000114440918 }, "errors": 0, "eval_tokens_total": 48, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1239.259736430645, "mean_watts": 137.57324051956618, "peak_watts": 263.89, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1239.259736430645, "mean_watts": 137.57324051956618, "min_watts": 119.72, "peak_watts": 263.89, "samples": 10, "seconds": 9.007999897003174 } }, "samples": 10, "seconds": 9.007999897003174, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.10559136449955986, "n": 48, "n_labelled": 48, "n_reached": 48, "p95_seconds": 0.10770713300007628, "prompt_tokens_total": 28918, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.5, "correct": 4, "gap": 0.05459529402031793, "hi": 0.6, "lo": 0.5, "mean_p": 0.5545952940203179, "n": 8, "sum_p": 4.436762352162543 }, { "accuracy": 0.7142857142857143, "correct": 5, "gap": -0.058995778877968696, "hi": 0.7, "lo": 0.6, "mean_p": 0.6552899354077456, "n": 7, "sum_p": 4.587029547854219 }, { "accuracy": 0.7, "correct": 7, "gap": 0.05546430803393465, "hi": 0.8, "lo": 0.7, "mean_p": 0.7554643080339346, "n": 10, "sum_p": 7.554643080339346 }, { "accuracy": 0.9230769230769231, "correct": 12, "gap": -0.08068933320492588, "hi": 0.9, "lo": 0.8, "mean_p": 0.8423875898719972, "n": 13, "sum_p": 10.951038668335965 }, { "accuracy": 0.8, "correct": 8, "gap": 0.16393574828108348, "hi": 1.0, "lo": 0.9, "mean_p": 0.9639357482810835, "n": 10, "sum_p": 9.639357482810835 } ], "unparsed": 0, "wall_seconds": 5.122459880996757 }, "apus-4b-low.kh": { "accuracy": 0.25, "brier": 0.8515104966414005, "correct": 12, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 48, "joules": 916.2154420006275, "joules_per_decision": 19.087821708346407, "mean_watts": 228.71079191301567, "net_joules_per_decision": 7.632184339869387, "peak_watts": 275.01, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 916.2154420006275, "mean_watts": 228.71079191301567, "min_watts": 121.42, "peak_watts": 275.01, "samples": 5, "seconds": 4.00600004196167 } }, "samples": 5, "seconds": 4.00600004196167 }, "errors": 0, "eval_tokens_total": 48, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1236.5911535811424, "mean_watts": 137.2617543502657, "peak_watts": 218.51, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1236.5911535811424, "mean_watts": 137.2617543502657, "min_watts": 121.87, "peak_watts": 218.51, "samples": 10, "seconds": 9.009000062942505 } }, "samples": 10, "seconds": 9.009000062942505, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.09470615849932074, "n": 48, "n_labelled": 48, "n_reached": 48, "p95_seconds": 0.09952337699724012, "prompt_tokens_total": 16837, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.38461538461538464, "correct": 5, "gap": -0.12171036620885517, "hi": 0.3, "lo": 0.2, "mean_p": 0.26290501840652947, "n": 13, "sum_p": 3.4177652392848827 }, { "accuracy": 0.07142857142857142, "correct": 1, "gap": 0.26750178566392113, "hi": 0.4, "lo": 0.3, "mean_p": 0.33893035709249253, "n": 14, "sum_p": 4.745024999294896 }, { "accuracy": 0.16666666666666666, "correct": 1, "gap": 0.30277429033627523, "hi": 0.5, "lo": 0.4, "mean_p": 0.4694409570029419, "n": 6, "sum_p": 2.8166457420176516 }, { "accuracy": 0.16666666666666666, "correct": 1, "gap": 0.39647320326314406, "hi": 0.6, "lo": 0.5, "mean_p": 0.5631398699298107, "n": 6, "sum_p": 3.378839219578864 }, { "accuracy": 0.4, "correct": 2, "gap": 0.2391744752228102, "hi": 0.7, "lo": 0.6, "mean_p": 0.6391744752228102, "n": 5, "sum_p": 3.195872376114051 }, { "accuracy": 0.0, "correct": 0, "gap": 0.7485536008434512, "hi": 0.8, "lo": 0.7, "mean_p": 0.7485536008434512, "n": 1, "sum_p": 0.7485536008434512 }, { "accuracy": 0.5, "correct": 1, "gap": 0.3481093669284656, "hi": 0.9, "lo": 0.8, "mean_p": 0.8481093669284656, "n": 2, "sum_p": 1.6962187338569312 }, { "accuracy": 1.0, "correct": 1, "gap": -0.003006461610207789, "hi": 1.0, "lo": 0.9, "mean_p": 0.9969935383897922, "n": 1, "sum_p": 0.9969935383897922 } ], "unparsed": 0, "wall_seconds": 4.596440386998438 } }, "tag": "counted", "tasks_ended": "2026-09-28T16:50:38Z", "tasks_started": "2026-09-28T16:44:30Z", "vendor_files_sha256": { "bridge": "7cb3e56d42651087e35b7a620b2407dc96801a09ae942bbe803cb202ee187e65", "openjet_runtime/__init__.py": "df6eb864cf6d0c2f512fb00f17eeaa11fd790afca33a0f7b2f609aeb1f2b3944", "openjet_runtime/candidate_projection.py": "84dc4746b5fb06ac6a9024dde3ba8414d901acf2a62d010b0d66f26acfaf74a6", "openjet_runtime/contracts.py": "d8e8e5270ecd6dab917d886dda2d684c24696b811faa968bb3c399ceef5e356a", "openjet_runtime/early_exit.py": "89b7751a927a6d1348a454fb8ee986395423ad5686d855fb3ccab255c95d2566", "openjet_runtime/runtime.py": "6e7b0b131cb14ab0d25cc8fd6c7fc41738b799cfe6de1ccdbae2d09d7c64313c" }, "versions": { "cuda_devices": 1, "cudnn": 92400, "device_name": "NVIDIA GeForce RTX 3090", "device_uuid_fp": "33a0b4acb8e8", "dists": [ "Jinja2==3.1.6", "MarkupSafe==3.0.3", "PyYAML==6.0.3", "Pygments==2.21.0", "annotated-doc==0.0.5", "anyio==4.15.1", "certifi==2026.7.22", "click==8.5.0", "cloudpickle==3.1.2", "cuda-bindings==13.4.2", "cuda-pathfinder==1.8.2", "cuda-toolkit==13.0.3.0", "filelock==4.0.1", "fsspec==2026.9.0", "h11==0.16.0", "hf-xet==1.6.0", "httpcore==1.0.9", "httpx==0.28.1", "huggingface_hub==1.32.0", "idna==3.20", "joblib==1.6.0", "markdown-it-py==4.2.0", "mdurl==0.1.2", "mpmath==1.3.0", "narwhals==2.26.0", "networkx==3.6.1", "numpy==2.5.3", "nvidia-cublas==13.1.1.3", "nvidia-cuda-cupti==13.0.85", "nvidia-cuda-nvrtc==13.0.88", "nvidia-cuda-runtime==13.0.96", "nvidia-cudnn-cu13==9.24.0.43", "nvidia-cufft==12.0.0.61", "nvidia-cufile==1.15.1.6", "nvidia-curand==10.4.0.35", "nvidia-cusolver==12.0.4.66", "nvidia-cusparse==12.6.3.3", "nvidia-cusparselt-cu13==0.8.1", "nvidia-nccl-cu13==2.30.7", "nvidia-nvjitlink==13.4.92", "nvidia-nvshmem-cu13==3.4.5", "nvidia-nvtx==13.0.85", "packaging==26.3", "pillow==12.3.0", "pip==26.2.1", "regex==2026.9.10", "rich==15.0.0", "safetensors==0.8.0", "scikit-learn==1.9.1", "scipy==1.18.1", "sentence-transformers==6.1.0", "setuptools==84.0.0", "shellingham==1.5.4", "sympy==1.14.0", "threadpoolctl==3.7.0", "tokenizers==0.23.2", "torch==2.14.0", "tqdm==4.70.1", "transformers==5.16.1", "triton==3.8.0", "typer==0.27.2", "typing_extensions==4.16.0" ], "huggingface_hub": "1.32.0", "jinja2": "3.1.6", "numpy": "2.5.3", "python": "3.14.4", "safetensors": "0.8.0", "tokenizers": "0.23.2", "torch": "2.14.0+cu130", "torch_cuda": "13.0", "transformers": "5.16.1" }, "warmups_s": [ [ "high", 0.557 ], [ "high", 0.166 ], [ "high", 0.164 ], [ "low", 0.115 ] ] }