{ "builder_check": { "of": 108, "s0_rebuilt_equal_kev9b": 108 }, "card": "RTX 3090 (fp 33a0b4acb8e8)", "depths": { "high": 32, "low": 16 }, "driver": [ "595.84", "NVIDIA GeForce RTX 3090", "24576", "16" ], "dry": 0, "ended": "2026-09-28T15:39:30Z", "env": { "CUDA_VISIBLE_DEVICES": "", "DO_NOT_TRACK": null, "GRADIO_ANALYTICS_ENABLED": "False", "HF_HOME": "/workshop/bench-scratch-apus-a1/sandbox-home/hf", "HF_HUB_DISABLE_TELEMETRY": "1", "HF_HUB_OFFLINE": "1", "ORT_DISABLE_TELEMETRY": "1", "TORCH_DISABLE_NATIVE_JIT": "1", "TRANSFORMERS_OFFLINE": "1", "VLLM_NO_USAGE_STATS": "1" }, "files": { "all_equal_pins": true, "bridge": "7cb3e56d42651087e35b7a620b2407dc96801a09ae942bbe803cb202ee187e65", "files": 39, "seconds": 9.8 }, "guard_heartbeat_age_s": 0.8, "h48_sha256": "", "hostos_sha256": "a6a74bba57250be1936825466c6614876d757cf81373fcf6fa0b48587cf0e3ad", "instrument_sha256": { "run.py": "", "run_addenda.py": "a34aca61833ed0c5860c91c7e2e227429e8ea9e2e50fec68d7b7fec15d89acab", "run_kev.py": "1fbce7a41746ed3d612a8a488e35dbebba789bc55086daa7d71ffce2cd010b05" }, "load_s": 3.7, "mem_available_bytes": 30952939520, "os": { "arch": "x86_64", "id": "ubuntu", "kernel": "7.0.0-31-generic", "pretty_name": "Ubuntu 26.04 LTS", "source": "/etc/os-release", "version_id": "26.04" }, "peak_memory": { "max_allocated_mib": 19977.09619140625, "max_reserved_mib": 23800.0 }, "pins_sha256": "e5a2bd17d01186b64bd23c8ed40381425ad019f3cab130bcb106a5078cfa80b5", "posture_after_release": { "compute_apps": 1, "empty": false, "memory_used_mib": 346.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P2", "temp_c": 61.0, "utc": "2026-09-28T15:39:30Z", "when": "after the model was released (this process still holds a context)" }, "posture_after_tasks": { "compute_apps": 1, "empty": false, "memory_used_mib": 24126.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P2", "temp_c": 61.0, "utc": "2026-09-28T15:39:30Z", "when": "after the tasks, model still resident" }, "posture_before": { "compute_apps": 0, "empty": true, "memory_used_mib": 1.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P8", "temp_c": 36.0, "utc": "2026-09-28T15:32:11Z", "when": "before the load" }, "posture_loaded": { "compute_apps": 1, "empty": false, "memory_used_mib": 18276.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P2", "temp_c": 39.0, "utc": "2026-09-28T15:32:27Z", "when": "after the load" }, "posture_resident": { "compute_apps": 1, "empty": false, "memory_used_mib": 18472.0, "persistence_read_inside_sandbox_not_of_record": "Disabled", "posture_ok": true, "power_limit_w": 300.0, "pstate": "P2", "temp_c": 43.0, "utc": "2026-09-28T15:32:28Z", "when": "after the warm-ups" }, "preflight": false, "residency": { "bar_mib": 17050.4, "compute_apps": 1, "growth_mib": 18471.0, "ok": true }, "runner_sha256": "015a72b6461d9dc874a69f248671adf5c44811d3ca10818d3cf420d1f3106262", "started": "2026-09-28T15:32:11Z", "summaries": { "apus-9b-high.ka": { "accuracy": 1.0, "brier": 6.371517637724394e-06, "correct": 37, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 37, "joules": 21073.521178435087, "joules_per_decision": 569.5546264441915, "mean_watts": 292.61463997333016, "net_joules_per_decision": 328.2802323774346, "peak_watts": 299.62, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 21073.521178435087, "mean_watts": 292.61463997333016, "min_watts": 116.15, "peak_watts": 299.62, "samples": 73, "seconds": 72.01800012588501 } }, "samples": 73, "seconds": 72.01800012588501 }, "errors": 0, "eval_tokens_total": 37, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1116.4828932726382, "mean_watts": 123.95724075738912, "peak_watts": 172.16, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1116.4828932726382, "mean_watts": 123.95724075738912, "min_watts": 116.21, "peak_watts": 172.16, "samples": 10, "seconds": 9.007000207901001 } }, "samples": 10, "seconds": 9.007000207901001, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 1.88302393900085, "n": 63, "n_labelled": 37, "n_reached": 37, "p95_seconds": 2.676891618000809, "prompt_tokens_total": 198260, "refused": 26, "refused_by_reason": { "kev-422": 16, "overlong": 10 }, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.6, "lo": 0.5, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.7, "lo": 0.6, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.8, "lo": 0.7, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.9, "lo": 0.8, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 1.0, "correct": 37, "gap": -0.0011218542470085024, "hi": 1.0, "lo": 0.9, "mean_p": 0.9988781457529915, "n": 37, "sum_p": 36.95849139286069 } ], "unparsed": 0, "wall_seconds": 72.44756734199837 }, "apus-9b-high.kc": { "accuracy": 0.6111111111111112, "brier": 0.5178344054525861, "correct": 66, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 108, "joules": 5674.679136641025, "joules_per_decision": 52.543325339268755, "mean_watts": 283.60633308961627, "net_joules_per_decision": 28.895788555773507, "peak_watts": 297.29, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 5674.679136641025, "mean_watts": 283.60633308961627, "min_watts": 109.26, "peak_watts": 297.29, "samples": 21, "seconds": 20.009000062942505 } }, "samples": 21, "seconds": 20.009000062942505 }, "errors": 0, "eval_tokens_total": 108, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1149.9020760047435, "mean_watts": 127.63926056192473, "peak_watts": 191.75, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1149.9020760047435, "mean_watts": 127.63926056192473, "min_watts": 109.18, "peak_watts": 191.75, "samples": 10, "seconds": 9.008999824523926 } }, "samples": 10, "seconds": 9.008999824523926, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.1825492084990401, "n": 108, "n_labelled": 108, "n_reached": 108, "p95_seconds": 0.2078083129999868, "prompt_tokens_total": 36905, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.3333333333333333, "correct": 3, "gap": 0.029211041864716958, "hi": 0.4, "lo": 0.3, "mean_p": 0.3625443751980503, "n": 9, "sum_p": 3.2628993767824523 }, { "accuracy": 0.13333333333333333, "correct": 2, "gap": 0.3147402420597313, "hi": 0.5, "lo": 0.4, "mean_p": 0.44807357539306464, "n": 15, "sum_p": 6.7211036308959695 }, { "accuracy": 0.3333333333333333, "correct": 2, "gap": 0.21531381803649202, "hi": 0.6, "lo": 0.5, "mean_p": 0.5486471513698253, "n": 6, "sum_p": 3.291882908218952 }, { "accuracy": 0.7, "correct": 7, "gap": -0.053095274369252876, "hi": 0.7, "lo": 0.6, "mean_p": 0.6469047256307471, "n": 10, "sum_p": 6.469047256307471 }, { "accuracy": 0.8333333333333334, "correct": 5, "gap": -0.08646168268189258, "hi": 0.8, "lo": 0.7, "mean_p": 0.7468716506514408, "n": 6, "sum_p": 4.481229903908645 }, { "accuracy": 0.5238095238095238, "correct": 11, "gap": 0.324828295515037, "hi": 0.9, "lo": 0.8, "mean_p": 0.8486378193245608, "n": 21, "sum_p": 17.82139420581578 }, { "accuracy": 0.8780487804878049, "correct": 36, "gap": 0.09031788189989243, "hi": 1.0, "lo": 0.9, "mean_p": 0.9683666623876973, "n": 41, "sum_p": 39.70303315789559 } ], "unparsed": 0, "wall_seconds": 20.023310936001508 }, "apus-9b-high.kd": { "accuracy": 0.9166666666666666, "brier": 0.1577693054899775, "correct": 44, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 48, "joules": 3382.5486593163014, "joules_per_decision": 70.46976373575627, "mean_watts": 281.71472207158763, "net_joules_per_decision": 40.592097905350876, "peak_watts": 299.44, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 3382.5486593163014, "mean_watts": 281.71472207158763, "min_watts": 112.61, "peak_watts": 299.44, "samples": 13, "seconds": 12.006999969482422 } }, "samples": 13, "seconds": 12.006999969482422 }, "errors": 0, "eval_tokens_total": 48, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1076.043883857727, "mean_watts": 119.44098971470883, "peak_watts": 165.5, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1076.043883857727, "mean_watts": 119.44098971470883, "min_watts": 112.4, "peak_watts": 165.5, "samples": 10, "seconds": 9.009000062942505 } }, "samples": 10, "seconds": 9.009000062942505, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.2507132230002753, "n": 48, "n_labelled": 48, "n_reached": 48, "p95_seconds": 0.2517374099988956, "prompt_tokens_total": 28918, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 1.0, "correct": 1, "gap": -0.4687906266262438, "hi": 0.6, "lo": 0.5, "mean_p": 0.5312093733737562, "n": 1, "sum_p": 0.5312093733737562 }, { "accuracy": 0.0, "correct": 0, "gap": 0.679178699175393, "hi": 0.7, "lo": 0.6, "mean_p": 0.679178699175393, "n": 1, "sum_p": 0.679178699175393 }, { "accuracy": 1.0, "correct": 1, "gap": -0.29421497216298875, "hi": 0.8, "lo": 0.7, "mean_p": 0.7057850278370112, "n": 1, "sum_p": 0.7057850278370112 }, { "accuracy": 1.0, "correct": 4, "gap": -0.14910030055842605, "hi": 0.9, "lo": 0.8, "mean_p": 0.850899699441574, "n": 4, "sum_p": 3.403598797766296 }, { "accuracy": 0.926829268292683, "correct": 38, "gap": 0.057341553969542325, "hi": 1.0, "lo": 0.9, "mean_p": 0.9841708222622253, "n": 41, "sum_p": 40.351003712751236 } ], "unparsed": 0, "wall_seconds": 12.069553311001073 }, "apus-9b-high.kf": { "accuracy": 1.0, "brier": 0.0003665292827295962, "correct": 40, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 40, "joules": 1601.4556116366387, "joules_per_decision": 40.03639029091597, "mean_watts": 266.64262411719426, "net_joules_per_decision": 18.80756319608637, "peak_watts": 298.78, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1601.4556116366387, "mean_watts": 266.64262411719426, "min_watts": 123.0, "peak_watts": 298.78, "samples": 7, "seconds": 6.00600004196167 } }, "samples": 7, "seconds": 6.00600004196167 }, "errors": 0, "eval_tokens_total": 40, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1273.7295919811727, "mean_watts": 141.38412884789705, "peak_watts": 248.13, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1273.7295919811727, "mean_watts": 141.38412884789705, "min_watts": 123.53, "peak_watts": 248.13, "samples": 10, "seconds": 9.008999824523926 } }, "samples": 10, "seconds": 9.008999824523926, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.16992022200065549, "n": 40, "n_labelled": 40, "n_reached": 40, "p95_seconds": 0.17202939100025105, "prompt_tokens_total": 11156, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.6, "lo": 0.5, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.7, "lo": 0.6, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.8, "lo": 0.7, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.9, "lo": 0.8, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 1.0, "correct": 40, "gap": -0.0029960429864084315, "hi": 1.0, "lo": 0.9, "mean_p": 0.9970039570135916, "n": 40, "sum_p": 39.88015828054366 } ], "unparsed": 0, "wall_seconds": 6.8088932179998665 }, "apus-9b-high.kh": { "accuracy": 0.6458333333333334, "brier": 0.4200292732845395, "correct": 31, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 48, "joules": 2441.052811398506, "joules_per_decision": 50.85526690413554, "mean_watts": 271.0172998411566, "net_joules_per_decision": 27.253399043141076, "peak_watts": 298.85, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 2441.052811398506, "mean_watts": 271.0172998411566, "min_watts": 110.55, "peak_watts": 298.85, "samples": 10, "seconds": 9.006999969482422 } }, "samples": 10, "seconds": 9.006999969482422 }, "errors": 0, "eval_tokens_total": 48, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1133.0154270124435, "mean_watts": 125.77880106208491, "peak_watts": 207.57, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1133.0154270124435, "mean_watts": 125.77880106208491, "min_watts": 110.51, "peak_watts": 207.57, "samples": 10, "seconds": 9.007999897003174 } }, "samples": 10, "seconds": 9.007999897003174, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.18744788549884106, "n": 48, "n_labelled": 48, "n_reached": 48, "p95_seconds": 0.2087297350008157, "prompt_tokens_total": 16837, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.0, "correct": 0, "gap": 0.3649604217456012, "hi": 0.4, "lo": 0.3, "mean_p": 0.3649604217456012, "n": 3, "sum_p": 1.0948812652368036 }, { "accuracy": 0.25, "correct": 1, "gap": 0.20220873696702907, "hi": 0.5, "lo": 0.4, "mean_p": 0.45220873696702907, "n": 4, "sum_p": 1.8088349478681163 }, { "accuracy": 0.0, "correct": 0, "gap": 0.5412402352701529, "hi": 0.6, "lo": 0.5, "mean_p": 0.5412402352701529, "n": 2, "sum_p": 1.0824804705403057 }, { "accuracy": 0.0, "correct": 0, "gap": 0.6625544487849659, "hi": 0.7, "lo": 0.6, "mean_p": 0.6625544487849659, "n": 4, "sum_p": 2.6502177951398638 }, { "accuracy": 0.6, "correct": 3, "gap": 0.1405378303400111, "hi": 0.8, "lo": 0.7, "mean_p": 0.7405378303400111, "n": 5, "sum_p": 3.7026891517000555 }, { "accuracy": 0.6666666666666666, "correct": 6, "gap": 0.19668326573346873, "hi": 0.9, "lo": 0.8, "mean_p": 0.8633499324001354, "n": 9, "sum_p": 7.770149391601218 }, { "accuracy": 1.0, "correct": 21, "gap": -0.02377111115995567, "hi": 1.0, "lo": 0.9, "mean_p": 0.9762288888400443, "n": 21, "sum_p": 20.50080666564093 } ], "unparsed": 0, "wall_seconds": 9.073853967001924 }, "apus-9b-high.kj": { "accuracy": 0.7815126050420168, "brier": 0.21501065096355546, "correct": 93, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 119, "joules": 13802.992814205885, "joules_per_decision": 115.99153625383097, "mean_watts": 293.5994897336036, "net_joules_per_decision": 57.786289274534106, "peak_watts": 300.27, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 13802.992814205885, "mean_watts": 293.5994897336036, "min_watts": 120.59, "peak_watts": 300.27, "samples": 48, "seconds": 47.01300001144409 } }, "samples": 48, "seconds": 47.01300001144409 }, "errors": 0, "eval_tokens_total": 119, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1327.2958065879345, "mean_watts": 147.32998083190327, "peak_watts": 297.91, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1327.2958065879345, "mean_watts": 147.32998083190327, "min_watts": 120.92, "peak_watts": 297.91, "samples": 10, "seconds": 9.009000062942505 } }, "samples": 10, "seconds": 9.009000062942505, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.37659359800090897, "n": 119, "n_labelled": 119, "n_reached": 119, "p95_seconds": 0.6788207550016523, "prompt_tokens_total": 111435, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.3333333333333333, "correct": 1, "gap": 0.22547235125003745, "hi": 0.6, "lo": 0.5, "mean_p": 0.5588056845833708, "n": 3, "sum_p": 1.6764170537501124 }, { "accuracy": 0.16666666666666666, "correct": 1, "gap": 0.46843782283249247, "hi": 0.7, "lo": 0.6, "mean_p": 0.6351044894991591, "n": 6, "sum_p": 3.8106269369949546 }, { "accuracy": 0.4, "correct": 2, "gap": 0.32685958863889986, "hi": 0.8, "lo": 0.7, "mean_p": 0.7268595886388999, "n": 5, "sum_p": 3.6342979431944995 }, { "accuracy": 0.0, "correct": 0, "gap": 0.8617415789722439, "hi": 0.9, "lo": 0.8, "mean_p": 0.8617415789722439, "n": 4, "sum_p": 3.4469663158889756 }, { "accuracy": 0.8811881188118812, "correct": 89, "gap": 0.11263152913588581, "hi": 1.0, "lo": 0.9, "mean_p": 0.993819647947767, "n": 101, "sum_p": 100.37578444272447 } ], "unparsed": 0, "wall_seconds": 47.84326578100081 }, "apus-9b-high.kperm": { "accuracy": 0.6111111111111112, "brier": 0.5178344054525861, "correct": 66, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 108, "joules": 36187.086477931734, "joules_per_decision": 335.06561553640495, "mean_watts": 296.5449732082081, "net_joules_per_decision": 175.89754033250776, "peak_watts": 299.27, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 36187.086477931734, "mean_watts": 296.5449732082081, "min_watts": 122.15, "peak_watts": 299.27, "samples": 123, "seconds": 122.02899980545044 } }, "samples": 123, "seconds": 122.02899980545044 }, "errors": 0, "eval_tokens_total": 0, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1268.9515507912636, "mean_watts": 140.86940112126607, "peak_watts": 273.56, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1268.9515507912636, "mean_watts": 140.86940112126607, "min_watts": 122.54, "peak_watts": 273.56, "samples": 10, "seconds": 9.007999897003174 } }, "samples": 10, "seconds": 9.007999897003174, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.18618651275028242, "n": 108, "n_labelled": 108, "n_reached": 108, "p95_seconds": 0.2129242153338661, "prompt_tokens_total": 36905, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.3333333333333333, "correct": 3, "gap": 0.029211041864716958, "hi": 0.4, "lo": 0.3, "mean_p": 0.3625443751980503, "n": 9, "sum_p": 3.2628993767824523 }, { "accuracy": 0.13333333333333333, "correct": 2, "gap": 0.3147402420597313, "hi": 0.5, "lo": 0.4, "mean_p": 0.44807357539306464, "n": 15, "sum_p": 6.7211036308959695 }, { "accuracy": 0.3333333333333333, "correct": 2, "gap": 0.21531381803649202, "hi": 0.6, "lo": 0.5, "mean_p": 0.5486471513698253, "n": 6, "sum_p": 3.291882908218952 }, { "accuracy": 0.7, "correct": 7, "gap": -0.053095274369252876, "hi": 0.7, "lo": 0.6, "mean_p": 0.6469047256307471, "n": 10, "sum_p": 6.469047256307471 }, { "accuracy": 0.8333333333333334, "correct": 5, "gap": -0.08646168268189258, "hi": 0.8, "lo": 0.7, "mean_p": 0.7468716506514408, "n": 6, "sum_p": 4.481229903908645 }, { "accuracy": 0.5238095238095238, "correct": 11, "gap": 0.324828295515037, "hi": 0.9, "lo": 0.8, "mean_p": 0.8486378193245608, "n": 21, "sum_p": 17.82139420581578 }, { "accuracy": 0.8780487804878049, "correct": 36, "gap": 0.09031788189989243, "hi": 1.0, "lo": 0.9, "mean_p": 0.9683666623876973, "n": 41, "sum_p": 39.70303315789559 } ], "unparsed": 0, "wall_seconds": 122.70109918800154 }, "apus-9b-low.kc": { "accuracy": 0.2777777777777778, "brier": 0.9197522835276776, "correct": 30, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 108, "joules": 2993.8104999113084, "joules_per_decision": 27.720467591771374, "mean_watts": 272.0162232737393, "net_joules_per_decision": 13.009173565555558, "peak_watts": 289.09, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 2993.8104999113084, "mean_watts": 272.0162232737393, "min_watts": 125.34, "peak_watts": 289.09, "samples": 12, "seconds": 11.00599980354309 } }, "samples": 12, "seconds": 11.00599980354309 }, "errors": 0, "eval_tokens_total": 108, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1300.3896459519863, "mean_watts": 144.35942060618876, "peak_watts": 266.85, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1300.3896459519863, "mean_watts": 144.35942060618876, "min_watts": 125.82, "peak_watts": 266.85, "samples": 10, "seconds": 9.007999897003174 } }, "samples": 10, "seconds": 9.007999897003174, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.10157239899854176, "n": 108, "n_labelled": 108, "n_reached": 108, "p95_seconds": 0.11486395499741775, "prompt_tokens_total": 36905, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.14285714285714285, "correct": 1, "gap": 0.12463615310732085, "hi": 0.3, "lo": 0.2, "mean_p": 0.2674932959644637, "n": 7, "sum_p": 1.872453071751246 }, { "accuracy": 0.14285714285714285, "correct": 3, "gap": 0.211442356350882, "hi": 0.4, "lo": 0.3, "mean_p": 0.35429949920802484, "n": 21, "sum_p": 7.440289483368522 }, { "accuracy": 0.4444444444444444, "correct": 8, "gap": 0.017353897335051993, "hi": 0.5, "lo": 0.4, "mean_p": 0.4617983417794964, "n": 18, "sum_p": 8.312370152030935 }, { "accuracy": 0.2777777777777778, "correct": 5, "gap": 0.27478648987368126, "hi": 0.6, "lo": 0.5, "mean_p": 0.552564267651459, "n": 18, "sum_p": 9.946156817726262 }, { "accuracy": 0.28, "correct": 7, "gap": 0.3708611413111923, "hi": 0.7, "lo": 0.6, "mean_p": 0.6508611413111923, "n": 25, "sum_p": 16.271528532779808 }, { "accuracy": 0.16666666666666666, "correct": 1, "gap": 0.5692119178774857, "hi": 0.8, "lo": 0.7, "mean_p": 0.7358785845441523, "n": 6, "sum_p": 4.415271507264914 }, { "accuracy": 0.375, "correct": 3, "gap": 0.47868009966231084, "hi": 0.9, "lo": 0.8, "mean_p": 0.8536800996623108, "n": 8, "sum_p": 6.829440797298487 }, { "accuracy": 0.4, "correct": 2, "gap": 0.5437721043531791, "hi": 1.0, "lo": 0.9, "mean_p": 0.9437721043531792, "n": 5, "sum_p": 4.718860521765896 } ], "unparsed": 0, "wall_seconds": 11.171307734999573 }, "apus-9b-low.kd": { "accuracy": 0.8958333333333334, "brier": 0.18195088016287306, "correct": 43, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 48, "joules": 1555.3057215046883, "joules_per_decision": 32.402202531347676, "mean_watts": 259.00178049363643, "net_joules_per_decision": 15.673426956191264, "peak_watts": 297.04, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1555.3057215046883, "mean_watts": 259.00178049363643, "min_watts": 118.99, "peak_watts": 297.04, "samples": 7, "seconds": 6.005000114440918 } }, "samples": 7, "seconds": 6.005000114440918 }, "errors": 0, "eval_tokens_total": 48, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1204.5386640691756, "mean_watts": 133.71876974264927, "peak_watts": 211.75, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1204.5386640691756, "mean_watts": 133.71876974264927, "min_watts": 119.25, "peak_watts": 211.75, "samples": 10, "seconds": 9.007999897003174 } }, "samples": 10, "seconds": 9.007999897003174, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.13296526950034604, "n": 48, "n_labelled": 48, "n_reached": 48, "p95_seconds": 0.1334890309990442, "prompt_tokens_total": 28918, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.3, "lo": 0.2, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.4, "lo": 0.3, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.5, "lo": 0.4, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 1.0, "correct": 2, "gap": -0.42633918707979257, "hi": 0.6, "lo": 0.5, "mean_p": 0.5736608129202074, "n": 2, "sum_p": 1.1473216258404149 }, { "accuracy": 0.6666666666666666, "correct": 2, "gap": -0.011476950103103234, "hi": 0.7, "lo": 0.6, "mean_p": 0.6551897165635634, "n": 3, "sum_p": 1.9655691496906902 }, { "accuracy": 0.7142857142857143, "correct": 5, "gap": 0.028695466035785766, "hi": 0.8, "lo": 0.7, "mean_p": 0.7429811803215001, "n": 7, "sum_p": 5.2008682622505 }, { "accuracy": 1.0, "correct": 3, "gap": -0.13967107076092933, "hi": 0.9, "lo": 0.8, "mean_p": 0.8603289292390707, "n": 3, "sum_p": 2.580986787717212 }, { "accuracy": 0.9393939393939394, "correct": 31, "gap": 0.031945228479431154, "hi": 1.0, "lo": 0.9, "mean_p": 0.9713391678733706, "n": 33, "sum_p": 32.05419253982123 } ], "unparsed": 0, "wall_seconds": 6.422894843999529 }, "apus-9b-low.kh": { "accuracy": 0.3125, "brier": 0.8118914653937003, "correct": 15, "energy": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "decisions": 48, "joules": 1235.2790157198906, "joules_per_decision": 25.73497949416439, "mean_watts": 246.80900026347516, "net_joules_per_decision": 11.250698395177217, "peak_watts": 286.07, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1235.2790157198906, "mean_watts": 246.80900026347516, "min_watts": 121.72, "peak_watts": 286.07, "samples": 6, "seconds": 5.004999876022339 } }, "samples": 6, "seconds": 5.004999876022339 }, "errors": 0, "eval_tokens_total": 48, "floored_rows": 0, "idle": { "cards": [ "GPU-fp:33a0b4acb8e8" ], "joules": 1251.3029934525491, "mean_watts": 138.91019180282615, "peak_watts": 213.62, "per_card": { "GPU-fp:33a0b4acb8e8": { "joules": 1251.3029934525491, "mean_watts": 138.91019180282615, "min_watts": 122.12, "peak_watts": 213.62, "samples": 10, "seconds": 9.007999897003174 } }, "samples": 10, "seconds": 9.007999897003174, "what": "the draw of these cards immediately before the arm, with the model resident and no work in flight" }, "letters_floored_total": 0, "median_seconds": 0.10452065600111382, "n": 48, "n_labelled": 48, "n_reached": 48, "p95_seconds": 0.11407554099787376, "prompt_tokens_total": 16837, "refused": 0, "refused_by_reason": {}, "reliability": [ { "accuracy": null, "correct": 0, "gap": null, "hi": 0.1, "lo": 0.0, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": null, "correct": 0, "gap": null, "hi": 0.2, "lo": 0.1, "mean_p": null, "n": 0, "sum_p": 0.0 }, { "accuracy": 0.14285714285714285, "correct": 1, "gap": 0.11972543226837307, "hi": 0.3, "lo": 0.2, "mean_p": 0.2625825751255159, "n": 7, "sum_p": 1.8380780258786114 }, { "accuracy": 0.18181818181818182, "correct": 2, "gap": 0.17033062921521702, "hi": 0.4, "lo": 0.3, "mean_p": 0.35214881103339885, "n": 11, "sum_p": 3.873636921367387 }, { "accuracy": 0.3, "correct": 3, "gap": 0.14532258556554556, "hi": 0.5, "lo": 0.4, "mean_p": 0.44532258556554555, "n": 10, "sum_p": 4.453225855655456 }, { "accuracy": 0.3333333333333333, "correct": 1, "gap": 0.23123963439923706, "hi": 0.6, "lo": 0.5, "mean_p": 0.5645729677325704, "n": 3, "sum_p": 1.6937189031977111 }, { "accuracy": 0.3333333333333333, "correct": 2, "gap": 0.29726712927482996, "hi": 0.7, "lo": 0.6, "mean_p": 0.6306004626081633, "n": 6, "sum_p": 3.7836027756489794 }, { "accuracy": 0.3333333333333333, "correct": 2, "gap": 0.39420559736705446, "hi": 0.8, "lo": 0.7, "mean_p": 0.7275389307003878, "n": 6, "sum_p": 4.365233584202326 }, { "accuracy": 0.6666666666666666, "correct": 2, "gap": 0.19292447108971178, "hi": 0.9, "lo": 0.8, "mean_p": 0.8595911377563784, "n": 3, "sum_p": 2.5787734132691353 }, { "accuracy": 1.0, "correct": 2, "gap": -0.06549715759493335, "hi": 1.0, "lo": 0.9, "mean_p": 0.9345028424050666, "n": 2, "sum_p": 1.8690056848101333 } ], "unparsed": 0, "wall_seconds": 5.040717875002883 } }, "tag": "counted", "tasks_ended": "2026-09-28T15:39:30Z", "tasks_started": "2026-09-28T15:32:28Z", "vendor_files_sha256": { "bridge": "7cb3e56d42651087e35b7a620b2407dc96801a09ae942bbe803cb202ee187e65", "openjet_runtime/__init__.py": "df6eb864cf6d0c2f512fb00f17eeaa11fd790afca33a0f7b2f609aeb1f2b3944", "openjet_runtime/candidate_projection.py": "84dc4746b5fb06ac6a9024dde3ba8414d901acf2a62d010b0d66f26acfaf74a6", "openjet_runtime/contracts.py": "d8e8e5270ecd6dab917d886dda2d684c24696b811faa968bb3c399ceef5e356a", "openjet_runtime/early_exit.py": "89b7751a927a6d1348a454fb8ee986395423ad5686d855fb3ccab255c95d2566", "openjet_runtime/runtime.py": "6e7b0b131cb14ab0d25cc8fd6c7fc41738b799cfe6de1ccdbae2d09d7c64313c" }, "versions": { "cuda_devices": 1, "cudnn": 92400, "device_name": "NVIDIA GeForce RTX 3090", "device_uuid_fp": "33a0b4acb8e8", "dists": [ "Jinja2==3.1.6", "MarkupSafe==3.0.3", "PyYAML==6.0.3", "Pygments==2.21.0", "annotated-doc==0.0.5", "anyio==4.15.1", "certifi==2026.7.22", "click==8.5.0", "cloudpickle==3.1.2", "cuda-bindings==13.4.2", "cuda-pathfinder==1.8.2", "cuda-toolkit==13.0.3.0", "filelock==4.0.1", "fsspec==2026.9.0", "h11==0.16.0", "hf-xet==1.6.0", "httpcore==1.0.9", "httpx==0.28.1", "huggingface_hub==1.32.0", "idna==3.20", "joblib==1.6.0", "markdown-it-py==4.2.0", "mdurl==0.1.2", "mpmath==1.3.0", "narwhals==2.26.0", "networkx==3.6.1", "numpy==2.5.3", "nvidia-cublas==13.1.1.3", "nvidia-cuda-cupti==13.0.85", "nvidia-cuda-nvrtc==13.0.88", "nvidia-cuda-runtime==13.0.96", "nvidia-cudnn-cu13==9.24.0.43", "nvidia-cufft==12.0.0.61", "nvidia-cufile==1.15.1.6", "nvidia-curand==10.4.0.35", "nvidia-cusolver==12.0.4.66", "nvidia-cusparse==12.6.3.3", "nvidia-cusparselt-cu13==0.8.1", "nvidia-nccl-cu13==2.30.7", "nvidia-nvjitlink==13.4.92", "nvidia-nvshmem-cu13==3.4.5", "nvidia-nvtx==13.0.85", "packaging==26.3", "pillow==12.3.0", "pip==26.2.1", "regex==2026.9.10", "rich==15.0.0", "safetensors==0.8.0", "scikit-learn==1.9.1", "scipy==1.18.1", "sentence-transformers==6.1.0", "setuptools==84.0.0", "shellingham==1.5.4", "sympy==1.14.0", "threadpoolctl==3.7.0", "tokenizers==0.23.2", "torch==2.14.0", "tqdm==4.70.1", "transformers==5.16.1", "triton==3.8.0", "typer==0.27.2", "typing_extensions==4.16.0" ], "huggingface_hub": "1.32.0", "jinja2": "3.1.6", "numpy": "2.5.3", "python": "3.14.4", "safetensors": "0.8.0", "tokenizers": "0.23.2", "torch": "2.14.0+cu130", "torch_cuda": "13.0", "transformers": "5.16.1" }, "warmups_s": [ [ "high", 0.549 ], [ "high", 0.169 ], [ "high", 0.169 ], [ "low", 0.123 ] ] }