#!/usr/bin/env python3 """Arm 2's tables. Reports in, markdown out — calls nothing, invents nothing. python3 tables_arm2.py rows/ > TABLES-ARM2.md Separate from `tables.py` on purpose, and the reason is not tidiness: arm 1's results are already published above the results line in README.md, and the file that drew them must keep drawing exactly them. This file reads the same directory, draws arm 2, and reaches into arm 1's reports ONLY to read numbers arm 2 is compared against — never to redraw them. THE GATE IS EVALUATED HERE, from README §A2.6, and the floor is **derived from arm 1's own report file**, not typed: the pre-registration says "arm 1's adapter on (c), minus two points", so this file reads `jev-readout.c.report.json` and subtracts. A number nobody typed cannot be a number somebody adjusted. """ from __future__ import annotations import json import os import sys from tables import (ARM_ORDER as ARM1_ORDER, TASK_NAME, confusion_c, load_all, num, pct, reliability_table) ARM2_ORDER = ["openjev-fp8-readout", "openjev-fp8-generate"] GATE_TASK = "c" GATE_MARGIN_POINTS = 2.0 GATE_REFERENCE_ARM = "jev-readout" # arm 1's adapter, the thing to match CARDS = {"GPU-46890836-d8f7-e868-1977-c1ce28db0d7a": "card 0", "GPU-7aa0be10-974f-6430-52fe-09018a0e2e08": "card 1"} def card(uuid: str) -> str: return "%s `%s…`" % (CARDS.get(uuid, "?"), uuid[:16]) def gate_table(reports) -> tuple[str, bool | None]: """README §A2.6: OpenJev's readout on (c), against arm 1's adapter minus two.""" ref = (reports.get((GATE_REFERENCE_ARM, GATE_TASK)) or {}).get("summary", {}) got = (reports.get(("openjev-fp8-readout", GATE_TASK)) or {}).get("summary", {}) base, jev = ref.get("accuracy"), got.get("accuracy") out = ["| gated task | arm 1's adapter readout (ollama, one card) | " "the floor (that − 2) | OpenJev-FP8 readout (vLLM, two cards) | " "delta (points) | verdict |", "|---|---|---|---|---|---|"] if base is None or jev is None: out.append("| %s | %s | — | %s | — | **NOT MEASURED** |" % (TASK_NAME[GATE_TASK], pct(base), pct(jev))) return "\n".join(out), None floor = base - GATE_MARGIN_POINTS / 100 ok = jev >= floor out.append("| %s | %s (%d / %d) | **%s** | %s (%d / %d) | %+.1f | **%s** |" % (TASK_NAME[GATE_TASK], pct(base), ref["correct"], ref["n_labelled"], pct(floor), pct(jev), got["correct"], got["n_labelled"], 100 * (jev - base), "PASS" if ok else "FAIL")) return "\n".join(out), ok def coverage_table(reports) -> str: """What the 16,384-token ceiling refused, per task, before anything was scored.""" out = ["| task | items in the kit | reached | refused as overlong | scored against a label |", "|---|---|---|---|---|"] for t in ("a", "b", "c"): r = reports.get((ARM2_ORDER[0], t)) if not r: continue s = r["summary"] out.append("| %s | %d | %d | %d | %d |" % (TASK_NAME[t], s["n"], s["n_reached"], s["refused_by_reason"].get("overlong", 0), s["n_labelled"] if t != "b" else 0)) return "\n".join(out) def accuracy_table(reports) -> str: out = ["| task | arm | accuracy | correct / scored | Brier | unparsed | " "rows with a floored letter | letters floored |", "|---|---|---|---|---|---|---|---|"] for t in ("a", "b", "c"): for arm in ARM2_ORDER: r = reports.get((arm, t)) if not r: continue s = r["summary"] if t == "b": acc = "**not a rate** — %d of %d reached rows carry a label" % ( s["n_labelled"], s["n_reached"]) got = "%d / %d" % (s["correct"], s["n_labelled"]) if s["n_labelled"] else "—" else: acc = pct(s["accuracy"]) got = "%d / %d" % (s["correct"], s["n_labelled"]) out.append("| %s | `%s` | %s | %s | %s | %d | %d | %d |" % (TASK_NAME[t] if arm == ARM2_ORDER[0] else "", arm, acc, got, num(s["brier"], 4), s["unparsed"], s["floored_rows"], s["letters_floored_total"])) return "\n".join(out) def energy_table(reports) -> str: """Both boards, each on its own clock, summed — never a mean of two means.""" out = ["| task | arm | median s | p95 s | prompt tokens | written tokens | " "card 0 mean W | card 1 mean W | both, J / decision | " "net of idle, J / decision | idle W (both) |", "|---|---|---|---|---|---|---|---|---|---|---|"] for t in ("a", "b", "c"): for arm in ARM2_ORDER: r = reports.get((arm, t)) if not r: continue s, e, i = r["summary"], r["energy"], r["idle"] per = e.get("per_card") or {} w = [num((per.get(u) or {}).get("mean_watts"), 1) for u in sorted(CARDS, key=lambda x: CARDS[x])] out.append("| %s | `%s` | %s | %s | %s | %s | %s | %s | %s | %s | %s |" % (TASK_NAME[t] if arm == ARM2_ORDER[0] else "", arm, num(s["median_seconds"], 3), num(s["p95_seconds"], 3), "{:,}".format(s["prompt_tokens_total"]), "{:,}".format(s["eval_tokens_total"]), w[0], w[1], num(e.get("joules_per_decision"), 1), num(e.get("net_joules_per_decision"), 1), num(i.get("mean_watts"), 1))) return "\n".join(out) def cross_runtime_table(reports) -> str: """The comparison the brief asked for, with its caveat welded to its header. Arm 1's base was NOT served under vLLM (README §A2.3.4), so this is not a like-for-like latency test and the header says so in every place it is drawn. It is here because the alternative — printing arm 2's seconds with no reference at all — tells a reader even less. """ out = ["| task | arm | server · cards | model | median s | p95 s | " "decisions it is a median of |", "|---|---|---|---|---|---|---|"] order = ARM1_ORDER + ARM2_ORDER for t in ("a", "b", "c"): for arm in order: r = reports.get((arm, t)) if not r: continue s, A = r["summary"], r["arm"] n_cards = len(A.get("cards") or [A.get("card_uuid")]) server = ("ollama" if A.get("runtime", "ollama") == "ollama" else "vLLM 0.29.0") out.append("| %s | `%s` | %s · %d card%s | `%s` | %s | %s | %d |" % (TASK_NAME[t] if arm == order[0] else "", arm, server, n_cards, "" if n_cards == 1 else "s", A["model"], num(s["median_seconds"], 3), num(s["p95_seconds"], 3), s.get("n_reached", s["n"]))) return "\n".join(out) def readout_vs_generate(rows) -> str: """The same model, the same prompt, paired item by item: read or written.""" out = ["| task | paired items | median readout s | median generate s | " "median delta | share | same choice |", "|---|---|---|---|---|---|---|"] import statistics for t in ("a", "b", "c"): rx, ry = rows.get(("openjev-fp8-readout", t)), rows.get(("openjev-fp8-generate", t)) if not rx or not ry or len(rx) != len(ry): continue pairs = [(p, q) for p, q in zip(rx, ry) if not p.get("refused") and not q.get("refused")] if not pairs: continue assert [p["id"] for p, _ in pairs] == [q["id"] for _, q in pairs] d = [q["seconds"] - p["seconds"] for p, q in pairs] mx = statistics.median(p["seconds"] for p, _ in pairs) my = statistics.median(q["seconds"] for _, q in pairs) agree = sum(1 for p, q in pairs if p["choice"] == q["choice"]) out.append("| %s | %d | %s | %s | %+.3f s | %s | %d / %d |" % (TASK_NAME[t], len(pairs), num(mx, 3), num(my, 3), statistics.median(d), pct(statistics.median(d) / my), agree, len(pairs))) return "\n".join(out) def cross_arm_agreement(rows) -> str: """Where two arms are the same items, how often they make the same call. Paired BY POSITION and asserted against the id and the label sequence — the kit's task-(c) ids collide (README §R6) and arm 1 was bitten by keying on them. Refused rows drop out of both sides of a pair. """ out = ["| task | pair | agree | of comparable | rate |", "|---|---|---|---|---|"] names = ARM1_ORDER + ARM2_ORDER for t in ("a", "b", "c"): have = [a for a in names if (a, t) in rows] for i, x in enumerate(have): for y in have[i + 1:]: if not ({x, y} & set(ARM2_ORDER)): continue # arm 1 against itself is arm 1's own table rx, ry = rows[(x, t)], rows[(y, t)] if len(rx) != len(ry) or [r.get("label") for r in rx] != [ r.get("label") for r in ry]: out.append("| %s | `%s` vs `%s` | — | — | **not comparable** |" % (TASK_NAME[t], x, y)) continue pairs = [(p, q) for p, q in zip(rx, ry) if not p.get("refused") and not q.get("refused")] agree = sum(1 for p, q in pairs if p["choice"] == q["choice"]) out.append("| %s | `%s` vs `%s` | %d | %d | %s |" % (TASK_NAME[t], x, y, agree, len(pairs), pct(agree / len(pairs)) if pairs else "—")) return "\n".join(out) def phb_table(path: str) -> str: """The §A2.8 receipt, drawn from the microbench's own JSON.""" if not os.path.exists(path): return "_not measured_" d = json.load(open(path, encoding="utf-8")) out = ["| what one all-reduce carries | bytes on the wire | median | p95 | " "bus bandwidth |", "|---|---|---|---|---|"] for r in d["results"]: out.append("| %s (%d token%s × 5,120 × bf16) | %s | %s | %s | %s |" % (r["what"], r["tokens"], "" if r["tokens"] == 1 else "s", "%.1f KiB" % (r["bytes"] / 1024) if r["bytes"] < 2 ** 20 else "%.1f MiB" % (r["bytes"] / 2 ** 20), "%.1f µs" % r["median_us"] if r["median_us"] < 1000 else "%.1f ms" % (r["median_us"] / 1000), "%.1f µs" % r["p95_us"] if r["p95_us"] < 1000 else "%.1f ms" % (r["p95_us"] / 1000), "%.2f GB/s" % r["busbw_GBs"])) dc = d["decode_cost"] out.append("") out.append("**The floor this link puts under one written token: %.1f ms** " "— %d all-reduces (2 per layer × 64 layers) at %.1f µs each, before " "the model computes anything. torch %s, NCCL %s, measured %s." % (dc["per_output_token_ms"], dc["allreduces_per_output_token"], dc["one_allreduce_us"], d["torch"], d["nccl"], d["stamp"])) return "\n".join(out) def residency_table(reports) -> str: out = ["| arm | task | card 0 MiB after | card 1 MiB after | both grew | " "no other board moved | ok |", "|---|---|---|---|---|---|---|"] for arm in ARM2_ORDER: for t in ("a", "b", "c"): r = reports.get((arm, t)) if not r: continue p = r["pin"] per = p.get("memory_after_per_card_mib") or {} vals = [per.get(u) for u in sorted(CARDS, key=lambda x: CARDS[x])] out.append("| `%s` | %s | %s | %s | %s | %s | %s |" % (arm, t, vals[0], vals[1], p["named_grew"], p["others_quiet"], p.get("resident"))) return "\n".join(out) def main(argv=None) -> int: rows_dir = (argv or sys.argv[1:])[0] here = os.path.dirname(os.path.abspath(__file__)) reports, rows = load_all(rows_dir) gate, ok = gate_table(reports) print("## The gate — arm 2\n\n" + gate + "\n") print("Verdict: **%s**\n" % ( "ACCEPTED" if ok else "REJECTED — the rung gets no threshold table" if ok is False else "NOT MEASURED")) print("## Coverage — what the 16,384-token ceiling refused\n\n" + coverage_table(reports) + "\n") print("_A refused item is in the row file with its measured token count and " "out of every rate. It was never truncated: a truncated docent page can " "turn a genuinely on-page question into an unanswerable one, which is " "arm 1 §8's ruling and it stands._\n") print("## Accuracy\n\n" + accuracy_table(reports) + "\n") print("## Latency and energy — BOTH boards, summed\n\n" + energy_table(reports) + "\n") print("## Reading the letter against writing it, on the same model\n\n" + readout_vs_generate(rows) + "\n") print("## Latency ACROSS RUNTIMES — not a like-for-like comparison\n\n" "_Arm 1 ran on **ollama, one card, a Q4\\_K\\_M GGUF of a 26B-A4B**; arm 2 runs " "on **vLLM 0.29.0, two cards tensor-parallel, an FP8 checkpoint of a different " "model family**. Arm 1's base was NOT served under vLLM (README §A2.3.4), so " "**the apples-to-apples latency comparison is not made**. These rows sit in one " "table because a reader needs the scale; they do not sit in one table because " "they are the same measurement._\n\n" + cross_runtime_table(reports) + "\n") print("## Agreement with arm 1's arms\n\n" + cross_arm_agreement(rows) + "\n") print("## The PHB receipt — what the host bridge costs (README §A2.8)\n\n" + phb_table(os.path.join(here, "receipts", "phb-allreduce.json")) + "\n") for arm in ARM2_ORDER: for t in ("a", "c"): if (arm, t) in reports: print("### Reliability — `%s`, task %s\n" % (arm, t)) print(reliability_table(reports, arm, t) + "\n") for arm in ARM2_ORDER: cm = confusion_c(rows, arm) if cm: print("### Task (c) confusion — `%s`\n\n%s\n" % (arm, cm)) print("## Residency — both boards, per arm\n\n" + residency_table(reports) + "\n") return 0 if __name__ == "__main__": raise SystemExit(main())