#!/usr/bin/env python3 """Reports in, markdown out. Calls nothing, measures nothing, invents nothing. python3 tables.py rows/ > TABLES.md Every number printed here is read from a `*.report.json` or a `*.jsonl` written by `run.py`; this file does arithmetic on them and nothing else. It is separate from the harness so a table can be redrawn after the cards are cold, and so a reader can see that the drawing cannot reach the instrument. THE GATE IS EVALUATED HERE, from the pre-registration in README.md §5, and it is evaluated the way it was written: the adapter's readout against the BASE READOUT's accuracy minus two points, on (a) and (c), and on nothing else. A task the pre-registration declared unlabelled prints its cost and its agreement and gets no accuracy column at all. """ from __future__ import annotations import collections import json import os import sys GATE_TASKS = ("a", "c") GATE_MARGIN_POINTS = 2.0 ARM_ORDER = ["jev-readout", "base-readout", "jev-generate", "base-generate"] TASK_NAME = {"a": "(a) does the article answer this question?", "b": "(b) which section is the answer in?", "c": "(c) which of the six guests said this line?"} def load_all(rows_dir: str): reports, rows = {}, {} for f in sorted(os.listdir(rows_dir)): p = os.path.join(rows_dir, f) if f.endswith(".report.json"): r = json.load(open(p, encoding="utf-8")) reports[(r["arm"]["name"], r["task"])] = r elif f.endswith(".jsonl"): arm, task = f[:-6].rsplit(".", 1) rows[(arm, task)] = [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()] return reports, rows def pct(x, nd=1): return "—" if x is None else "%.*f %%" % (nd, 100 * x) def num(x, nd=2, unit=""): return "—" if x is None else "%.*f%s" % (nd, x, unit) def accuracy_table(reports) -> str: out = ["| task | arm | accuracy | correct / labelled | Brier | unparsed | floored rows |", "|---|---|---|---|---|---|---|"] for t in ("a", "b", "c"): for arm in ARM_ORDER: r = reports.get((arm, t)) if not r: continue s = r["summary"] if t == "b": # THE PRE-REGISTRATION DECLARED THIS TASK UNLABELLED and a rate # over its one labelled row would read as one. It gets a count, # never a percentage, however many rows happen to carry a label. acc = "**not a rate** — %d of %d rows carry a label" % ( s["n_labelled"], s["n"]) got = "%d / %d" % (s["correct"], s["n_labelled"]) if s["n_labelled"] else "—" elif s["n_labelled"]: acc, got = pct(s["accuracy"]), "%d / %d" % (s["correct"], s["n_labelled"]) else: acc, got = "not labelled", "—" out.append("| %s | `%s` | %s | %s | %s | %d | %d |" % (TASK_NAME[t] if arm == ARM_ORDER[0] else "", arm, acc, got, num(s["brier"], 4), s["unparsed"], s["floored_rows"])) return "\n".join(out) def cost_table(reports) -> str: out = ["| task | arm | median s | p95 s | prompt tokens | written tokens | " "mean W | J / decision | J / decision net of idle | idle W |", "|---|---|---|---|---|---|---|---|---|---|"] for t in ("a", "b", "c"): for arm in ARM_ORDER: r = reports.get((arm, t)) if not r: continue s, e, i = r["summary"], r["energy"], r["idle"] out.append("| %s | `%s` | %s | %s | %s | %s | %s | %s | %s | %s |" % (TASK_NAME[t] if arm == ARM_ORDER[0] else "", arm, num(s["median_seconds"], 3), num(s["p95_seconds"], 3), "{:,}".format(s["prompt_tokens_total"]), "{:,}".format(s["eval_tokens_total"]), num(e.get("mean_watts"), 1), num(e.get("joules_per_decision"), 1), num(e.get("net_joules_per_decision"), 1), num(i.get("mean_watts"), 1))) return "\n".join(out) def gate_table(reports) -> tuple[str, dict]: base = {t: (reports.get(("base-readout", t)) or {}).get("summary", {}).get("accuracy") for t in GATE_TASKS} jev = {t: (reports.get(("jev-readout", t)) or {}).get("summary", {}).get("accuracy") for t in GATE_TASKS} out = ["| gated task | base readout | adapter readout | delta (points) | " "floor (base − 2) | verdict |", "|---|---|---|---|---|---|"] verdicts = {} assert "b" not in GATE_TASKS, "task (b) is unlabelled and cannot be gated" for t in GATE_TASKS: b, j = base[t], jev[t] if b is None or j is None: out.append("| %s | — | — | — | — | **NOT MEASURED** |" % TASK_NAME[t]) verdicts[t] = None continue delta = 100 * (j - b) ok = delta >= -GATE_MARGIN_POINTS verdicts[t] = ok out.append("| %s | %s | %s | %+.1f | %s | **%s** |" % (TASK_NAME[t], pct(b), pct(j), delta, pct(b - 0.02), "PASS" if ok else "FAIL")) return "\n".join(out), verdicts def reliability_table(reports, arm: str, task: str) -> str: r = reports.get((arm, task)) if not r or not r["summary"]["n_labelled"]: return "_no labelled rows_" out = ["| bin | n | mean confidence | accuracy | gap |", "|---|---|---|---|---|"] for b in r["summary"]["reliability"]: if not b["n"]: continue out.append("| %.1f–%.1f | %d | %s | %s | %+.3f |" % (b["lo"], b["hi"], b["n"], num(b["mean_p"], 3), pct(b["accuracy"]), b["gap"])) return "\n".join(out) #: The same map `kit/build_kit.py` filters self-names with. Repeated here on #: purpose: this file must be readable without the builder open, and if the two #: ever disagree the disagreement is visible rather than silent. NAME_TOKENS = { "darwin": ["darwin", "charles"], "hypatia": ["hypatia"], "ibn_sina": ["ibn sina", "avicenna"], "sagan": ["sagan", "carl"], "socrates": ["socrates", "socratic"], "einstein": ["einstein", "albert"], } def elimination_baseline(kit_dir: str) -> tuple[float, str]: """What a reader scores on task (c) WITHOUT hearing a voice at all. 103 of the 108 lines address another guest by name, and a line's own speaker was filtered out before sampling — so anyone who simply crosses off every guest the line names and guesses among the rest already beats the 16.7 % six-way chance. That is real information in the text and it is not cheating, but it is not voice either, and an arm's accuracy has to be read against it rather than against chance. The number below is the expected accuracy of exactly that strategy: the mean of 1 / (guests not named) over the set. """ items = json.load(open(os.path.join(kit_dir, "task_c.json"), encoding="utf-8")) shares, named = [], 0 for it in items: low = it["text"].lower() out = {g for g in it["options"] if g != it["label"] and any(t in low for t in NAME_TOKENS[g])} shares.append(1.0 / (len(it["options"]) - len(out))) named += bool(out) exp = sum(shares) / len(shares) return exp, ("%d of %d lines name at least one other guest; crossing those off " "and guessing among the rest is worth **%s**, against a six-way " "chance of %s" % (named, len(items), pct(exp), pct(1 / 6))) def task_a_breakdown(rows) -> str: """The yes/no split, because 42 yes against 21 no hides a degenerate arm. An arm that answered "yes" to everything scores 42 of 63 — 66.7 % — and an accuracy column alone would let that read as most of the way there. The abstain half is the half the docent actually needs, so it gets its own column. """ out = ["| arm | on-page answered yes (of 42) | off-page answered no (of 21) | " "accuracy | always-yes would score |", "|---|---|---|---|---|"] for arm in ARM_ORDER: r = rows.get((arm, "a")) if not r: continue on = [x for x in r if x["kind"] == "on-page"] off = [x for x in r if x["kind"] == "off-page"] hit_on = sum(1 for x in on if x["choice"] == "yes") hit_off = sum(1 for x in off if x["choice"] == "no") out.append("| `%s` | %d (%s) | %d (%s) | %s | %s |" % (arm, hit_on, pct(hit_on / len(on)) if on else "—", hit_off, pct(hit_off / len(off)) if off else "—", pct((hit_on + hit_off) / len(r)), pct(len(on) / len(r)))) return "\n".join(out) def agreement_table(rows) -> str: """Where there is no label, two arms agreeing is the only thing measurable. PAIRED BY POSITION, NOT BY ID, and that is a fix rather than a shortcut. The kit frozen for the 2026-09-21 run gives task (c) ids built from (guest, edition, course, line_index) — and every wall entry is edition 1, so two different lines that happen to sit at the same course and index in different dinners collide: 108 distinct texts under 50 distinct ids. Keying this table by id silently scored 50 of the 108 rows and printed the rate as if it were all of them. Nothing else read the id, so accuracy, Brier, latency and energy are untouched; `kit/build_kit.py` now puts the entry in the id so a later kit has none of this. Every arm walks the kit in file order, so row i of one arm and row i of another are the same item — asserted here against the label sequence rather than assumed, because an assumption is what produced the defect. """ out = ["| task | pair | agree | of | rate |", "|---|---|---|---|---|"] for t in ("a", "b", "c"): names = [a for a in ARM_ORDER if (a, t) in rows] for i, x in enumerate(names): for y in names[i + 1:]: rx, ry = rows[(x, t)], rows[(y, t)] if len(rx) != len(ry) or [r.get("label") for r in rx] != [ r.get("label") for r in ry]: out.append("| %s | `%s` vs `%s` | — | — | **not comparable** |" % (TASK_NAME[t] if i == 0 else "", x, y)) continue agree = sum(1 for p, q in zip(rx, ry) if p["choice"] == q["choice"]) out.append("| %s | `%s` vs `%s` | %d | %d | %s |" % (TASK_NAME[t] if (x, y) == (names[0], names[1]) else "", x, y, agree, len(rx), pct(agree / len(rx)))) return "\n".join(out) def confusion_c(rows, arm: str) -> str: r = rows.get((arm, "c")) if not r: return "" guests = sorted({x["label"] for x in r}) cm = collections.Counter((x["label"], x["choice"]) for x in r) out = ["| said by ↓ / read as → | " + " | ".join("`%s`" % g for g in guests) + " |", "|---" * (len(guests) + 1) + "|"] for g in guests: out.append("| `%s` | " % g + " | ".join( ("**%d**" if g == h else "%d") % cm[(g, h)] for h in guests) + " |") return "\n".join(out) def main(argv=None) -> int: rows_dir = (argv or sys.argv[1:])[0] reports, rows = load_all(rows_dir) gate, verdicts = gate_table(reports) print("## The gate\n\n" + gate + "\n") print("Verdict: **%s**\n" % ( "ACCEPTED" if all(verdicts.get(t) for t in GATE_TASKS) else "REJECTED — the rung gets no threshold table")) print("## Accuracy\n\n" + accuracy_table(reports) + "\n") print("_Brier here is the MULTI-CLASS form — the squared error summed over every " "option — so on a two-option task it is twice the familiar binary Brier and " "must not be read against a published binary number. Lower is better; 0 is " "perfect. A generating arm has no probabilities, so its Brier is just its " "error rate doubled and its reliability table is one bin at 1.0: that is a " "property of writing a letter instead of scoring one, and it is the point._\n") print("## Task (a), split — the abstain half is the half that matters\n\n" + task_a_breakdown(rows) + "\n") print("## Latency, tokens and energy\n\n" + cost_table(reports) + "\n") print("## Agreement between arms\n\n" + agreement_table(rows) + "\n") kit = os.path.join(os.path.dirname(os.path.abspath(__file__)), "kit") if os.path.exists(os.path.join(kit, "task_c.json")): _, note = elimination_baseline(kit) print("## Task (c): the floor an arm has to beat\n\n" + note + "\n") for arm in ARM_ORDER: for t in ("a", "c"): if (arm, t) in reports: print("### Reliability — `%s`, task %s\n" % (arm, t)) print(reliability_table(reports, arm, t) + "\n") for arm in ("jev-readout", "base-readout"): cm = confusion_c(rows, arm) if cm: print("### Task (c) confusion — `%s`\n\n%s\n" % (arm, cm)) # residency, printed for every arm so no row is taken on trust print("## Residency, per arm\n") print("| arm | task | card 0 MiB after | named grew | others quiet | ok |") print("|---|---|---|---|---|---|") for (arm, t), r in sorted(reports.items()): p = r["pin"] print("| `%s` | %s | %s | %s | %s | %s |" % (arm, t, p.get("memory_after_named_mib"), p["named_grew"], p["others_quiet"], p.get("resident"))) return 0 if __name__ == "__main__": raise SystemExit(main())