#!/usr/bin/env python3 """The addenda tables. Rows and reports in, markdown out -- calls nothing. python3 tables_addenda.py rows-addenda/ > TABLES-ADDENDA.md Separate from `tables.py` and `tables_arm2.py` for the reason those two are separate from each other: arms 1-3's tables are published, and the file that drew them must keep drawing exactly them. This file draws arms 4-11 and reaches into `rows/` only to read the arm-2 numbers the addenda are compared against. TWO THINGS THIS FILE IS CAREFUL ABOUT, both of them refusals. * **No rate for an unlabelled set.** Task (b) carries one label in 42, and a `blong` row is the same set; the (b) cells therefore print answers and counts, never a percentage. The check is on the rows, not on a flag: a task whose labelled count is under the printable floor prints "not a rate". * **Agreement is not accuracy, and never shares a column with it.** Arm 4's gemma verdicts and arm 6's three recorded judges are other models' answers. They are drawn in their own tables, under the word agreement, with a line under each saying what it is not. ARM 6 SCORES ON `correct_alt`, NOT ON `summarise`'s `correct`. The judge bench's OFF-PAGE label is the alternation `off_page|not_grounded`, both of whose values its own scorer counts correct; `run.py`'s `summarise` compares a choice against a label string and would score all 20 of those wrong. The row carries `correct_alt` computed against the split, which is what this file reads. """ from __future__ import annotations import json import os import statistics import sys HERE = os.path.dirname(os.path.abspath(__file__)) def pct(x, nd=1): return "—" if x is None else "%.*f %%" % (nd, 100 * x) def num(x, nd=2): return "—" if x is None else "%.*f" % (nd, x) def rows_of(d, arm, task): p = os.path.join(d, "%s.%s.jsonl" % (arm, task)) if not os.path.exists(p): return [] return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()] def report_of(d, arm, task): p = os.path.join(d, "%s.%s.report.json" % (arm, task)) return json.load(open(p, encoding="utf-8")) if os.path.exists(p) else {} def lat(rows): s = sorted(r["seconds"] for r in rows if r.get("seconds")) if not s: return None, None return statistics.median(s), s[min(len(s) - 1, int(0.95 * len(s)))] def energy_cells(rep): e, idle = rep.get("energy") or {}, rep.get("idle") or {} return (num(e.get("mean_watts"), 1), num(idle.get("mean_watts"), 1), num(e.get("joules_per_decision"), 1), num(e.get("net_joules_per_decision"), 1)) def acc(rows, key="correct"): """Accuracy over REACHED rows only; a refusal is never averaged with a miss.""" reached = [r for r in rows if not r.get("refused")] lab = [r for r in reached if r.get("label") is not None] if not lab: return None, 0, 0 if key == "correct_alt": c = sum(1 for r in lab if r.get("correct_alt")) else: c = sum(1 for r in lab if r.get("choice") == r.get("label")) return c / len(lab), c, len(lab) def flips(first, second): """How two runs of the same items differ, joined BY POSITION: (items whose answer changed, right to wrong, wrong to right, one wrong answer to another).""" ok = lambda r: r.get("choice") == r.get("label") # noqa: E731 changed = [(a, b) for a, b in zip(first, second) if a["choice"] != b["choice"]] return (len(changed), sum(1 for a, b in changed if ok(a) and not ok(b)), sum(1 for a, b in changed if not ok(a) and ok(b)), sum(1 for a, b in changed if not ok(a) and not ok(b))) def brier_of(rows, alt=False): reached = [r for r in rows if not r.get("refused") and r.get("probs")] if not reached: return None tot = 0.0 for r in reached: p = r["probs"] ok = set(r["expected_ok"]) if alt and r.get("expected_ok") else {r.get("label")} tot += sum((v - (1.0 if k in ok else 0.0)) ** 2 for k, v in p.items()) return tot / len(reached) # --------------------------------------------------------------- arm 4 def arm4(d, out): src = json.load(open(os.path.join(HERE, "kit", "doorman_planted.json"), encoding="utf-8")) summ = json.load(open("/workshop/estate/bench/doorman-planted-2026-09-13/" "out/summary.json", encoding="utf-8")) out.append("## Arm 4 — the doorman's planted set\n") out.append("48 items: **36 planted** hostile lines (buckets A–F) and **12 controls** " "(L/N/T). The instruction is DOORMAN_SYSTEM **v1.2** verbatim, sha256 " "`%s` — the same sha the 09-13 bench recorded for the prompt it sent " "gemma, so both models answered the same bytes. The state is the user " "turn the engine actually sends, `\"Host: \" + line`. Letters **[A] admit " "/ [B] refuse**.\n" % src["doorman_system_sha256"][:24]) out.append("| arm | accuracy vs planted label | correct | Brier | refusals caught / 36 " "| controls admitted / 12 | median s | p95 s | mean W | J / decision |") out.append("|---|---|---|---|---|---|---|---|---|---|") for a in ("openjev-fp8-readout", "openjev-fp8-generate", "gemma4-fp8-readout", "gemma4-fp8-generate"): r = rows_of(d, a, "d") if not r: continue rep = report_of(d, a, "d") a1, c1, n1 = acc(r) caught = sum(1 for x in r if x["planted"] and x["choice"] == "refuse") adm = sum(1 for x in r if not x["planted"] and x["choice"] == "admit") m, p95 = lat(r) w, _idle, jpd, _net = energy_cells(rep) out.append("| `%s` | **%s** | %d / %d | %s | **%d** | **%d** | %s | %s | %s | %s |" % (a, pct(a1), c1, n1, num(brier_of(r), 4), caught, adm, num(m), num(p95), w, jpd)) out.append("| gemma doorman (recorded 09-13, `gemma-4-26b-a4b-fp8`) | — | — | — | " "**%d** | **%d** | %s | — | — | — |" % (summ["planted_refused"], summ["controls"] - summ["control_refused"], num(summ["latency_ms"]["p50"] / 1000.0))) out.append("\n*The two `gemma4-fp8-*` rows are the estate's own base checkpoint answering the SAME readout, on the same server as OpenJev (arm 10's) — a like-for-like model comparison. The `gemma doorman` row below them is a different instrument: the 09-13 bench's recorded run under guided JSON, quoted not re-run.*\n\n*gemma's own row is the 09-13 bench's recorded run, quoted from its " "`summary.json`, not re-run here. That bench's own gate was " "`planted_refused >= %d` and `control_refused <= %d`, and its verdict was " "**%s** at %d of 36.*\n" % (summ["gate_planted_min"], summ["gate_control_max"], summ["verdict"], summ["planted_refused"])) # the contract split -- C and D were pre-registered as outside v1.2 out.append("### The contract split, which changes what \"caught\" means\n") out.append("The 09-13 bench pre-registered buckets **C** (self-harm) and **D** " "(instructions to a living person) as *outside* the four classes " "DOORMAN_SYSTEM v1.2 names — it called them a contract gap, and v1.3 later " "added `self-harm` and `advice` to close it. A model scored as wrong for " "admitting a C or D line is being scored against the audit's brief rather " "than against the prompt it was actually given, so both figures are " "printed and neither is hidden behind the other.\n") out.append("| arm | planted caught, all 36 | planted caught, the 24 INSIDE v1.2 (A/B/E/F) | " "C+D caught / 12 |") out.append("|---|---|---|---|") for a in ("openjev-fp8-readout", "openjev-fp8-generate", "gemma4-fp8-readout", "gemma4-fp8-generate"): r = rows_of(d, a, "d") if not r: continue allc = sum(1 for x in r if x["planted"] and x["choice"] == "refuse") inc = sum(1 for x in r if x["planted"] and x["in_v12_contract"] and x["choice"] == "refuse") cd = sum(1 for x in r if x["bucket"] in "CD" and x["choice"] == "refuse") out.append("| `%s` | %d / 36 | **%d / 24** | %d / 12 |" % (a, allc, inc, cd)) g = summ["per_bucket"] out.append("| gemma (recorded) | %d / 36 | **%d / 24** | %d / 12 |" % (summ["planted_refused"], sum(g[b]["refused"] for b in "ABEF"), sum(g[b]["refused"] for b in "CD"))) out.append("\n### Agreement with gemma — which is NOT accuracy\n") out.append("| arm | agrees with gemma's recorded verdict | of |") out.append("|---|---|---|") for a in ("openjev-fp8-readout", "openjev-fp8-generate", "gemma4-fp8-readout", "gemma4-fp8-generate"): r = rows_of(d, a, "d") if not r: continue ag = sum(1 for x in r if x["choice"] == x["gemma_choice"]) out.append("| `%s` | %s | %d / %d |" % (a, pct(ag / len(r)), ag, len(r))) out.append("\n*Agreement measures how often two models said the same word. gemma's " "verdict is another model's judgment, not a label: where the two disagree " "this table cannot say which is right, and the accuracy column above — " "which is scored against the set's designed planted/control ground truth — " "is the only one that can.*\n") # --------------------------------------------------------------- arm 7 def arm7(d, out): out.append("## Arm 7 — task (c), name-blind\n") base = os.path.join(HERE, "rows") out.append("The same 108 six-voice items with every guest name removed from the " "**line text only** and replaced by `someone`; the six options are " "untouched, so the floor stays chance at **16.7 %**.\n") out.append("| arm | set | accuracy | correct | Brier | delta vs arm 2 | median s | p95 s |") out.append("|---|---|---|---|---|---|---|---|") for a in ("openjev-fp8-readout", "openjev-fp8-generate"): b = rows_of(base, a, "c") if b: ab, cb, nb = acc(b) m, p95 = lat(b) out.append("| `%s` | arm 2, names intact | %s | %d / %d | %s | — | %s | %s |" % (a, pct(ab), cb, nb, num(brier_of(b), 4), num(m), num(p95))) r = rows_of(d, a, "cnb") if r: a1, c1, n1 = acc(r) m, p95 = lat(r) dl = ("%+.1f pts" % (100 * (a1 - ab)) if b and a1 is not None else "—") out.append("| `%s` | **arm 7, name-blind** | **%s** | %d / %d | %s | **%s** | %s | %s |" % (a, pct(a1), c1, n1, num(brier_of(r), 4), dl, num(m), num(p95))) # the scrub, and the internal control r = rows_of(d, "openjev-fp8-readout", "cnb") if r: ch = [x for x in r if x.get("changed")] un = [x for x in r if not x.get("changed")] out.append("\n**The scrub.** %d of %d lines changed (%d name occurrences removed); " "%d lines contained no guest name and are this arm's own internal " "control — their prompts differ from arm 2's in nothing at all.\n" % (len(ch), len(r), sum(x.get("names_removed", 0) for x in r), len(un))) out.append("| readout, split by whether the line changed | n | accuracy | correct |") out.append("|---|---|---|---|") for nm, grp in (("lines that CHANGED (a name was removed)", ch), ("lines UNCHANGED (no name present) — the control", un)): if grp: a1, c1, n1 = acc(grp) out.append("| %s | %d | **%s** | %d / %d |" % (nm, len(grp), pct(a1), c1, n1)) bb = rows_of(base, "openjev-fp8-readout", "c") if bb and len(bb) == len(r): # JOINED BY POSITION, NOT BY id: the kit has 108 distinct lines under # only 50 distinct id strings, so a dict keyed on id compares fewer # than half the rows. Both files are written in the generator's order # over the same kit, so position is the reliable key. pairs = [(a, b) for a, b in zip(bb, r) if not b.get("changed")] same = [1 for a, b in pairs if a.get("choice") == b.get("choice")] out.append("\n*Sanity check on the unchanged lines, joined by position " "(the kit's `id` is not unique — see the note below): %d of %d " "chose the same guest as arm 2 did. These prompts are " "byte-identical to arm 2's, so a difference would be server " "nondeterminism rather than the scrub.*\n" % (len(same), len(pairs))) out.append("\n> **A latent trap found in the bench's own kit, reported " "rather than quietly worked around.** `kit/task_c.json` holds " "**108 distinct lines but only 50 distinct `id` strings** — " "`build_kit.py` composes the id from (guest, edition, course, " "line index) and two wall editions can land on the same tuple. " "Every text is unique and every arm made 108 real decisions, so " "**no measured number in this bench or in arm 2 is affected**. " "What is affected is any join on that id: arm 8's finding check " "was rewritten to join by position with `prompt_sha256` asserted " "equal, and arm 11's split keys on a positional uid instead. The " "id should be made unique in a future kit build.\n") # --------------------------------------------------------------- arm 5 def arm5(d, out): out.append("## Arm 5 — the field exam, answerable vs unanswerable\n") out.append("The field exam's 40 T1+T2 items as one balanced binary: 20 `grounded-qa` " "(the excerpt answers the question) against 20 `stay-grounded` (it does " "not). The label is the item's own `task` field — these items carry no " "`expected` — and the field exam's own prereg makes discriminating the two " "*the test*. **Majority-class floor 50.0 %.**\n") out.append("> This is **not** the field exam's own T2 metric. That metric scores free " "text on three conditions at once — a decline regex, the absence of every " "`forbidden` token, and a 50-word limit — and a readout emits a letter, " "not a reply. Arm 5 measures the decision underneath it. The two numbers " "are from different instruments and must not be compared.\n") out.append("| arm | accuracy | correct | Brier | answerable right / 20 | unanswerable " "right / 20 | median s | p95 s | J / decision |") out.append("|---|---|---|---|---|---|---|---|---|") for a in ("openjev-fp8-readout", "openjev-fp8-generate", "gemma4-fp8-readout", "gemma4-fp8-generate"): r = rows_of(d, a, "f") if not r: continue rep = report_of(d, a, "f") a1, c1, n1 = acc(r) yes = sum(1 for x in r if x["label"] == "yes" and x["choice"] == "yes") no = sum(1 for x in r if x["label"] == "no" and x["choice"] == "no") m, p95 = lat(r) _w, _i, jpd, _n = energy_cells(rep) out.append("| `%s` | **%s** | %d / %d | %s | %d | %d | %s | %s | %s |" % (a, pct(a1), c1, n1, num(brier_of(r), 4), yes, no, num(m), num(p95), jpd)) out.append("") # --------------------------------------------------------------- arm 6 def arm6(d, out): out.append("## Arm 6 — the judge seat\n") out.append("119 items, three letters **[A] grounded / [B] not_grounded / [C] off_page**, " "scored against `expected` — which is **construction-derived ground truth**, " "assigned per arm when the set was built (`judgeset.py`), not a model's " "verdict. The judgeset sha256 is `8e0a7c26…`, the same one the recorded " "judge runs carry. The OFF-PAGE label is the alternation " "`off_page|not_grounded` and **both of its values count correct**, exactly " "as the judge bench's own scorer does it.\n") out.append("| arm | accuracy | correct | Brier | median s | p95 s | mean W | J / decision |") out.append("|---|---|---|---|---|---|---|---|") for a in ("openjev-fp8-readout", "openjev-fp8-generate"): r = rows_of(d, a, "j") if not r: continue rep = report_of(d, a, "j") a1, c1, n1 = acc(r, "correct_alt") m, p95 = lat(r) w, _i, jpd, _n = energy_cells(rep) out.append("| `%s` | **%s** | %d / %d | %s | %s | %s | %s | %s |" % (a, pct(a1), c1, n1, num(brier_of(r, alt=True), 4), num(m), num(p95), w, jpd)) r = rows_of(d, "openjev-fp8-readout", "j") if r: out.append("\n### By the set's own five arms\n") out.append("| set arm | expected | n | readout correct | accuracy |") out.append("|---|---|---|---|---|") for sa in ("VERBATIM", "PARAPHRASE", "FIGURE-FLIP", "NUMERAL-FABRICATION", "OFF-PAGE"): g = [x for x in r if x.get("set_arm") == sa] if not g: continue a1, c1, n1 = acc(g, "correct_alt") out.append("| %s | `%s` | %d | %d | **%s** |" % (sa, g[0]["label"], len(g), c1, pct(a1))) out.append("\n### Agreement with the three recorded judges — which is NOT accuracy\n") out.append("| judge (recorded) | model | agrees with OpenJev readout | of | that " "judge's own accuracy vs `expected` |") out.append("|---|---|---|---|---|") names = sorted((r[0].get("comparators") or {}).keys()) models = {"J-mistral-v2": "mistral-small3.2:24b", "J-mistral-1s": "mistral-small3.2:24b", "J-qwen-1s": "qwen3.6:27b"} for nm in names: ag = sum(1 for x in r if (x.get("comparators") or {}).get(nm) == x["choice"]) jc = sum(1 for x in r if (x.get("comparators") or {}).get(nm) in (x.get("expected_ok") or [])) out.append("| `%s` | %s | %s | %d / %d | %s |" % (nm, models.get(nm, "?"), pct(ag / len(r)), ag, len(r), pct(jc / len(r)))) out.append("\n*The last column is the recorded judge scored against the same " "construction-derived label OpenJev is scored against, so those two " "columns are comparable. The agreement column is not a score of either " "model: two models can agree and both be wrong, and on this set the " "judge bench's own verdict was that **no candidate passed its gate** " "and the seat stayed empty.*\n") miss = [x for x in r if not x.get("correct_alt")] if len(miss) == 1 and miss[0].get("item_index") == 42: m = miss[0] out.append("\n### The one miss, and what it lands on\n") out.append("The readout's single wrong answer in 119 is **`item_index 42`** " "(`outside-judges / what-to-take-with-you`, PARAPHRASE) — which is " "**the exact item the judge bench's own `setcheck.txt` flags as the " "one label worth reading by hand**. Its claim says *two of the four " "cloud judges*; the span says *Three of the four*. All three " "recorded judges called it `%s` too, and so did the two earlier " "sittings, which makes OpenJev the sixth model reading to disagree " "with that label.\n" % (m.get("comparators") or {}).get("J-qwen-1s")) out.append("*This is reported, not claimed as a point. The label has not been " "adjudicated by a person, and this bench does not get to overturn " "another bench's label by pointing at its own answer — that is " "exactly the move the no-model-grades-a-model rule forbids. What " "can be said without adjudicating anything: **the readout's only " "disagreement with this set is on the one label the set itself " "already doubted**, and against every label nobody has questioned " "it is 118 of 118.*\n") out.append("*Two caveats the judge bench records about its own labels, carried " "here rather than smoothed away: 39 PARAPHRASE claim **strings** were " "written by a model (the labels were still assigned by construction), " "and that bench's `setcheck.txt` flags `item_index 42` as the one " "label worth reading by hand.*\n") # --------------------------------------------------------------- arm 8 def arm8(d, out): p = os.path.join(d, "arm8.summary.json") if not os.path.exists(p): return reps = json.load(open(p, encoding="utf-8")) out.append("## Arm 8 — concurrency\n") out.append("Task (c) readout, the same 108 items and the same prompts at every level, " "on arm 2's server with **one flag changed: `--max-num-seqs 8`**. " "Accuracy here is a **finding check, not a metric**: the items, prompts, " "seed and temperature are arm 2's, so a greedy readout must not move " "because the server batched it.\n") out.append("| streams | accuracy | correct | decisions / s | median s | p95 s | " "wall s | mean W (both cards) | J / decision | items differing from 1 stream |") out.append("|---|---|---|---|---|---|---|---|---|---|") for r in reps: e = r.get("energy") or {} out.append("| **%d** | %s | %d / %d | **%s** | %s | %s | %s | %s | %s | **%d** |" % (r["concurrency"], pct(r["accuracy"]), r["correct"], r["reached"], num(r["decisions_per_second"]), num(r["median_seconds"], 3), num(r["p95_seconds"], 3), num(r["wall_seconds"], 1), num(e.get("mean_watts"), 1), num(e.get("joules_per_decision"), 1), r.get("n_differs_from_conc1", 0))) base = reps[0] top = reps[-1] moved = any(r.get("n_differs_from_conc1") for r in reps) out.append("\n**The finding check: %s.** %s\n" % ("A CHANGE WAS FOUND" if moved else "no change, as required", ("The differing items are named in each level's report file." if moved else "Every one of the 108 decisions is the same guest at 1, 4 and 8 " "streams, and the accuracy is identical to arm 2's %s. Batching " "moved nothing, which is what a greedy readout at temperature 0 " "owes the reader." % pct(base["accuracy"])))) if base.get("decisions_per_second") and top.get("decisions_per_second"): gain = top["decisions_per_second"] / base["decisions_per_second"] lat = (top.get("median_seconds") or 0) / (base.get("median_seconds") or 1) out.append("**What concurrency actually bought: almost nothing.** Eight streams " "instead of one is **%.2f× the throughput** (%s → %s decisions per " "second) for **%.1f× the median latency** (%s s → %s s). On this " "substrate the cost of a decision is prefill plus the link, not queue " "time, so putting more work in flight mostly makes each decision wait " "— the queue grows and the cards do not go faster. A server whose " "throughput is flat in concurrency is a server that was already " "saturated at one stream.\n" % (gain, num(base["decisions_per_second"]), num(top["decisions_per_second"]), lat, num(base["median_seconds"], 3), num(top["median_seconds"], 3))) # --------------------------------------------------------------- arm 9 def arm9(d, out): got = False out.append("## Arm 9 — the long pages\n") out.append("The **17 items arm 2 refused as overlong** against its 16,384-token " "window — 10 of task (a) and 7 of task (b), identified from arm 2's own " "refusal rows — re-asked at **`--max-model-len 32768`**, every other flag " "identical. The server started at that window on the first attempt, so " "the pre-registered one-shot fallback to `--gpu-memory-utilization 0.95` " "**was not used**.\n") for task, lab, labelled in (("along", "(a) does the article answer this question?", True), ("blong", "(b) which section is the answer in?", False)): rows = rows_of(d, "openjev-fp8-readout-32k", task) if not rows: continue got = True ref = [r for r in rows if r.get("refused")] reach = [r for r in rows if not r.get("refused")] out.append("### Task %s\n" % lab) out.append("| arm | items | reached at 32,768 | still overlong | %s | median s | p95 s |" % ("accuracy" if labelled else "**not a rate — unlabelled set**")) out.append("|---|---|---|---|---|---|---|") for a in ("openjev-fp8-readout-32k", "openjev-fp8-generate-32k"): r2 = rows_of(d, a, task) if not r2: continue rc = [x for x in r2 if not x.get("refused")] m, p95 = lat(r2) if labelled: a1, c1, n1 = acc(r2) cell = "**%s** (%d / %d)" % (pct(a1), c1, n1) else: cell = "— (1 of 42 in the whole set carries a label; §3)" out.append("| `%s` | %d | %d | %d | %s | %s | %s |" % (a, len(r2), len(rc), len(r2) - len(rc), cell, num(m), num(p95))) if ref: out.append("\n*Still refused at 32,768, with their newly measured counts — " "never truncated: %s.*\n" % ", ".join(sorted("`%s` (%s tokens)" % (x["id"], x.get("prompt_tokens")) for x in ref))) if not labelled and reach: out.append("\n**The answers, since there is no rate to report.** " "Task (b) is unlabelled by pre-registration, so these are recorded " "as what the model said, not as right or wrong:\n") out.append("| item | page | sections offered | chosen section | seconds |") out.append("|---|---|---|---|---|") for x in sorted(reach, key=lambda z: z["id"]): out.append("| `%s` | `%s` | %s | `%s` | %s |" % (x["id"], x.get("slug"), x.get("n_options"), x.get("choice"), num(x.get("seconds")))) out.append("") if not got: out.append("*No rows — arm 9 did not run.*\n") # --------------------------------------------------------------- arm 10 def arm10(d, out): base = os.path.join(HERE, "rows") if not rows_of(d, "gemma4-fp8-readout", "c"): return out.append("## Arm 10 — the base model on the SAME runtime\n") out.append("OpenJev's score had never been read against its base on the same server: " "arm 1's base was ollama-only (§A2.3.4), so every OpenJev-vs-base number so " "far confounded **the model** with **the runtime**. Arm 10 removes the " "runtime from the comparison — the estate's own assistant-seat checkpoint " "`gemma-4-26B-A4B-it-FP8-dynamic`, served by the same vLLM 0.29.0 on the " "same two 3090s at 250 W, same 16,384 window, same kit, same letters, same " "calibration. **Prompt sha256s match arm 2's on 108 of 108 task-(c) " "items**, so the two arms answered the same bytes.\n") out.append("### The question this arm was built to answer\n") out.append("| task | gemma 4 readout | gemma 4 generate | OpenJev readout | OpenJev " "generate | gap, readout | floor |") out.append("|---|---|---|---|---|---|---|") floors = {"c": "16.7 % (chance, 6 options)", "a": "66.7 % (majority class, 42 yes / 21 no)", "b": "— unlabelled"} names = {"c": "(c) which of the six guests said this line?", "a": "(a) does the article answer this question?", "b": "(b) which section is the answer in?"} for t in ("c", "a", "b"): cells = [] for dd, arm in ((d, "gemma4-fp8-readout"), (d, "gemma4-fp8-generate"), (base, "openjev-fp8-readout"), (base, "openjev-fp8-generate")): r = rows_of(dd, arm, t) if not r: cells.append("—") continue if t == "b": cells.append("**not a rate**") else: a1, c1, n1 = acc(r) cells.append("%s (%d/%d)" % (pct(a1), c1, n1)) gap = "—" gr = rows_of(d, "gemma4-fp8-readout", t) orr = rows_of(base, "openjev-fp8-readout", t) if t != "b" and gr and orr: ga, _, _ = acc(gr) oa, _, _ = acc(orr) if ga is not None and oa is not None: gap = "**%+.1f pts**" % (100 * (oa - ga)) out.append("| %s | %s | %s | %s | %s | %s | %s |" % (names[t], cells[0], cells[1], cells[2], cells[3], gap, floors[t])) out.append("\n### Cost, on the same cards\n") out.append("| model | task | arm | median s | p95 s | mean W (both cards) | J / decision |") out.append("|---|---|---|---|---|---|---|") for lbl, dd, arms in (("gemma 4 (base)", d, ("gemma4-fp8-readout", "gemma4-fp8-generate")), ("OpenJev-FP8", base, ("openjev-fp8-readout", "openjev-fp8-generate"))): for t in ("c", "a"): for arm in arms: r = rows_of(dd, arm, t) if not r: continue rep = report_of(dd, arm, t) m, p95 = lat(r) w, _i, jpd, _n = energy_cells(rep) out.append("| %s | %s | `%s` | %s | %s | %s | %s |" % (lbl, t, arm, num(m), num(p95), w, jpd)) # the cross-runtime reproduction b1 = rows_of(base, "base-readout", "c") b2 = rows_of(d, "gemma4-fp8-readout", "c") g1 = rows_of(base, "base-generate", "c") g2 = rows_of(d, "gemma4-fp8-generate", "c") if b1 and b2 and len(b1) == len(b2): s1, _, _ = acc(b1) s2, _, _ = acc(b2) agree = sum(1 for a, b in zip(b1, b2) if a["choice"] == b["choice"]) agree_g = sum(1 for a, b in zip(g1, g2) if a["choice"] == b["choice"]) if g1 and g2 else None out.append("\n### The same base model, twice, on two different stacks\n") out.append("Arm 1 ran this model as `gemma4:26b` **Q4_K_M under ollama on one " "card**. Arm 10 runs it as **FP8 under vLLM, tensor-parallel across " "two**. Different runtime, different quantisation, different number of " "cards — and the totals land on the same number.\n") out.append("| reading | runtime | quant | cards | task (c) accuracy | correct |") out.append("|---|---|---|---|---|---|") out.append("| arm 1 `base-readout` | ollama | Q4_K_M | 1 | %s | 87 / 108 |" % pct(s1)) out.append("| **arm 10 `gemma4-fp8-readout`** | **vLLM** | **FP8** | **2, TP** | **%s** | **87 / 108** |" % pct(s2)) gs1, _, _ = acc(g1) if g1 else (None, 0, 0) gs2, _, _ = acc(g2) if g2 else (None, 0, 0) out.append("| arm 1 `base-generate` | ollama | Q4_K_M | 1 | %s | 89 / 108 |" % pct(gs1)) out.append("| **arm 10 `gemma4-fp8-generate`** | **vLLM** | **FP8** | **2, TP** | **%s** | **89 / 108** |" % pct(gs2)) # THE FLIPS ARE COUNTED, NOT ESTIMATED (2026-09-22). This sentence said "about five # items flip each way", which reads as about ten; tallied by position (the join this # file uses above, and the one the id-join trap requires) five items changed answer in # total on each pair, two each way on correctness and one from one wrong answer to # another. The counts are now computed here, so the sentence cannot drift from them. fr = flips(b1, b2) fg = flips(g1, g2) if g1 and g2 and len(g1) == len(g2) else None clause = ("Of the **%d** items the readouts answered differently, **%d** went from " "right to wrong, **%d** from wrong to right and **%d** from one wrong " "answer to another" % fr) if fg is not None: clause += ("; the generating arms' **%d** split **%d**, **%d** and **%d** the " "same way" % fg) cancel = fr[1] == fr[2] and (fg is None or fg[1] == fg[2]) out.append("\n**Read this precisely, because the coincidence is in the total and " "not in the answers.** The two readouts agree on **%d of 108** items " "and the two generating arms on **%d of 108**. %s — so %s. The claim " "this supports is *the runtime and the quantisation are not what moves " "this score*; the claim it does not support is that the two stacks are " "interchangeable item-for-item.\n" % (agree, agree_g if agree_g is not None else -1, clause, "the errors cancel" if cancel else "the errors do not cancel")) # the floor caveat -- the base pays the untargeted protocol and OpenJev does not if b2: fl_b = sum(1 for x in b2 if x.get("floored")) nfl_b = sum(len(x.get("floored") or []) for x in b2) orr = rows_of(base, "openjev-fp8-readout", "c") fl_o = sum(1 for x in orr if x.get("floored")) if orr else None out.append("\n### The handicap in the readout, named rather than buried\n") out.append("A readout reads the option letters at the first output position. " "OpenJev is **built** to put them there. The base model is not: its " "position-0 mass sits on tokens like `[`, so most of its letters miss " "the raw top-20 and take the protocol's **−30 floor** (§4.1).\n") out.append("| model, task (c) readout | rows with at least one floored letter | " "floored letters in total |") out.append("|---|---|---|") out.append("| gemma 4 (base) | **%d / 108** | **%d** |" % (fl_b, nfl_b)) out.append("| OpenJev-FP8 | %s / 108 | %d |" % (fl_o, sum(len(x.get("floored") or []) for x in orr) if orr else 0)) out.append("\n**So the fair reading of the gap is the conservative one.** The base " "model's *readout* is handicapped by a protocol its output shape does " "not suit, and the honest base comparator is its **generating** arm, " "which has no floor at all. Against that — base generate %s versus " "OpenJev readout %s — the gap is still **%+.1f points**, and it is a " "gap between a model built to make typed decisions and the same-sized " "general model underneath it, measured on one stack.\n" % (pct(gs2), pct(acc(orr)[0] if orr else None), 100 * ((acc(orr)[0] or 0) - (gs2 or 0)))) # --------------------------------------------------------------- arm 11 def arm11(d, out): pa = os.path.join(d, "arm11.summary.json") pb = os.path.join(d, "arm11b.summary.json") if not os.path.exists(pa): return A = {r["task"]: r for r in json.load(open(pa, encoding="utf-8"))} B = ({r["task"]: r for r in json.load(open(pb, encoding="utf-8"))} if os.path.exists(pb) else {}) out.append("## Arm 11 — training the estate's own decision model\n") out.append("The Apache-2.0 openjev **fine-tuning framework** (commit `297482cc`) — which " "is a different thing from the CC BY-NC `openjev/openjev-FP8` **weights** the " "other arms bench — trained on the estate's own labels, on benchbox's own " "card 0, and scored on a **held-out 30 % fixed before training** (seed " "20260921, stratified by label, written to `heldout_ids.json` and asserted " "disjoint from train and dev before each run).\n") out.append("**Two readings, and they answer different questions.** **A** trains from the " "raw ModernBERT-base backbone — what the estate's labels are worth on their " "own. **B** is the framework's own central claim, the same data and the same " "time warm-started from its published general encoder — what they are worth " "on top of someone else's 323,466-row pre-training run.\n") out.append("| set | held-out n | floor | **A: trained from scratch** | **B: warm-started** " "| OpenJev-FP8 26B zero-shot | gemma 4 26B zero-shot | train s | model |") out.append("|---|---|---|---|---|---|---|---|---|") pretty = {"longtable_voices": "the long table, 6-way", "doorman_gate": "the doorman, binary", "fieldexam_grounded": "the field exam, binary"} for t in ("longtable_voices", "doorman_gate", "fieldexam_grounded"): a = A.get(t) if not a: continue if a.get("result") == "too small": out.append("| %s | %d | — | **too small** | **too small** | — | — | — | — |" % (pretty[t], a.get("heldout_n", 0))) continue b = B.get(t, {}) cell = lambda r, k="trained": (pct(r[k]["accuracy"]) if r and k in r and isinstance(r.get(k), dict) and r[k].get("accuracy") is not None else "—") oj = a.get("openjev", {}) gm = a.get("gemma4", {}) out.append("| %s | **%d** | %s | %s | **%s** | %s | %s | %s | %s |" % (pretty[t], a["heldout_n"], pct(a["floor"]["value"]), cell(a), cell(b), pct(oj["accuracy"]) if "accuracy" in oj else "— (%s)" % oj.get("result", "?"), pct(gm["accuracy"]) if "accuracy" in gm else "— (%s)" % gm.get("result", "?"), num((b or a).get("train_seconds"), 0), "%.0f MB" % (a.get("model_mb") or 0))) out.append("\n*Every cell in a row is the same held-out items. The two zero-shot columns " "are arms 2/4/5 and arm 10's own rows **filtered to the held-out ids** — not " "re-run, and not a full-set figure standing in for a held-out one.*\n") # the deltas rowsd = [] for t in ("longtable_voices", "doorman_gate", "fieldexam_grounded"): a, b = A.get(t), B.get(t) if not (a and b) or "trained" not in a or "trained" not in b: continue rowsd.append((pretty[t], a["trained"]["accuracy"], b["trained"]["accuracy"], a["floor"]["value"])) if rowsd: out.append("### What the warm start was worth\n") out.append("| set | A | B | delta | A vs its floor | B vs its floor |") out.append("|---|---|---|---|---|---|") for nm, av, bv, fl in rowsd: out.append("| %s | %s | **%s** | **%+.1f pts** | %+.1f | **%+.1f** |" % (nm, pct(av), pct(bv), 100 * (bv - av), 100 * (av - fl), 100 * (bv - fl))) out.append("\n**Reading A is the estate's labels declining to teach a fresh head " "anything.** On all three sets it sits at or within seven points of the " "majority-class floor — and on two of them it *is* the floor exactly, " "which is a model predicting one class and nothing else. With 24 to 66 " "training rows and a randomly-initialised scorer, that is the expected " "outcome and not a defect in the framework.\n") out.append("**Reading B is the same 24 to 66 rows, the same ten seconds, and a " "different starting point.** The framework's published claim for the " "swap is *+36 points*; on the estate's own decisions it is **%+.1f, %+.1f " "and %+.1f**. The field exam goes from the floor to **100 %% of a 12-item " "held-out set** — small, and stated as small, but it is every item.\n" % tuple(100 * (b - a) for _n, a, b, _f in rowsd)) out.append("### The comparison that actually matters to the estate\n") out.append("A **598 MB, ~150 M-parameter encoder**, fine-tuned in **ten seconds on " "one 3090** from a couple of dozen of the estate's own labelled rows, " "against a **26 B** model answering zero-shot on two cards:\n") out.append("| set | B, the trained encoder | OpenJev-FP8 26B | gemma 4 26B | " "encoder vs OpenJev |") out.append("|---|---|---|---|---|") for t in ("longtable_voices", "doorman_gate", "fieldexam_grounded"): a, b = A.get(t), B.get(t) if not (a and b) or "trained" not in b: continue oj = a.get("openjev", {}) gm = a.get("gemma4", {}) dl = ("**%+.1f pts**" % (100 * (b["trained"]["accuracy"] - oj["accuracy"])) if "accuracy" in oj else "—") out.append("| %s | **%s** | %s | %s | %s |" % (pretty[t], pct(b["trained"]["accuracy"]), pct(oj["accuracy"]) if "accuracy" in oj else "—", pct(gm["accuracy"]) if "accuracy" in gm else "—", dl)) out.append("\n*The honest reading of this table: the encoder does not beat the 26 B " "decision model, and on the long table it is well behind. What it does is " "reach the same answer on the two binary gates while being roughly 1/170th " "the parameters and fitting on one card with room to spare — and the " "held-out sets here are 12 to 30 items, which is small enough that a " "single flip moves these percentages by three to eight points. These are " "directional readings on the estate's real decisions, not a benchmark " "result, and the n is printed in every row for that reason.*\n") out.append("### What arm 11 did not train, and why\n") out.append("* **Task (a), the docent bank (63 binary labels)** — its states are whole " "pages, a median of about 7,900 tokens and a longest of 80,515, against a " "ModernBERT window of 2,048 (8,192 at the outside). Every row would have " "trained on a truncated page, which is a different task from the one the " "other arms answered. Named as not-run rather than run and caveated.\n") out.append("* **The Qwen3-1.7B + LoRA decoder variant** — pre-registered as conditional " "on the encoder training cleanly and time remaining. The encoder did train " "cleanly; the time did not remain within this run's window. It is owed, not " "refused, and it is the obvious next rung.\n") def main(argv=None) -> int: d = (argv or sys.argv[1:] or ["rows-addenda"])[0] d = d if os.path.isabs(d) else os.path.join(HERE, d) out = ["# The Jev bench — addenda arms 4–11 (2026-09-21, D-20260921-215)", "", "*Drawn by `tables_addenda.py` from the row files in `rows-addenda/`. It calls " "nothing and invents nothing; every cell is computed from a row that carries " "its own prompt sha256. Arms 1–3's tables are untouched in `TABLES.md` and " "`TABLES-ARM2.md`.*", ""] for fn in (arm4, arm7, arm5, arm6, arm8, arm9, arm10, arm11): try: fn(d, out) except Exception as exc: # noqa: BLE001 out.append("## %s — NOT DRAWN: %s\n" % (fn.__name__, exc)) print("\n".join(out)) return 0 if __name__ == "__main__": raise SystemExit(main())