#!/usr/bin/env python3 """Gaps 4 and 5's tables. Rows in, markdown out — calls nothing, invents nothing. python3 tables_gaps_card.py rows-gaps-card/ > TABLES-GAPS-CARD.md Separate from `tables.py`, `tables_arm2.py`, `tables_addenda.py` and the gaps 1-3 lane's `tables_gaps.py` for the reason all of those are separate from each other: a published table's drawing code must keep drawing exactly that table. This file draws gaps 4 and 5 and reaches into `rows/` only to read the arm-2 numbers gap 4 is compared against. WHAT THIS FILE IS CAREFUL ABOUT, each one a rule the bench already paid for. * **Nothing joins on `id`.** `kit/task_c.json` holds 108 lines under only 50 distinct `id` strings (the latent trap arm 7 found). Gap 4's rows carry a positional `uid` and this file joins on that, and on `prompt_sha256` where the join has to be proven rather than trusted. * **A refused row is never averaged with a wrong one.** Every rate prints the denominator it actually scored. * **The card's own figures are quoted, never verified.** OpenJev's 2.3 % was measured on its own set through its own targeted extraction; this bench is on the untargeted path. The two numbers sit in the same table under a line that says they are not the same measurement. * **Every derived number states its n.** 36 items is a small set and the reader is told so in the table, not in a footnote nobody scrolls to. """ from __future__ import annotations import collections import itertools import json import os import statistics import sys HERE = os.path.dirname(os.path.abspath(__file__)) LETTERS6 = ["A", "B", "C", "D", "E", "F"] #: Quoted from OpenJev's model card, and quoted is all they are: a different #: set, a different extraction path, a different option count. CARD_TUNED_REORDER = 0.023 CARD_UNTUNED_REORDER = 0.185 def pct(x, nd=1): return "—" if x is None else "%.*f %%" % (nd, 100 * x) def num(x, nd=2): return "—" if x is None else "%.*f" % (nd, x) #: The site's own title suffixes, stripped for display only. Every article's #: ends "· strata→signal research" and most doors carry a shorter form; #: printing them in a narrow cell costs the words that distinguish one page from #: another. The ROW keeps the title whole — this trims the CELL. SUFFIXES = (" \u00b7 strata\u2192signal research", " \u00b7 strata \u2192 signal", " \u00b7 strata\u2192signal") def short(title, width=58): t = title or "" for s in SUFFIXES: if t.endswith(s): t = t[:-len(s)] break return t if len(t) <= width else t[:width].rsplit(" ", 1)[0] + "\u2026" def rows_of(d, arm, task): p = os.path.join(d, "%s.%s.jsonl" % (arm, task)) if not os.path.exists(p): return [] return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()] def report_of(d, arm, task): p = os.path.join(d, "%s.%s.report.json" % (arm, task)) return json.load(open(p, encoding="utf-8")) if os.path.exists(p) else {} def lat(rows): s = sorted(r["seconds"] for r in rows if r.get("seconds")) if not s: return None, None return statistics.median(s), s[min(len(s) - 1, int(0.95 * len(s)))] def acc(rows, key="correct"): """Accuracy over REACHED, LABELLED rows. Returns (rate, correct, n). `key="correct_alt"` is the judge set's own scorer: its OFF-PAGE label is the alternation `off_page|not_grounded` and BOTH values count correct, which is how arm 6 scored it and therefore the only way a Kev row can be put in the same column as an OpenJev one. """ lab = [r for r in rows if not r.get("refused") and r.get("label") is not None] if not lab: return None, 0, 0 if key == "correct_alt": c = sum(1 for r in lab if r.get("correct_alt")) else: c = sum(1 for r in lab if r.get("choice") == r.get("label")) return c / len(lab), c, len(lab) def brier_of(rows): live = [r for r in rows if not r.get("refused") and r.get("probs") and r.get("label") is not None] if not live: return None return statistics.fmean( sum((p - (1.0 if k == r["label"] else 0.0)) ** 2 for k, p in r["probs"].items()) for r in live) def energy(rep): e, idle = rep.get("energy") or {}, rep.get("idle") or {} return (e.get("mean_watts"), idle.get("mean_watts"), e.get("joules_per_decision"), e.get("net_joules_per_decision")) def reliability_table(rows, title): """The bench's ten-bin reliability table, drawn only where it has rows.""" live = [r for r in rows if not r.get("refused") and r.get("probs") and r.get("label") is not None] if not live: return [] bins = [{"n": 0, "sum_p": 0.0, "correct": 0} for _ in range(10)] for r in live: p = max(r["probs"].values()) b = bins[min(9, int(p * 10))] b["n"] += 1 b["sum_p"] += p b["correct"] += int(r["choice"] == r["label"]) out = ["", "### %s" % title, "", "| confidence bin | n | mean confidence | accuracy | gap |", "|---|---|---|---|---|"] for i, b in enumerate(bins): if not b["n"]: continue mp, a = b["sum_p"] / b["n"], b["correct"] / b["n"] out.append("| %.1f–%.1f | %d | %s | %s | %+.3f |" % (i / 10, (i + 1) / 10, b["n"], pct(mp), pct(a), mp - a)) return out # ---------------------------------------------------------------- gap 4 def by_item(rows): """{uid: {k: row}} — joined POSITIONALLY, never on the kit's `id`.""" out = collections.defaultdict(dict) for r in rows: if r.get("refused"): continue out[r["uid"]][r["order_k"]] = r return out def reorder_stats(rows): """Every gap-4 figure, computed exactly as the pre-registration defines it.""" items = by_item(rows) full = {u: ks for u, ks in items.items() if len(ks) == 6} moved = sum(1 for ks in full.values() if len({r["choice"] for r in ks.values()}) > 1) pairs = disagree = 0 for ks in full.values(): for a, b in itertools.combinations(sorted(ks), 2): pairs += 1 disagree += int(ks[a]["choice"] != ks[b]["choice"]) # per-guest probability movement, over every (item, guest) pair ranges, mads = [], [] for ks in full.values(): guests = sorted(ks[0]["probs"]) if 0 in ks else sorted(next(iter(ks.values()))["probs"]) for g in guests: ps = [ks[k]["probs"][g] for k in sorted(ks)] m = statistics.fmean(ps) ranges.append(max(ps) - min(ps)) mads.append(statistics.fmean(abs(p - m) for p in ps)) letters = collections.Counter(r["chosen_letter"] for r in rows if not r.get("refused") and r.get("chosen_letter")) label_letters = collections.Counter(r["label_letter"] for r in rows if not r.get("refused")) per_k = {} for k in range(6): ks = [r for r in rows if not r.get("refused") and r.get("order_k") == k] per_k[k] = acc(ks) + (brier_of(ks),) + lat(ks) return {"items_full": len(full), "items_moved": moved, "moved_rate": moved / len(full) if full else None, "pairs": pairs, "disagree": disagree, "pair_rate": disagree / pairs if pairs else None, "mean_range": statistics.fmean(ranges) if ranges else None, "mean_mad": statistics.fmean(mads) if mads else None, "n_guest_cells": len(ranges), "letters": letters, "label_letters": label_letters, "per_k": per_k} def kit_order_check(rows, arm2_path): """Do the k=0 prompts reproduce arm 2's, byte for byte? Joined by position.""" if not os.path.exists(arm2_path): return None arm2 = [json.loads(l) for l in open(arm2_path, encoding="utf-8") if l.strip()] k0 = sorted((r for r in rows if r.get("order_k") == 0), key=lambda r: r["uid"]) if len(k0) != len(arm2): return {"n": min(len(k0), len(arm2)), "match": None, "note": "row counts differ: %d here, %d in arm 2" % (len(k0), len(arm2))} same = sum(1 for a, b in zip(k0, arm2) if a.get("prompt_sha256") == b.get("prompt_sha256")) agree = sum(1 for a, b in zip(k0, arm2) if a.get("choice") == b.get("choice")) return {"n": len(arm2), "match": same, "agree": agree} def gap4_section(d): out = ["## Gap 4 — reorder consistency", "", "The same **108** six-voice items from task (c), asked **six times**: the " "kit's own option order (`k = 0`, byte-identical to arm 2's prompts) plus " "five seeded shuffles, `random.Random(20260921 + k)`. **648 decisions per " "arm.** The question is whether the chosen **guest** moves — not whether " "the chosen letter does, which it must.", ""] any_rows = False for arm in ("openjev-fp8-readout", "openjev-fp8-generate"): rows = rows_of(d, arm, "creorder") if not rows: continue any_rows = True s = reorder_stats(rows) rep = report_of(d, arm, "creorder") mw, iw, jd, njd = energy(rep) chk = kit_order_check(rows, os.path.join(HERE, "rows", "openjev-fp8-%s.c.jsonl" % arm.split("-")[-1])) out += ["### `%s`" % arm, "", "| figure | value | of | what it is |", "|---|---|---|---|", "| **items whose answer moved at all** | **%s** | %d / %d items | " "the chosen guest is not the same under all six orders |" % (pct(s["moved_rate"]), s["items_moved"], s["items_full"]), "| **order-pairs that disagree** | **%s** | %d / %d pairs | " "the 15 pairs of the six orders, per item — **the figure comparable " "to the card's 2.3 %%** |" % (pct(s["pair_rate"], 2), s["disagree"], s["pairs"]), "| mean per-guest probability **range** | %s | %d (item, guest) cells | " "max − min of that guest's probability over the six orders |" % (num(s["mean_range"], 4), s["n_guest_cells"]), "| mean per-guest probability **deviation** | %s | %d (item, guest) cells | " "mean absolute deviation from that guest's own six-order mean |" % (num(s["mean_mad"], 4), s["n_guest_cells"]), "| mean W, both cards | %s | — | the arm's own 1 Hz sampler, summed |" % num(mw, 1), "| J / decision (net of idle) | %s (%s) | %d decisions | " "idle read on both boards for ten seconds before the arm (%s W) |" % (num(jd, 1), num(njd, 1), (rep.get("summary") or {}).get("n_reached", 0), num(iw, 1)), ""] out += ["**Accuracy, per presentation.** Same items, same labels, only the " "option order differs.", "", "| order | accuracy | correct | Brier | median s | p95 s |", "|---|---|---|---|---|---|"] for k in range(6): a, c, n, br, med, p95 = s["per_k"][k] name = "**k = 0** — the kit's order (arm 2's)" if k == 0 else "k = %d" % k out.append("| %s | %s | %d / %d | %s | %s | %s |" % (name, pct(a), c, n, num(br, 4), num(med), num(p95))) accs = [s["per_k"][k][0] for k in range(6) if s["per_k"][k][0] is not None] if len(accs) > 1: out += ["", "*Spread across the six orders: **%s to %s**, a range of " "%.1f points.*" % (pct(min(accs)), pct(max(accs)), 100 * (max(accs) - min(accs)))] if chk: out += ["", "**The k = 0 control, checked rather than asserted.** %s" % ("%d of %d prompts sha256-identical to arm 2's, and the same " "guest chosen on %d of %d — a difference there would be " "server nondeterminism, not the shuffle." % (chk["match"], chk["n"], chk.get("agree", 0), chk["n"]) if chk.get("match") is not None else chk["note"])] out += ["", "**Positional bias — which letter was chosen, over all %d " "presentations.** A content-blind model would sit at 16.7 %% in " "every column." % sum(s["letters"].values()), "", "| letter | times chosen | share | times the CORRECT answer sat there | share |", "|---|---|---|---|---|"] tot = sum(s["letters"].values()) or 1 ltot = sum(s["label_letters"].values()) or 1 for L in LETTERS6: out.append("| %s | %d | %s | %d | %s |" % (L, s["letters"].get(L, 0), pct(s["letters"].get(L, 0) / tot), s["label_letters"].get(L, 0), pct(s["label_letters"].get(L, 0) / ltot))) out += ["", "*The right-hand pair is the schedule, not a result: it is where " "the correct answer happened to land once the seeds had shuffled. " "Read the left-hand share against it, not against 16.7 % alone.*", ""] out += reliability_table(rows, "Reliability — `%s`, all six orders" % arm) out += [""] if not any_rows: return ["## Gap 4 — reorder consistency", "", "*No rows. The arm did not run; see the refusal recorded in " "`GAPS-CARD.md`.*", ""] out += ["### The card's figure, beside this one — and they are not the same measurement", "", "| reading | set | extraction | reorder rate |", "|---|---|---|---|", "| OpenJev's model card, tuned | the card's own | targeted " "(`allowed_token_ids`) | **%s** |" % pct(CARD_TUNED_REORDER, 1), "| OpenJev's model card, untuned | the card's own | targeted | %s |" % pct(CARD_UNTUNED_REORDER, 1), "| **this bench**, `openjev-fp8-readout` | the long table, 108 items, " "6 options | untargeted, `top_logprobs = 20` | **see the table above** |", "", "*Quoted, not verified. A rate measured on a different set, with a " "different number of options, through a different extraction path is a " "different number that happens to share a name. What this bench can say " "is what this model did on this workshop's decision.*", "", "**The base model's reorder rate is owed, and was not measured.** The " "control would be the same six orders on `gemma-4-26B-A4B-it-FP8-dynamic` " "— arm 10's checkpoint, which is the 27 GB this run deleted to make room " "for OpenJev (`GAPS-CARD.md` §0). benchbox cannot hold both. So there is no " "base column here, and the figure above is a rate for one model rather " "than a comparison between two.", ""] return out # ---------------------------------------------------------------- gap 5 def gap5_section(d): out = ["## Gap 5 — a screenshot decision", "", "36 screenshots at 1,280 × 800 of public pages this workshop publishes — " "**24 research-hub article pages** and **12 product front doors** — put to " "the same checkpoint arm 2 measured, served with **one flag changed**: " "`--limit-mm-per-prompt '{\"image\":1}'` instead of `'{\"image\":0}'`. " "The letters, the calibration, the temperature and the seed are arm 2's.", "", "> **36 items is a small set and every table below prints its n.** A " "reading here says what this model did on these 36 pictures; it decides " "nothing about screenshot decisions in general.", ""] tasks = [("shot_title", "(i) which of these six titles is this page?", 1 / 6, "chance, 6 options"), ("shot_kind", "(ii) is this a front door or an article?", 24 / 36, "majority class, 24 of 36 are articles")] any_rows = False head = ["| arm | decision | accuracy | correct | floor | Brier | median s | p95 s |", "|---|---|---|---|---|---|---|---|"] body = [] for task, label, floor, floor_why in tasks: for arm in ("openjev-fp8-readout-mm", "openjev-fp8-generate-mm"): rows = rows_of(d, arm, task) if not rows: continue any_rows = True a, c, n = acc(rows) med, p95 = lat(rows) body.append("| `%s` | %s | **%s** | %d / %d | %s (%s) | %s | %s | %s |" % (arm, label, pct(a), c, n, pct(floor), floor_why, num(brier_of(rows), 4), num(med), num(p95))) if not any_rows: return ["## Gap 5 — a screenshot decision", "", "*No rows. The arm did not run; see the refusal recorded in " "`GAPS-CARD.md`.*", ""] out += head + body + [""] # cost, including what the picture cost in tokens out += ["### What a picture costs", "", "`prompt_tokens` is the server's own count for the call **with** the " "image. `text_tokens` is the same server's `/tokenize` asked the **text " "alone** under the same chat template, with no image attached — so the " "image column is a subtraction of two measured numbers, not an estimate.", "", "| arm | decision | n | mean prompt tokens | mean text tokens | " "**mean image tokens** | mean W, both cards | J / decision (net) |", "|---|---|---|---|---|---|---|---|"] for task, label, _f, _w in tasks: for arm in ("openjev-fp8-readout-mm", "openjev-fp8-generate-mm"): rows = [r for r in rows_of(d, arm, task) if not r.get("refused")] if not rows: continue rep = report_of(d, arm, task) mw, _iw, jd, njd = energy(rep) pt = [r["prompt_tokens"] for r in rows if r.get("prompt_tokens")] tt = [r["text_tokens"] for r in rows if r.get("text_tokens")] it = [r["image_tokens"] for r in rows if r.get("image_tokens") is not None] out.append("| `%s` | %s | %d | %s | %s | **%s** | %s | %s (%s) |" % (arm, label, len(rows), num(statistics.fmean(pt), 0) if pt else "—", num(statistics.fmean(tt), 0) if tt else "—", num(statistics.fmean(it), 0) if it else "—", num(mw, 1), num(jd, 1), num(njd, 1))) out += [""] # the split that says where the errors are out += ["### Where the misses are, by kind of page", "", "| arm | decision | articles (24) | front doors (12) |", "|---|---|---|---|"] for task, label, _f, _w in tasks: for arm in ("openjev-fp8-readout-mm", "openjev-fp8-generate-mm"): rows = rows_of(d, arm, task) if not rows: continue cells = [] for kind in ("article", "front_door"): sub = [r for r in rows if r.get("kind") == kind] a, c, n = acc(sub) cells.append("%s (%d / %d)" % (pct(a), c, n)) out.append("| `%s` | %s | %s | %s |" % (arm, label, cells[0], cells[1])) out += [""] # every wrong answer, named — 36 items is small enough to show whole for task, label, _f, _w in tasks: rows = rows_of(d, "openjev-fp8-readout-mm", task) wrong = [r for r in rows if not r.get("refused") and r.get("label") is not None and r["choice"] != r["label"]] if not rows: continue out += ["### Every miss on %s — `openjev-fp8-readout-mm`" % label, ""] if not wrong: out += ["*None. All %d items correct.*" % len([r for r in rows if not r.get("refused")]), ""] continue out += ["| page | kind | the page's own title | chose | confidence |", "|---|---|---|---|---|"] for r in wrong: out.append("| `%s` | %s | %s | `%s` | %s |" % (r["id"], r["kind"], short(r.get("page_title")), r["choice"], pct(r.get("confidence")))) out += [""] for task, label, _f, _w in tasks: out += reliability_table(rows_of(d, "openjev-fp8-readout-mm", task), "Reliability — `openjev-fp8-readout-mm`, %s" % label) out += [""] return out # ---------------------------------------------------------------- arm 12 ROWS = os.path.join(HERE, "rows") ROWS_ADD = os.path.join(HERE, "rows-addenda") #: Each of arm 12's five sets, and WHERE the two comparators' rows already live. #: (label, floor, floor_why, scorer, kev task, [(name, dir, arm, task)]) SETS = [ ("task (c) — which of the six guests said this line?", 1 / 6, "chance, 6 options", "correct", "kc", [("OpenJev-FP8 (arm 2, vLLM, TP=2)", ROWS, "openjev-fp8-readout", "c"), ("gemma 4 26B (arm 10, same runtime)", ROWS_ADD, "gemma4-fp8-readout", "c")]), ("task (a) — does the article answer this question?", 42 / 63, "majority class", "correct", "ka", [("OpenJev-FP8 (arm 2, vLLM, TP=2)", ROWS, "openjev-fp8-readout", "a"), ("gemma 4 26B (arm 10, same runtime)", ROWS_ADD, "gemma4-fp8-readout", "a")]), ("the doorman's planted set (arm 4)", 36 / 48, "majority class", "correct", "kd", [("OpenJev-FP8 (arm 4)", ROWS_ADD, "openjev-fp8-readout", "d"), ("gemma 4 26B (arm 10)", ROWS_ADD, "gemma4-fp8-readout", "d")]), ("the field exam, answerable or not (arm 5)", 0.5, "majority class", "correct", "kf", [("OpenJev-FP8 (arm 5)", ROWS_ADD, "openjev-fp8-readout", "f"), ("gemma 4 26B (arm 10)", ROWS_ADD, "gemma4-fp8-readout", "f")]), ("the judge seat (arm 6)", None, "—", "correct_alt", "kj", [("OpenJev-FP8 (arm 6)", ROWS_ADD, "openjev-fp8-readout", "j")]), ] KEV_ARMS = [("kev-9b", "**Kev-9B** — Apache-2.0, one 3090"), ("kev-4b", "**Kev-4B** — Apache-2.0, one 3090")] #: Quoted from Kev's own cards and README. A different set, a different task mix #: and a different readout: context, never a number ours passes or misses. KEV_PUBLISHED = [("Kev-9B, locked test (its own suite)", "0.852", "0.237"), ("Kev-4B, locked test (its own suite)", "0.837", "0.255"), ("hosted Jev, the same development items", "0.857", "0.211")] def arm12_section(d): have = [(a, lab) for a, lab in KEV_ARMS if any(rows_of(d, a, s[4]) for s in SETS)] if not have: return ["## Arm 12 — the Kev family, Apache-2.0", "", "*No rows. The arm did not run; see the refusal recorded in " "`GAPS-CARD.md`.*", ""] out = ["## Arm 12 — the Kev family, Apache-2.0", "", "*What this arm asks, for a reader who scrolled straight here: every " "OpenJev number in this bench is fenced — **CC BY-NC 4.0, bench-only, " "those weights never become a seat**. Kev is the same category of " "model, a typed decision read off a pointer head with no generation, " "under **Apache-2.0** on an Apache-2.0 base. So the question is not " "which is better. It is whether a model this workshop **could** deploy " "lands within a few points of one it cannot.*", "", "Kev-9B is ~19 GB in bf16 and fits **one 3090 whole**, so arm 12 has " "no tensor-parallel and no all-reduce — the host bridge arm 2 paid " "6.4 ms a token for is simply absent. It is also a different server " "(transformers + FastAPI, not vLLM). **Every latency comparison below " "is cross-runtime and cross-topology**, and the decisions are what is " "comparable.", ""] for title, floor, floor_why, scorer, task, comparators in SETS: rows_by = {a: rows_of(d, a, task) for a, _ in have} if not any(rows_by.values()): continue # THE SHARED DENOMINATOR. On task (a) Kev could not reach every item # (its server's own 8,192-token state window — see the refusal block # below), so the comparators are restricted to the ids Kev ACTUALLY # ANSWERED before their rates are printed. A 100 % over 53 items and a # 100 % over 38 are not the same claim, and putting them in one column # without saying so would be the quiet kind of wrong. reached = None any_kev = next((r for r in rows_by.values() if r), []) if any(x.get("refused") for x in any_kev): reached = {x["id"] for x in any_kev if not x.get("refused")} out += ["### %s" % title, "", "| model | accuracy | correct | Brier | median s | p95 s | floor |", "|---|---|---|---|---|---|---|"] for a, lab in have: rows = rows_by.get(a) or [] if not rows: continue r, c, n = acc(rows, scorer) med, p95 = lat(rows) out.append("| %s | **%s** | %d / %d | %s | %s | %s | %s |" % (lab, pct(r), c, n, num(brier_of(rows), 4), num(med), num(p95), pct(floor) if floor else "—")) for name, base, arm, t in comparators: rows = rows_of(base, arm, t) if not rows: continue if reached is not None: rows = [x for x in rows if x.get("id") in reached] name += " — **restricted to the items Kev reached**" r, c, n = acc(rows, scorer) med, p95 = lat(rows) out.append("| %s | %s | %d / %d | %s | %s | %s | %s |" % (name, pct(r), c, n, num(brier_of(rows), 4), num(med), num(p95), pct(floor) if floor else "—")) if reached is not None: out += refusal_block(rows_by, len(any_kev), len(reached)) out += [""] out += ["### Cost, and it is not a like-for-like latency", "", "| model | set | n | median s | p95 s | mean W | idle W | J / decision (net) |", "|---|---|---|---|---|---|---|---|"] for a, lab in have: for title, _f, _w, _s, task, _c in SETS: rows = [r for r in rows_of(d, a, task) if not r.get("refused")] if not rows: continue rep = report_of(d, a, task) mw, iw, jd, njd = energy(rep) med, p95 = lat(rows) out.append("| %s | %s | %d | %s | %s | %s | %s | %s (%s) |" % (lab, title.split(" — ")[0], len(rows), num(med), num(p95), num(mw, 1), num(iw, 1), num(jd, 1), num(njd, 1))) out += ["", "*Arm 12's watts are **one** board; arm 2's and arm 10's are the " "sum of two. A joules-per-decision figure here is not the same " "quantity as one in the OpenJev tables and must not be subtracted " "from it.*", ""] out += ["### Kev's own published numbers, quoted", "", "| reading | accuracy | Brier |", "|---|---|---|"] for name, a, b in KEV_PUBLISHED: out.append("| %s | %s | %s |" % (name, a, b)) out += ["", "*Quoted, never verified — a different set, a different task mix " "and a different readout. Kev's published Brier is for the **raw " "logits**; the arms above run **as served**, with the checkpoint's " "own fitted temperature applied. The raw pass below is the only " "Brier in this file comparable with the column on the right.*", ""] out += raw_vs_served(d) out += kev_permute_section(d) return out def refusal_block(rows_by, total, reached_n): """What an arm could not reach, why, and in whose words.""" counts, example = collections.Counter(), None for rows in rows_by.values(): for r in rows: if r.get("refused"): counts[r["refused"]] += 1 if r["refused"] == "kev-422" and example is None: example = r.get("server_message") break # one arm's refusals: they match if not counts: return [] out = ["", "**What Kev could not reach, and why — %d of %d items.**" % (total - reached_n, total), "", "| reason | items | what it is |", "|---|---|---|"] if counts.get("overlong"): out.append("| `overlong` | %d | inherited from arm 2's own refusal rows, " "so the set started where arm 2's did (pre-registered) |" % counts["overlong"]) if counts.get("kev-422"): out.append("| `kev-422` | %d | **the server refused the request**: " "`kev.serve` runs with `INFER_MAX_STATE = INFER_MAX_BRANCH = " "8192` and `encode` rejects a question when " "`len(branch) > max_branch - len(state)`, so a page that " "fills the 8,192-token state window leaves a budget of zero " "and even a 56-token question is refused |" % counts["kev-422"]) for k, v in counts.items(): if k not in ("overlong", "kev-422"): out.append("| `%s` | %d | — |" % (k, v)) if example: out += ["", "*The server's own sentence, carried on every such row: " "`%s`*" % example.split(": ", 1)[-1].strip()] out += ["", "**This is a real limit, not a harness problem, and it is the " "finding this set produced.** The estate's docent pages run from " "1,772 to 82,500 tokens. A decision model that can hold 8,192 " "tokens of state cannot be asked *does this article answer this " "question?* about most of them without truncating the article — " "which this bench forbids (arm 1 §8), because truncating a page can " "turn a genuinely on-page question into an unanswerable one. Kev " "answered every page it could hold, and refused the rest in its own " "words rather than guessing at a fragment."] return out def raw_vs_served(d): """The calibration column: same answers, different Brier, both printed.""" out, seen = ["### Calibration — the same answers, a different Brier", "", "| model | task (c) | accuracy | correct | Brier |", "|---|---|---|---|---|"], False for base, lab in (("kev-9b", "Kev-9B"), ("kev-4b", "Kev-4B")): for suffix, how in (("", "as served (the checkpoint's own T)"), ("-raw", "`KEV_TEMPERATURE=1.0` (raw logits)")): rows = rows_of(d, base + suffix, "kc") if not rows: continue seen = True r, c, n = acc(rows) out.append("| %s | %s | %s | %d / %d | %s |" % (lab, how, pct(r), c, n, num(brier_of(rows), 4))) if not seen: return [] out += ["", "*Kev's temperature never changes an answer — the argmax is " "identical — so a pair of rows here that differ in accuracy would be " "a finding about the server, not about calibration.*", ""] return out def kev_permute_section(d): """Kev's own reorder number, beside gap 4's, measured the same way.""" out, seen = ["### Reorder consistency — Kev's own `/permute`, beside gap 4", "", "| model | how | items whose answer moved | order-pairs that disagree | " "mean per-option spread |", "|---|---|---|---|---|"], False for base, lab in (("kev-9b", "**Kev-9B**"), ("kev-4b", "**Kev-4B**")): rows = [r for r in rows_of(d, base, "kperm") if not r.get("refused")] if not rows: continue seen = True moved = sum(1 for r in rows if r.get("argmax_stable") is False) pairs = dis = 0 spreads = [] for r in rows: ch = r.get("choices_by_order") or [] for i, j in itertools.combinations(range(len(ch)), 2): pairs += 1 dis += int(ch[i] != ch[j]) spreads.extend((r.get("spread") or {}).values()) out.append("| %s | its own `/permute`, `n_perm = 6`, seed 20260921 | " "**%s** (%d / %d) | **%s** (%d / %d) | %s |" % (lab, pct(moved / len(rows)), moved, len(rows), pct(dis / pairs, 2) if pairs else "—", dis, pairs, num(statistics.fmean(spreads), 4) if spreads else "—")) for arm, lab in (("openjev-fp8-readout", "OpenJev-FP8 (gap 4)"),): rows = rows_of(d, arm, "creorder") if not rows: continue s = reorder_stats(rows) out.append("| %s | gap 4's own six orders, seed 20260921 | **%s** (%d / %d) | " "**%s** (%d / %d) | %s |" % (lab, pct(s["moved_rate"]), s["items_moved"], s["items_full"], pct(s["pair_rate"], 2), s["disagree"], s["pairs"], num(s["mean_range"], 4))) out.append("| OpenJev's model card, tuned / untuned | its own set, targeted " "extraction | — | %s / %s | — |" % (pct(CARD_TUNED_REORDER), pct(CARD_UNTUNED_REORDER))) out.append("| Kev-4B's model card (Qwen3 generation) | its own suites | — | " "6 % (its \"option-order flip rate\"; Jev 0 %) | — |") if not seen: return [] out += ["", "*The same statistic on different draws, and that is the whole " "caveat. Kev shuffles inside its own server with its own seeded " "`random.Random`; gap 4 shuffles with ours. The six orders are " "therefore not the same six — so this compares two rates measured " "the same way, not two runs of one schedule. The spread column is " "`max − min` of an option's probability across the orders, which is " "gap 4's per-guest range defined identically.*", ""] return out def residency_section(d): """What switching the eyes on cost, read from the arms' own pin blocks. WHY `ok` IS NOT THE COLUMN. `verify_pin`'s `ok` is a GROWTH test — it wants the named boards to grow and every other board to stay quiet — and it was written for arm 1, where the bench itself caused the load. Under vLLM the server is already up and the weights are already resident before the first task starts, so growth is legitimately zero and `ok` is structurally False for every task but the one that happens to allocate something. Printing that as a failed pin would be a false red. The field that carries the meaning here is `resident` — every named board above 2 GB AND no other board busy — and the growth column is printed beside it rather than instead of it, because on one row it is the measurement: the vision tower allocating on first use. """ out = ["## Residency — what switching the eyes on cost", "", "Both boards, by UUID, read by each arm's own `verify_pin` before and " "after. Arm 2 served this checkpoint with `'{\"image\":0}'`; gap 5 " "served it with `'{\"image\":1}'` and nothing else changed.", "", "| arm · task | posture | card 0 MiB | card 1 MiB | growth this task | " "other boards | resident |", "|---|---|---|---|---|---|---|"] seen = False refs = [(os.path.join(HERE, "rows"), "openjev-fp8-readout", "c", "arm 2 — `'{\"image\":0}'`"), (d, "openjev-fp8-readout", "creorder", "gap 4 — `'{\"image\":0}'`"), (d, "openjev-fp8-readout-mm", "shot_title", "**gap 5 — `'{\"image\":1}'`, first image**"), (d, "openjev-fp8-readout-mm", "shot_kind", "gap 5 — `'{\"image\":1}'`, second task")] for base, arm, task, posture in refs: rep = report_of(base, arm, task) pin = rep.get("pin") or {} per = pin.get("memory_after_per_card_mib") or {} if not per: continue seen = True cards = sorted(per) growth = pin.get("growth_mib") or {} out.append("| `%s` · %s | %s | %s | %s | %s | %s | %s |" % (arm, task, posture, num(per[cards[0]], 0), num(per[cards[1]], 0) if len(cards) > 1 else "—", " / ".join("%+d" % round(growth.get(c, 0)) for c in cards), "quiet" if pin.get("others_quiet") else "**BUSY**", "yes" if pin.get("resident") else "**NO**")) if not seen: return [] out += ["", "**The flag does not change the footprint; it changes how the " "budget is divided.** `--gpu-memory-utilization 0.90` is a ceiling, " "and vLLM fills it either way — so the steady-state figures above are " "within a few hundred MiB of each other. What the eyes actually cost " "is visible in the server's own startup lines, per card:", "", "| posture | available KV cache | GPU KV cache size | max concurrency at 16,384 tokens |", "|---|---|---|---|", "| `'{\"image\":0}'` (arm 2's, and gap 4's) | 5.23 GiB | 132,285 tokens | 8.07× |", "| `'{\"image\":1}'` (gap 5's) | **4.86 GiB** | **123,183 tokens** | **7.52×** |", "", "*Switching the eyes on cost **0.37 GiB of KV cache per card** — " "9,102 fewer tokens of context budget, about 6.9 % — plus the " "**+242 MiB per card** the vision tower allocated the first time an " "image arrived, which is the growth row above. TP=2 held on both " "boards in both postures.*", ""] return out def main(argv=None) -> int: d = (argv or sys.argv[1:] or ["rows-gaps-card"])[0] out = ["# The Jev bench — gaps 4 and 5 (2026-09-22)", "", "*What this is, for a reader who scrolled straight here: the Jev bench " "(`README.md`, benchbox, 2026-09-21) measured a decision model — " "`openjev/openjev-FP8`, CC BY-NC 4.0, bench-only — against the generating " "seat on this workshop's own decisions, and its draft article listed what " "it did not know. Two of those lines were claims on the model's own card. " "**Gap 4** shuffles the options; **gap 5** switches the model's eyes on. " "The pre-registration for both, written before any row here existed, is " "`GAPS-CARD.md`.*", "", "*Drawn by `tables_gaps_card.py` from the row files in `%s`. It calls " "nothing and invents nothing; every cell is computed from a row that " "carries its own prompt sha256. Gaps 1–3 are a different lane's " "`TABLES-GAPS.md` and are not touched here.*" % d, ""] out += gap4_section(d) out += ["---", ""] out += gap5_section(d) out += ["---", ""] out += arm12_section(d) out += ["---", ""] out += residency_section(d) print("\n".join(out)) return 0 if __name__ == "__main__": raise SystemExit(main())