#!/usr/bin/env python3 """The Jev bench, GAPS 4 and 5 — the two claims on the model card nothing tested. python3 run_gaps_card.py --arm openjev-fp8-readout --task creorder python3 run_gaps_card.py --arm openjev-fp8-readout-mm --task shot_title WHY THIS FILE EXISTS AND `run.py` IS NOT EDITED, for the third time in this bench: `run.py` is the measured record of arms 1-3 and `run_addenda.py` is the record of arms 4-11. A file that edited either would make their rows unreproducible against the code that claims to have produced them. So this module IMPORTS `run.py` and registers new task generators into its `TASKS` table. Every row here is built by arm 2's own `decide()`, scored by arm 2's own `summarise()`, measured by arm 2's own `Boards` sampler and pinned by its own `verify_pin` — **the tasks are new, the instrument is not.** THE TWO GAPS, both of them claims on OpenJev's own model card that this bench had measured nothing about, and both of them named in the draft article's "What this page does not know": * **GAP 4 — reorder consistency.** The card claims the chosen answer moves in 2.3 % of cases when the options are shuffled, against 18.5 % untuned. Every arm in this bench so far asked task (c) with ONE option order: the kit holds exactly one `options` tuple per item (verified: 108 items, 1 distinct order), so no arm could see the effect at all. Gap 4 asks the same 108 items six times — the original order plus five seeded shuffles — and asks whether the chosen GUEST moves, not whether the chosen LETTER does. * **GAP 5 — a screenshot decision.** Both Jev and OpenJev claim web and screenshot decisions, and `openjev/openjev-FP8` is a `Qwen3_5ForConditionalGeneration` — a vision-language model. Arm 2 served it with `--limit-mm-per-prompt '{"image":0}'`, which switched the eyes off. Gap 5 serves the same checkpoint with **one image per prompt allowed** and asks it two decisions about 36 screenshots of pages this workshop publishes. THE IMAGE HOOK, and it is deliberately the smallest one that exists. `run.py`'s `call_vllm` builds the request body, times the call and normalises the reply; `decide` reads the letters, applies the calibration and writes the row. None of that should be forked to add a picture. So this module patches exactly one function — `run.post`, the JSON POST underneath both — to add an `image_url` part to the user turn when an image is attached, and only on `/v1/chat/completions`. `/tokenize` is therefore still asked the TEXT alone, which is how each row gets its text-only token count and, by difference, what the picture cost in tokens. """ from __future__ import annotations import base64 import contextlib import hashlib import json import os import random import sys HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import run as R # noqa: E402 -- arm 2's instrument, unedited GAPS_KIT = os.path.join(HERE, "kit-gaps-card") #: Gap 4's shuffles. SIX presentations per item: k=0 is the kit's own order, #: which is arm 2's, and k=1..5 are `random.Random(20260921 + k)`. Six orders #: give 15 ordered-pair comparisons per item, which is the figure the card's #: 2.3 % is a rate of. ORDERS = 6 SEED = 20260921 # ----------------------------------------------------------- the image hook _ATTACHED: list[str | None] = [None] _POST = R.post def _post_with_image(url: str, body: dict, timeout: float) -> dict: """`run.post`, plus a picture on the user turn when one is attached. The guard is on the URL for a reason: `/tokenize` must keep seeing the text alone, so `text_tokens` is a real text-only count and `image_tokens` can be stated as a difference rather than guessed. Everything else about the call — the body vLLM is sent, the clock around it, the normalisation of its reply — is `run.py`'s and is not touched here. """ img = _ATTACHED[0] if img and url.endswith("/v1/chat/completions"): for msg in body.get("messages") or []: if msg.get("role") == "user" and isinstance(msg.get("content"), str): msg["content"] = [ {"type": "image_url", "image_url": {"url": img}}, {"type": "text", "text": msg["content"]}, ] return _POST(url, body, timeout) R.post = _post_with_image @contextlib.contextmanager def attached_image(data_url: str): """Attach one image to every `/v1/chat/completions` call inside the block.""" _ATTACHED[0] = data_url try: yield finally: _ATTACHED[0] = None def data_url(raw: bytes) -> str: return "data:image/png;base64," + base64.b64encode(raw).decode("ascii") def load_gaps_kit(name: str): with open(os.path.join(GAPS_KIT, name + ".json"), encoding="utf-8") as fh: return json.load(fh) # ------------------------------------------- GAP 4: task (c), six orders def permutations_for(items: list[dict]) -> list[list[list[str]]]: """The six presentations of every item, decided here and written to the row. `k = 0` is the kit's own order untouched, so those 108 prompts are byte-identical to arm 2's (verified before the run: 108 of 108 sha256 matches against `rows/openjev-fp8-readout.c.jsonl`) and the pass is this arm's own control rather than a fresh measurement of a different prompt. `k = 1..5` draw from `random.Random(20260921 + k)` walking the items in kit order, so the whole schedule is reproducible from two integers. """ out = [] for k in range(ORDERS): rng = random.Random(SEED + k) per_item = [] for item in items: order = list(item["options"]) if k: rng.shuffle(order) per_item.append(order) out.append(per_item) return out def task_creorder(arm, _articles, _overlong): """GAP 4 — the same 108 six-voice items under six option orders. THE SHUFFLE IS OF THE WHOLE PRESENTATION, not of the letter block alone. `run.py`'s `task_c` builds the state from the item's option order ("A dinner table with six guests: ...") and the letters from the same list, so a kit that had sampled a different order would have moved both. Moving only the letters would measure a prompt this bench never asks; moving both is exactly the prompt arm 2 would have had under a different draw. What rides on every row so the analysis needs nothing but the file: `uid` (POSITIONAL — `kit/task_c.json` holds 108 lines under only 50 distinct `id` strings, the latent trap arm 7 found, so nothing here may join on `id`), `order_k`, the presented `permutation`, and `chosen_letter` — derived from the permutation rather than read back from the text, because `choice` is a guest and the positional-bias question is about the letter. """ roster = {g["id"]: g["name"] for g in R.load("manifest")["c_roster"]} items = R.load("task_c") schedule = permutations_for(items) for k in range(ORDERS): for idx, item in enumerate(items): order = schedule[k][idx] options = [(gid, roster[gid]) for gid in order] row = R.decide(arm, "A dinner table with six guests: %s." % ", ".join(roster[g] for g in order), R.INSTR_C % item["text"], options) pos = order.index(row["choice"]) if row.get("choice") in order else None row.update(id=item["id"], uid="c%03d" % idx, task="creorder", label=item["label"], order_k=k, permutation=order, is_kit_order=(order == list(item["options"])), chosen_letter=(R.LETTERS[pos] if pos is not None else None), label_letter=R.LETTERS[order.index(item["label"])], text_sha256=R.sha(item["text"])) yield row # ------------------------------------------- GAP 5: two screenshot decisions #: Decision (ii)'s two letters, in this fixed order for every item, so the #: letter a row reports means the same thing in every row. The descriptions are #: what the two kinds of page ARE on this site, not a paraphrase of the label. KIND_OPTIONS = [ ("front_door", "The top page of one of this site's products — its front door."), ("article", "A research article page on this site's research hub."), ] INSTR_TITLE = ("The state is a screenshot of one web page. Which of these six " "titles is the title of the page in the screenshot?") INSTR_KIND = ("The state is a screenshot of one web page. Is this page a " "product's front door, or a research article?") #: The text half of a picture decision. `readout_prompt` wants a state, and a #: screenshot is not text — so the state SAYS that the picture is the state and #: the picture rides beside it on the same user turn. The sentence is identical #: for all 72 decisions, so nothing in it can separate one item from another. SHOT_STATE = "The screenshot attached to this message." def _shot_rows(arm, task: str, instructions: str, options_for, label_for): """One decision per screenshot, the picture read off disk at call time. The image the model is shown is hashed AT THE CALL and the hash compared with the one the frozen manifest recorded. A file that changed between the build and the run is a `refused-kit-moved` row carrying both hashes, not a silently different measurement — which is the same rule §A2.5 applies to an overlong page: written down, excluded from every rate, never worked around. """ kit = load_gaps_kit("shots_manifest") for page in kit["pages"]: with open(os.path.join(GAPS_KIT, page["png"]), "rb") as fh: raw = fh.read() seen = hashlib.sha256(raw).hexdigest() options = options_for(page, kit) if seen != page["png_sha256"]: row = R.refusal("kit-moved", {"png_sha256_at_call": seen, "png_bytes_at_call": len(raw)}) else: with attached_image(data_url(raw)): row = R.decide(arm, SHOT_STATE, instructions, options) # The TEXT-ONLY count, from the same server's own /tokenize under # the same template — the picture is not attached to that call, so # the difference below is what the picture cost in tokens and not # an estimate of it. try: row["text_tokens"] = R.count_tokens_vllm( arm, R.readout_prompt(SHOT_STATE, instructions, options)) except R.SeatError as exc: # reported, never silently None row["text_tokens"] = None row["tokenize_error"] = str(exc)[:200] if row.get("prompt_tokens") and row.get("text_tokens"): row["image_tokens"] = row["prompt_tokens"] - row["text_tokens"] row.update(id=page["id"], uid=page["id"], task=task, label=label_for(page), kind=page["kind"], url=page["url"], page_title=page["title"], png_sha256=page["png_sha256"], png_sha256_at_call=seen, png_bytes=len(raw), viewport="%dx%d" % (page["width"], page["height"]), option_keys=[k for k, _ in options]) yield row def task_shot_title(arm, _articles, _overlong): """GAP 5 (i) — which of these six titles is this page? Floor 16.7 %. The six options are the page's own `` and five distractors drawn, by seed, from the other pages OF THE SAME KIND in this set (`build_shots.py` says why: every article's title ends "· strata→signal research" and no front door's does, so a cross-kind distractor could be discarded on punctuation without looking at the picture). The option key is the page's short id and the description is its title — the same `(key, description)` shape task (c) gives a guest, and the key names the same page the title does, so it can carry no information about which option is right. """ def options_for(page, kit): titles = {p["id"]: p["title"] for p in kit["pages"]} return [(pid, titles[pid]) for pid in page["title_options"]] return _shot_rows(arm, "shot_title", INSTR_TITLE, options_for, lambda page: page["title_label"]) def task_shot_kind(arm, _articles, _overlong): """GAP 5 (ii) — front door or article? Majority-class floor 66.7 % (24/36). Not run through the yes/no calibration. `READOUT_NOUL_T` is the card's constant for a `noul` yes/no decision; this is a two-way choice between two named kinds, which is arm 4's shape (admit/refuse) and arm 4 did not apply it either. """ return _shot_rows(arm, "shot_kind", INSTR_KIND, lambda page, kit: KIND_OPTIONS, lambda page: page["kind"]) # --------------------------------------------------------------- the arms #: GAP 5's arms. The SAME checkpoint, the same two cards, the same window and #: the same letters as arm 2 — with one serving flag changed, #: `--limit-mm-per-prompt '{"image":1}'`, which is the whole subject of the gap. #: They are separate arm names because their rows were measured under a #: different serving posture and a row must never claim a posture it did not #: run under; the VRAM difference that flag costs is read from both arms' own #: pin blocks and reported. for _name, _mode in (("readout", "readout"), ("generate", "generate")): _a = R.ARMS["openjev-fp8-%s" % _name] R.ARMS["openjev-fp8-%s-mm" % _name] = R.Arm( "openjev-fp8-%s-mm" % _name, _a.model, _mode, runtime="vllm", host=_a.host, cards=_a.cards, num_ctx=_a.num_ctx, quant=_a.quant, top_logprobs=_a.top_logprobs, note="GAP 5: arm 2's model and flags with --limit-mm-per-prompt " "'{\"image\":1}' — the eyes switched on; CC BY-NC 4.0, bench-only") R.TASKS["creorder"] = task_creorder R.TASKS["shot_title"] = task_shot_title R.TASKS["shot_kind"] = task_shot_kind R.EXPECT_MINUTES.update(creorder=10, shot_title=4, shot_kind=4) def main(argv=None) -> int: import argparse ap = argparse.ArgumentParser() ap.add_argument("--arm", action="append", required=True, choices=sorted(R.ARMS)) ap.add_argument("--task", action="append", required=True, choices=sorted(R.TASKS)) ap.add_argument("--out", default=os.path.join(HERE, "rows-gaps-card")) a = ap.parse_args(argv) for name in a.arm: R.run_arm(R.ARMS[name], a.task, a.out) return 0 if __name__ == "__main__": raise SystemExit(main())