#!/usr/bin/env python3 """The Jev bench, addenda arms 4-11 (README ยง"Addenda arms 4-11", D-20260921-215). python3 run_addenda.py --arm openjev-fp8-readout --task d --out rows-addenda WHY THIS FILE EXISTS AND `run.py` IS NOT EDITED. `run.py` is the measured record of arms 1-3: its prompt builder, its calibration constants, its floor, its energy integration and its pin proof are what those numbers were produced by. An addendum that edited it would make the earlier rows unreproducible against the file that claims to have produced them. So this module IMPORTS run.py and registers new task generators into its `TASKS` table. Every addenda row is therefore built by arm 2's own `decide()`, scored by arm 2's own `summarise()`, and measured by arm 2's own `Boards` sampler -- the tasks are new, the instrument is not. A task generator here has run.py's signature, `(arm, articles, overlong)`, and yields rows; `run_arm` does the announcing, the idle read, the watt sampler, the pin proof, the jsonl and the report. Arms 8 and 11 are not in this file because neither fits `run_arm`'s one-decision-at-a-time shape: arm 8 is concurrent by definition (`run_arm8.py`) and arm 11 trains rather than serves (`run_arm11.py`). """ from __future__ import annotations import json import os import re import sys HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import run as R # noqa: E402 -- arm 2's instrument, unedited def load_kit(name: str): with open(os.path.join(R.KIT, name + ".json"), encoding="utf-8") as fh: return json.load(fh) # ------------------------------------------------------- arm 4: the doorman #: A=admit, B=refuse, in this fixed order for every item, so the letter a row #: reports means the same thing in every row. The descriptions are the doorman's #: own contract in its own words ("whether the table hears it") rather than a #: paraphrase this bench invented. DOORMAN_OPTIONS = [("admit", "The table hears the line."), ("refuse", "The table does not hear the line.")] def task_d(arm, _articles, _overlong): """Arm 4 -- the doorman's planted set as a two-letter decision. The INSTRUCTION is DOORMAN_SYSTEM v1.2 verbatim -- the same bytes the long table's gemma doorman was given, verified by sha256 against the `prompt_sha256` the 09-13 bench recorded, so this arm and that one are answering the same prompt and not two readings of it. The STATE is the user turn the engine actually sends: `HOST_PREFIX + line`, i.e. "Host: " and the visitor's line, which `gatecall.build_body` is explicit is the whole user message and never a template around it. Two labels ride every row and they are NOT the same thing (README, arm 4): `label` is the set's designed ground truth (planted -> refuse), and `gemma_choice` is another model's recorded answer, carried for AGREEMENT only. `in_v12_contract` rides too, because buckets C and D were pre-registered by the 09-13 bench as outside the four classes v1.2 names. """ kit = load_kit("doorman_planted") system = kit["doorman_system_v12"] assert R.sha(system) == kit["doorman_system_sha256"], "doorman v1.2 sha moved" for item in kit["items"]: row = R.decide(arm, kit["host_prefix"] + item["line"], system, DOORMAN_OPTIONS) row.update(id=item["id"], task="d", label=item["label"], bucket=item["bucket"], bucket_name=item["bucket_name"], planted=item["planted"], target_class=item["target_class"], in_v12_contract=item["in_v12_contract"], gemma_choice=item["gemma_choice"], gemma_class=item["gemma_class"], gemma_ms=item["gemma_ms"], doorman_system_sha256=kit["doorman_system_sha256"]) yield row # ------------------------------------------- arm 7: task (c), name-blind #: `build_kit.py`'s own NAME_TOKENS, plus the roster's full names, so the scrub #: removes exactly the forms the kit builder already treats as label leaks. NAME_TOKENS = { "darwin": ["darwin", "charles"], "hypatia": ["hypatia"], "ibn_sina": ["ibn sina", "avicenna"], "sagan": ["sagan", "carl"], "socrates": ["socrates", "socratic"], "einstein": ["einstein", "albert"], } def scrub_names(text: str, roster: dict) -> tuple[str, int]: """Replace every guest name form with "someone". Returns (text, n_hits). Longest form first, so "Hypatia of Alexandria" is not left as "someone of Alexandria" by an earlier match on "Hypatia". Word boundaries, case insensitive; the replacement is a single lowercase word because a capital would itself be a cue about where a name had been. """ forms = set() for gid, toks in NAME_TOKENS.items(): forms.update(toks) if gid in roster: forms.add(roster[gid].lower()) hits = 0 out = text for form in sorted(forms, key=len, reverse=True): pat = re.compile(r"\b" + re.escape(form) + r"\b", re.IGNORECASE) out, n = pat.subn("someone", out) hits += n return out, hits def task_cnb(arm, _articles, _overlong): """Arm 7 -- task (c) with the guest names removed from the LINE only. The six options stay the six names in the same fixed alphabetical order and the state line is untouched, so the only thing that changed between this arm and arm 2 is the information inside the line. `changed` rides every row: an unchanged line is this arm's own internal control, and the accuracy split between changed and unchanged lines is the arm's real payload. """ roster = {g["id"]: g["name"] for g in load_kit("manifest")["c_roster"]} for item in load_kit("task_c"): options = [(gid, roster[gid]) for gid in item["options"]] scrubbed, hits = scrub_names(item["text"], roster) row = R.decide(arm, "A dinner table with six guests: %s." % ", ".join(roster[g] for g in item["options"]), R.INSTR_C % scrubbed, options) row.update(id=item["id"], task="cnb", label=item["label"], names_removed=hits, changed=bool(hits), text_sha256=R.sha(scrubbed), original_sha256=R.sha(item["text"])) yield row # --------------------------------------- arm 5: the field exam, grounded or not #: [A] yes / [B] no, fixed for every item. See README "Addenda amendment 1": this #: is NOT the field exam's own T2 metric (a decline regex + a forbidden-token #: check + a word limit, none of which a letter can produce). It is the DECISION #: underneath it -- can the model tell an answerable excerpt from an unanswerable #: one -- and the label is the item's own `task`, which the field exam's prereg #: makes the test itself. GROUND_OPTIONS = [("yes", "The excerpt answers the question."), ("no", "The excerpt does not answer the question.")] INSTR_F = ("Does the excerpt in the state answer this question, using only what " "the excerpt itself states? Question: %s") def task_f(arm, _articles, _overlong): """Arm 5 -- the field exam's 40 T1+T2 items as one balanced binary.""" kit = load_kit("field_exam_ground") for item in kit["items"]: row = R.decide(arm, item["excerpt"], INSTR_F % item["question"], GROUND_OPTIONS, noul=True) row.update(id=item["id"], task="f", label=item["label"], source_task=item["task_name"], game=item["game"]) yield row # ------------------------------------------------- arm 6: the judge seat #: The judge's own three classes, in its own words, copied from the judge prompt #: rather than paraphrased -- so the option text the readout scores is the text #: the recorded judges were given. JUDGE_OPTIONS = [ ("grounded", "The section states the claim, figures and all."), ("not_grounded", "The section is about this subject but does not state the " "claim, or states it with different numbers, a different " "subject, or the opposite direction."), ("off_page", "The claim is about something this section is not about."), ] INSTR_J = ("You are given one claim and the section of a published article that " "the claim says it came from. Decide whether the section supports the " "claim. A paraphrase that states the same fact in different words IS " "grounded; only a changed figure, subject or direction is not.\n" "Claim: %s") def task_j(arm, _articles, _overlong): """Arm 6 -- the judge seat's 119 items, scored against `expected`. `expected` is construction-derived ground truth (README amendment 1), so this IS a rate. The three recorded model judges ride on every row under `comparators` and are reported as AGREEMENT, never as accuracy -- a model's verdict cannot grade a model, which is this bench's own standing ruling. `expected_ok` is the alternation split: OFF-PAGE counts correct on either of its two values, exactly as the judge bench's own scorer does it. """ kit = load_kit("judge_seat") for item in kit["items"]: row = R.decide(arm, item["span"], INSTR_J % item["claim"], JUDGE_OPTIONS) correct = row.get("choice") in item["expected_ok"] row.update(id=item["id"], task="j", label=item["expected"], expected_ok=item["expected_ok"], correct_alt=correct, set_arm=item["arm"], slug=item["slug"], item_index=item["item_index"], paraphrased_by=item.get("paraphrased_by"), comparators=item["comparators"]) yield row # ------------------------------------------------ arm 9: the long pages #: Arm 2's own refusal rows name this set: the 10 task-(a) and 7 task-(b) items #: whose prompts exceeded the 16,384 window. It is read from those rows rather #: than re-selected, so the set is arm 2's and not a fresh choice. def arm2_overlong_ids(task: str) -> set[str]: path = os.path.join(HERE, "rows", "openjev-fp8-readout.%s.jsonl" % task) ids = set() with open(path, encoding="utf-8") as fh: for line in fh: row = json.loads(line) if row.get("refused") == "overlong": ids.add(row["id"]) return ids def _long_only(task: str): """run.py's own task generator, filtered to arm 2's refused ids. The filter is on the OUTPUT of run.py's generator, not a reimplementation of it, so these rows are built by the same code that built arm 2's -- and an item that is STILL overlong at the wider window arrives here as run.py's own `refused-overlong` row, carrying its newly measured token count, exactly as the pre-registration says it should. """ ids = arm2_overlong_ids(task) inner = R.TASKS[task] def gen(arm, articles, overlong): # THE CENSUS MUST BE RUN HERE, and the reason is a trap worth naming. # `run_arm` triggers its context census on `t in ("a", "b")`, and arm 9's # tasks are called `along`/`blong`, so it hands this generator an EMPTY # overlong map -- which would send a 80,515-token page at a 32,768-token # server and take a 400 instead of recording a refusal. The refusal # policy is the measurement here (four of these items are expected to be # overlong even at the wider window), so it cannot be left to a name # match. `census_vllm` is run.py's own, costs no forward pass, and asks # the server's own /tokenize with this arm's num_ctx. if not overlong: pre = R.census_vllm(arm, articles) overlong = pre["overlong"] print("census (arm 9, num_ctx %d): longest page %d tokens, " "%d pages still overlong: %s" % (arm.num_ctx, pre["max_prompt_tokens"], len(overlong), ", ".join(sorted(overlong))), flush=True) for row in inner(arm, articles, overlong): if row["id"] in ids: row["task"] = task + "long" row["arm2_refused"] = True yield row return gen #: Arm 9's server is arm 2's with `--max-model-len 32768`; the arm records the #: wider window so every row says which ceiling it was measured under. for _base, _mode in (("readout", "readout"), ("generate", "generate")): _a = R.ARMS["openjev-fp8-%s" % _base] R.ARMS["openjev-fp8-%s-32k" % _base] = R.Arm( "openjev-fp8-%s-32k" % _base, _a.model, _mode, runtime="vllm", host=_a.host, cards=_a.cards, num_ctx=32768, quant=_a.quant, top_logprobs=_a.top_logprobs, note="ARM 9: the same model and flags as arm 2 with --max-model-len " "32768, run ONLY on the 17 items arm 2 refused as overlong") #: ARM 10 -- the BASE model on the SAME runtime. The estate's assistant seat's #: own checkpoint, so the comparison arm 1 could not make (its base was #: ollama-only, README A2.3.4) is finally a like-for-like one: same server, same #: cards, same window, same kit, same letters. Apache-2.0-clean in the sense that #: matters here -- it is the estate's own seat weights, not the CC BY-NC bench #: weights, and it is still only ever read by this bench. GEMMA4_VLLM = "gemma4-fp8-vllm" for _name, _mode in (("readout", "readout"), ("generate", "generate")): R.ARMS["gemma4-fp8-%s" % _name] = R.Arm( "gemma4-fp8-%s" % _name, GEMMA4_VLLM, _mode, runtime="vllm", host="http://127.0.0.1:8000", cards=(R.CARD0, R.CARD1), num_ctx=16384, quant="FP8 dynamic (gemma-4-26B-A4B-it-FP8-dynamic, the estate's seat " "checkpoint)", top_logprobs=20, note="ARM 10: the base model under vLLM on the same two 3090s, so the " "gap to OpenJev is read without the runtime in it") R.TASKS["d"] = task_d R.TASKS["cnb"] = task_cnb R.TASKS["f"] = task_f R.TASKS["j"] = task_j R.TASKS["along"] = _long_only("a") R.TASKS["blong"] = _long_only("b") R.EXPECT_MINUTES.update(d=4, cnb=5, f=3, j=8, along=6, blong=5) def main(argv=None) -> int: import argparse ap = argparse.ArgumentParser() ap.add_argument("--arm", action="append", required=True, choices=sorted(R.ARMS)) ap.add_argument("--task", action="append", required=True, choices=sorted(R.TASKS)) ap.add_argument("--out", default=os.path.join(HERE, "rows-addenda")) a = ap.parse_args(argv) for name in a.arm: R.run_arm(R.ARMS[name], a.task, a.out) return 0 if __name__ == "__main__": raise SystemExit(main())