#!/usr/bin/env python3 """The Jev bench, ARM 12 — the Kev family, Apache-2.0, one card each. python3 run_kev.py --arm kev-9b --task kc --task ka --task kd --task kf --task kj python3 run_kev.py --arm kev-9b --task kperm WHY ARM 12 EXISTS, and it is not an accuracy question. Every OpenJev number in this bench is fenced: CC BY-NC 4.0, bench-only, those weights never become a seat. Kev is the same category of model — a typed decision read off a pointer head, no generation — under **Apache-2.0**, on an Apache-2.0 base. So a Kev result is a result about a model this workshop could actually deploy. The pre-registration is `GAPS-CARD.md` §"Arm 12", written before the first call. WHY THIS FILE IS NOT `run_addenda.py`. Kev does not speak OpenAI. It serves TypeSafe's `/v1/systemone` contract: typed questions in, a probability distribution per question out, no letters and no logprobs at a first output position. `run.py`'s `decide` is a letter readout and mapping Kev onto it would either fake letters or fork the reader. So this module brings its own `kev_decide` — and takes everything *around* the decision from `run.py` unchanged: `run_arm` still does the announcement, the ten-second idle read, the 1 Hz `nvidia-smi` sampler by UUID, the residency proof, the jsonl, the report and `summarise`. The rows land in the same shape as every other arm's, so the tables can put them in one column. WHAT EVERY ROW KEEPS THAT THIS BENCH HAS NOT NEEDED BEFORE: `kev_response`, the server's own answer body, verbatim. Kev's readout is its own — a pointer head with a built-in temperature — and a row that kept only our summary of it would not let anyone re-derive the number. The raw body is the receipt. THE CALIBRATION, stated because it moves a column. Each checkpoint carries a fitted temperature in `head.pt` (9B: 2.30, 4B: 2.14) and every loader applies it by default. It never changes an answer, so ACCURACY is identical either way — but BRIER is not, and Kev's own published Brier is for the raw logits. The headline arms run as served (calibrated, which is what a seat would get); the `-raw` arms are the same model with `KEV_TEMPERATURE=1.0` and exist so exactly one Brier in our tables is comparable with the card's. """ from __future__ import annotations import json import os import sys import time import urllib.error import urllib.request HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import run as R # noqa: E402 -- the instrument, unedited import run_addenda as A # noqa: E402 -- arms 4/5/6's own option text KEV_HOST = "http://127.0.0.1:8009" SEED = 20260921 N_PERM = 6 # --------------------------------------------------------------- the call def kev_post(url: str, body: dict, timeout: float = 900.0) -> dict: """POST JSON and raise the SERVER'S OWN SENTENCE, not the status line. Kev answers an invalid request with `422` and a body that names the field; a bench that printed only "422 Unprocessable Entity" would send someone hunting the harness for a fact the server had already stated. Same rule `run.post` follows, and the same reason. """ req = urllib.request.Request( url, method="POST", data=json.dumps(body).encode("utf-8"), headers={"Content-Type": "application/json"}) try: with urllib.request.urlopen(req, timeout=timeout) as fh: return json.loads(fh.read().decode("utf-8")) except urllib.error.HTTPError as exc: detail = "" try: detail = exc.read().decode("utf-8", "replace")[:600] except Exception: # noqa: BLE001 - best effort pass raise R.SeatError("HTTP %s: %s" % (exc.code, detail)) from exc except (urllib.error.URLError, OSError, ValueError) as exc: raise R.SeatError("%s: %s" % (exc.__class__.__name__, exc)) from exc def kev_decide(arm, state, instructions, options, *, noul_keys=None) -> dict: """One typed decision through `/v1/systemone`, in this bench's row shape. `options` is the bench's own `[(key, description), ...]` — the same list every other arm's letters were built from — and it crosses into Kev's `criteria` mapping unchanged, which is why its `option_text` ("name: desc") renders the same words our `[A] name: desc` did. `noul_keys` turns a two-way decision into Kev's `noul`, which is its own primitive rather than a two-option choice: the answer is a single probability of *true*, so the pair is named here — `noul_keys = (yes, no)` — and the probability is spread back over OUR two keys, so `label` comparison and Brier work exactly as they do for every other arm. ONE QUESTION PER REQUEST. Kev can answer many questions off one prefill and that is a real feature of it; it is not what this arm measures, and bundling would make every latency row a different quantity from arm 2's. """ qid = "q" if noul_keys: yes_key, no_key = noul_keys by_key = dict(options) question = {"type": "noul", "instructions": instructions, "criteria": {"true": by_key.get(yes_key), "false": by_key.get(no_key)}} else: question = {"type": "choice", "instructions": instructions, "criteria": {k: d for k, d in options}} body = {"state": state, "model": "kev-latest", "questions": {qid: question}} started = time.monotonic() try: out = kev_post(arm.host + "/v1/systemone", body) except R.SeatError as exc: # THE PRE-REGISTERED 422, now actually implemented. GAPS-CARD.md §"Arm # 12" says a request the server answers 422 is "recorded with the # server's own message, excluded from every rate" — the first run # raised instead, which killed the arm four items into task (a) and is # the reason this branch exists. # # What the 422 IS, on this model: `kev/serve.py` serves with # INFER_MAX_STATE = INFER_MAX_BRANCH = 8192, and `kev/model.py::encode` # rejects a question when `len(branch) > max_branch - len(state)`. A # docent page that fills the 8,192-token state window therefore leaves a # budget of ZERO and even a 56-token question is refused. The state # would also have been silently truncated to fit, which this bench's own # rule forbids (arm 1 §8) — so the refusal is the honest outcome and not # a workaround to be routed around. msg = str(exc) if "HTTP 422" not in msg: raise row = R.refusal("kev-422", {"server_message": msg[:300], "n_options": len(options), "mode": "kev-" + question["type"]}) row["seconds"] = None return row seconds = time.monotonic() - started answer = (out.get("answers") or {}).get(qid) or {} row: dict = {"mode": "kev-" + question["type"], "prompt_sha256": R.sha(json.dumps(body, sort_keys=True, ensure_ascii=False)), "n_options": len(options), "kev_response": out} if noul_keys: p_yes = float(answer.get("noul")) probs = {noul_keys[0]: p_yes, noul_keys[1]: 1.0 - p_yes} ordered = [probs[k] for k, _ in options] row["kev_noul"] = p_yes else: probs = {k: float((answer.get("probabilities") or {}).get(k, 0.0)) for k, _ in options} ordered = [probs[k] for k, _ in options] best = max(probs, key=probs.__getitem__) row.update(choice=answer.get("choice") if not noul_keys else best, probs=probs, confidence=(answer.get("confidence") if answer.get("confidence") is not None else R.choice_confidence(ordered)), seconds=seconds, prompt_tokens=(out.get("usage") or {}).get("input_tokens"), eval_tokens=(out.get("usage") or {}).get("output_tokens"), server_latency_ms=out.get("latency_ms"), prompt_eval_ms=None, total_ms=seconds * 1e3, done_reason=None) # Kev returns the argmax itself; if its own answer and our argmax over its # own probabilities ever disagree, that is a finding about the server and it # is written down rather than resolved silently in favour of either. if not noul_keys and row["choice"] != best: row["argmax_disagrees"] = {"server": row["choice"], "ours": best} return row # --------------------------------------------------------------- the tasks def _overlong_ids(task: str) -> set[str]: """The items ARM 2 refused, read from arm 2's own rows and not re-chosen. Task (a)'s states reach 80,515 tokens and no 24 GB card holds that beside 19 GB of weights. Inheriting arm 2's refusal set — as arm 9 did — means arm 12's task-(a) denominator is the SAME 53 items arm 2 and arm 10 scored, so the three rows compare cell for cell instead of over three different sets. """ path = os.path.join(HERE, "rows", "openjev-fp8-readout.%s.jsonl" % task) ids = set() with open(path, encoding="utf-8") as fh: for line in fh: row = json.loads(line) if row.get("refused") == "overlong": ids.add(row["id"]) return ids def task_kc(arm, _articles, _overlong): """Task (c) — which of the six guests said this line? Six-way, floor 16.7 %.""" roster = {g["id"]: g["name"] for g in R.load("manifest")["c_roster"]} for idx, item in enumerate(R.load("task_c")): options = [(gid, roster[gid]) for gid in item["options"]] row = kev_decide(arm, "A dinner table with six guests: %s." % ", ".join(roster[g] for g in item["options"]), R.INSTR_C % item["text"], options) row.update(id=item["id"], uid="c%03d" % idx, task="kc", label=item["label"], text_sha256=R.sha(item["text"])) yield row def task_ka(arm, articles, _overlong): """Task (a) — does the article answer this question? `noul`, the same 53.""" over = _overlong_ids("a") for item in R.load("task_a"): art = articles[item["slug"]] if item["id"] in over: row = R.refusal("overlong", { "why": "inherited from arm 2's own refusal rows so the " "denominator matches (GAPS-CARD.md, arm 12)"}) else: row = kev_decide(arm, art["state"], R.INSTR_A % item["question"], R.A_OPTIONS, noul_keys=("yes", "no")) row.update(id=item["id"], task="ka", slug=item["slug"], label=item["label"], kind=item["kind"], state_sha256=art["state_sha256"]) yield row def task_kd(arm, _articles, _overlong): """The doorman's planted set (arm 4) — `noul`, DOORMAN_SYSTEM v1.2 verbatim.""" kit = A.load_kit("doorman_planted") system = kit["doorman_system_v12"] assert R.sha(system) == kit["doorman_system_sha256"], "doorman v1.2 sha moved" for item in kit["items"]: row = kev_decide(arm, kit["host_prefix"] + item["line"], system, A.DOORMAN_OPTIONS, noul_keys=("admit", "refuse")) row.update(id=item["id"], task="kd", label=item["label"], bucket=item["bucket"], bucket_name=item["bucket_name"], planted=item["planted"], target_class=item["target_class"], in_v12_contract=item["in_v12_contract"], gemma_choice=item["gemma_choice"], doorman_system_sha256=kit["doorman_system_sha256"]) yield row def task_kf(arm, _articles, _overlong): """The field exam (arm 5) — answerable or not, `noul`, floor 50 %.""" for item in A.load_kit("field_exam_ground")["items"]: row = kev_decide(arm, item["excerpt"], A.INSTR_F % item["question"], A.GROUND_OPTIONS, noul_keys=("yes", "no")) row.update(id=item["id"], task="kf", label=item["label"], source_task=item["task_name"], game=item["game"]) yield row def task_kj(arm, _articles, _overlong): """The judge seat (arm 6) — three classes, scored on arm 6's own alternation.""" for item in A.load_kit("judge_seat")["items"]: row = kev_decide(arm, item["span"], A.INSTR_J % item["claim"], A.JUDGE_OPTIONS) row.update(id=item["id"], task="kj", label=item["expected"], expected_ok=item["expected_ok"], correct_alt=row.get("choice") in item["expected_ok"], set_arm=item["arm"], slug=item["slug"], item_index=item["item_index"], comparators=item["comparators"]) yield row def task_kperm(arm, _articles, _overlong): """Kev's OWN reorder number, through its own `/v1/systemone/permute`. `n_perm = 6`, `seed = 20260921`: order 0 is the kit's own and five are shuffled — the shape gap 4 gave OpenJev. The server returns each order's probabilities, an `argmax_stable` flag and a per-option `spread` (max − min across the orders), which is gap 4's "mean per-guest probability range" defined identically. THE TWO ARE THE SAME STATISTIC ON DIFFERENT DRAWS. Kev shuffles with its own `random.Random(seed)` inside the server; gap 4 shuffles with ours. The six orders are therefore not the same six, and the tables say so: this compares two rates measured the same way, not two runs of one schedule. `choice` on the row is order 0's — the kit's own order — so this task's accuracy is the accuracy of the unshuffled question and can be read against `kc` on the same items. """ roster = {g["id"]: g["name"] for g in R.load("manifest")["c_roster"]} for idx, item in enumerate(R.load("task_c")): options = [(gid, roster[gid]) for gid in item["options"]] request = {"state": "A dinner table with six guests: %s." % ", ".join(roster[g] for g in item["options"]), "model": "kev-latest", "questions": {"q": {"type": "choice", "instructions": R.INSTR_C % item["text"], "criteria": {k: d for k, d in options}}}} started = time.monotonic() out = kev_post(arm.host + "/v1/systemone/permute", {"request": request, "question": "q", "n_perm": N_PERM, "seed": SEED}) seconds = time.monotonic() - started runs = out.get("runs") or [] first = runs[0] if runs else {} choices = [r.get("choice") for r in runs] row = {"id": item["id"], "uid": "c%03d" % idx, "task": "kperm", "label": item["label"], "mode": "kev-permute", "n_options": len(options), "n_perm": len(runs), "prompt_sha256": R.sha(json.dumps(request, sort_keys=True, ensure_ascii=False)), "choice": first.get("choice"), "probs": first.get("probabilities"), "confidence": R.choice_confidence( list((first.get("probabilities") or {"x": 1.0}).values())), "seconds": seconds / max(1, len(runs)), # per ORDER, so the # number is a decision's cost and not a batch of six "seconds_all_orders": seconds, "argmax_stable": out.get("argmax_stable"), "spread": out.get("spread"), "choices_by_order": choices, "orders": [r.get("order") for r in runs], "probabilities_by_order": [r.get("probabilities") for r in runs], "prompt_tokens": None, "eval_tokens": None, "kev_response": out} yield row # --------------------------------------------------------------- the arms #: ONE CARD EACH. Kev-9B is ~19 GB in bf16 and fits a 3090 whole, so this arm has #: no tensor-parallel and no all-reduce — the wire arm 2 paid 6.4 ms a token for #: is simply absent. Card 0 by UUID; card 1 must stay quiet and `verify_pin` #: refuses in both directions. def register(name: str, run: str, card: str = R.CARD0, raw: bool = False): R.ARMS[name] = R.Arm( name, "kev-latest", "readout", runtime="kev", host=KEV_HOST, cards=(card,), num_ctx=0, quant="bf16 (LoRA r=16 + pointer head)", note="ARM 12: %s, Apache-2.0, served by kev.serve on ONE card%s" % (run, " with KEV_TEMPERATURE=1.0 (raw logits)" if raw else " as served (the checkpoint's own fitted temperature)")) register("kev-9b", "jaredpalmer/kev-9b") register("kev-4b", "jaredpalmer/kev-4b") register("kev-9b-raw", "jaredpalmer/kev-9b", raw=True) register("kev-4b-raw", "jaredpalmer/kev-4b", raw=True) R.TASKS.update(kc=task_kc, ka=task_ka, kd=task_kd, kf=task_kf, kj=task_kj, kperm=task_kperm) R.EXPECT_MINUTES.update(kc=6, ka=20, kd=3, kf=3, kj=8, kperm=20) def main(argv=None) -> int: import argparse ap = argparse.ArgumentParser() ap.add_argument("--arm", action="append", required=True, choices=sorted(R.ARMS)) ap.add_argument("--task", action="append", required=True, choices=sorted(R.TASKS)) ap.add_argument("--out", default=os.path.join(HERE, "rows-gaps-card")) a = ap.parse_args(argv) # the server's own statement of what it loaded, once, into the receipts try: with urllib.request.urlopen(KEV_HOST + "/v1/models", timeout=60) as fh: print("MODELS " + fh.read().decode("utf-8"), flush=True) except Exception as exc: # noqa: BLE001 - reported print("MODELS unavailable: %s" % exc, flush=True) for name in a.arm: R.run_arm(R.ARMS[name], a.task, a.out) return 0 if __name__ == "__main__": raise SystemExit(main())