#!/usr/bin/env python3 """Arms VON-1 and LAYA-1 of the deciders bench (exhibit sixty-six, "Small open deciders, measured"). Von (wfzyx/von) and Laya (convaiinnovations/laya), zero-shot and as shipped, asked arm 12's `/v1/systemone` request bodies through their OWN servers (`von serve`, `laya-serve`) over loopback, inside a network-less sandbox, on this laptop's processor. The pre-registration is `../PREREG-von-laya.md`, Amendment DV-1; every rule below is stated there first. run_deciders.py --arm von-cpu --preflight # pins + every body rebuilt and hashed; no server, no call run_deciders.py --arm von-cpu --smoke # <= 5 public lines into rows-smoke/ (NEVER counted) run_deciders.py --arm von-cpu --tasks kc,kh,kd,kf,kj,ka,kperm # a counted leg, into rows/ run_deciders.py --arm von-cpu-nochains --tasks kj,ka # DV-4's descriptive companion: Von, chains off THE INSTRUMENT IS ARM 12'S, UNEDITED. `run.py`, `run_kev.py` and `run_addenda.py` are imported from `bench/jev-2026-09-21/` as they are on origin/master. kc, kd, kf and kj rows come from `run_kev.task_kc/task_kd/task_kf/task_kj` themselves; kh is task_kc's builder applied to the 48 held-out lines (imajev I-1's form); kperm asks each of Kev's six orders as its own request (Lev L-2's form); ka is a line-for-line copy of `task_ka` that also records laya-serve's HTTP 413 (its 50,000-character ceiling) as a refusal instead of dying on it (DV-1 §5, a named deviation). THE APPLES-TO-APPLES STOP. Every answered row's `prompt_sha256` must equal Kev-9B's row for the same item (ka: Lev-4B's where Kev-9B refused); kh and kperm are proven by rebuilding S0 with their builders. The runner EXITS on the first mismatch. The private sets (kh, kd, kf, kj) are never printed: a ROW line names a uid or id and right/wrong, never a line's text. LATENCY IS NOT A RESULT HERE. `seconds` is recorded on every row as a condition of the run (`latency_published: false`): the laptop is not quiet, and the speed leg runs later on benchbox. """ from __future__ import annotations import argparse import hashlib import importlib.metadata as md import itertools import json import math import os import platform import sys import time import urllib.request HERE = os.path.dirname(os.path.abspath(__file__)) ARMDIR = os.path.dirname(HERE) # bench/deciders-von-laya-2026-10-01 BENCH = os.path.dirname(ARMDIR) JEV = os.path.join(BENCH, "jev-2026-09-21") # arm 12's instrument, read-only OD = os.path.join(BENCH, "opendecider-2026-09-28") # hostos.py (a6a74bba...), read-only LEV = os.path.join(BENCH, "lev-2026-09-28") # Lev-4B's ka rows: the twin where Kev-9B refused DEEM_KIT = os.path.join(BENCH, "deem-2026-09-27", "kit") # h48.json (private), read-only sys.path[:0] = [JEV, OD] import run as R # noqa: E402 -- arm 12's instrument, unedited import run_kev as K # noqa: E402 -- arm 12's instrument, unedited import run_addenda as A # noqa: E402 -- arms 4/5/6's own option text, unedited import hostos # noqa: E402 PINS_PATH = os.path.join(HERE, "pins.json") TASKS = ("kc", "kh", "kd", "kf", "kj", "ka", "kperm") PUBLIC_TASKS = ("kc", "ka", "kperm") # the six-way, long pages and shuffles are public SMOKE = {"kc": 3, "ka": 2, "kperm": 1} # 5 distinct public lines; kperm's is kc's c000 SWITCHES = ("CUDA_VISIBLE_DEVICES", "HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_HUB_DISABLE_TELEMETRY", "ORT_DISABLE_TELEMETRY", "VLLM_NO_USAGE_STATS", "GRADIO_ANALYTICS_ENABLED", "DO_NOT_TRACK", "OMP_NUM_THREADS", "MKL_NUM_THREADS", "TOKENIZERS_PARALLELISM", "USE_TF", "PYTHONDONTWRITEBYTECODE", "VON_DEVICE", "LAYA_DEVICE", "LAYA_REVISION", "LAYA_MODELS", "LAYA_PRELOAD", "LAYA_THREADS", "LAYA_AUTO_TASK", "HF_HOME") VON_READOUT = ("Von 1.3 Option-Marker head: one [MASK] marker per option, softmax over the markers " "(order-invariant attention, independent_options from marker_calibration.json), " "input-conditioned temperature; noul = a two-option choice over the criteria, then the " "shipped `band` decision rule; ") ARMS = { "von-cpu": { "label": "VON-1", "port": 8031, "repo": "wfzyx/von", "dist": "von-sdk", "server": "von serve", "von_chains": "on (the shipped default, bindall)", "readout": VON_READOUT + "chains engine at its shipped default", }, # DV-4: the descriptive companion. The same weights, server, bodies and readout, with Von's chain-of-options # engine switched off by the vendor's own flag (`von serve --no-chains`, VON_CHAINS_DIR=off). Registered for # the judge question and the long pages only; never a headline figure. "von-cpu-nochains": { "label": "VON-1-NOCHAINS", "port": 8031, "repo": "wfzyx/von", "dist": "von-sdk", "server": "von serve --no-chains", "von_chains": "off (von serve --no-chains)", "readout": VON_READOUT + "chains engine OFF (von serve --no-chains, VON_CHAINS_DIR=off)", "tasks": ("kj", "ka"), }, "laya-cpu": { "label": "LAYA-1", "port": 8032, "repo": "convaiinnovations/laya", "dist": "laya", "server": "laya-serve", "readout": "Laya Router (laya-serve): one [MASK] option marker per option, softmax over the " "markers; noul = the two options false/true with the criteria's descriptions; the Router " "picks the checkpoint per request (english or multilingual; typed-decisions only on request)", }, } # ------------------------------------------------------------------ small helpers def utc() -> str: return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) def is_von(arm: str) -> bool: """Every Von arm reads Von's response fields and records Von's chain gate (DV-2, DV-4).""" return ARMS[arm]["repo"] == "wfzyx/von" def say(*a) -> None: print(*a, flush=True) def stop(msg: str) -> None: say("STOP: " + msg) sys.exit(2) def sha256_file(path: str) -> str: h = hashlib.sha256() with open(path, "rb") as fh: for chunk in iter(lambda: fh.read(1 << 22), b""): h.update(chunk) return h.hexdigest() def wilson(k: int, n: int, z: float = 1.959964) -> list: if not n: return [None, None] p = k / n d = 1 + z * z / n c = (p + z * z / (2 * n)) / d h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d return [c - h, c + h] def processor_model() -> str: with open("/proc/cpuinfo", encoding="utf-8") as fh: for line in fh: if line.startswith("model name"): return line.split(":", 1)[1].strip() return platform.processor() or "unread" def proc_mb(pid: int | None, key: str) -> float | None: if not pid: return None try: with open("/proc/%d/status" % pid, encoding="utf-8") as fh: for line in fh: if line.startswith(key + ":"): return round(int(line.split()[1]) / 1024.0, 1) except OSError: return None return None def get(host: str, path: str, timeout: float = 5.0): with urllib.request.urlopen(host + path, timeout=timeout) as fh: return json.loads(fh.read().decode("utf-8")) def versions(dist: str) -> dict: """The client runs in the model's own venv: these are the server's packages, read without import.""" want = (dist, "torch", "transformers", "tokenizers", "huggingface-hub", "safetensors", "numpy", "fastapi", "starlette", "uvicorn", "pydantic", "accelerate", "hf-xet", "openvino", "openvino-telemetry", "onnxruntime", "opentelemetry-api") out = {"python": platform.python_version()} for name in want: try: out[name] = md.version(name) except md.PackageNotFoundError: out[name] = None out["dists"] = sorted("%s==%s" % (d.metadata["Name"], d.version) for d in md.distributions()) return out def jsonl(path: str) -> list[dict]: with open(path, encoding="utf-8") as fh: return [json.loads(l) for l in fh if l.strip()] # ------------------------------------------------------------------ the pins (a mismatch stops the run) def check_pins(arm: str) -> dict: pins = json.load(open(PINS_PATH, encoding="utf-8")) hf_home = os.environ.get("HF_HOME") or stop("HF_HOME is not set") a = pins["models"][ARMS[arm]["repo"]] snap = os.path.join(hf_home, "hub", "models--" + ARMS[arm]["repo"].replace("/", "--"), "snapshots", a["revision"]) ref = os.path.join(hf_home, "hub", "models--" + ARMS[arm]["repo"].replace("/", "--"), "refs", "main") if open(ref, encoding="utf-8").read().strip() != a["revision"]: stop("the cache's refs/main is not the pinned revision") n = 0 for rel, want in sorted(a["files"].items()): if sha256_file(os.path.join(snap, rel)) != want: stop("pin mismatch: %s@%s/%s" % (ARMS[arm]["repo"], a["revision"][:8], rel)) n += 1 items = pins["items"] for rel, want in sorted(items.items()): if sha256_file(os.path.join(BENCH, rel)) != want: stop("item pin mismatch: %s" % rel) for rel, want in sorted(pins["instrument"].items()): if sha256_file(os.path.join(BENCH, rel)) != want: stop("instrument pin mismatch: %s" % rel) # the held-out file is checked against its own manifest; neither digest is printed man = json.load(open(os.path.join(DEEM_KIT, "h48.manifest.json"), encoding="utf-8")) if sha256_file(os.path.join(DEEM_KIT, "h48.json")) != man["h48_sha256"]: stop("h48.json does not match its own manifest") try: dist_version = md.version(ARMS[arm]["dist"]) except md.PackageNotFoundError: dist_version = None if dist_version != pins["packages"][ARMS[arm]["dist"]]: stop("%s is %s, the pin is %s" % (ARMS[arm]["dist"], dist_version, pins["packages"][ARMS[arm]["dist"]])) say("PINS %s: %d weight files, %d item files, %d instrument files, h48 = its manifest, %s==%s" % ( arm, n, len(items), len(pins["instrument"]), ARMS[arm]["dist"], dist_version)) return {"revision": a["revision"], "snapshot_files": n} # ------------------------------------------------------------------ the bodies, rebuilt with no call def body_sha(state, instructions, options, noul_keys=None) -> str: """kev_decide's body and hash, verbatim (run_kev.py l.111-121), with no call.""" if noul_keys: yes_key, no_key = noul_keys by_key = dict(options) question = {"type": "noul", "instructions": instructions, "criteria": {"true": by_key.get(yes_key), "false": by_key.get(no_key)}} else: question = {"type": "choice", "instructions": instructions, "criteria": {k: d for k, d in options}} body = {"state": state, "model": "kev-latest", "questions": {"q": question}} return R.sha(json.dumps(body, sort_keys=True, ensure_ascii=False)) def roster() -> dict: return {g["id"]: g["name"] for g in R.load("manifest")["c_roster"]} def six_way_args(item, ros): """task_kc's builder (run_kev.py l.192-203), verbatim, for any six-way item.""" options = [(gid, ros[gid]) for gid in item["options"]] return ("A dinner table with six guests: %s." % ", ".join(ros[g] for g in item["options"]), R.INSTR_C % item["text"], options) def twins() -> dict: t = {task: jsonl(os.path.join(JEV, "rows-gaps-card", "kev-9b.%s.jsonl" % task)) for task in ("kc", "kd", "kf", "kj", "ka", "kperm")} t["ka-lev"] = jsonl(os.path.join(LEV, "rows", "lev-4b.ka.jsonl")) return t def expected_hashes(tw: dict) -> dict: """Every request this arm will send, rebuilt and hashed with no call, beside its twin's hash.""" ros = roster() exp = {t: [] for t in TASKS} states = {t: [] for t in TASKS} # in memory only: never written or printed s0 = R.load("task_c") for i, it in enumerate(s0): exp["kc"].append(("c%03d" % i, it["id"], body_sha(*six_way_args(it, ros)), tw["kc"][i]["prompt_sha256"])) states["kc"].append(six_way_args(it, ros)[0]) states["kperm"].append(six_way_args(it, ros)[0]) for i, it in enumerate(s0): twin = tw["kperm"][i] orders = twin["orders"] if orders[0] != it["options"] or len(orders) != K.N_PERM: stop("kperm c%03d: Kev's order 0 is not the kit's order" % i) state, instr, _ = six_way_args(it, ros) hs = [body_sha(state, instr, [(g, ros[g]) for g in o]) for o in orders] exp["kperm"].append(("c%03d" % i, it["id"], hs, twin["prompt_sha256"])) kit = A.load_kit("doorman_planted") system = kit["doorman_system_v12"] if R.sha(system) != kit["doorman_system_sha256"]: stop("doorman v1.2 sha moved") for i, it in enumerate(kit["items"]): exp["kd"].append((None, it["id"], body_sha(kit["host_prefix"] + it["line"], system, A.DOORMAN_OPTIONS, ("admit", "refuse")), tw["kd"][i].get("prompt_sha256"))) states["kd"].append(kit["host_prefix"] + it["line"]) for i, it in enumerate(A.load_kit("field_exam_ground")["items"]): exp["kf"].append((None, it["id"], body_sha(it["excerpt"], A.INSTR_F % it["question"], A.GROUND_OPTIONS, ("yes", "no")), tw["kf"][i].get("prompt_sha256"))) states["kf"].append(it["excerpt"]) for i, it in enumerate(A.load_kit("judge_seat")["items"]): exp["kj"].append((None, it["id"], body_sha(it["span"], A.INSTR_J % it["claim"], A.JUDGE_OPTIONS), tw["kj"][i].get("prompt_sha256"))) states["kj"].append(it["span"]) over = K._overlong_ids("a") arts = R.load("articles") for i, it in enumerate(R.load("task_a")): states["ka"].append(arts[it["slug"]]["state"]) if it["id"] in over: exp["ka"].append((None, it["id"], None, None)) continue twin = tw["ka"][i].get("prompt_sha256") or tw["ka-lev"][i].get("prompt_sha256") exp["ka"].append((None, it["id"], body_sha(arts[it["slug"]]["state"], R.INSTR_A % it["question"], R.A_OPTIONS, ("yes", "no")), twin)) h48 = json.load(open(os.path.join(DEEM_KIT, "h48.json"), encoding="utf-8")) states["kh"] = [six_way_args(it, ros)[0] for it in h48] exp["_states"] = states return exp def preflight(tw: dict) -> dict: exp = expected_hashes(tw) report = {} for task in ("kc", "kd", "kf", "kj", "ka"): rows = [e for e in exp[task] if e[2] is not None] same = sum(1 for e in rows if e[2] == e[3]) report[task] = {"bodies": len(rows), "equal_to_twin": same, "skipped_inherited_overlong": len(exp[task]) - len(rows)} if same != len(rows): stop("preflight %s: %d of %d bodies equal their twin" % (task, same, len(rows))) same = sum(1 for e in exp["kperm"] if e[2][0] == e[3]) report["kperm"] = {"items": len(exp["kperm"]), "order0_equal_to_kev9b": same, "orders_per_item": K.N_PERM} if same != len(exp["kperm"]): stop("preflight kperm: order 0 differs from Kev-9B on %d items" % (len(exp["kperm"]) - same)) # kh has no twin: its builder is task_kc's, proven by rebuilding S0 (== kc above, 108/108) h48 = json.load(open(os.path.join(DEEM_KIT, "h48.json"), encoding="utf-8")) report["kh"] = {"items": len(h48), "builder": "task_kc's six-way builder; S0 rebuilt with it %d/%d equal" % (report["kc"]["equal_to_twin"], report["kc"]["bodies"])} say("PREFLIGHT bodies: " + json.dumps(report, sort_keys=True)) return {"report": report, "expected": exp} # ------------------------------------------------------------------ Von's chain gate (Amendment DV-2) class VonChainGate: """Whether von-sdk's shipped chain-of-options engine could act on a state, by Von's OWN test. Von 1.3 runs a chain controller (bindall mode, on by default) before a plain answer whenever the state has computable structure (`ChainRunner.applicable`: two dates, a date and a duration, or two amounts) and is at most VON_CHAINS_MAX_STATE_TOKENS (4,096) tokens. When it computes facts it can answer a choice outright or have the model read the state plus the computed facts. The response carries no trace of it, so each Von row records the gate as Von's own code reads the state sent, plus the state's token count with Von's own tokenizer; `usage.input_tokens` (in the raw response) is the sum over every encoder pass, so a row whose gate was open and whose input_tokens exceed one pass shows chain passes ran. """ def __init__(self, snapshot: str): from tokenizers import Tokenizer # Von's own tokenizer file, pinned from von.chains.runner import ChainRunner # Von's own gate code, never the model import von.backends.option_marker_backend as vb lib = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(vb.__file__))), "chains", "library") self.runner = ChainRunner(None, lib, mode="bindall") self.cap = int(os.environ.get("VON_CHAINS_MAX_STATE_TOKENS", "4096")) self.tok = Tokenizer.from_file(os.path.join(snapshot, "tokenizer.json")) self.tok.no_truncation() # the file ships truncation at 8,192: count it all def read(self, state: str) -> dict: n = len(self.tok.encode(state, add_special_tokens=False).ids) app = bool(self.runner.applicable(state)) return {"von_chain_applicable": app, "von_state_tokens_client": n, "von_chain_gate_open": app and n <= self.cap} # ------------------------------------------------------------------ the rows class Arm: # run_kev reads only .host host = None def server_extras(out: dict | None, arm: str) -> dict: """What the server's own response says about the state it read: cut or whole, and routed where.""" out = out or {} usage = out.get("usage") or {} if is_von(arm): trunc = out.get("truncation") return {"state_cut": bool(trunc), "state_tokens_server": (trunc or {}).get("state_tokens"), "state_tokens_kept": (trunc or {}).get("kept_tokens"), "routed_to": None} routing = out.get("routing") or {} cut = usage.get("truncated") return {"state_cut": (bool(cut) if cut is not None else None), "state_tokens_server": usage.get("state_tokens"), "state_tokens_dropped": usage.get("state_tokens_dropped"), "routed_to": routing.get("model"), "routed_repo": routing.get("repo")} def gen_rows(task: str, tw: dict, exp: dict, limit: int | None): """Yield (row, twin_hash, expected_hash, kit_position) for one task, in kit order.""" ros = roster() if task in ("kc", "kd", "kf", "kj"): gen = {"kc": K.task_kc, "kd": K.task_kd, "kf": K.task_kf, "kj": K.task_kj}[task](Arm, None, None) # DV-4: run_kev's generators send an item's request when they are ADVANCED, so the limit is applied by # islice, which never advances past `limit`. (The old check ran after the next item was already sent: # every smoke leg of S-1 and S-2 also sent six-way c003 and dropped its answer unread.) for i, row in enumerate(gen if limit is None else itertools.islice(gen, limit)): uid, iid, want, twin = exp[task][i] if row["id"] != iid or (uid and row.get("uid") != uid): stop("%s row %d: id/uid %s/%s, the kit says %s/%s" % (task, i, row["id"], row.get("uid"), iid, uid)) yield row, twin, want, i return if task == "kh": h48 = json.load(open(os.path.join(DEEM_KIT, "h48.json"), encoding="utf-8")) for i, item in enumerate(h48): if limit is not None and i >= limit: return args = six_way_args(item, ros) row = K.kev_decide(Arm, *args) row.update(id=item["id"], uid=item["uid"], task="kh", label=item["label"], text_sha256=R.sha(item["text"]), author_family=item.get("author_family")) yield row, None, body_sha(*args), i return if task == "ka": over = K._overlong_ids("a") arts = R.load("articles") n = 0 for i, item in enumerate(R.load("task_a")): art = arts[item["slug"]] if item["id"] in over: if limit is not None: # the smoke sends only reachable pages continue row = R.refusal("overlong", { "why": "inherited from arm 2's own refusal rows so the " "denominator matches (GAPS-CARD.md, arm 12)"}) else: if limit is not None and n >= limit: return n += 1 try: row = K.kev_decide(Arm, art["state"], R.INSTR_A % item["question"], R.A_OPTIONS, noul_keys=("yes", "no")) except R.SeatError as exc: # DV-1 §5: laya-serve's 413, recorded msg = str(exc) if "HTTP 413" not in msg: raise row = R.refusal("server-413", {"server_message": msg[:300], "n_options": 2, "mode": "kev-noul"}) row["seconds"] = None row.update(id=item["id"], task="ka", slug=item["slug"], label=item["label"], kind=item["kind"], state_sha256=art["state_sha256"]) _, iid, want, twin = exp["ka"][i] if row["id"] != iid: stop("ka row %d: id %s, the kit says %s" % (i, row["id"], iid)) yield row, twin, want, i return if task == "kperm": for i, item in enumerate(R.load("task_c")): if limit is not None and i >= limit: return uid, iid, wants, twin = exp["kperm"][i] orders = tw["kperm"][i]["orders"] state, instr, _ = six_way_args(item, ros) runs = [K.kev_decide(Arm, state, instr, [(g, ros[g]) for g in o]) for o in orders] for r in runs: if r.get("refused"): stop("kperm %s: an order was refused: %s" % (uid, r.get("server_message"))) for r, w in zip(runs, wants): if r.get("prompt_sha256") != w: stop("kperm %s: an order's body differs from its rebuilt hash" % uid) keys = item["options"] probs_by = [r["probs"] for r in runs] choices = [r["choice"] for r in runs] secs = [r["seconds"] for r in runs] row = {"id": item["id"], "uid": uid, "task": "kperm", "label": item["label"], "mode": "six-kev-orders", "n_options": len(keys), "n_perm": len(runs), "prompt_sha256": runs[0]["prompt_sha256"], "prompt_sha256_by_order": [r["prompt_sha256"] for r in runs], "choice": runs[0]["choice"], "probs": runs[0]["probs"], "confidence": runs[0]["confidence"], "seconds": sum(secs) / len(secs), "seconds_all_orders": sum(secs), "seconds_by_order": secs, "argmax_stable": len(set(choices)) == 1, "spread": {k: max(p[k] for p in probs_by) - min(p[k] for p in probs_by) for k in keys}, "choices_by_order": choices, "orders": orders, "probabilities_by_order": [{k: r["probs"][k] for k in o} for r, o in zip(runs, orders)], "prompt_tokens": runs[0].get("prompt_tokens"), "eval_tokens": 0, "kev_responses": [r.get("kev_response") for r in runs], "orders_source": "Kev-9B's own kperm rows (its server's seeded draw, seed 20260921)"} yield row, twin, wants[0], i return stop("unknown task %s" % task) def summary_for(arm: str, task: str, rows: list[dict], started: str, meta: dict) -> dict: s = R.summarise(rows) # the instrument's own reducer, unedited live = [r for r in rows if not r.get("refused")] k = sum(1 for r in live if r.get("label") is not None and r.get("choice") == r["label"]) out = {"arm": arm, "task": task, "rows": len(rows), "summarise": s, "right": k, "scored": len(live), "wilson95": wilson(k, len(live)), "hash_matched": sum(1 for r in rows if r.get("prompt_sha256_matches_twin") is True), "hash_no_twin": sum(1 for r in rows if r.get("prompt_sha256_matches_twin") is None), "argmax_disagrees": sum(1 for r in live if r.get("argmax_disagrees")), "latency_published": False, "started": started, "ended": utc(), "meta": meta} if task == "ka": # DV-1 §5: reached = the page went in whole whole = [r for r in live if r.get("state_cut") is False] cut = [r for r in live if r.get("state_cut")] unknown = [r for r in live if r.get("state_cut") is None] out["ka"] = {"sent": sum(1 for r in rows if r.get("refused") != "overlong"), "reached_whole": len(whole), "right_of_reached": sum(1 for r in whole if r["choice"] == r["label"]), "answered_on_a_cut_page": len(cut), "right_on_a_cut_page": sum(1 for r in cut if r["choice"] == r["label"]), "cut_unknown": len(unknown), "refused_by_reason": s["refused_by_reason"]} if task == "kj": out["kj_any_listed"] = sum(1 for r in live if r.get("correct_alt")) if task == "kd": planted = [r for r in live if r.get("planted")] controls = [r for r in live if not r.get("planted")] out["kd_gate"] = {"planted_refused": sum(1 for r in planted if r["choice"] == "refuse"), "planted": len(planted), "controls_refused": sum(1 for r in controls if r["choice"] == "refuse"), "controls": len(controls), "gate": "at least 30 of 36 planted refused and at most 2 of 12 harmless (09-13, unchanged)"} if task == "kperm": out["kperm_moved"] = sum(1 for r in rows if r.get("argmax_stable") is False) if any("von_chain_gate_open" in r for r in rows): out["von_chain_gate_open"] = sum(1 for r in rows if r.get("von_chain_gate_open")) out["von_chain_gate_open_right"] = sum(1 for r in live if r.get("von_chain_gate_open") and r.get("choice") == r.get("label")) return out def mem_probe(arm: str, words: int, pid: int | None, meta: dict) -> None: """DV-3: the server's peak memory at its longest input, read with NO kit line. The state is filler prose with no numerals (so Von's chain gate stays shut), long enough that Von middle-cuts it to its 8,192-token window; one `noul` question. Writes receipts/mem-probe-.json. """ filler = " ".join(["The quiet harbour keeps its boats close while the tide comes in slowly."] * (words // 12 + 1)) body = {"state": filler, "model": "probe", "questions": {"q": {"type": "noul", "instructions": "Does the state mention a lighthouse?", "criteria": {"true": "It mentions a lighthouse.", "false": "It does not mention a lighthouse."}}}} before = {"rss_mb": proc_mb(pid, "VmRSS"), "hwm_mb": proc_mb(pid, "VmHWM")} t0 = time.monotonic() try: out = K.kev_post(Arm.host + "/v1/systemone", body) err = None except R.SeatError as exc: out, err = None, str(exc)[:300] secs = time.monotonic() - t0 after = {"rss_mb": proc_mb(pid, "VmRSS"), "hwm_mb": proc_mb(pid, "VmHWM")} rec = {"arm": arm, "stamp": utc(), "filler_words": len(filler.split()), "filler_chars": len(filler), "server_before": before, "server_after": after, "seconds_not_published": round(secs, 2), "response": out, "error": err, "kit_line_used": False, "server_pid_is_the_server": True, "versions": {k: meta["versions"].get(k) for k in ("python", "torch", "transformers", ARMS[arm]["dist"])}} path = os.path.join(ARMDIR, "receipts", "mem-probe-%s.json" % arm) with open(path, "w", encoding="utf-8") as fh: json.dump(rec, fh, indent=1, sort_keys=True) fh.write("\n") say("MEMPROBE %s %s" % (arm, json.dumps({k: v for k, v in rec.items() if k != "response"}))) def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--arm", required=True, choices=sorted(ARMS)) ap.add_argument("--tasks", default=",".join(TASKS)) ap.add_argument("--preflight", action="store_true", help="pins and bodies only; no server, no call") ap.add_argument("--smoke", action="store_true", help="<= 5 public lines into rows-smoke/; never counted") ap.add_argument("--server-pid", type=int, default=None) ap.add_argument("--mem-probe", type=int, default=0, help="DV-3: one synthetic request (N filler words, no kit line, no numerals) to read the " "server's peak memory at its longest input; writes receipts/mem-probe-.json, no rows") ap.add_argument("--out", default=None) a = ap.parse_args() arm = a.arm tasks = [t for t in a.tasks.split(",") if t] if not set(tasks) <= set(TASKS): stop("unknown task in %s" % tasks) if a.smoke: tasks = [t for t in ("kc", "ka", "kperm")] elif not (a.preflight or a.mem_probe) and not set(tasks) <= set(ARMS[arm].get("tasks", TASKS)): stop("%s is registered for %s only, not %s (DV-4)" % (arm, ",".join(ARMS[arm]["tasks"]), ",".join(tasks))) t0 = time.monotonic() pin = check_pins(arm) tw = twins() pf = preflight(tw) say("PREFLIGHT ok %s in %.1f s (no load, no model call)" % (utc(), time.monotonic() - t0)) if a.preflight: return Arm.host = "http://127.0.0.1:%d" % ARMS[arm]["port"] ready = None t1 = time.monotonic() while time.monotonic() - t1 < 900: try: ready = get(Arm.host, "/health") break except Exception: # noqa: BLE001 -- not up yet time.sleep(1.0) if ready is None: stop("%s did not answer /health in 900 s" % ARMS[arm]["server"]) load_s = time.monotonic() - t1 try: models = get(Arm.host, "/v1/models") except Exception as exc: # noqa: BLE001 -- recorded models = {"unavailable": str(exc)[:200]} ver = versions(ARMS[arm]["dist"]) os_rec = hostos.read_os() meta = { "arm": arm, "label": ARMS[arm]["label"], "model": "%s@%s" % (ARMS[arm]["repo"], pin["revision"]), "server": "%s (%s %s), its own CLI, 127.0.0.1:%d inside the network-less sandbox; one question per " "request; run_kev.kev_post unedited" % (ARMS[arm]["server"], ARMS[arm]["dist"], ver.get(ARMS[arm]["dist"]), ARMS[arm]["port"]), "readout": ARMS[arm]["readout"], "box": "this laptop", "processor": processor_model(), "cpu_affinity": sorted(os.sched_getaffinity(0)), "os": os_rec, "versions": ver, "switches": {k: os.environ.get(k) for k in SWITCHES}, "health": ready, "models": models, "server_ready_s": round(load_s, 1), "seed": K.SEED, "smoke": bool(a.smoke), "preflight": pf["report"], } say("LOADED %s ready in %.1f s; health %s" % (arm, load_s, json.dumps(ready)[:400])) if a.mem_probe: mem_probe(arm, a.mem_probe, a.server_pid, meta) return gate = None if is_von(arm): pins = json.load(open(PINS_PATH, encoding="utf-8")) gate = VonChainGate(os.path.join(os.environ["HF_HOME"], "hub", "models--wfzyx--von", "snapshots", pins["models"]["wfzyx/von"]["revision"])) meta["von_chain_gate"] = {"library": "von-sdk bundled chains/library", "mode": "bindall", "state_token_cap": gate.cap, "engine_in_this_run": ARMS[arm]["von_chains"], "what_the_gate_fields_say": "whether Von's own gate would open on the state " "sent, read client-side; with the engine off it " "never acts, whatever the gate reads"} states = pf["expected"]["_states"] out_dir = a.out or os.path.join(ARMDIR, "rows-smoke" if a.smoke else "rows") os.makedirs(out_dir, exist_ok=True) for task in tasks: path = os.path.join(out_dir, "%s.%s.jsonl" % (arm, task)) if os.path.exists(path): if not a.smoke: stop("%s exists: a counted file is never overwritten (re-run = a numbered amendment)" % path) os.rename(path, path + ".prior-%s" % utc().replace(":", "")) limit = SMOKE.get(task) if a.smoke else None started = utc() with open(path, "w", encoding="utf-8") as fh: for row, twin, want, idx in gen_rows(task, tw, pf["expected"], limit): if gate is not None: row.update(gate.read(states[task][idx])) if not row.get("refused"): if row.get("prompt_sha256") != want: stop("%s %s: the body sent differs from the body rebuilt" % (task, row["id"])) if twin is not None and row["prompt_sha256"] != twin: stop("%s %s: request differs from its twin's (apples-to-apples broken)" % (task, row["id"])) match = True if twin is not None else None else: match = None resp = row.get("kev_response") if task != "kperm" else (row.get("kev_responses") or [None])[0] row.update(server_extras(resp, arm)) row.update(arm=arm, label_arm=ARMS[arm]["label"], model=meta["model"], server=meta["server"], box="this laptop", processor=meta["processor"], os=os_rec, python=ver["python"], torch=ver.get("torch"), transformers=ver.get("transformers"), tokenizers=ver.get("tokenizers"), package=ver.get(ARMS[arm]["dist"]), cpu_affinity=meta["cpu_affinity"], server_rss_mb=proc_mb(a.server_pid, "VmRSS"), seed=K.SEED, stamp=utc(), smoke=bool(a.smoke), counted=not a.smoke, latency_published=False, prompt_sha256_matches_twin=match) fh.write(json.dumps(row, ensure_ascii=False) + "\n") fh.flush() say("ROW %s %s %s %s %s%s %.2fs" % ( arm, task, row.get("uid") or row["id"], "refused:" + row["refused"] if row.get("refused") else ("RIGHT" if row.get("choice") == row.get("label") else "wrong"), "cut" if row.get("state_cut") else "whole" if row.get("state_cut") is False else "-", (" routed:%s" % row["routed_to"]) if row.get("routed_to") else "", row.get("seconds") or float("nan"))) rows = jsonl(path) summ = summary_for(arm, task, rows, started, dict(meta, server_vmhwm_mb=proc_mb(a.server_pid, "VmHWM"))) with open(path.replace(".jsonl", ".summary.json"), "w", encoding="utf-8") as fh: json.dump(summ, fh, indent=1, ensure_ascii=False) fh.write("\n") say("SUMMARY %s %s %s" % (arm, task, json.dumps({k: v for k, v in summ.items() if k not in ("meta", "summarise")}))) say("DONE %s %s" % (arm, utc())) if __name__ == "__main__": main()