#!/usr/bin/env python3 """DV-5's table script: Von and Laya's SPEED readings on benchbox, read from the committed rows. It only reads. tables_speed.py --json bench/deciders-von-laya-2026-10-01/TABLES-speed-von-laya.json What this file is, for a reader who opened it cold. Exhibit sixty-six prints, for each small decider, the median seconds a six-way line (one request at a time), its p95 by nearest rank, and on a graphics card the memory it added at load and its highest reading (EVIDENCE-v2 tables 9 and 9b). This script computes the same figures for Von and Laya from DV-5's rows (rows-speed/), the arm summaries (*.timed.speed.json) and the stop guard's 2 Hz card telemetry (receipts-speed/timed-telemetry.jsonl), and adds the p90 the brief asked for. Every figure carries the file it came from at its commit. Counts and seconds only: no line text, no digest, no UUID. The registered headline is pass 1 (DV-5: the page's form, one pass after three warm-ups); passes 2 and 3 and the pooled 324 rows are printed beside it. The quantile rule is R.summarise's: sorted[round(q * (n - 1))]. Answer agreement with the accuracy legs (rows/von-cpu.kc.jsonl, rows/laya-cpu.kc.jsonl, this laptop's processor) is descriptive: the same request bodies, so a different answer is numerics (device, precision, kernels), never a different question. """ from __future__ import annotations import argparse import datetime as dt import glob import json import os import statistics import subprocess import sys ARM_DIR = "bench/deciders-von-laya-2026-10-01" ARMS = ("speed-von-cpu", "speed-laya-cpu", "speed-von-gpu", "speed-laya-gpu") ACCURACY = {"speed-von-cpu": "rows/von-cpu.kc.jsonl", "speed-von-gpu": "rows/von-cpu.kc.jsonl", "speed-laya-cpu": "rows/laya-cpu.kc.jsonl", "speed-laya-gpu": "rows/laya-cpu.kc.jsonl"} CARD_TOTAL_MIB = 24576.0 def jsonl(path: str) -> list[dict]: with open(path, encoding="utf-8") as fh: return [json.loads(l) for l in fh if l.strip()] def nearest_rank(secs, q): s = sorted(secs) return s[min(len(s) - 1, int(round(q * (len(s) - 1))))] if s else None def stats(secs) -> dict: return {"n": len(secs), "median_s": round(statistics.median(secs), 4), "p90_s": round(nearest_rank(secs, .90), 4), "p95_s": round(nearest_rank(secs, .95), 4), "mean_s": round(statistics.fmean(secs), 4), "min_s": round(min(secs), 4), "max_s": round(max(secs), 4)} class Cite: def __init__(self, root: str): self.root = root def __call__(self, rel: str) -> str: path = os.path.join(self.root, rel) st = subprocess.run(["git", "-C", self.root, "status", "--porcelain", "--", rel], capture_output=True, text=True).stdout.strip() if st: return "%s@UNTRACKED" % rel if st.startswith("??") else "%s@MODIFIED" % rel h = subprocess.run(["git", "-C", self.root, "log", "-1", "--format=%h", "--", rel], capture_output=True, text=True).stdout.strip() if not h or not os.path.exists(path): return "%s@MISSING" % rel return "%s@%s" % (rel, h) def ts(utc: str) -> float: return dt.datetime.strptime(utc, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=dt.timezone.utc).timestamp() def highest(tel: list[dict], t0: float, t1: float) -> dict: win = [r for r in tel if t0 <= r["t"] <= t1 and r.get("mem_mib") is not None] if not win: return {"samples": 0} top = max(win, key=lambda r: r["mem_mib"]) first = min(r["t"] for r in win if r["mem_mib"] == top["mem_mib"]) return {"samples": len(win), "highest_mib": top["mem_mib"], "highest_first_seen_utc": dt.datetime.fromtimestamp(first, dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")[:-4] + "Z", "headroom_mib": CARD_TOTAL_MIB - top["mem_mib"], "power_w_max": max((r.get("power_w") or 0) for r in win), "temp_c_max": max((r.get("temp_c") or 0) for r in win)} def agreement(rows: list[dict], acc: list[dict]) -> dict: by = {r["uid"]: r for r in acc} same = sum(1 for r in rows if by[r["uid"]]["choice"] == r["choice"]) diff = max(abs(r["probs"][k] - by[r["uid"]]["probs"][k]) for r in rows for k in r["probs"]) return {"same_choice": same, "of": len(rows), "largest_probability_difference": round(diff, 6), "changed_uids": sorted(r["uid"] for r in rows if by[r["uid"]]["choice"] != r["choice"])} def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("root") ap.add_argument("--json", required=True) a = ap.parse_args() root = os.path.abspath(a.root) cite = Cite(root) base = os.path.join(root, ARM_DIR) tel_rel = os.path.join(ARM_DIR, "receipts-speed", "timed-telemetry.jsonl") tel = jsonl(os.path.join(root, tel_rel)) if os.path.exists(os.path.join(root, tel_rel)) else [] out = {"what": "Von and Laya, what it takes to run: median seconds a six-way line, one request at a time, " "p90 and p95 by nearest rank, and memory; benchbox's processor and its RTX 3090 at 300 W " "(Amendment DV-5). Counts and seconds only.", "head": subprocess.run(["git", "-C", root, "rev-parse", "--short=8", "HEAD"], capture_output=True, text=True).stdout.strip(), "script": cite(os.path.join(ARM_DIR, "instrument", "speed", "tables_speed.py")), "quantile_rule": "nearest rank: sorted[round(q * (n - 1))] (R.summarise's p95 form)", "headline": "pass 1 (rep1): the page's form, one pass of S0's 108 lines after three discarded warm-ups", "rows": []} for arm in ARMS: summ_rel = os.path.join(ARM_DIR, "rows-speed", "%s.timed.speed.json" % arm) if not os.path.exists(os.path.join(root, summ_rel)): out["rows"].append({"arm": arm, "status": "no timed summary"}) continue s = json.load(open(os.path.join(root, summ_rel), encoding="utf-8")) m = s["meta"] passes, pooled, files = [], [], {"summary": cite(summ_rel)} acc_rows = jsonl(os.path.join(base, ACCURACY[arm])) for p in s["passes"]: rel = os.path.join(ARM_DIR, "rows-speed", p["file"]) rows = jsonl(os.path.join(root, rel)) secs = [r["seconds"] for r in rows] pooled += secs passes.append({"pass": p["rep"], "started": p["started"], "ended": p["ended"], "wall_seconds": p["wall_seconds"], **stats(secs), "agreement_with_accuracy_leg": agreement(rows, acc_rows)}) files[p["rep"]] = cite(rel) gpu = m["device"] == "cuda" load = s["load"] mem = {"server_vmhwm_mb_after_warmups": load.get("server_hwm_mb_after_warmups"), "server_vmhwm_mb_end_of_pass_1": s["passes"][0].get("server_hwm_mb_at_end"), "server_vmhwm_mb_final_after_the_longest_input": s.get("server_hwm_mb_final")} if gpu: # DV-6: the runner's stamps are whole seconds (truncated), and its server is still up through the # summary's last second, so the window runs to the END of that second. The next arm starts later. t0 = ts(load["card_before_server"]["utc"]) t1 = ts(s["ended"]) + 1.0 points = [load["card_after_warmups"]["memory_used_mib"]] points += [(p.get("card_at_end") or {}).get("memory_used_mib") for p in s["passes"]] points.append((s["probe"].get("card_after") or {}).get("memory_used_mib")) points = [x for x in points if x is not None] mem.update({"card_before_server_mib": load["card_before_server"]["memory_used_mib"], "card_after_warmups_mib": load["card_after_warmups"]["memory_used_mib"], "grew_at_load_mib": load["grew_at_load_mib"], "card_after_each_pass_mib": [(p.get("card_at_end") or {}).get("memory_used_mib") for p in s["passes"]], "card_after_the_longest_input_mib": (s["probe"].get("card_after") or {}).get("memory_used_mib"), "telemetry_window_utc": [load["card_before_server"]["utc"], s["ended"] + " + 1 s"], **highest(tel, t0, t1)}) mem["highest_below_a_runner_reading"] = bool(points) and mem.get("highest_mib", -1) < max(points) files["telemetry"] = cite(tel_rel) elif tel: # DV-6: a processor arm never touches the card; the guard's telemetry over its passes and its longest # input shows what the card held meanwhile (expect the idle 1 MiB). w = highest(tel, ts(s["passes"][0]["started"]), ts(s["ended"]) + 1.0) mem["card_during_the_processor_arm"] = {"samples": w.get("samples"), "highest_mib": w.get("highest_mib"), "power_w_max": w.get("power_w_max")} files["telemetry"] = cite(tel_rel) logs = sorted(glob.glob(os.path.join(base, "receipts-speed", "serve-%s-timed-*.log" % arm))) banner = None for lg in logs[-1:]: for line in open(lg, encoding="utf-8", errors="replace"): if line.startswith("Starting Von Decision Server"): banner = line.strip().split(" on http")[0] break files["server_log"] = cite(os.path.relpath(lg, root)) out["rows"].append({ "arm": arm, "label": m["label"], "model": m["model"], "box": m["box"], "processor": m["processor"], "device": m["device"], "card": m["card"], "power_limit_w": m["power_limit_w"], "precision": m["precision"], "server": m["server"], "package": m["versions"].get( "von-sdk" if "von" in arm else "laya"), "torch": m["versions"].get("torch"), "threads": m["threads"], "cpu_affinity": m["cpu_affinity"], "os": m["os"].get("pretty_name"), "kernel": m["os"].get("kernel"), "device_seen": m["device_seen"], "von_banner": banner, "headline": dict(passes[0], pass_is="the registered headline"), "passes": passes, "pooled_324": stats(pooled), "load": {"server_ready_s": load["server_ready_s"], "warmup_seconds": load["warmup_seconds"]}, "longest_input": {k: s["probe"].get(k) for k in ("filler_words", "filler_chars", "seconds", "usage", "truncation", "error")}, "memory": mem, "files": files}) with open(os.path.join(root, a.json), "w", encoding="utf-8") as fh: json.dump(out, fh, indent=1, sort_keys=True) fh.write("\n") w = sys.stdout.write w("| arm | device | precision | median s | p90 s | p95 s | passes 2, 3 median s | memory |\n|---|---|---|---:|---:|---:|---|---|\n") for r in out["rows"]: if "headline" not in r: w("| %s | %s |\n" % (r["arm"], r["status"])) continue h, mm = r["headline"], r["memory"] memtxt = ("grew %s MiB, highest %s MiB, server VmHWM %s MB" % (mm.get("grew_at_load_mib"), mm.get("highest_mib"), mm.get("server_vmhwm_mb_final_after_the_longest_input")) if r["device"] == "cuda" else "server VmHWM %s MB (after the longest input)" % mm.get("server_vmhwm_mb_final_after_the_longest_input")) w("| %s | %s | %s | %.3f | %.3f | %.3f | %s | %s |\n" % ( r["arm"], (r["card"] or r["processor"]) + (" @ %s W" % r["power_limit_w"] if r["power_limit_w"] else ""), r["precision"].split(" (")[0], h["median_s"], h["p90_s"], h["p95_s"], ", ".join("%.3f" % p["median_s"] for p in r["passes"][1:]), memtxt)) if __name__ == "__main__": main()