"""Leg 2b — the runtime-dividend A/B (AB-LEG-PREREG.md), window 2026-08-21. DISCLOSED DEVIATION (recorded in results, prereg unamended): the-gpu-box has no docker and no podman, so Arm OLD runs the official v0.32.9 release BINARY as an isolated second instance (own port, own scratch model store) instead of the registered container. Same substance — old runtime, same model, same posture, same box, same hour — different vehicle. Run ON THE GPU BOX (scp'd there): stages a scratch store with gemma4:26b copied from the main store, serves 0.32.9 on :11435, benches OLD then NEW, kills the old instance, writes receipts. Never touches the system service. """ # SANITIZED AT PUBLICATION (2026-08-21): estate-internal box names and # home paths in this runner are replaced with public aliases; quoted # clock strings, where any, are re-expressed in UTC. Logic untouched. import json, os, shutil, subprocess, time, urllib.request, hashlib, signal OLD_DIR = os.path.expanduser("~/ab-leg/old-runtime") SCRATCH = os.path.expanduser("~/ab-leg/scratch-models") OLD_HOST = "http://127.0.0.1:11436" NEW_HOST = "http://127.0.0.1:11434" MODEL = "gemma4:26b" Q1 = ("In the board game Catan (Settlers of Catan), when you roll a 7 and must move " "the robber, can you choose to move it back to the desert hex? Answer in plain " "english for a family game night, in under 120 words.") OUT = os.path.expanduser("~/ab-leg/ab-runs.json") def find_store(): for cand in ("/usr/share/ollama/.ollama/models", os.path.expanduser("~/.ollama/models")): if os.path.isdir(os.path.join(cand, "manifests")): return cand out = subprocess.run(["systemctl", "show", "ollama", "-p", "Environment"], capture_output=True, text=True).stdout for tok in out.split(): if "OLLAMA_MODELS=" in tok: return tok.split("=", 1)[1] raise SystemExit("model store not found") def stage_scratch(store): os.makedirs(SCRATCH, exist_ok=True) man_src = os.path.join(store, "manifests/registry.ollama.ai/library/gemma4/26b") man_dst = os.path.join(SCRATCH, "manifests/registry.ollama.ai/library/gemma4") os.makedirs(man_dst, exist_ok=True) shutil.copy2(man_src, os.path.join(man_dst, "26b")) man = json.load(open(man_src)) digests = [l["digest"] for l in man.get("layers", [])] + [man["config"]["digest"]] os.makedirs(os.path.join(SCRATCH, "blobs"), exist_ok=True) total = 0 for d in digests: fn = d.replace(":", "-") src = os.path.join(store, "blobs", fn) dst = os.path.join(SCRATCH, "blobs", fn) if not os.path.exists(dst): shutil.copy2(src, dst) total += os.path.getsize(src) return total def call(host, npred=220, keep="5m"): body = json.dumps({"model": MODEL, "prompt": Q1, "stream": False, "think": False, "options": {"temperature": 0, "seed": 0, "num_ctx": 32768, "num_predict": npred}, "keep_alive": keep}).encode() req = urllib.request.Request(host + "/api/generate", body, {"Content-Type": "application/json"}) with urllib.request.urlopen(req, timeout=600) as r: return json.load(r) def bench(host, arm): rows = [] call(host) # warm-up, unscored time.sleep(2) for i in range(10): d = call(host) time.sleep(2) rows.append({"arm": arm, "i": i, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "eval_count": d["eval_count"], "eval_duration_ns": d["eval_duration"], "prompt_eval_count": d["prompt_eval_count"], "decode_tok_s": round(d["eval_count"] / (d["eval_duration"] / 1e9), 2), "reply_sha256": hashlib.sha256(d["response"].encode()).hexdigest()}) return rows def main(): store = find_store() copied = stage_scratch(store) print(f"scratch staged: {copied/1e9:.1f} GB") env = dict(os.environ, OLLAMA_HOST="127.0.0.1:11436", OLLAMA_MODELS=SCRATCH, OLLAMA_KEEP_ALIVE="5m") old = subprocess.Popen([os.path.join(OLD_DIR, "bin/ollama"), "serve"], env=env, stdout=open(os.path.expanduser("~/ab-leg/old-serve.log"), "w"), stderr=subprocess.STDOUT) try: for _ in range(60): try: v = json.load(urllib.request.urlopen(OLD_HOST + "/api/version", timeout=3)) print("old runtime up:", v); break except Exception: time.sleep(2) else: raise SystemExit("old runtime never came up") rows = bench(OLD_HOST, "OLD-0.32.9") # release old arm's model then kill the instance call(OLD_HOST, npred=1, keep=0) finally: old.send_signal(signal.SIGTERM) try: old.wait(timeout=30) except Exception: old.kill() print("old instance stopped") newv = json.load(urllib.request.urlopen(NEW_HOST + "/api/version", timeout=5)) rows += bench(NEW_HOST, f"NEW-{newv.get('version','current')}") call(NEW_HOST, npred=1, keep=0) # release the visitor on the main runtime json.dump({"prereg": "AB-LEG-PREREG.md (sha pinned in PREREG-INDEX.txt)", "deviation": "no docker/podman on the box; official v0.32.9 release binary as an isolated second instance (own port+store) — substance preserved, vehicle changed, disclosed", "new_version": newv, "rows": rows}, open(OUT, "w"), indent=1) import statistics for arm in ("OLD-0.32.9", rows[-1]["arm"]): r = sorted(x["decode_tok_s"] for x in rows if x["arm"] == arm) print(f"{arm}: median {statistics.median(r):.2f} range {r[0]:.2f}-{r[-1]:.2f}") shas = {a: {x["reply_sha256"] for x in rows if x["arm"] == a} for a in {x["arm"] for x in rows}} print("determinism per arm:", {a: len(s) for a, s in shas.items()}) if __name__ == "__main__": main()