#!/usr/bin/env python3 """recount_von_laya.py: the page's second recount, for the two models added on 2026-10-01, Von 1.3 and Laya. python3 recount_von_laya.py `recount_v2.py`, beside this file, re-derives every count of the page's first fourteen models, and `recount_on_the_kit.py` runs it over this kit. Neither knows the two models added on 2026-10-01. This script re-derives theirs from their rows in this kit, with recount_v2.py's own functions, and checks every figure against `recount-von-laya.json`, their figures of record. It runs recount_on_the_kit.py's own check first, so this one command checks the whole page: 1. THE FIRST FOURTEEN. It loads recount_on_the_kit.py and recount_v2.py from beside this file, each pinned by the sha256 this kit's index.json publishes for it, and refuses to run if either differs. It runs recount_v2.py over this kit and the Reading the answer kit, statement by statement, exactly as recount_on_the_kit.py does, and compares every figure with recount-v2.json the same way (recount_on_the_kit.py prints the detail). 2. VON 1.3 AND LAYA, from what this kit holds for them, in deciders-von-laya-2026-10-01/: the six-way (`.kc`), the long pages (`.ka`, with Von 1.3's chains-off companion's) and the six orders (`.kperm`). It scores them with recount_v2.py's score, task_a, reorder_perm, keyed and mcnemar. A long page counts as reached only when the server's own response says the page went in whole (the pre-registration's rule DV1.5). 3. THE PAGE'S OWN CHECKS, WIDENED BY THE TWO: recount_v2.py's checked_holm (the correction for nineteen tests), checked_resample_by_dinner, checked_tests_in_each_option_order and checked_paired_shuffle_counts, called with its own lists (PAGE_NB_TESTS, KEV_ORDER_FILES, MOVED_PAIRS) widened for the call and restored after. The seventeen earlier entries must read as recount-v2.json does. recount_v2.py's own text for the correction says "seventeen", the count it was written for; its `tests` field is the number corrected for, here 19. 4. POST HOC, as the page labels it (computed after the rows were seen, no bar): the six-order means, the split by writer, one fixed guess's score, the named-guest and first-offered pulls, exact ties, the long pages beside Kev-9B's and Lev-4B's, the chains-off companion, each server's own account of the inputs it cut, and the checkpoint Laya's Router chose. 5. THE SPEED LEG (the pre-registration's Amendment DV-5), from rows-speed/, each arm's *.timed.speed.json and receipts-speed/timed-telemetry.jsonl: the median seconds a six-way line, its p90 and p95 by nearest rank (sorted[round(q * (n - 1))]), the three passes, the answers beside the accuracy legs', what each server grew the card by and its highest reading. Checked against TABLES-speed-von-laya.json. Then the page's own readings of them, which no file records: the higher of the telemetry's highest and the runner's own reading after its longest input, and how far passes 2 and 3 moved from pass 1. 6. THE HELD-BACK SETS. The held-out lines, the doorman, the field exam and the judge question are held back for every model on the page, and their rows are not here. Their figures are in recount-von-laya.json; this script reports them as held back and never compares them. Standard library only; no model and no network. Like recount_on_the_kit.py it builds a tree of symbolic links in your system's temporary folder and leaves it there with its output, recount-von-laya-on-the-kit.json, whose path it prints first; delete that folder when you are done. Exit 0 when no re-derived figure differs from its record. """ from __future__ import annotations import sys sys.dont_write_bytecode = True # leave the kit as it came: no __pycache__ beside the scripts import contextlib # noqa: E402 import copy # noqa: E402 import datetime as dt # noqa: E402 import glob # noqa: E402 import hashlib # noqa: E402 import importlib.util # noqa: E402 import io # noqa: E402 import json # noqa: E402 import os # noqa: E402 import re # noqa: E402 import statistics # noqa: E402 from collections import Counter # noqa: E402 HERE = os.path.dirname(os.path.abspath(__file__)) #: the two scripts this one runs, pinned by the sha256 the kit's index.json publishes for each PINS = {"recount_v2.py": "1874539a8e9936e00b5ed141dda706eb023d757b8ff0440fe515d62ff6a53f89", "recount_on_the_kit.py": "31744e8d7115c0f76a54b6eac270df75c72b83d95d65d8e18e05a74be7f494b3"} RECORD = "recount-von-laya.json" OUTPUT = "recount-von-laya-on-the-kit.json" FOLDER = "deciders-von-laya-2026-10-01" BENCH = "bench/" + FOLDER ROWS = BENCH + "/rows" ARMS = ("von-cpu", "laya-cpu") COMPANION = "von-cpu-nochains" J = "bench/jev-2026-09-21/" TASK_C = J + "kit/task_c.json" KEV9B_KA = J + "rows-gaps-card/kev-9b.ka.jsonl" KEV9B_KPERM = J + "rows-gaps-card/kev-9b.kperm.jsonl" LEV4B_KA = "bench/lev-2026-09-28/rows/lev-4b.ka.jsonl" LATENCY_KEYS = ("median_s", "p95_s", "timed_rows") POST_HOC = "descriptive, computed after the rows were seen; not registered, no bar" TASK_A_NOTE = ("the recount's task_a counts any answer as reached; a truncating server answers cut pages, so DV1.5's " "long_pages is the registered reading") #: guest names as the public roster spells them, matched whole-word in the public six-way lines; Ibn Sina also by his #: Latin name. Two readings: surnames only, and surnames plus the roster's first names GUEST_NAMES = {"darwin": ["Darwin"], "einstein": ["Einstein"], "hypatia": ["Hypatia"], "ibn_sina": ["Ibn Sina", "Avicenna"], "sagan": ["Sagan"], "socrates": ["Socrates"]} GUEST_NAMES_WIDE = {"darwin": ["Darwin", "Charles"], "einstein": ["Einstein", "Albert"], "hypatia": ["Hypatia"], "ibn_sina": ["Ibn Sina", "Avicenna"], "sagan": ["Sagan", "Carl"], "socrates": ["Socrates"]} PUBLIC_SETS = {"von-cpu": ("kc", "ka", "kperm"), "laya-cpu": ("kc", "ka", "kperm"), COMPANION: ("ka",)} SET_NAMES = {"kc": "six-way", "kh": "held-out", "kd": "doorman", "kf": "field exam", "kj": "judge question", "ka": "long pages", "kperm": "six orders"} SPEED_ARMS = ("speed-von-cpu", "speed-laya-cpu", "speed-von-gpu", "speed-laya-gpu") SPEED_ACCURACY = {"speed-von-cpu": "von-cpu", "speed-von-gpu": "von-cpu", "speed-laya-cpu": "laya-cpu", "speed-laya-gpu": "laya-cpu"} SPEED_TABLES = BENCH + "/TABLES-speed-von-laya.json" TELEMETRY = BENCH + "/receipts-speed/timed-telemetry.jsonl" CARD_TOTAL_MIB = 24576.0 #: keys whose values name a file (and the commit it was read at), and keys whose values are prose: neither is a figure, #: and neither is compared CITE_KEYS = {"src", "kit", "kit_lock", "kit_order_src", "tables_src", "files"} TEXT_KEYS = {"what", "label", "note", "rule", "note_on_holm_what", "dinner_of_each_line", "pass_is"} #: the record's own provenance, which is not a figure of the two models: reported, never compared PROVENANCE = [r"^what$", r"^checkout_head$", r"^base(\.|$)", r"^privacy_stop(\.|$)", r"^recount(\.|$)", r"^cut(\.|$)", r"^chains_pair\.(head|rows)$", r"^routing\.what$"] _KH = "the held-out lines' rows (.kh) are held back" _KD = "the doorman's rows (.kd) are held back" _KF = "the field exam's rows (.kf) are held back" _KJ = "the judge question's rows (.kj) are held back" #: the record's figures that read held-back rows, each with the reason it cannot be re-derived here HELD = [ (r"^registered\.arms\.[^.]+\.(h48|s0_minus_h48)(\.|$)", _KH), (r"^registered\.arms\.[^.]+\.door(\.|$)", _KD), (r"^registered\.arms\.[^.]+\.field(\.|$)", _KF), (r"^registered\.arms\.[^.]+\.judge(\.|$)", _KJ), (r"^registered\.(equal_to_TABLES_von_laya|mismatches)$", "the check against TABLES-von-laya.json, the reduction's table, which this kit does not carry"), (r"^checked\.heldout_interval_widths(\.|$)", _KH), (r"^checked\.bound_on_0_of_12(\.|$)", _KD), (r"^checked\.wilson(\.|$)", "the field exam's and the judge question's rows (.kf, .kj) are held back"), (r"^checked_old_entries_unchanged\.heldout_", _KH), (r"^post_hoc\.door_by_kind(\.|$)", _KD), (r"^descriptive_extra\.answer_spread\.[^.]+\.kd(\.|$)", _KD), (r"^descriptive_extra\.answer_spread\.[^.]+\.kf(\.|$)", _KF), (r"^descriptive_extra\.answer_spread\.[^.]+\.kj(\.|$)", _KJ), (r"^descriptive_extra\.answer_spread_chains_off\.kj(\.|$)", _KJ), (r"^descriptive_extra\.constant_answer_floors\.door_", _KD), (r"^descriptive_extra\.constant_answer_floors\.field_", _KF), (r"^descriptive_extra\.constant_answer_floors\.judge_", _KJ), (r"^descriptive_extra\.paired_reads_beside_kev9b_and_lev4b\.[^.]+\.kf\|", _KF), (r"^descriptive_extra\.paired_reads_beside_kev9b_and_lev4b\.[^.]+\.kj\|", _KJ), (r"^descriptive_extra\.chains_recheck\.kj(\.|$)", _KJ), (r"^input_cuts\.arms\.[^.]+\.kh(\.|$)", _KH), (r"^input_cuts\.arms\.[^.]+\.kd(\.|$)", _KD), (r"^input_cuts\.arms\.[^.]+\.kf(\.|$)", _KF), (r"^input_cuts\.arms\.[^.]+\.kj(\.|$)", _KJ), (r"^chains_pair\.pairs\.kj(\.|$)", _KJ), (r"^routing\.arms\.laya-cpu\.kh(\.|$)", _KH), (r"^routing\.arms\.laya-cpu\.kd(\.|$)", _KD), (r"^routing\.arms\.laya-cpu\.kf(\.|$)", _KF), (r"^routing\.arms\.laya-cpu\.kj(\.|$)", _KJ), ] #: figures the record counted over every set, held-back ones included: re-derived over the public sets, and reported #: as partial where the two differ, never compared as a figure PARTIAL = [r"^descriptive_extra\.probability_decimals_seen\.", r"^input_cuts\.row_state_cut_disagrees_with_the_response$", r"^routing\.arms\.laya-cpu\.all_sets(\.|$)"] # ------------------------------------------------------------------ loading, pinned def sha256(path: str) -> str: with open(path, "rb") as fh: return hashlib.sha256(fh.read()).hexdigest() def verify_pins() -> None: """Refuse, before anything runs, unless both scripts beside this one are the files index.json publishes.""" for name, want in PINS.items(): if sha256(os.path.join(HERE, name)) != want: raise SystemExit("recount_von_laya: %s is not the file this kit's index.json publishes (its sha256 " "differs); nothing was run" % name) def load(name: str): spec = importlib.util.spec_from_file_location(name[:-3], os.path.join(HERE, name)) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod def rel(arm: str, task: str) -> str: return "%s/%s.%s.jsonl" % (ROWS, arm, task) def live(rows): return [r for r in rows if not r.get("refused")] def no_latency(d: dict) -> dict: return {k: v for k, v in d.items() if k not in LATENCY_KEYS} def drop_cites(x): if isinstance(x, dict): return {k: drop_cites(v) for k, v in x.items() if k not in CITE_KEYS} if isinstance(x, list): return [drop_cites(v) for v in x] return x # ------------------------------------------------------------------ 1. the first fourteen (recount_on_the_kit.py's run) def run_base(rok, rta: str): """recount_on_the_kit.py's own run and comparison; returns its module, the scratch root and the tallies.""" root = rok.build_root(rta) source = open(os.path.join(HERE, "recount_v2.py"), encoding="utf-8").read() C = rok.load_recount() with contextlib.redirect_stdout(io.StringIO()): stopped = rok.run(C, root, os.path.join(root, "recount-on-the-kit.json"), source) mine = C.__dict__["R"] recorded = json.load(open(os.path.join(HERE, "recount-v2.json"), encoding="utf-8")) totals = {"equal": 0, "partial": 0, "different": 0, "held back": 0, "not run": 0} notes = [] for section in sorted(k for k in recorded if k != "ref"): tally = dict.fromkeys(totals, 0) rok.compare(mine.get(section, {}), recorded[section], section, tally, notes) for k in totals: totals[k] += tally[k] return C, root, recorded, {"totals": totals, "statements_that_could_not_run": len(stopped), "different": [t for k, t in notes if k == "different"]} # ------------------------------------------------------------------ 2. the registered readings, public sets def long_pages_dv15(C, ka): """DV1.5: reached = the server's own response says the page went in whole.""" sent = [r for r in ka if r.get("refused") != "overlong"] answered = live(sent) whole = [r for r in answered if r.get("state_cut") is False] cut = [r for r in answered if r.get("state_cut") is True] return {"rule": "DV1.5: reached = the server's own response says the page went in whole", "rows": len(ka), "inherited_overlong": len(ka) - len(sent), "sent": len(sent), "refused_by_reason": dict(Counter(r["refused"] for r in sent if r.get("refused"))), "answered": len(answered), "reached_whole": len(whole), "right_of_reached": sum(C.is_right(r) for r in whole), "answered_on_a_cut_page": len(cut), "right_on_a_cut_page": sum(C.is_right(r) for r in cut), "cut_unknown": len(answered) - len(whole) - len(cut)} def arm_readings(C, src, arm: str) -> dict: kc, ka, kp = src.rows(rel(arm, "kc")), src.rows(rel(arm, "ka")), src.rows(rel(arm, "kperm")) s0 = no_latency(C.score(live(kc), "choice")) return {"s0": dict(s0, refused_rows=len(kc) - len(live(kc)), src=src.cite(rel(arm, "kc"))), "task_a_any_answer": dict(no_latency(C.task_a(ka)), src=src.cite(rel(arm, "ka")), note=TASK_A_NOTE), "long_pages": dict(long_pages_dv15(C, ka), src=src.cite(rel(arm, "ka"))), "reorder": dict(C.reorder_perm(kp), src=src.cite(rel(arm, "kperm")))} def server_413s(src) -> dict: """Laya's long-page refusals: how many, over how many distinct pages, and the limit and length its server said.""" rows = [r for r in src.rows(rel("laya-cpu", "ka")) if r.get("refused") == "server-413"] said = [re.search(r"\((\d+) > (\d+) chars\)", r.get("server_message") or "") for r in rows] return {"refused": len(rows), "distinct_pages": len({r["state_sha256"] for r in rows}), "page_characters": sorted({int(m.group(1)) for m in said if m}), "server_limit_characters": sorted({int(m.group(2)) for m in said if m})} def six_orders(src) -> dict: """Whether each model's six orders are, line by line, the six Kev's server generated for Kev-9B.""" def sig(rel_path): out = [] for r in src.rows(rel_path): o = json.loads(r["orders"]) if isinstance(r["orders"], str) else r["orders"] out.append((r["uid"], [list(x) for x in o])) return sorted(out) kev = sig(KEV9B_KPERM) return {"the_six_orders_are_kev9bs": {arm: sig(rel(arm, "kperm")) == kev for arm in ARMS}} # ------------------------------------------------------------------ 3. the page's own checks, widened def paired_interval(C, K, arm: str, ref: str = "naive-bayes") -> dict: a_map, b_map = K[arm], K[ref] a = sum(1 for k in a_map if a_map[k] and b_map[k]) b = sum(1 for k in a_map if a_map[k] and not b_map[k]) c = sum(1 for k in a_map if b_map[k] and not a_map[k]) d = len(a_map) - a - b - c n = len(a_map) out = {"model_right": a + b, "word_counter_right": a + c, "n": n, "both_right": a, "only_model_right": b, "only_word_counter_right": c, "neither_right": d, "edge_lines": b - c, "edge_points": 100 * (b - c) / n} for key, corrected in (("phi_as_computed", False), ("phi_continuity_corrected", True)): lo, hi, phi = C.newcombe_paired(a, b, c, d, corrected) out[key] = {"phi": phi, "points": [100 * lo, 100 * hi], "lines": [n * lo, n * hi]} return out def widen_and_check(C, src, R: dict, readings: dict, K: dict): R2 = copy.deepcopy(R) mc = {} for arm, r in readings.items(): R2["s0"][arm] = r["s0"] R2["reorder"][arm] = r["reorder"] for ref in ("naive-bayes", "kev-9b"): x, y, p = C.mcnemar(K[arm], K[ref]) mc["%s|%s" % (arm, ref)] = R2["mcnemar"]["%s|%s" % (arm, ref)] = { "only_first_right": x, "only_second_right": y, "p": p} old = (list(C.PAGE_NB_TESTS), dict(C.KEV_ORDER_FILES), list(C.MOVED_PAIRS)) C.PAGE_NB_TESTS = old[0] + list(ARMS) C.KEV_ORDER_FILES = dict(old[1], **{arm: rel(arm, "kperm") for arm in ARMS}) C.MOVED_PAIRS = old[2] + [(ref, arm) for arm in ARMS for ref in ("lev-4b-gpu", "kev-9b", "apus-9b-high")] try: checked = {"label": C.CHECKED, "holm": C.checked_holm(R2), "resample_by_dinner": C.checked_resample_by_dinner(src, R2, K), "tests_in_each_option_order": C.checked_tests_in_each_option_order(src, R2, K), "paired_shuffle_counts": C.checked_paired_shuffle_counts(src, R2)} finally: C.PAGE_NB_TESTS, C.KEV_ORDER_FILES, C.MOVED_PAIRS = old checked["paired_interval_vs_word_counter"] = {arm: paired_interval(C, K, arm) for arm in ARMS} return mc, checked def old_entries_unchanged(record_v2: dict, checked: dict) -> dict: """The seventeen earlier tests, the three page shares and the earlier shuffle pairs read as recount-v2.json does (file cites set aside: this kit's are "kit").""" rc = drop_cites(record_v2["checked"]) new = drop_cites(checked) out = {} old = rc["resample_by_dinner"]["paired_against_the_word_counter"] out["resample_paired_old_17_equal"] = all( old[m] == new["resample_by_dinner"]["paired_against_the_word_counter"][m] for m in old) out["resample_single_shares_equal"] = (rc["resample_by_dinner"]["single_shares"] == new["resample_by_dinner"]["single_shares"]) out["each_order_old_models_equal"] = all( rc["tests_in_each_option_order"]["models"][m] == new["tests_in_each_option_order"]["models"][m] for m in rc["tests_in_each_option_order"]["models"]) out["each_order_page_four_equal"] = (rc["tests_in_each_option_order"]["the_page_four"] == new["tests_in_each_option_order"]["the_page_four"]) out["shuffle_old_pairs_equal"] = all(rc["paired_shuffle_counts"]["pairs"][p] == new["paired_shuffle_counts"]["pairs"][p] for p in rc["paired_shuffle_counts"]["pairs"]) h_old, h_new = rc["holm"]["rows"], new["holm"]["rows"] out["holm_old_17_raw_p_equal"] = all(h_old[m]["p"] == h_new[m]["p"] for m in h_old) out["holm_verdicts_after_widening"] = { "told_apart_after_holm_17": rc["holm"]["told_apart_after_holm"], "told_apart_after_holm_19": new["holm"]["told_apart_after_holm"], "old_17_verdict_changed": sorted(m for m in h_old if h_old[m]["told_apart_after_holm"] != h_new[m]["told_apart_after_holm"]), "largest_holm_p_told_apart_17": rc["holm"]["largest_holm_p_told_apart"], "largest_holm_p_told_apart_19": new["holm"]["largest_holm_p_told_apart"]} return out # ------------------------------------------------------------------ 4. post hoc, public sets def post_hoc(C, src, readings: dict, K: dict) -> dict: out = {"label": POST_HOC, "six_order": {}} kit_opts = C.read_json(src, C.KIT_LOCK)["sets"]["c"]["options_in_kit_order"] for arm in ARMS: per = readings[arm]["reorder"]["right_by_order"] out["six_order"][arm] = {"schedule": "K", "right_by_order": per, "mean": statistics.mean(per), "fewest": min(per), "most": max(per), "kit_order_is_order_0": C.kit_order_first(src, [rel(arm, "kperm")], False, kit_opts), "order_0_equals_s0_count": per[0] == readings[arm]["s0"]["k"], "kit_order_rank_from_lowest": sorted(per).index(per[0]) + 1, "src": src.cite(rel(arm, "kperm"))} lock = sorted(C.read_json(src, C.KIT_LOCK)["sets"]["c"]["items_list"], key=lambda it: it["kit_index"]) fam = {it["text_sha256"]: it["author_family"] for it in lock} keys = {f: [k for k in K["kev-9b"] if fam[k[1]] == f] for f in ("gemma", "mistral")} out["author_family"] = {"kit_lock": src.cite(C.KIT_LOCK), "n": {f: len(v) for f, v in keys.items()}, "counts": {}, "against_the_word_counter": {}} for arm in ARMS + ("naive-bayes", "kev-9b"): out["author_family"]["counts"][arm] = {f: sum(K[arm][k] for k in keys[f]) for f in keys} for arm in ARMS: for f in keys: x, y, p = C.mcnemar(K[arm], K["naive-bayes"], keys[f]) out["author_family"]["against_the_word_counter"]["%s|%s" % (arm, f)] = { "only_model_right": x, "only_word_counter_right": y, "p": p} return out def answer_spread(src, arm: str, tasks) -> dict: return {t: dict(sorted(Counter(str(r.get("choice")) for r in live(src.rows(rel(arm, t)))).items())) for t in tasks} def constant_floors(src) -> dict: """What one answer given to every question scores, from the labels alone (public sets only here).""" kc = src.rows(rel("von-cpu", "kc")) ka = [r for r in src.rows(rel("von-cpu", "ka")) if r.get("refused") != "overlong"] whole_von = [r for r in ka if r.get("state_cut") is False] return {"six_way_any_one_guest": dict(Counter(r["label"] for r in kc)), "long_pages_sent_labels": dict(Counter(r["label"] for r in ka)), "long_pages_von_whole_labels": dict(Counter(r["label"] for r in whole_von))} def position_and_name_pulls(src) -> dict: out = {} for arm in ARMS: kc, kp = src.rows(rel(arm, "kc")), src.rows(rel(arm, "kperm")) per_order = [] for o in range(len(kp[0]["choices_by_order"])): first = sum(1 for r in kp if r["choices_by_order"][o] == r["orders"][o][0]) top_guest, top_n = Counter(r["choices_by_order"][o] for r in kp).most_common(1)[0] per_order.append({"order": o + 1, "first_offered_guest": kp[0]["orders"][o][0], "answer_is_the_first_offered": first, "most_chosen_guest": top_guest, "most_chosen_count": top_n}) right_by_label = Counter(r["label"] for r in kc if r["choice"] == r["label"]) out[arm] = {"kit_order_first_offered": "darwin", "kc_answer_is_darwin": sum(1 for r in kc if r["choice"] == "darwin"), "kc_right_by_speaker": dict(sorted(right_by_label.items())), "kperm_by_order": per_order, "src": [src.cite(rel(arm, "kc")), src.cite(rel(arm, "kperm"))]} return out def named_guest_pull(src, names_by_guest: dict) -> dict: """On the public six-way lines that name a guest (read, never printed), how often each model picks a guest the line names. Rows join the kit by position, checked label by label: the kit's `id` repeats.""" items = json.load(open(os.path.join(src.root, TASK_C), encoding="utf-8")) pats = {g: re.compile(r"\b(?:%s)\b" % "|".join(re.escape(n) for n in names)) for g, names in names_by_guest.items()} named = [{g for g, p in pats.items() if p.search(it["text"])} for it in items] out = {"kit": src.cite(TASK_C), "names_matched": names_by_guest, "lines": len(items), "lines_naming_a_guest": sum(1 for v in named if v), "lines_naming_their_own_speaker": sum(1 for it, v in zip(items, named) if it["label"] in v), "lines_naming_two_or_more_guests": sum(1 for v in named if len(v) > 1)} for arm in ARMS: kc = src.rows(rel(arm, "kc")) if [(r["id"], r["label"]) for r in kc] != [(it["id"], it["label"]) for it in items]: raise SystemExit("recount_von_laya: %s's six-way rows are not in the kit's order" % arm) hit = [(r, v) for r, v in zip(kc, named) if v] rank = Counter() for r in kc: p = r["probs"] rank[sorted(p, key=lambda g: -p[g]).index(r["label"]) + 1] += 1 out[arm] = {"on_lines_naming_a_guest": len(hit), "picks_a_named_guest": sum(1 for r, v in hit if r["choice"] in v), "right_on_those_lines": sum(1 for r, v in hit if r["choice"] == r["label"]), "picks_a_named_guest_on_lines_naming_one": sum(1 for r, v in hit if len(v) == 1 and r["choice"] in v), "lines_naming_one": sum(1 for r, v in hit if len(v) == 1), "true_speaker_rank_in_its_probabilities": dict(sorted(rank.items())), "src": src.cite(rel(arm, "kc"))} return out def kperm_ties(src) -> dict: out = {} for arm in ARMS: tied = lines_with_a_tie = moved_with_a_tie = 0 rows = src.rows(rel(arm, "kperm")) for r in rows: t = 0 for p in r["probabilities_by_order"]: m = max(p.values()) t += sum(1 for v in p.values() if v == m) > 1 tied += t lines_with_a_tie += t > 0 moved_with_a_tie += t > 0 and len(set(r["choices_by_order"])) > 1 out[arm] = {"order_answers": sum(len(r["probabilities_by_order"]) for r in rows), "order_answers_with_a_tied_top": tied, "lines_with_a_tied_order": lines_with_a_tie, "moved_lines_with_a_tied_order": moved_with_a_tie} return out def probability_decimals(src) -> dict: """The most decimals in any probability the server returned on a public choice row (the six-way here; the record also read the judge question's rows).""" out = {} for arm in ARMS: most = 0 for r in live(src.rows(rel(arm, "kc"))): for v in (r.get("probs") or {}).values(): s = repr(v) if "e" not in s and "." in s: most = max(most, len(s.split(".")[1])) out[arm] = most return out def paired_long_pages(C, src) -> dict: """The long pages each model took whole, beside Kev-9B's and Lev-4B's, joined by id (exact McNemar).""" out = {} kev = {r["id"]: C.is_right(r) for r in src.rows(KEV9B_KA) if r.get("choice") is not None} lev = {r["id"]: C.is_right(r) for r in src.rows(LEV4B_KA) if r.get("choice") is not None} for arm in ARMS: whole = {r["id"]: C.is_right(r) for r in live(src.rows(rel(arm, "ka"))) if r.get("state_cut") is False} o = {} if whole: both = sorted(set(whole) & set(kev)) x, y, p = C.mcnemar({k: whole[k] for k in both}, {k: kev[k] for k in both}) xl, yl, pl = C.mcnemar({k: whole[k] for k in both if k in lev}, {k: lev[k] for k in both if k in lev}) o["ka_whole|kev-9b"] = {"reached_here": len(whole), "kev_reached": len(kev), "in_both": len(both), "same_ids": set(whole) == set(kev), "reached_whole_here_only": sorted(set(whole) - set(kev)), "reached_by_kev_only": sorted(set(kev) - set(whole)), "right_here": sum(whole[k] for k in both), "kev_right": sum(kev[k] for k in both), "only_here": x, "only_kev": y, "p": p, "lev_on_these": {"right": sum(lev[k] for k in both if k in lev), "only_here": xl, "only_lev": yl, "p": pl}, "src": [src.cite(rel(arm, "ka")), src.cite(KEV9B_KA), src.cite(LEV4B_KA)]} out[arm] = o return out def chains_recheck_ka(src) -> dict: on = {r["id"]: r for r in live(src.rows(rel("von-cpu", "ka")))} off = {r["id"]: r for r in live(src.rows(rel(COMPANION, "ka")))} both = sorted(set(on) & set(off)) changed = [k for k in both if on[k]["choice"] != off[k]["choice"]] return {"ka": {"answered_in_both": len(both), "changed_answers": len(changed), "changed_on_gate_open": sum(1 for k in changed if on[k].get("von_chain_gate_open")), "changed_on_gate_shut": sum(1 for k in changed if not on[k].get("von_chain_gate_open"))}} def chains_pair_ka(C, src) -> dict: """Amendment DV-4's pairing of Von 1.3 with its chain engine on and off, on the long pages, by the definitions of instrument/chains_pair.py: joined by id, split by Von's own chain gate, and compared only within the same page reading in both runs (whole in both, or cut in both).""" def group(on, off, ids): right_on = {i: bool(C.is_right(on[i])) for i in ids} right_off = {i: bool(C.is_right(off[i])) for i in ids} changed = sorted(i for i in ids if on[i].get("choice") != off[i].get("choice")) dps = [] for i in ids: pa, pb = on[i].get("probs"), off[i].get("probs") if isinstance(pa, dict) and isinstance(pb, dict) and set(pa) == set(pb): dps.append(max(abs(float(pa[k]) - float(pb[k])) for k in pa)) x, y, p = C.mcnemar(right_on, right_off) if ids else (0, 0, None) return {"rows": len(ids), "same_answer": len(ids) - len(changed), "changed_answer": len(changed), "changed_ids": changed, "right_with_chains": sum(right_on.values()), "right_without_chains": sum(right_off.values()), "both_right": sum(1 for i in ids if right_on[i] and right_off[i]), "right_only_with_chains": x, "right_only_without_chains": y, "both_wrong": sum(1 for i in ids if not right_on[i] and not right_off[i]), "mcnemar_exact_p_DESCRIPTIVE": p, "max_abs_probability_difference": max(dps) if dps else None} def reading(r): return "whole" if r.get("state_cut") is False else "cut" if r.get("state_cut") is True else "unknown" on_rows, off_rows = src.rows(rel("von-cpu", "ka")), src.rows(rel(COMPANION, "ka")) on, off = {r["id"]: r for r in on_rows}, {r["id"]: r for r in off_rows} refused = {} for name, rows in (("von-cpu", on_rows), (COMPANION, off_rows)): refused[name] = dict(Counter(r["refused"] for r in rows if r.get("refused"))) both = [i for i in on if not on[i].get("refused") and not off[i].get("refused")] out = {"items": len(on), "answered_in_both": len(both), "refused_by_reason": refused, "groups": {}} for gate_name, want_open in (("gate_open", True), ("gate_shut", False)): ids = [i for i in both if bool(on[i].get("von_chain_gate_open")) is want_open] mixed = [i for i in ids if reading(on[i]) != reading(off[i])] out["groups"]["%s.page_read_differently" % gate_name] = {"rows": len(mixed), "ids": sorted(mixed)} for page in ("whole", "cut"): sub = [i for i in ids if reading(on[i]) == reading(off[i]) == page] out["groups"]["%s.%s_in_both" % (gate_name, page)] = group(on, off, sub) return {"pairs": {"ka": out}} def input_cuts(src) -> dict: """Each server's own account of the states it cut (Laya: usage.truncated, usage.state_tokens and usage.state_tokens_dropped; Von 1.3: truncation.kept_tokens and truncation.state_tokens), on the public sets.""" def cut_fields(arm, r): resp = r.get("kev_response") or {} if arm.startswith("von"): t = resp.get("truncation") if t: return True, t["kept_tokens"], t["state_tokens"], (resp.get("usage") or {}).get("input_tokens") return False, None, None, (resp.get("usage") or {}).get("input_tokens") u = resp.get("usage") or {} full = u.get("state_tokens") kept = full - u.get("state_tokens_dropped", 0) if isinstance(full, int) else None return u.get("truncated") is True, kept, full, u.get("input_tokens") def spread(xs): return {"min": min(xs), "median": statistics.median(xs), "max": max(xs), "n": len(xs)} if xs else None out = {"arms": {}} disagree = 0 for arm, sets in PUBLIC_SETS.items(): out["arms"][arm] = {} for t in sets: rows = src.rows(rel(arm, t)) rows_live = live(rows) cut_rows, whole_rows, kept, full, rest, share = [], [], [], [], [], [] for r in rows_live: c, k, f, inp = cut_fields(arm, r) if c != bool(r.get("state_cut")) and r.get("state_cut") is not None: disagree += 1 if c: cut_rows.append(r) kept.append(k) full.append(f) share.append(100.0 * k / f) if isinstance(inp, int): rest.append(inp - k) else: whole_rows.append(r) e = {"set": SET_NAMES[t], "rows": len(rows), "refused": len(rows) - len(rows_live), "answered": len(rows_live), "answered_on_a_cut_state": len(cut_rows), "answered_whole": len(whole_rows), "src": src.cite(rel(arm, t))} if cut_rows: e.update({"kept_tokens": spread(kept), "full_state_tokens": spread(full), "kept_share_pct_median": round(statistics.median(share), 1), "rest_of_the_window_tokens": spread(rest), "right_on_cut": sum(1 for r in cut_rows if r.get("choice") == r.get("label")), "right_on_whole": sum(1 for r in whole_rows if r.get("choice") == r.get("label"))}) if not arm.startswith("von"): e["input_tokens_on_cut_rows"] = sorted({cut_fields(arm, r)[3] for r in cut_rows}) out["arms"][arm][t] = e out["row_state_cut_disagrees_with_the_response"] = disagree return out def routing(src) -> dict: out = {"arms": {"laya-cpu": {}}} total = Counter() answered = 0 for t in PUBLIC_SETS["laya-cpu"]: rows = src.rows(rel("laya-cpu", t)) rows_live = live(rows) e = {"rows": len(rows), "answered": len(rows_live), "routed_to": dict(sorted(Counter(str(r.get("routed_to")) for r in rows_live).items()))} if t == "kperm": e["orders_routed_to"] = dict(sorted(Counter( str((x.get("routing") or {}).get("model")) for r in rows_live for x in r["kev_responses"]).items())) out["arms"]["laya-cpu"][t] = e total.update(e["routed_to"]) answered += len(rows_live) out["arms"]["laya-cpu"]["all_sets"] = {"answered": answered, "routed_to": dict(sorted(total.items()))} return out # ------------------------------------------------------------------ 5. the speed leg def jsonl(path: str) -> list[dict]: with open(path, encoding="utf-8") as fh: return [json.loads(line) for line in fh if line.strip()] def src_rows(root: str, rel_path: str) -> list[dict]: return jsonl(os.path.join(root, rel_path)) def nearest_rank(secs, q): s = sorted(secs) return s[min(len(s) - 1, int(round(q * (len(s) - 1))))] def speed_stats(secs) -> dict: return {"n": len(secs), "median_s": round(statistics.median(secs), 4), "p90_s": round(nearest_rank(secs, .90), 4), "p95_s": round(nearest_rank(secs, .95), 4), "mean_s": round(statistics.fmean(secs), 4), "min_s": round(min(secs), 4), "max_s": round(max(secs), 4)} def ts(utc: str) -> float: return dt.datetime.strptime(utc, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=dt.timezone.utc).timestamp() def highest(tel: list[dict], t0: float, t1: float) -> dict: win = [r for r in tel if t0 <= r["t"] <= t1 and r.get("mem_mib") is not None] if not win: return {"samples": 0} top = max(win, key=lambda r: r["mem_mib"]) first = min(r["t"] for r in win if r["mem_mib"] == top["mem_mib"]) return {"samples": len(win), "highest_mib": top["mem_mib"], "highest_first_seen_utc": dt.datetime.fromtimestamp(first, dt.timezone.utc).strftime( "%Y-%m-%dT%H:%M:%S.%fZ")[:-4] + "Z", "headroom_mib": CARD_TOTAL_MIB - top["mem_mib"], "power_w_max": max((r.get("power_w") or 0) for r in win), "temp_c_max": max((r.get("temp_c") or 0) for r in win)} def agreement(rows: list[dict], acc: list[dict]) -> dict: by = {r["uid"]: r for r in acc} diff = max(abs(r["probs"][k] - by[r["uid"]]["probs"][k]) for r in rows for k in r["probs"]) return {"same_choice": sum(1 for r in rows if by[r["uid"]]["choice"] == r["choice"]), "of": len(rows), "largest_probability_difference": round(diff, 6), "changed_uids": sorted(r["uid"] for r in rows if by[r["uid"]]["choice"] != r["choice"])} def speed(C, root: str) -> tuple[list[dict], dict, list[str]]: """Each timed arm, read the way the pinned table script reads it; and the page's own readings.""" base = os.path.join(root, BENCH) tel = jsonl(os.path.join(root, TELEMETRY)) rows_out, unrounded, checks = [], {}, [] for arm in SPEED_ARMS: s = json.load(open(os.path.join(base, "rows-speed", "%s.timed.speed.json" % arm), encoding="utf-8")) m = s["meta"] acc = src_rows(root, rel(SPEED_ACCURACY[arm], "kc")) passes, pooled, medians, right = [], [], [], [] for p in s["passes"]: rows = jsonl(os.path.join(base, "rows-speed", p["file"])) secs = [r["seconds"] for r in rows] pooled += secs medians.append(statistics.median(secs)) right.append(sum(1 for r in rows if r["choice"] == r["label"])) if C.p95(secs) != nearest_rank(secs, .95): checks.append("%s %s: recount_v2.p95 and the nearest rank differ" % (arm, p["rep"])) passes.append({"pass": p["rep"], "started": p["started"], "ended": p["ended"], "wall_seconds": p["wall_seconds"], **speed_stats(secs), "agreement_with_accuracy_leg": agreement(rows, acc)}) if C.p95(pooled) != nearest_rank(pooled, .95): checks.append("%s pooled: recount_v2.p95 and the nearest rank differ" % arm) unrounded[arm] = {"medians_s": medians, "right_by_pass": right, "right_by_pass_in_the_runs_own_summary": [p["summarise"]["correct"] for p in s["passes"]]} load = s["load"] mem = {"server_vmhwm_mb_after_warmups": load.get("server_hwm_mb_after_warmups"), "server_vmhwm_mb_end_of_pass_1": s["passes"][0].get("server_hwm_mb_at_end"), "server_vmhwm_mb_final_after_the_longest_input": s.get("server_hwm_mb_final")} if m["device"] == "cuda": points = [load["card_after_warmups"]["memory_used_mib"]] points += [(p.get("card_at_end") or {}).get("memory_used_mib") for p in s["passes"]] points.append((s["probe"].get("card_after") or {}).get("memory_used_mib")) points = [x for x in points if x is not None] mem.update({"card_before_server_mib": load["card_before_server"]["memory_used_mib"], "card_after_warmups_mib": load["card_after_warmups"]["memory_used_mib"], "grew_at_load_mib": load["grew_at_load_mib"], "card_after_each_pass_mib": [(p.get("card_at_end") or {}).get("memory_used_mib") for p in s["passes"]], "card_after_the_longest_input_mib": (s["probe"].get("card_after") or {}).get( "memory_used_mib"), "telemetry_window_utc": [load["card_before_server"]["utc"], s["ended"] + " + 1 s"], **highest(tel, ts(load["card_before_server"]["utc"]), ts(s["ended"]) + 1.0)}) mem["highest_below_a_runner_reading"] = bool(points) and mem.get("highest_mib", -1) < max(points) if load["card_after_warmups"]["memory_used_mib"] - load["card_before_server"]["memory_used_mib"] \ != load["grew_at_load_mib"]: checks.append("%s: grew at load is not after-warmups less before-server" % arm) else: w = highest(tel, ts(s["passes"][0]["started"]), ts(s["ended"]) + 1.0) mem["card_during_the_processor_arm"] = {"samples": w.get("samples"), "highest_mib": w.get("highest_mib"), "power_w_max": w.get("power_w_max")} banner = None for lg in sorted(glob.glob(os.path.join(base, "receipts-speed", "serve-%s-timed-*.log" % arm)))[-1:]: for line in open(lg, encoding="utf-8", errors="replace"): if line.startswith("Starting Von Decision Server"): banner = line.strip().split(" on http")[0] break rows_out.append({ "arm": arm, "label": m["label"], "model": m["model"], "processor": m["processor"], "device": m["device"], "card": m["card"], "power_limit_w": m["power_limit_w"], "precision": m["precision"], "server": m["server"], "package": m["versions"].get("von-sdk" if "von" in arm else "laya"), "torch": m["versions"].get("torch"), "threads": m["threads"], "cpu_affinity": m["cpu_affinity"], "os": m["os"].get("pretty_name"), "kernel": m["os"].get("kernel"), "device_seen": m["device_seen"], "von_banner": banner, "headline": dict(passes[0], pass_is="the registered headline"), "passes": passes, "pooled_324": speed_stats(pooled), "load": {"server_ready_s": load["server_ready_s"], "warmup_seconds": load["warmup_seconds"]}, "longest_input": {k: s["probe"].get(k) for k in ("filler_words", "filler_chars", "seconds", "usage", "truncation", "error")}, "memory": mem}) by = {r["arm"]: r for r in rows_out} page = {"what": "the page's own readings of the speed leg, which no file records as such: the card's highest by " "the page's rule (the higher of the telemetry's top reading in the arm's window and the runner's " "own reading after its longest input), what that lifts the telemetry's figure by, how far passes " "2 and 3 moved each median from pass 1, and the window of the timed passes", "highest_by_the_page_rule_mib": {}, "lift_over_the_telemetry_window_mib": {}, "medians_unrounded_s": {a: unrounded[a]["medians_s"] for a in SPEED_ARMS}, "right_by_pass": {a: unrounded[a]["right_by_pass"] for a in SPEED_ARMS}, "right_by_pass_in_the_runs_own_summary": {a: unrounded[a]["right_by_pass_in_the_runs_own_summary"] for a in SPEED_ARMS}, "same_choice_as_the_accuracy_leg_by_pass": { a: [p["agreement_with_accuracy_leg"]["same_choice"] for p in by[a]["passes"]] for a in SPEED_ARMS}, "passes_2_and_3_moved_the_median_pct": { a: [100 * (x - unrounded[a]["medians_s"][0]) / unrounded[a]["medians_s"][0] for x in unrounded[a]["medians_s"][1:]] for a in SPEED_ARMS}, "timed_window_utc": [by[SPEED_ARMS[0]]["passes"][0]["started"], json.load(open(os.path.join(base, "rows-speed", "%s.timed.speed.json" % SPEED_ARMS[-1]), encoding="utf-8"))["ended"]]} page["passes_2_and_3_largest_move_pct"] = max(abs(x) for v in page["passes_2_and_3_moved_the_median_pct"].values() for x in v) for a in ("speed-von-gpu", "speed-laya-gpu"): mm = by[a]["memory"] page["highest_by_the_page_rule_mib"][a] = max(mm["highest_mib"], mm["card_after_the_longest_input_mib"]) page["lift_over_the_telemetry_window_mib"][a] = page["highest_by_the_page_rule_mib"][a] - mm["highest_mib"] for a in SPEED_ARMS: if page["right_by_pass"][a] != page["right_by_pass_in_the_runs_own_summary"][a]: checks.append("%s: the right count by pass differs from the run's own summary" % a) return rows_out, page, checks # ------------------------------------------------------------------ comparing with the record def classify(path: str): for pat in PROVENANCE: if re.search(pat, path): return ("provenance", None) for pat, why in HELD: if re.search(pat, path): return ("held", why) for pat in PARTIAL: if re.search(pat, path): return ("partial", None) return (None, None) def leaves(x) -> int: if isinstance(x, dict): return sum(leaves(v) for k, v in x.items() if k not in CITE_KEYS and k not in TEXT_KEYS) if isinstance(x, list): return sum(leaves(v) for v in x) if x else 1 return 1 def kind(v) -> str: return "true/false" if isinstance(v, bool) else "numbers" if isinstance(v, (int, float)) else "other values" def compare(rec, mine, path: str, t: dict, mode=None) -> None: """Every leaf of the record against the value re-derived at the same path. `t` collects the tallies.""" if isinstance(rec, dict): for k, v in rec.items(): if k in CITE_KEYS or k in TEXT_KEYS: continue sub = "%s.%s" % (path, k) if path else k cls, why = classify(sub) if mode is None else (mode, None) if cls == "provenance": t["provenance"] += leaves(v) continue if cls == "held": t["held back"][why] = t["held back"].get(why, 0) + leaves(v) continue if not isinstance(mine, dict) or k not in mine: t["not run"].append(sub) continue if cls == "partial" and mode is None: if drop_cites(mine[k]) == drop_cites(v): t["equal_partial"] += leaves(v) else: t["partial"].append("%s: recorded %s, from this kit's public rows %s" % (sub, json.dumps(drop_cites(v), sort_keys=True), json.dumps(drop_cites(mine[k]), sort_keys=True))) continue compare(v, mine[k], sub, t, mode) return if isinstance(rec, list) and isinstance(mine, list) and len(rec) == len(mine) and rec: for i, (a, b) in enumerate(zip(rec, mine)): compare(a, b, "%s[%d]" % (path, i), t, mode) return if rec == mine and type(rec) is type(mine): t["equal"][kind(rec)] = t["equal"].get(kind(rec), 0) + 1 elif isinstance(rec, (int, float)) and isinstance(mine, (int, float)) and not isinstance(rec, bool) \ and not isinstance(mine, bool) and rec == mine: t["equal"]["numbers"] = t["equal"].get("numbers", 0) + 1 else: t["different"].append("%s: recorded %s, re-derived %s" % (path, json.dumps(rec), json.dumps(mine))) def new_tally() -> dict: return {"equal": {}, "equal_partial": 0, "different": [], "partial": [], "held back": {}, "not run": [], "provenance": 0} # ------------------------------------------------------------------ main def main() -> int: if len(sys.argv) != 2 or not os.path.isfile(os.path.join(sys.argv[1], "kit", "task_c.json")): raise SystemExit(__doc__.split("\n\n")[1]) rta = os.path.abspath(sys.argv[1]) verify_pins() rok = load("recount_on_the_kit.py") # it loads recount_v2.py from beside it, pinned above too C, root, record_v2, base = run_base(rok, rta) out_path = os.path.join(root, OUTPUT) print("recount_von_laya: the page's whole recount, over this kit and the Reading the answer kit") print(" output: %s" % out_path) b = base["totals"] print(" 1. the first fourteen models, recount_v2.py as recount_on_the_kit.py runs it, against recount-v2.json:") print(" {:,} equal, {:,} partial, {:,} different, {:,} held back, {:,} not run (recount_on_the_kit.py prints " "the detail)".format(b["equal"], b["partial"], b["different"], b["held back"], b["not run"])) src = C.__dict__["src"] R, K = C.__dict__["R"], C.__dict__["K"] record = json.load(open(os.path.join(HERE, RECORD), encoding="utf-8")) readings = {arm: arm_readings(C, src, arm) for arm in ARMS} for arm in ARMS: K[arm] = C.keyed(src, rel(arm, "kc"), "choice") mc, checked = widen_and_check(C, src, R, readings, K) mine = { "registered": {"arms": readings, "mcnemar": mc}, "checked": checked, "checked_old_entries_unchanged": old_entries_unchanged(record_v2, checked), "post_hoc": post_hoc(C, src, readings, K), "descriptive_extra": { "label": POST_HOC, "answer_spread": {arm: answer_spread(src, arm, ("kc", "ka")) for arm in ARMS}, "answer_spread_chains_off": answer_spread(src, COMPANION, ("ka",)), "constant_answer_floors": constant_floors(src), "six_way_position_and_name": position_and_name_pulls(src), "six_way_named_guest_pull": named_guest_pull(src, GUEST_NAMES), "six_way_named_guest_pull_with_first_names": named_guest_pull(src, GUEST_NAMES_WIDE), "kperm_ties": kperm_ties(src), "probability_decimals_seen": probability_decimals(src), "paired_reads_beside_kev9b_and_lev4b": paired_long_pages(C, src), "chains_recheck": chains_recheck_ka(src)}, "input_cuts": input_cuts(src), "chains_pair": chains_pair_ka(C, src), "routing": routing(src), "server_413": server_413s(src), "six_orders": six_orders(src), } mine = json.loads(json.dumps(mine, sort_keys=True)) # compared as the record is read: keys as text, no tuples t = new_tally() by_section = {} for section in sorted(k for k in record): before = (sum(t["equal"].values()) + t["equal_partial"], len(t["different"]), len(t["partial"]), sum(t["held back"].values()), len(t["not run"])) cls, why = classify(section) if cls == "provenance": t["provenance"] += leaves(record[section]) continue compare(record[section], mine.get(section, {}), section, t) after = (sum(t["equal"].values()) + t["equal_partial"], len(t["different"]), len(t["partial"]), sum(t["held back"].values()), len(t["not run"])) by_section[section] = [a - b for a, b in zip(after, before)] speed_rows, page, speed_checks = speed(C, root) tables = json.load(open(os.path.join(root, SPEED_TABLES), encoding="utf-8")) ts_ = new_tally() mine_by_arm = {r["arm"]: r for r in speed_rows} for rec_row in tables["rows"]: rec = {k: v for k, v in rec_row.items() if k != "box"} # a machine's name, which the kit's rules rewrite compare(rec, json.loads(json.dumps(mine_by_arm.get(rec_row["arm"], {}))), "speed.%s" % rec_row["arm"], ts_, mode="figure") ts_["different"] += speed_checks print(" 2. Von 1.3 and Laya, added 2026-10-01, re-derived from this kit's rows against %s:" % RECORD) print(" %-30s %7s %10s %8s %10s %8s" % ("section", "equal", "different", "partial", "held back", "not run")) for section, (eq, df, pa, hb, nr) in by_section.items(): print(" %-30s %7d %10d %8d %10d %8d" % (section, eq, df, pa, hb, nr)) eq_kinds = ", ".join("{:,} {}".format(n, k) for k, n in sorted(t["equal"].items())) print(" equal: %s; and %d figures the record counted over every set, which these public rows match" % (eq_kinds, t["equal_partial"])) h = mine["checked"]["holm"] print(" the correction for %d tests (recount_v2.py's own text says seventeen, the count it was written for): " "%d told apart, every row below %d behind: %s" % (h["tests"], h["told_apart_after_holm"], h["the_page_reads"]["below"], h["the_page_reads"]["every_row_below_it_behind_after_holm"])) print(" 3. their speed leg against %s: %s equal (%s), %d different" % (SPEED_TABLES[len("bench/"):], "{:,}".format(sum(ts_["equal"].values())), ", ".join("{:,} {}".format(n, k) for k, n in sorted(ts_["equal"].items())), len(ts_["different"]))) hp = page["highest_by_the_page_rule_mib"] print(" the page's readings: card B's highest, the higher of the telemetry's and the runner's own: Von 1.3 " "{:,.0f} MiB, Laya {:,.0f} MiB (Laya's {:,.0f} MiB above the telemetry's window); passes 2 and 3 moved a " "median by at most {:.2f} per cent; timed {} to {}".format( hp["speed-von-gpu"], hp["speed-laya-gpu"], page["lift_over_the_telemetry_window_mib"]["speed-laya-gpu"], page["passes_2_and_3_largest_move_pct"], page["timed_window_utc"][0], page["timed_window_utc"][1])) if t["held back"]: print(" held back (the rows are not in this kit; the figures are in %s):" % RECORD) for why, n in sorted(t["held back"].items()): print(" %-104s %5s" % (why, "{:,}".format(n))) if t["partial"]: print(" partial (the record also counted held-back rows):") for line in t["partial"]: print(" " + line) different = base["different"] + t["different"] + ts_["different"] if t["not run"]: print(" NOT RUN (a figure of the record this script did not re-derive):") for line in t["not run"]: print(" " + line) if different: print(" DIFFERENT:") for line in different: print(" " + line) result = {"what": "recount_von_laya.py's run over this kit: the first fourteen models' tallies, the two models' " "figures re-derived from their rows, and their speed leg; counts only", "base": base, "two_models": mine, "speed": {"rows": speed_rows, "page": page}, "tally": {"two_models": t, "two_models_by_section": by_section, "speed": ts_}} with open(out_path, "w", encoding="utf-8") as fh: json.dump(result, fh, indent=1, sort_keys=True, default=str) fh.write("\n") bad = bool(different or t["not run"]) print("recount_von_laya: {} (the first fourteen: {:,} equal; Von 1.3 and Laya: {:,} equal, {:,} held back; their " "speed leg: {:,} equal)".format("no re-derived figure differs from its record" if not bad else "%d figure(s) DIFFER or were not run" % (len(different) + len(t["not run"])), b["equal"], sum(t["equal"].values()) + t["equal_partial"], sum(t["held back"].values()), sum(ts_["equal"].values()))) return 1 if bad else 0 if __name__ == "__main__": sys.exit(main())