#!/usr/bin/env python3 """DV-4's descriptive pairing: Von with its chain engine ON (VON-1, as shipped) beside OFF (VON-1-NOCHAINS). python3 chains_pair.py [--rows bench/deciders-von-laya-2026-10-01/rows] [--recount-sha256 ] [--json out.json] Registered in Amendment DV-4, before either arm's judge or long-page rows existed. DESCRIPTIVE ONLY: no bar and no test is attached to anything here. It joins the two arms' rows by item id on the judge question (kj) and the long pages (ka), and splits them by Von's own chain gate (recorded on every Von row, DV-2): on a gate-open row the engine could act in VON-1; on a gate-shut row it could not, so the two runs are expected to give the same answer there, and the largest probability difference is printed to show it. Long pages are compared only within the same page reading in BOTH runs (whole in both, or cut in both; DV1.5). For each group: rows, the same answer, a changed answer (by id), the 2 x 2 of right-with-chains by right-without, and the page recount's exact McNemar p beside it, labelled descriptive. Standard library and the pinned recount only (loaded and digest-checked by `tables_von_laya.load_recount`); no model, no network; counts and item ids only, never a line's text or a digest. """ from __future__ import annotations import argparse import json import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from tables_von_laya import load_recount # noqa: E402 -- the pinned recount, digest-checked ON, OFF = "von-cpu", "von-cpu-nochains" SETS = ("kj", "ka") def max_abs_dp(a: dict, b: dict) -> float | None: """The largest difference between the two runs' probabilities for one item (None if either lacks them).""" pa, pb = a.get("probs"), b.get("probs") if not isinstance(pa, dict) or not isinstance(pb, dict) or set(pa) != set(pb): return None return max(abs(float(pa[k]) - float(pb[k])) for k in pa) def page_reading(r: dict) -> str: return "whole" if r.get("state_cut") is False else "cut" if r.get("state_cut") is True else "unknown" def group_reading(C, on: dict, off: dict, ids: list) -> dict: right_on = {i: bool(C.is_right(on[i])) for i in ids} right_off = {i: bool(C.is_right(off[i])) for i in ids} changed = sorted(i for i in ids if on[i].get("choice") != off[i].get("choice")) dps = [d for d in (max_abs_dp(on[i], off[i]) for i in ids) if d is not None] x, y, p = C.mcnemar(right_on, right_off) if ids else (0, 0, None) return {"rows": len(ids), "same_answer": len(ids) - len(changed), "changed_answer": len(changed), "changed_ids": changed, "right_with_chains": sum(right_on.values()), "right_without_chains": sum(right_off.values()), "both_right": sum(1 for i in ids if right_on[i] and right_off[i]), "right_only_with_chains": x, "right_only_without_chains": y, "both_wrong": sum(1 for i in ids if not right_on[i] and not right_off[i]), "mcnemar_exact_p_DESCRIPTIVE": p, "max_abs_probability_difference": max(dps) if dps else None} def pair(C, src, rows_rel: str, task: str) -> dict: on_rows = src.rows("%s/%s.%s.jsonl" % (rows_rel, ON, task)) off_rows = src.rows("%s/%s.%s.jsonl" % (rows_rel, OFF, task)) on = {r["id"]: r for r in on_rows} off = {r["id"]: r for r in off_rows} if len(on) != len(on_rows) or len(off) != len(off_rows) or set(on) != set(off): sys.exit("STOP: %s: the two runs do not hold the same item ids once each" % task) for i in on: if bool(on[i].get("von_chain_gate_open")) != bool(off[i].get("von_chain_gate_open")): sys.exit("STOP: %s %s: the two runs read Von's gate differently on the same state" % (task, i)) refused = {name: {} for name in (ON, OFF)} for name, rows in ((ON, on_rows), (OFF, off_rows)): for r in rows: if r.get("refused"): refused[name][r["refused"]] = refused[name].get(r["refused"], 0) + 1 both = [i for i in on if not on[i].get("refused") and not off[i].get("refused")] out = {"items": len(on), "answered_in_both": len(both), "refused_by_reason": refused, "groups": {}} for gate_name, want_open in (("gate_open", True), ("gate_shut", False)): ids = [i for i in both if bool(on[i].get("von_chain_gate_open")) is want_open] if task == "ka": # DV1.5: like is compared with like mixed = [i for i in ids if page_reading(on[i]) != page_reading(off[i])] out["groups"]["%s.page_read_differently" % gate_name] = {"rows": len(mixed), "ids": sorted(mixed)} for page in ("whole", "cut"): sub = [i for i in ids if page_reading(on[i]) == page_reading(off[i]) == page] out["groups"]["%s.%s_in_both" % (gate_name, page)] = group_reading(C, on, off, sub) else: out["groups"][gate_name] = group_reading(C, on, off, ids) out["files"] = {ON: src.cite("%s/%s.%s.jsonl" % (rows_rel, ON, task)), OFF: src.cite("%s/%s.%s.jsonl" % (rows_rel, OFF, task))} return out def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("root", help="an estate checkout (the rows' commit)") ap.add_argument("--rows", default="bench/deciders-von-laya-2026-10-01/rows") ap.add_argument("--json", default=None) ap.add_argument("--recount-sha256", default=None) a = ap.parse_args() C, got = load_recount(a.recount_sha256) src = C.Src(a.root) res = {"what": "DV-4: Von with its chain engine on (as shipped) beside off; descriptive, no bar, no test", "recount_v2_sha256": got, "head": src.head, "rows": a.rows, "pairs": {}} for task in SETS: have = [os.path.exists(os.path.join(a.root, "%s/%s.%s.jsonl" % (a.rows, arm, task))) for arm in (ON, OFF)] res["pairs"][task] = pair(C, src, a.rows, task) if all(have) else {"not run": dict(zip((ON, OFF), have))} text = json.dumps(res, indent=1, sort_keys=True) if a.json: with open(a.json, "w", encoding="utf-8") as fh: fh.write(text + "\n") print(text) if __name__ == "__main__": main()