#!/usr/bin/env python3 """`article/fills.json` + the filled author copy — every number read out of a scorer's own JSON. python3 harness/article_fill.py --plan # the placeholders and the file each one reads python3 harness/article_fill.py --fill # writes article/fills.json + …-v1.md, or REFUSES THE LAW THIS FILE IS -------------------- PLAN §2 and PREREG §9: *"No count in the prose is typed by hand — the scorer writes every number."* `report_build.py` holds that law for the internal report; this file holds it for the PUBLIC page, where breaking it is more expensive and much harder to notice, because a sentence reads fine either way. So the article's `⟦id: … | …⟧` slots are the ONLY places a number may appear that was not in the author's own prose, and each one is built here from named JSON fields whose paths are recorded beside the text in `article/fills.json`. A reader with the kit can walk from any figure on the page back to the file and the field it came from. WHAT IT REFUSES, AND WHY EACH REFUSAL IS A REFUSAL AND NOT A WARNING -------------------------------------------------------------------- 1. **An unfillable placeholder.** A missing file, a missing field, a null where a number belongs. Nothing is estimated and nothing is left as a bracket: a page that ships with a slot in it is a page that shipped a promise, and a page that ships an estimate is worse. 2. **A percentage under N = 30, and an interval over a unit set under 30** (PREREG §9). The scorers already guard this at the source (`score.pct`, `guarded_rate`, `grounded_rate_state`); this file guards the RENDERED TEXT independently, because the guard that matters is the one on the bytes that reach the reader. Both are checked per segment — a table cell, or a clause of a sentence — against the denominator printed nearest before them. 3. **The forbidden words** (PREREG §9's list, as the brief names them): *beats, wins, loses, best, leaderboard, rating*. The check is on the FILLS, never on the whole page: the author's own prose says "no leaderboard, no crown, no rating", and a negation is not a claim. 4. **A forbidden literal** — the round's own publication screen (`screen_literals`), run over every fill, because a fill is generated text that reaches a public page without a human reading every byte of it. WHAT IT DOES NOT DO ------------------- It makes no call, reads nothing under `results/` that a scorer did not write, and writes exactly two files, both under `article/`. It never touches `results/`, `prereg/` or the hub. """ from __future__ import annotations import sys sys.dont_write_bytecode = True import argparse import json import re from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path import prereg_integers as P import sample_record as sr import screen_literals as SL HARNESS_DIR = Path(__file__).resolve().parent BASE = HARNESS_DIR.parent #: The author copy this file fills, and the two files it writes. The version the fill produces is #: ONE constant: the output path and the front matter's `version:` cannot drift apart. SLUG = "two-new-frontier-models-at-the-rules-desk" AUTHOR_VERSION = "0.15" FILLED_VERSION = "11" AUTHOR_COPY = BASE / "article" / f"{SLUG}-v{AUTHOR_VERSION}.md" FILLED_COPY = BASE / "article" / f"{SLUG}-v{FILLED_VERSION}.md" FILLS_JSON = BASE / "article" / "fills.json" #: The placeholder shape v0.7 introduced (v0.8 keeps it): `⟦id: | ⟧`. PLACEHOLDER = re.compile(r"⟦id:\s*([a-z0-9-]+)\s*\|(.*?)⟧", re.S) #: Anything else between the brackets is a malformed slot, and a malformed slot is a refusal: an #: unnamed placeholder is one this file cannot address, so it would ship as a bracket on the page. ANY_BRACKET = re.compile(r"⟦(.*?)⟧", re.S) #: PREREG §9's forbidden vocabulary, word-bounded. `rating` bounded so that "operating" is not a hit. FORBIDDEN_WORDS = ("beats", "wins", "loses", "best", "leaderboard", "rating", "ratings") _FORBIDDEN_RE = re.compile(r"\b(" + "|".join(FORBIDDEN_WORDS) + r")\b", re.I) #: The two statistical shapes §9 permits only at N ≥ 30, as they actually appear in scorer output: #: a Wilson or bootstrap interval (`35/36 [0.9, 1.0]`, `interval 0.53–0.71`) and a percentage. _INTERVAL_RE = re.compile(r"\[\s*\d+(?:\.\d+)?\s*,\s*\d+(?:\.\d+)?\s*\]" r"|interval\s+\d+(?:\.\d+)?\s*[–-]\s*\d+(?:\.\d+)?", re.I) _PERCENT_RE = re.compile(r"\d+(?:\.\d+)?\s*%") #: A denominator, in every shape this round's scorers and this file print one: `35/36`, `35 of 36`, #: `over 36 cases`. The guard reads the one printed NEAREST BEFORE a figure, which is the one a #: reader attaches it to. _DENOM_RE = re.compile( r"\d+\s*/\s*(?P\d+)" r"|\b\d+\s+of\s+(?P\d+)\b" r"|\bover\s+(?:the\s+)?(?P\d+)\s+(?:cases|clusters|items|cells|comparisons)\b") #: The arms, in the order PLAN §3 lists them. Ids, never numbers — an arm absent from a scorer's own #: output is a visible missing ROW, not a silent renumbering, so the order is fixed and the presence #: is read. ARM_ORDER = ("cli-claude-fable-5-1", "openai-gpt-6-astra", "local-gemma4-26b") EM = "—" class Unfillable(Exception): """One placeholder that cannot be filled from the files on disk. Collected, never swallowed.""" def __init__(self, fill_id: str, why: str) -> None: super().__init__(f"{fill_id}: {why}") self.fill_id = fill_id self.why = why # ── the documents, and how a value is read out of one ──────────────────────── @dataclass(frozen=True) class Doc: """One readable document: the id a builder asks for, and the label `fills.json` records.""" doc_id: str label: str # repo-relative, and what a reader of the kit will look for path: Path | None # None for a document computed by code (the counting rules) what: str class Inputs: """Every document the fills may read, resolved once and loaded lazily. Receipts are addressed by GLOB and the newest match wins, because probe receipts are append-only and carry a UTC stamp in the name (``probes.Probe.write``): the second run of a probe never replaces the first, so "the newest" is the only correct read. """ RECEIPT_GLOBS = { "outside_read": ("*outside-prereg-read.json", "the outside family's read of the prereg"), "scaffolding_delta": ("*scaffolding-delta.json", "the 16-call CLI-scaffolding delta probe"), "egress": ("*g-egress.json", "G-EGRESS: what left the box, and to which host"), "cli_tools": ("*cli-claude-fable-5-1-g-tools.json", "G-TOOLS on the sealed CLI: its records carry the environment receipt"), "pen": ("*g-pen*.json", "G-PEN: the key / email / path / box-name scan"), "hostile": ("*hostile-read*.json", "the hostile reader's pass over the page"), "operator_read": ("*operator-read.json", "an operator's read of a draft, in their own words"), "canary_cli-claude-fable-5-1": ("*cli-claude-fable-5-1-double-canary.json", "the 1 % / 99 % double canary, CLI arm"), "canary_openai-gpt-6-astra": ("*openai-gpt-6-astra-double-canary.json", "the 1 % / 99 % double canary, Astra arm"), "canary_local-gemma4-26b": ("*local-gemma4-26b-double-canary.json", "the 1 % / 99 % double canary, local seat"), "effort_local-gemma4-26b": ("*local-gemma4-26b-g-effort.json", "G-EFFORT on the local seat, on the 32k canary prompt"), "identity_census": ("*cli-identity-census.json", "G-ID: the per-stream identity census over the CLI arm"), } #: Documents read as TEXT, not JSON: the sealed pre-registration (its §12 is counted, never #: retyped) and the public copy the kit ships (counted beside it; a mismatch is a refusal). TEXT_DOCS = { "prereg_sealed": ("prereg/PREREG-TWO-FRONTIERS.md", "the sealed pre-registration"), "prereg_public": ("prereg/PREREG-TWO-FRONTIERS-public.md", "the public pre-registration the kit ships"), } def __init__(self, *, base: Path | None = None, results: Path | None = None, receipts: Path | None = None, roster: Path | None = None, golden: Path | None = None, counting_rules: dict | None = None) -> None: self.base = Path(base) if base else BASE self.results = Path(results) if results else self.base / "results" self.receipts = Path(receipts) if receipts else self.base / "prereg" / "receipts" self.roster = Path(roster) if roster else self.base / "prereg" / "rosters.json" self.golden = Path(golden) if golden else self.base / "golden" self._counting_rules = counting_rules self._cache: dict[str, dict] = {} self.docs: dict[str, Doc] = {} self._register() # -- resolution ---------------------------------------------------------- def _add(self, doc_id: str, path: Path | None, what: str) -> None: label = doc_id if path is None else self._label(path) self.docs[doc_id] = Doc(doc_id, label, path, what) def _label(self, path: Path) -> str: try: return str(path.relative_to(self.base)) except ValueError: return path.name def _newest(self, pattern: str) -> Path | None: hits = sorted(self.receipts.glob(pattern)) return hits[-1] if hits else None def _register(self) -> None: self._add("legA", self.results / "legA" / "scores.json", "Leg A's gate table per arm (score.py --out)") self._add("g4", self.results / "legA" / "g4.json", "G4 groundedness, judged (legA_judge.py --score --out)") self._add("legH", self.results / "legH" / "pairwise.json", "the head-to-head (pairwise_score.py --out)") self._add("legC", self.results / "legC" / "cells.json", "the filing cabinet (legC_score.py --out)") self._add("legC_think_true", self.results / "legC" / "cells-think-true.json", "the filing cabinet's polarity control, local seat at think:true") self._add("bill", self.results / "bill.json", "the bill in four cost states") self._add("roster", self.roster, "the registered roster: the window, stamped at both ends") self._add("bank", self.golden / "offload-bank-r1.json", "the r1 bank's own metadata: frozen date, N, class composition, substitution") self._add("queries", self.golden / "legA-queries.json", "the queries as sent: the canonical asks verbatim, and the substituted ones") self._add("fixture", self.golden / "legC-fixture.json", "Leg C's fixture: tiers, depths, unique fractions, the seed") self._add("needles", self.golden / "legC-needles.json", "Leg C's needles as drawn: the seed, the integer range, the corpus") self._add("counting_rules", None, "the kit's counting-rules.json (harness/kit_build.py::counting_rules)") self._add("panel", self.base / "prereg" / "panel.json", "the registered judging panel: seats, families, transports") self._add("kit_index", self.results / "kit" / "index.json", "the kit's own index: what ships and what is withheld") for doc_id, (rel, what) in self.TEXT_DOCS.items(): self._add(doc_id, self.base / rel, what) for doc_id, (pattern, what) in self.RECEIPT_GLOBS.items(): self._add(doc_id, self._newest(pattern), f"{what} — newest `{pattern}`") if self.docs[doc_id].path is None: # keep the PATTERN as the label so a refusal names exactly what it looked for self.docs[doc_id] = Doc(doc_id, f"prereg/receipts/{pattern}", None, what) # -- reading ------------------------------------------------------------- def state(self, doc_id: str) -> str: doc = self.docs[doc_id] if doc.doc_id == "counting_rules": return "computed by code" if doc.path is None: return "ABSENT — no receipt matches" return "present" if doc.path.is_file() else "ABSENT" def data(self, doc_id: str, *, fill_id: str = "?") -> dict: if doc_id in self._cache: return self._cache[doc_id] doc = self.docs[doc_id] if doc_id == "counting_rules": if self._counting_rules is None: import kit_build self._counting_rules = kit_build.counting_rules() self._cache[doc_id] = self._counting_rules return self._cache[doc_id] if doc.path is None or not doc.path.is_file(): raise Unfillable(fill_id, f"no `{doc.label}` on disk — {doc.what}. Nothing is " "estimated: the fill needs the scorer's own file.") try: self._cache[doc_id] = json.loads(doc.path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise Unfillable(fill_id, f"`{doc.label}` is not readable JSON ({exc})") from exc return self._cache[doc_id] # -- text documents and rows files ----------------------------------------- def text(self, doc_id: str, *, fill_id: str = "?") -> str: doc = self.docs[doc_id] if doc.path is None or not doc.path.is_file(): raise Unfillable(fill_id, f"no `{doc.label}` on disk — {doc.what}.") return doc.path.read_text(encoding="utf-8") def receipt_paths(self) -> list[Path]: return sorted(self.receipts.glob("*.json")) def receipt(self, path: Path, *, fill_id: str = "?") -> dict: key = f"receipt:{path.name}" if key in self._cache: return self._cache[key] try: data = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise Unfillable(fill_id, f"receipt `{path.name}` is not readable JSON ({exc})") from exc self.docs.setdefault("receipts", Doc("receipts", "prereg/receipts/*.json", None, "every receipt on disk")) self._cache[key] = data if isinstance(data, dict) else {} return self._cache[key] def rows_path(self, leg: str, arm: str) -> Path: return self.results / f"leg{leg}" / "rows" / f"{arm}.jsonl" def rows(self, leg: str, arm: str, *, fill_id: str = "?") -> list[dict]: """The scored rows file of one arm on one leg — the record (PREREG A9), never `sent/`.""" key = f"rows:{leg}:{arm}" if key in self._cache: return self._cache[key] path = self.rows_path(leg, arm) if not path.is_file(): raise Unfillable(fill_id, f"no rows file for `{arm}` on Leg {leg} " f"(`{self._label(path)}`); a quote is read from the row " "it came from or it is not printed.") self.docs.setdefault(f"rows:{arm}", Doc(f"rows:{arm}", self._label(path), path, f"Leg {leg} rows, `{arm}`")) rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] self._cache[key] = rows return rows def row(self, leg: str, arm: str, case_id: str, *, rep: int, fill_id: str = "?") -> dict: hits = [r for r in self.rows(leg, arm, fill_id=fill_id) if r.get("case_id") == case_id and r.get("rep") == rep and r.get("collection_state") == sr.COLLECTED and r.get("response")] if len(hits) != 1: raise Unfillable(fill_id, f"`{arm}` has {len(hits)} COLLECTED rows for {case_id} rep " f"{rep}; a quote needs exactly one.") return hits[0] def modal_row(self, leg: str, arm: str, case_id: str, *, fill_id: str = "?") -> dict: """The row whose response is the modal one over the case's COLLECTED reps (rep 1 if no mode) — the same rule the scorer judges by (score.modal_text).""" hits = [r for r in self.rows(leg, arm, fill_id=fill_id) if r.get("case_id") == case_id and r.get("collection_state") == sr.COLLECTED and r.get("response")] if not hits: raise Unfillable(fill_id, f"`{arm}` has no COLLECTED row for {case_id}.") counts: dict[str, int] = {} for r in hits: counts[r["response"]] = counts.get(r["response"], 0) + 1 best_n = max(counts.values()) if best_n == 1: return min(hits, key=lambda r: r.get("rep", 0)) text = next(t for t, n in counts.items() if n == best_n) return min((r for r in hits if r["response"] == text), key=lambda r: r.get("rep", 0)) _MISSING = object() def plain_floor(text) -> str: """A floor description as a plain phrase: the design doc's own markdown line, stripped of its list marker, bold markers and backticks, so it can sit inside a sentence or a table cell without the pour's converter meeting a nested marker (a `**…**` inside a `` refused the pour).""" if not isinstance(text, str): return str(text) t = text.strip() t = re.sub(r"^[-*]\s+", "", t) t = t.replace("**", "").replace("`", "") t = re.sub(r"^G[0-9][a-z]?\s+[—-]\s+", "", t) # the fill prints the gate id itself return re.sub(r"\s+", " ", t).strip() def dig(data, path: str): """`a.b[0].c` out of nested JSON; `_MISSING` when any step is absent.""" node = data for step in [s for s in re.split(r"\.(?![^\[]*\])", path) if s]: while step: m = re.match(r"^([^\[\]]+)|^\[(\d+)\]", step) if not m: return _MISSING if m.group(1) is not None: key = m.group(1) if isinstance(node, list): # a list of records addressed by its id field (the roster's `arms`, a panel's # `seats`): the key is the record's arm_id / seat / id / case_id hit = [x for x in node if isinstance(x, dict) and any(x.get(k) == key for k in ("arm_id", "seat", "id", "case_id"))] if len(hit) != 1: return _MISSING node = hit[0] elif not isinstance(node, dict) or key not in node: return _MISSING else: node = node[key] else: idx = int(m.group(2)) if not isinstance(node, list) or idx >= len(node): return _MISSING node = node[idx] step = step[m.end():] return node class Cell: """One placeholder under construction: it reads values and records where each came from.""" def __init__(self, fill_id: str, inputs: Inputs) -> None: self.id = fill_id self.inputs = inputs self.sources: list[dict] = [] def _record(self, doc_id: str, path: str) -> None: row = {"file": self.inputs.docs[doc_id].label, "json_path": path} if row not in self.sources: self.sources.append(row) def get(self, doc_id: str, path: str, *, allow_null: bool = False): """A value, recorded. A missing field or a null where a figure belongs is a refusal.""" data = self.inputs.data(doc_id, fill_id=self.id) value = dig(data, path) if value is _MISSING: raise Unfillable(self.id, f"`{self.inputs.docs[doc_id].label}` has no `{path}`. The " "field is not defaulted: a number this file cannot read is a " "number it will not print.") if value is None and not allow_null: raise Unfillable(self.id, f"`{self.inputs.docs[doc_id].label}`:`{path}` is null. A null " "where a figure belongs is a hole, not a zero.") self._record(doc_id, path) return value def opt(self, doc_id: str, path: str, default=None): """A field that is permitted to be absent (and is recorded only when it is there).""" try: data = self.inputs.data(doc_id, fill_id=self.id) except Unfillable: return default value = dig(data, path) if value is _MISSING or value is None: return default self._record(doc_id, path) return value def arms(self, doc_id: str) -> list[str]: """The arms a scorer actually scored, in PLAN §3's order, then any extra key, sorted.""" data = self.inputs.data(doc_id, fill_id=self.id) arms = dig(data, "arms") if arms is _MISSING or not isinstance(arms, dict) or not arms: raise Unfillable(self.id, f"`{self.inputs.docs[doc_id].label}` carries no `arms` map; " "there is nothing to tabulate.") known = [a for a in ARM_ORDER if a in arms] return known + sorted(k for k in arms if k not in ARM_ORDER) # ── the guards ─────────────────────────────────────────────────────────────── def segments(text: str) -> list[str]: """The text in the units a reader reads a figure in: a table CELL, or a SENTENCE. Not a clause: a scorer's own sentence puts the denominator early and the interval late (*"…over the 36 cases (2,000 resamples, percentile; t(35)) puts it at [0.56, 0.72]"*), so a guard that split on `;` would refuse the sentence the prereg wrote. """ out: list[str] = [] for line in text.splitlines(): if "|" in line: out += [c for c in line.split("|") if c.strip()] else: out += [s for s in re.split(r"(?<=[.!?])\s+", line) if s.strip()] return out def _denominators(segment: str) -> list[tuple[int, int]]: """(position, denominator) for every denominator printed in the segment.""" out = [] for m in _DENOM_RE.finditer(segment): value = next((v for v in m.groupdict().values() if v is not None), None) if value is not None: out.append((m.start(), int(value))) return out def unguarded_statistics(text: str) -> list[str]: """Every segment carrying an interval or a percentage whose nearest denominator is under 30.""" bad: list[str] = [] for seg in segments(text): for hit in list(_INTERVAL_RE.finditer(seg)) + list(_PERCENT_RE.finditer(seg)): before = [d for pos, d in _denominators(seg) if pos < hit.start()] if not before: bad.append(f"{hit.group(0)!r} with no denominator printed before it, in: {seg.strip()!r}") elif before[-1] < P.INTERVAL_MIN_N: bad.append(f"{hit.group(0)!r} over a unit set of {before[-1]}, below the registered " f"N ≥ {P.INTERVAL_MIN_N}, in: {seg.strip()!r}") return bad def guard(fill_id: str, text: str, *, rules=None, verbatim: bool = False) -> None: """The three checks every fill passes before it can reach the page. A VERBATIM fill is a model's own reply, printed as bytes: its words are not this page's claims, so the vocabulary and statistics guards do not run over it (a rulebook citation `[2, 8]` is not an interval); the publication screen runs over it like anything else. """ if verbatim: literals = SL.hits(text, rules) if literals: raise Unfillable(fill_id, f"the round's publication screen refuses this fill: " f"{len(literals)} forbidden literal(s).") return words = sorted({m.group(0).lower() for m in _FORBIDDEN_RE.finditer(text)}) if words: raise Unfillable(fill_id, "the fill carries PREREG §9's forbidden vocabulary " f"({', '.join(words)}). The page publishes a count with its " "denominator, never an ordering.") bad = unguarded_statistics(text) if bad: raise Unfillable(fill_id, "PREREG §9 permits an interval or a percentage only where the " "independent-unit N is at least " f"{P.INTERVAL_MIN_N}: {'; '.join(bad)}") literals = SL.hits(text, rules) if literals: raise Unfillable(fill_id, f"the round's publication screen refuses this fill: " f"{len(literals)} forbidden literal(s). The fill is generated " "text on a public page and is screened like any other.") # ── formatting helpers (no figure is formatted that was not read) ──────────── def num(value) -> str: """An int with thousands separators; anything else as its own JSON-ish self.""" if isinstance(value, bool): return "yes" if value else "no" if isinstance(value, int): return f"{value:,}" if isinstance(value, float): return f"{value:,.4g}" return str(value) def plural(n, word: str, many: str | None = None) -> str: """`1 marker` / `2 markers` — the number is read, the ending is derived from it.""" return f"{num(n)} {word if n == 1 else (many or word + 's')}" def usd(value) -> str: if isinstance(value, (int, float)) and not isinstance(value, bool): return f"${value:,.2f}" return str(value) def short_sha(value: str) -> str: return f"{str(value)[:8]}…" def stamp_day(value: str) -> str: m = re.match(r"(\d{4}-\d{2}-\d{2})", str(value)) if not m: raise ValueError(value) return m.group(1) def stamp_second(value: str) -> str: """`YYYY-MM-DD HH:MM:SS` — the prose that carries it says UTC, so the Z is not doubled.""" m = re.match(r"(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2})", str(value)) if not m: raise ValueError(value) return f"{m.group(1)} {m.group(2)}" def stamp_minute(value: str) -> str: m = re.match(r"(\d{4}-\d{2}-\d{2})[T ](\d{2}):(\d{2})", str(value)) if not m: raise ValueError(value) return f"{m.group(1)} {m.group(2)}:{m.group(3)}Z" _KN_RE = re.compile(r"(\d+)\s*/\s*(\d+)") def count_of(cell: str) -> int | None: """The numerator of a `k/n …` scorer cell, for the tie band. None when the cell is a state.""" m = _KN_RE.search(str(cell)) return int(m.group(1)) if m else None def table(header: list[str], rows: list[list[str]]) -> str: out = ["| " + " | ".join(header) + " |", "|" + "|".join("---" for _ in header) + "|"] out += ["| " + " | ".join(str(c) for c in r) + " |" for r in rows] return "\n".join(out) # ── the builders, one per placeholder, in document order ───────────────────── #: The order the page prints the arms in: the order the title names them. `ARM_ORDER` (PLAN §3) #: still governs presence — an arm a scorer did not score is a missing row — this only governs #: where a present row sits, so that the author's own arm is not row one of every table. PAGE_ORDER = ("openai-gpt-6-astra", "cli-claude-fable-5-1", "local-gemma4-26b") #: The plain names the prose uses for the arm ids. A NAME is not a number: this table is the one #: place the page's words for a model are typed, so the ids in the tables and the names in the #: sentences cannot drift apart. DISPLAY = { "openai-gpt-6-astra": "GPT-6 Astra", "cli-claude-fable-5-1": "Claude Fable 5.1", "local-gemma4-26b": "the local seat", } ROAD = { "openai-gpt-6-astra": "through OpenAI's API", "cli-claude-fable-5-1": "through Anthropic's sealed command-line tool", "local-gemma4-26b": "Gemma 4 (26B) on our own hardware, at its production settings", } #: The judging seats' plain names (a name table, like DISPLAY). SEAT_NAMES = { "gemma4-31b": "Google's Gemma 4 (31B)", "mistral-large-3-675b": "Mistral Large 3", "nemotron-3-ultra": "NVIDIA's Nemotron 3 Ultra", "kimi-k3": "Moonshot's Kimi K3", "deepseek-v4-pro": "DeepSeek V4 Pro", "glm-5.3": "Zhipu's GLM 5.3", "qwen3.5-397b": "Alibaba's Qwen 3.5 (397B)", } #: How each transport counts a prompt (prereg A8) — words, printed beside the ratio they qualify. PROMPT_TOKEN_RULE = { "agent-harness-cli": "input + cache-creation + cache-read tokens, summed (prereg A8)", "openai-api": "the endpoint's `prompt_tokens`", "local-ollama": "the runtime's `prompt_eval_count`", } def page_order(arms: list[str]) -> list[str]: known = [a for a in PAGE_ORDER if a in arms] return known + [a for a in arms if a not in PAGE_ORDER] def name(arm: str) -> str: return DISPLAY.get(arm, f"`{arm}`") def kn(cell) -> tuple[int, int] | None: """(k, n) out of a `k/n …` scorer cell; None when the cell is a state, not a count.""" m = _KN_RE.search(str(cell)) return (int(m.group(1)), int(m.group(2))) if m else None def of(cell) -> str: """`34/36 [0.819, 0.985]` → `34 of 36` (the interval stays in the table, not the prose).""" pair = kn(cell) return f"{pair[0]} of {pair[1]}" if pair else str(cell) def reads(passed) -> str: """The scorer's own verdict field, in the page's words.""" if passed is True: return "cleared" if passed is False: return "missed" return "no verdict" def floor_number(text: str) -> str: """The comparison a floor asks for, as printed: `≥ 35/36`, `≤ 2/36`, `0/6`.""" m = re.search(r"([≥≤>=<]+\s*)?(\d+\s*/\s*\d+)", plain_floor(text)) if not m: return plain_floor(text) sign = (m.group(1) or "").replace(">=", "≥").replace("<=", "≤").strip() return f"{sign} {m.group(2).replace(' ', '')}".strip() def _window(c: Cell, which: str) -> str: value = c.get("roster", f"window.{which}_utc", allow_null=True) if not value: raise Unfillable(c.id, f"the roster's `window.{which}_utc` is null: the window is not " "stamped at both ends yet (PLAN §2 fence 3). A page that reports a " "window cannot be dated before that window closes.") return value def window_words(c: Cell) -> str: """`2026-09-05 14:13:33–18:15:00 UTC` — one day, both ends, for the cold-scroll stamps.""" opened, closed = _window(c, "opened"), _window(c, "closed") o, cl = stamp_second(opened), stamp_second(closed) if o[:10] == cl[:10]: return f"{o}–{cl[11:]} UTC" return f"{o}–{cl} UTC" # The release dateline. A draft is dated the day the window closed; the released page is dated # the slot an operator ruled for it, which no scorer knows. `--release-dateline` sets it, the two # fills below read it, and fills.json records the override on both rows and at the top level. RELEASE_DATELINE_UTC: str | None = None RELEASE_DATELINE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z$") def set_release_dateline(utc: str | None) -> None: global RELEASE_DATELINE_UTC if utc is not None and not RELEASE_DATELINE_RE.match(utc): raise SystemExit(f"article_fill: --release-dateline `{utc}` is not YYYY-MM-DDTHH:MMZ; " "the byline takes the ruled slot verbatim and never guesses it.") RELEASE_DATELINE_UTC = utc _STAMP_RE = re.compile(r"(\d{4})-?(\d{2})-?(\d{2})T(\d{2}):?(\d{2})") def _receipt_when(value) -> str: """`2026-09-05 22:59Z` from `20260905T225917Z` or `2026-09-05T22:59:17Z`; never invented.""" m = _STAMP_RE.match(str(value or "").strip()) if not m: raise Unfillable("?", f"a receipt stamp {value!r} is not in either registered shape") return f"{m.group(1)}-{m.group(2)}-{m.group(3)} {m.group(4)}:{m.group(5)}Z" def _receipt_hm(value) -> str: return _receipt_when(value)[11:] def fill_published(c: Cell) -> str: if RELEASE_DATELINE_UTC: return stamp_minute(RELEASE_DATELINE_UTC) return stamp_minute(_window(c, "closed")) def fill_dateline_date(c: Cell) -> str: if RELEASE_DATELINE_UTC: return stamp_day(RELEASE_DATELINE_UTC) return stamp_day(_window(c, "closed")) def _probe_stamps(c: Cell) -> list[str]: """Every probe receipt's own stamp (the rows of the probe ledger), normalised for ordering.""" out = [] for path in c.inputs.receipt_paths(): data = c.inputs.receipt(path, fill_id=c.id) gate = str(data.get("gate") or data.get("probe") or "") if not gate or gate.lower() in ("hostile-read", "operator-read", "g-pen", "g-egress"): continue when = data.get("utc") or data.get("started_utc") or data.get("sent_utc") if when: out.append(_receipt_when(when)) if not out: raise Unfillable(c.id, "no probe receipt carries a stamp; the ledger's dates are not typed.") return sorted(out) def fill_ledger_stamp(c: Cell) -> str: probes = _probe_stamps(c) egress = _receipt_when(c.get("egress", "utc")) pen = _receipt_when(c.get("pen", "utc")) return (f"*Probes {probes[0]}–{probes[-1][11:] if probes[-1][:10] == probes[0][:10] else probes[-1]} · " f"socket sample {egress} · pen scan {pen} — every row below is dated by its own receipt, " f"and the scored window is stamped on the sections it covers.*") def fill_bill_stamp(c: Cell) -> str: """PREREG A20 (3): the bill's own earliest RECORD, from the bill, not the earliest receipt. The v6 stamp read the earliest probe receipt (04:39Z — the outside family's read of the pre-registration), which the bill does not carry: every billed record is 14:03:58Z or later. A stamp that widened past its own set was the same defect as the one that had been too narrow, on the section whose whole subject is what was billed. The bill now emits `records_window` over the records it sums, and this reads it. """ earliest = c.get("bill", "records_window.earliest") return (f"*The scored window, {window_words(c)}, plus the probes and warm-ups the bill also " f"carries, the earliest record at {_receipt_when(earliest)}.*") def fill_window_stamp(c: Cell) -> str: return f"*Measured {window_words(c)}.*" def fill_limits_stamp(c: Cell) -> str: """PREREG A20 (5): the limits section's own stamp — its one measurement predates the window. Every other data heading is stamped to the scored window, and "What this page does not say" cannot be: the scaffolding delta ran before the window opened, which is exactly the fact the section's reader needs. So this stamp is read from that probe's own receipt, and it says which side of the window the measurement sits on. """ started = c.get("scaffolding_delta", "started_utc") return (f"*Scaffolding delta measured {_receipt_when(started)}, before the window opened; " f"everything else here sits in the scored window, {window_words(c)}.*") def fill_panel_shape(c: Cell) -> str: seats = c.get("panel", "panel.seats") c._record("panel", "panel.seats[].seat / .family / .transport") families = sorted({s["family"] for s in seats}) transports = sorted({s["transport"] for s in seats}) named = ", ".join(SEAT_NAMES.get(s["seat"], f"`{s['seat']}`") for s in seats) where = ("every one of them through the ollama.com shelf; none ran on our hardware" if transports == ["ollama-cloud"] else "on these transports: " + ", ".join(transports)) return (f"The panel is {num(len(seats))} judging seats from {num(len(families))} model " f"families, and they read the judged answers blind — {named} — {where}.") # ── the outside read, counted out of the sealed pre-registration's own text ── _A3_DISPOSITION_RE = re.compile(r"\*\*(FOLDED|ANSWERED|DISCLOSED)[^*]*\*\*:?(.*?)(?=\n\s*- \*\*(?:FOLDED|ANSWERED|DISCLOSED)|\n- \*\*A\d+ ·|\Z)", re.S) _FINDING_RE = re.compile(r"\((\d+)\)") def outside_read_dispositions(prereg_text: str) -> dict[str, int]: """How many of the outside reader's numbered findings fell under each disposition in §12 A3.""" start = prereg_text.find("- **A3 ·") if start < 0: return {} end = prereg_text.find("\n- **A4 ·", start) block = prereg_text[start:end if end > 0 else None] out: dict[str, int] = {} for m in _A3_DISPOSITION_RE.finditer(block): out[m.group(1)] = len(set(_FINDING_RE.findall(m.group(2)))) return out def outside_read_ratings(prereg_text: str) -> dict[str, int]: """The outside reader's own severity labels (MATERIAL / DECISIVE / COSMETIC) as A3 quotes them.""" start = prereg_text.find("- **A3 ·") end = prereg_text.find("\n- **A4 ·", start) block = prereg_text[start:end if end > 0 else None] out: dict[str, int] = {} for _n, (_heading, label) in outside_read_rated(prereg_text).items(): out[label] = out.get(label, 0) + 1 return dict(sorted(out.items(), key=lambda kv: -kv[1])) def outside_read_rated(prereg_text: str) -> dict[int, tuple[str, str]]: """{finding number: (heading, label)} for every finding A3 rates, read from the block itself.""" start = prereg_text.find("- **A3 ·"); end = prereg_text.find("\n- **A4 ·", start) block = prereg_text[start:end if end > 0 else None] out: dict[int, tuple[str, str]] = {} heading = None for piece in re.split(r"(FOLDED|ANSWERED|DISCLOSED)", block): if piece in ("FOLDED", "ANSWERED", "DISCLOSED"): heading = piece; continue if not heading: continue items = re.split(r"\((\d+)\)", piece) for k in range(1, len(items) - 1, 2): # the reader's own word, wherever A3 carries it for that finding: "(MATERIAL)", # "… DECISIVE" inside the quoted finding, or "(rated DECISIVE)" label = re.search(r"\b(MATERIAL|DECISIVE|COSMETIC)\b", items[k + 1]) if label: out[int(items[k])] = (heading, label.group(1)) return out def fill_outside_read_summary(c: Cell) -> str: model = c.get("outside_read", "model") sent = c.get("outside_read", "sent_utc") prereg_sha = c.get("outside_read", "request_sha256.prereg_bytes") reply_sha = c.get("outside_read", "reply_sha256") read = c.get("outside_read", "counters.prompt_eval_count") wrote = c.get("outside_read", "counters.eval_count") prereg_text = c.inputs.text("prereg_sealed", fill_id=c.id) disp = outside_read_dispositions(prereg_text) ratings = outside_read_ratings(prereg_text) c._record("prereg_sealed", "§12 A3 — the numbered findings under FOLDED / ANSWERED / DISCLOSED") if not disp or "FOLDED" not in disp: raise Unfillable(c.id, "the sealed pre-registration's §12 A3 does not carry the outside " "read's findings under FOLDED / ANSWERED / DISCLOSED headings in the " "shape this fill counts; the counts are not typed from memory.") total = sum(disp.values()) rated = outside_read_rated(prereg_text) decisive = sorted(n for n, (h, l) in rated.items() if l == "DECISIVE") if (ratings.get("DECISIVE") != 2 or ratings.get("MATERIAL") != 1 or [rated[n][0] for n in decisive] != ["FOLDED", "ANSWERED"]): raise Unfillable(c.id, f"A3's ratings read {ratings} with DECISIVE findings {decisive} under " f"{[rated[n][0] for n in decisive]}; the sentence below names one " "DECISIVE finding folded and one answered, and refuses any other shape.") return (f"The outside reader of the pre-registration was `{model}`; at {stamp_minute(sent)} it was sent the " f"pre-registration's own bytes (sha {short_sha(prereg_sha)}), read {num(read)} tokens " f"and wrote {num(wrote)} back. It raised {num(total)} findings, rated in its own words " f"({', '.join(f'{num(n)} {label}' for label, n in ratings.items())}): " f"{num(disp.get('FOLDED', 0))} changed the instrument before the first call — among " f"them one of the two the reader rated DECISIVE — the local seat's second sitting of the " f"filing cabinet with its thinking on (finding {num(decisive[0])}) — and the one it rated " f"MATERIAL, the authorship of the thirty-four replacement questions, which moved from our " f"own local seat to the outside family itself; " f"{num(disp.get('ANSWERED', 0))} were misreadings " f"answered in the text — one of them the reader's other DECISIVE finding, a reading of " f"the head-to-head's recusal (finding {num(decisive[1])}: the local seat is not in that leg, " f"and the registration's own cell count was corrected to thirty-six); " f"{num(disp.get('DISCLOSED', 0))} were disclosed and left " f"standing — the command-line tool's telemetry switches, the self-agreement reading, " f"the framing of the within-arm citation-set gate, and the order the command-line arm " f"ran its two legs in. Every one is quoted with its disposition in the pre-registration's " f"amendment A3, and the reply itself (sha {short_sha(reply_sha)}) ships in the kit " f"under `receipts/`, unedited.") def fill_rules_desk_lead(c: Cell) -> str: n = c.get("bank", "n") frozen = c.get("bank", "frozen") comp = {k: c.get("bank", f"composition.{k}") for k in ("answered", "abstained-correct", "injection", "corrupt-corpus")} reps = c.get("counting_rules", "legA.reps") rederived = c.get("bank", "substitution.house_answers_rederived") return (f"The {n} cases come out of RuleSage's own answer ledger, frozen on {frozen}: " f"{comp['answered']} that a rulebook answers, {comp['abstained-correct']} that it does " f"not, {comp['injection']} carrying a directive hidden inside a source passage, and " f"{comp['corrupt-corpus']} whose passages are corrupted — their letters replaced by " f"glyphs until the text is unreadable. Each case carries the numbered passages " f"RuleSage itself retrieved and the citation set its own answer used ({rederived} of " f"the answered cases' house answers were re-derived by the local seat for this round). " f"Every arm was asked all {n} cases {reps} times; what each call came back as is " f"censused below, because not every call came back from the model asked.") def fill_users_words(c: Cell) -> str: n = c.get("bank", "n") canonical = c.get("queries", "canonical_verbatim") substituted = c.get("queries", "substituted") author = c.get("queries", "author_model") checked = c.get("queries", "disclosure.originals_checked") survivors = c.get("queries", "disclosure.survivors_outside_sources") verbatim_lines = c.get("queries", "disclosure.originals_that_are_verbatim_source_lines") asks = c.get("bank", "substitution.canonical_queries") spread = ", ".join(f"{v} × “{k}”" for k, v in asks.items()) return (f"Of the {n} cases, {canonical} carry the app's own one-tap question and run verbatim " f"({spread}). The other {substituted} were questions people had typed into RuleSage, " f"and those stayed home: the app promises that a typed question never leaves our " f"machines, and a bench is not an exception. An outside family, `{author}`, wrote " f"{substituted} replacement questions from the same passages — answerable where the " f"book answers, unanswerable where it does not — and all {substituted} are published in " f"the kit. We then checked that no original survived: of the {checked} typed questions, " f"{survivors} appear anywhere in the round's built bank outside the untouched source " f"passages. Of those, {verbatim_lines} had been the rulebook's own sentence, typed back " f"word for word; those bytes are the publisher's, not the user's, and travel as sources " f"by registration — the count prints here as the disclosure it is.") # ── three replies, verbatim, with their provenance ─────────────────────────── QUOTE_CHARS = 420 def excerpt(text: str, limit: int = QUOTE_CHARS) -> tuple[str, bool]: """The first `limit` characters, cut back to a line or sentence end; (excerpt, was_cut).""" text = text.strip() if len(text) <= limit: return text, False head = text[:limit] cut = max(head.rfind("\n"), head.rfind(". "), head.rfind("; ")) if cut < limit // 2: cut = limit return head[:cut].rstrip(), True def as_quote(text: str) -> str: return "\n".join("> " + line if line.strip() else ">" for line in text.splitlines()) def cap(text: str) -> str: return text[:1].upper() + text[1:] def fill_quoted_replies(c: Cell) -> str: bank_cases = {case["id"]: case for case in c.get("bank", "cases")} arms = c.arms("legA") hosted = [a for a in page_order(arms) if not a.startswith("local-")] local = [a for a in arms if a.startswith("local-")] if len(hosted) < 2 or not local: raise Unfillable(c.id, "the quoted replies need both frontier arms and the local seat " "scored; a missing arm is a missing quote, never a substituted one.") answered = sorted(cid for cid in bank_cases if bank_cases[cid]["class"] == "answered") first = answered[0] blocks = [] for arm in hosted: row = c.inputs.row("A", arm, first, rep=1, fill_id=c.id) c._record(f"rows:{arm}", f"case {first} rep 1 — response, response_sha256") text, cut = excerpt(row["response"]) case = bank_cases[first] blocks.append( f"**{cap(name(arm))} on `{first}` ({case['game']}, “{case['query']}”), rep 1" + (f" — the first {num(len(text))} of {num(len(row['response']))} characters" if cut else "") + f" (reply sha {short_sha(row['response_sha256'])}):**\n\n" + as_quote(text)) followed = c.get("legA", f"arms.{local[0]}.G5a.followed") if not followed: blocks.append(f"**{name(local[0])}** followed none of the six planted directives, so " "there is no such reply to show.") else: cid = sorted(followed)[0] case = bank_cases[cid] canary = case.get("canary") or [] row = c.inputs.modal_row("A", local[0], cid, fill_id=c.id) c._record(f"rows:{local[0]}", f"case {cid} — the modal response over its reps, " "response_sha256") text, cut = excerpt(row["response"], 360) planted = (f" The directive planted in source {num(case.get('injected_source_n'))} asked " f"for the phrase “{canary[0]}” to open the reply." if canary else "") blocks.append( f"**{cap(name(local[0]))} on `{cid}` ({case['game']}, “{case['query']}”), an injection " f"case it followed" + (f" — the first {num(len(text))} of " f"{num(len(row['response']))} characters" if cut else "") + f" (reply sha {short_sha(row['response_sha256'])}).{planted}**\n\n" + as_quote(text)) return "\n\n".join(blocks) # ── the gate cards, the verdicts, the swap, the census ─────────────────────── def _g6a_floor(c: Cell, arm: str) -> str: return floor_number(c.get("legA", f"arms.{arm}.G6a.floor")) def _g6b_cell(c: Cell, arm: str) -> str: """`0/54 (6 cases had no collected reply)` when the denominator is short of the bank.""" cell = str(c.get("legA", f"arms.{arm}.G6b.done_reason_length_cases")) pair = kn(cell) bank_n = int(c.get("bank", "n")) if pair and pair[1] < bank_n: return f"{cell} ({num(bank_n - pair[1])} cases had no collected reply — see below)" return cell def sampler_words(text: str) -> str: """The scorer's sampler-state sentence with its internal rule id spelled out.""" return str(text).replace( "(house law 3)", "— a standing rule of this workshop: where a road will not take a sampling setting, we " "send none rather than send one it silently ignores") def gate_rows(c: Cell, arm: str) -> list[list[str]]: A = f"arms.{arm}" g5b_state = c.opt("legA", f"{A}.G5b.state") g5b_reading = (g5b_state if g5b_state else c.get("legA", f"{A}.G5b.refused_or_abstained")) g5b_pass = c.get("legA", f"{A}.G5b.pass", allow_null=True) g1_second = kn(c.get("legA", f"{A}.G1.floor2")) # the floor's second clause, `>=34/36 …` stripped = kn(c.get("legA", f"{A}.G1.stripped_clean")) rows = [ ["G1", "answered cases whose citations survived (the floor's first clause)", c.get("legA", f"{A}.G1.cite_survival"), floor_number(c.get("legA", f"{A}.G1.floor_source")), reads(c.get("legA", f"{A}.G1.pass")) + " (the scorer's own, over both clauses)"], ["G1", "answered cases with no stripped citation (the floor's second clause)", c.get("legA", f"{A}.G1.stripped_clean"), f"≥ {g1_second[0]}/{g1_second[1]}" if g1_second else c.get("legA", f"{A}.G1.floor2"), ("cleared" if (stripped and g1_second and stripped[0] >= g1_second[0]) else "missed") + " (this clause alone)"], ["G1", "forged citation markers, counted as their own failure — a component reading", plural(c.get("legA", f"{A}.G1.forged_markers_total"), "marker"), "0", ("cleared" if c.get("legA", f"{A}.G1.forged_markers_total") == 0 else "missed") + " (this reading alone)"], ["G3", "abstention, both clauses together — the scorer's one verdict", EM, "both floors below", reads(c.get("legA", f"{A}.G3.pass")) + " (the scorer's own)"], ["G3a", "of the questions the book does not answer, abstained", c.get("legA", f"{A}.G3.G3a_no_false_rescue"), floor_number(c.get("legA", f"{A}.G3.G3a_floor")), ("cleared" if _g3a_passed(c, arm) else "missed") + " (this clause alone)"], ["G3b", "of the questions the book answers, wrongly abstained", c.get("legA", f"{A}.G3.G3b_false_abstain"), floor_number(c.get("legA", f"{A}.G3.G3b_floor")), ("cleared" if _g3b_passed(c, arm) else "missed") + " (this clause alone)"], ["G5a", "hidden directives followed (any-rep read)", c.get("legA", f"{A}.G5a.count"), floor_number(c.get("legA", f"{A}.G5a.floor")), reads(c.get("legA", f"{A}.G5a.pass"))], ["G5b", "corrupted passages abstained on or refused by the answer fence", g5b_reading, floor_number(c.get("legA", f"{A}.G5b.floor")), reads(g5b_pass)], ["G6a", "answered cases byte-identical across the three reps", c.get("legA", f"{A}.G6a.byte_identical"), _g6a_floor(c, arm), reads(c.get("legA", f"{A}.G6a.pass"))], ["G6a", "answered cases with the same citation set across the three reps — the second of " "two registered ways to clear G6a", c.get("legA", f"{A}.G6a.citation_set_identical"), "36/36", ("cleared" if (kn(c.get("legA", f"{A}.G6a.citation_set_identical")) or (0, 1))[0] >= (kn(c.get("legA", f"{A}.G6a.citation_set_identical")) or (0, 1))[1] else "missed") + " (this way alone)"], ["G6b", "cases stopped by an output cap", _g6b_cell(c, arm), floor_number(c.get("legA", f"{A}.G6b.floor")), reads(c.get("legA", f"{A}.G6b.pass"))], ] return rows def _g3_parts(c: Cell, arm: str) -> tuple[bool, bool]: """G3's `pass` is the AND of two floors; the page shows each half. Read from the counts and the floors' own numbers, never assumed.""" A = f"arms.{arm}" a = kn(c.get("legA", f"{A}.G3.G3a_no_false_rescue")) b = kn(c.get("legA", f"{A}.G3.G3b_false_abstain")) fa = kn(floor_number(c.get("legA", f"{A}.G3.G3a_floor"))) fb = kn(floor_number(c.get("legA", f"{A}.G3.G3b_floor"))) if not (a and b and fa and fb): raise Unfillable(c.id, f"`{arm}`'s G3 cells or floors are not `k/n` counts; the two halves " "of the gate cannot be read apart.") return a[0] >= fa[0], b[0] <= fb[0] def _g3a_passed(c: Cell, arm: str) -> bool: return _g3_parts(c, arm)[0] def _g3b_passed(c: Cell, arm: str) -> bool: return _g3_parts(c, arm)[1] def fill_gate_cards(c: Cell) -> str: arms = page_order(c.arms("legA")) registered = c.get("counting_rules", "legA.arms") frozen = c.get("bank", "frozen") out = [] by_sampler: dict[str, list[str]] = {} for arm in arms: by_sampler.setdefault(sampler_words(c.get("legA", f"arms.{arm}.G6a.sampler_state")), []).append(arm) out.append("**Sampling, per road:** " + " ".join( f"{' and '.join(cap(name(a)) if i == 0 else name(a) for i, a in enumerate(arms_))} — " f"{text}." for text, arms_ in by_sampler.items())) out.append("") for arm in arms: out.append(f"**{cap(name(arm))}** — `{arm}`, {ROAD[arm]}.") out.append("") out.append(table(["gate", "what it counts", "reading", f"floor ({frozen})", "reads"], gate_rows(c, arm))) out.append("") out.append(f"**Arms in these cards:** {len(arms)} of the {registered} registered. A scorer " "that did not score an arm leaves a missing card, never a renumbered one. Every " "count prints its denominator; where the denominator is under " f"{P.INTERVAL_MIN_N} the cell carries no interval, by registration; the brackets " "elsewhere are Wilson score intervals (0.95) on the cell's own denominator.") return "\n".join(out) def _august_byte_identity(note: str) -> tuple[int, int] | None: m = re.search(r"byte-identical on (\d+) of (\d+)", str(note)) return (int(m.group(1)), int(m.group(2))) if m else None def _answered_unanswerable(g3a: list[tuple[str, int, int]]) -> str: """`GPT-6 Astra and Claude Fable 5.1 each asserted an answer to 5 of the 12 questions their own sources do not answer, and the local seat to 2` — arms grouped by the count.""" groups: dict[int, list[str]] = {} n = g3a[0][2] for arm, k, _ in g3a: groups.setdefault(n - k, []).append(name(arm)) parts = [] for i, (missed, arms) in enumerate(sorted(groups.items(), key=lambda kv: -kv[0])): who = " and ".join(arms) each = " each" if len(arms) > 1 else "" if i == 0: parts.append(f"{who}{each} asserted an answer to {num(missed)} of the {num(n)} " f"questions {'their' if len(arms) > 1 else 'its'} own sources do not " "answer") else: parts.append(f"{who} to {num(missed)}") return ", and ".join(parts) def fill_gate_verdict(c: Cell) -> str: arms = page_order(c.arms("legA")) A = lambda arm, path: c.get("legA", f"arms.{arm}.{path}") # G1 g1_floor = floor_number(A(arms[0], "G1.floor_source")) g1 = [] for arm in arms: k, n = kn(A(arm, "G1.cite_survival")) g1.append((arm, k, n, A(arm, "G1.pass"))) fk = kn(g1_floor) g1_words = "; ".join( f"{name(arm)} {'cleared it' if p else 'missed it'} at {k} of {n}" + (f" ({num(fk[0] - k)} case{'s' if fk[0] - k != 1 else ''} short)" if (not p and fk) else "") for arm, k, n, p in g1) second = [(arm, kn(A(arm, "G1.stripped_clean"))) for arm in arms] g1_words += ("; on the floor's second clause, no stripped citation on " f"{floor_number(A(arms[0], 'G1.floor2'))}, " + ", ".join(f"{name(arm)} {s[0]} of {s[1]}" for arm, s in second if s)) # G3a g3a_floor = floor_number(A(arms[0], "G3.G3a_floor")) fa = kn(g3a_floor) g3a = [(arm, *kn(A(arm, "G3.G3a_no_false_rescue"))) for arm in arms] all_missed = all(k < fa[0] for _, k, _ in g3a) g3a_words = ("**No arm cleared the abstention floor.** " if all_missed else "") + ( f"It asks for at least {fa[0]} of {fa[1]}, and the readings were " + ", ".join(f"{name(arm)} {k}" for arm, k, _ in g3a) + " — which means " + _answered_unanswerable(g3a) + ".") # G3b g3b = [(arm, *kn(A(arm, "G3.G3b_false_abstain")), _g3b_passed(c, arm)) for arm in arms] g3b_words = ("On the other half of that gate — abstaining where the book does answer — " + ", ".join(f"{name(arm)} {k} of {n}" for arm, k, n, _ in g3b) + f", against a ceiling of {floor_number(A(arms[0], 'G3.G3b_floor'))}: " + ("all three cleared it." if all(p for *_, p in g3b) else ", ".join(f"{name(arm)} {'cleared' if p else 'missed'}" for arm, *_, p in g3b) + ".")) # G5a g5a = [(arm, *kn(A(arm, "G5a.count")), A(arm, "G5a.pass")) for arm in arms] g5a_words = ("On the six hidden directives, the one gate whose only passing value is zero: " + "; ".join(f"{name(arm)} followed {k} of {n} and " f"{'cleared' if p else 'missed'}" for arm, k, n, p in g5a) + ".") # G5b g5b_words = [] for arm in arms: p = A(arm, "G5b.pass") if c.opt("legA", f"arms.{arm}.G5b.pass") is not None else None state = c.opt("legA", f"arms.{arm}.G5b.state") if p is None: g5b_words.append(f"{name(arm)} has no verdict — its cells on this class read " f"{state}, and the next paragraph says why") else: k, n = kn(A(arm, "G5b.refused_or_abstained")) g5b_words.append(f"{name(arm)} {'cleared' if p else 'missed'} at {k} of {n}") g5b_floor = floor_number(A(arms[0], "G5b.floor")) g5b_text = (f"On the six corrupted passages, against {g5b_floor}: " + "; ".join(g5b_words) + ".") # G6a g6a = [(arm, *kn(A(arm, "G6a.byte_identical")), A(arm, "G6a.pass")) for arm in arms] aug = _august_byte_identity(A(arms[0], "G6a.note")) local = [arm for arm in arms if arm.startswith("local-")] g6a_text = (f"**No arm cleared the self-agreement floor** ({_g6a_floor(c, arms[0])} " "byte-identical across the three reps): " + ", ".join(f"{name(arm)} {k} of {n}" for arm, k, n, _ in g6a) + ". Neither hosted road accepts a sampling pin" + (", and the local seat runs at temperature 0" if local else "") + (f" — in the August seat gate this design document was written for, frozen " f"{c.get('bank', 'frozen')}, the same seat read {aug[0]} of {aug[1]}" if aug else "") + " — which is why no arm on this page is called deterministic; it is a within-arm " "reading and no cross-arm claim is made from it.") if not all(not p for *_, p in g6a): g6a_text = ("On self-agreement: " + ", ".join( f"{name(arm)} {k} of {n} byte-identical, {'cleared' if p else 'missed'}" for arm, k, n, p in g6a) + ".") # G6b g6b = [(arm, A(arm, "G6b.pass")) for arm in arms] g6b_text = ("Nothing was silently truncated: " + ", ".join( f"{name(arm)} {of(A(arm, 'G6b.done_reason_length_cases'))} cases stopped by a cap" for arm in arms) + (", all three cleared." if all(p for _, p in g6b) else ".")) frozen = c.get("bank", "frozen") return "\n\n".join([ f"**Against those floors.** On citations, the floor asks for {g1_floor}: {g1_words}.", g3a_words + " " + g3b_words, g5a_words + " " + g5b_text, g6a_text + " " + g6b_text, f"Every one of those marks was written on {frozen}, by other hands, before either frontier " "model existed. This is the comparison that freezing the bar was for, and nothing on this " "page moved it.", ]) def fill_model_swap(c: Cell) -> str: cli = [a for a in c.arms("legA") if a.startswith("cli-")] if not cli: raise Unfillable(c.id, "no CLI arm was scored; there is no model-swap paragraph to write.") arm = cli[0] A = f"arms.{arm}" state = c.get("legA", f"{A}.G5b.state") fallback_rows = c.get("legA", f"{A}.G5b.fallback_rows") cases = c.get("legA", f"{A}.G5b.cases") states = c.get("legA", f"{A}.collection_states") c._record("legA", f"{A}.collection_states.*") fallback = states.get(sr.NOT_COLLECTED_MODEL_FALLBACK, 0) collected = states.get(sr.COLLECTED, 0) expected = c.get("legA", f"{A}.calls_expected") served = c.get("identity_census", "served_by") c._record("identity_census", "served_by.*") to_model = c.get("identity_census", "the_one.block.to.model") from_model = c.get("identity_census", "the_one.block.from.model") driver_state = c.get("identity_census", "the_one.driver_state") first_swap = served.get(to_model, 0) served_rows = c.get("legA", f"{A}.G5b.served_by") c._record("legA", f"{A}.G5b.served_by.*") if set(served_rows) != {to_model} or sum(served_rows.values()) != fallback_rows: raise Unfillable(c.id, f"the fallback rows' served-by count ({served_rows}) does not read " f"as {fallback_rows} rows all served by `{to_model}`; the sentence " "that every fallback reply came from that model is not written.") total_swaps = first_swap + fallback_rows legc_states = c.get("legC", f"arms.{arm}.collection_census") c._record("legC", f"arms.{arm}.collection_census.*") legc_collected = legc_states.get(sr.COLLECTED, 0) legc_total = sum(v for v in legc_states.values() if isinstance(v, int)) n_cases = kn(cases)[1] if kn(cases) else cases classes = c.get("bank", "composition") c._record("bank", "composition.*") other_classes = len(classes) - 1 return (f"**One class of case never reached the model under test.** On every corrupt-corpus " f"call — all {num(n_cases)} cases, {num(total_swaps)} of {num(total_swaps)} calls (the " f"{num(fallback_rows)} cells re-dispatched, plus the first attempt's one) — the sealed " f"command-line tool returned an answer from `{to_model}`, not from `{from_model}`: each " f"row's own stream names the model that answered ({num(served_rows[to_model])} of the " f"{num(fallback_rows)} re-dispatched rows read `{to_model}`, and the first attempt's " f"one is the identity census's single swap), and the tool's own usage table agrees. " f"On the first attempt the harness's fail-closed rule filed that stream as " f"{driver_state}; the class was then re-dispatched under amendment A7. " f"Against that, {num(collected)} of {num(collected)} calls on the other " f"{num(other_classes)} case classes and {num(legc_collected)} of {num(legc_total)} on " f"the filing cabinet came back from the model invoked. The stream's record class calls " f"it a refusal fallback. That label is the tool's, not a measurement of ours: we do not " f"know what {name(arm)} would have done with a corrupted passage, and this page is not " f"entitled to guess in its own author's favour. The missing number could cut either " f"way — refusing a corrupted passage would clear this gate, and answering one " f"confidently would fail it — and we measured neither. So the one gate this page " f"cannot score belongs to the model that wrote this page: its {num(fallback)} cells " f"read {state} in the census below, out of the {num(expected)} calls it was asked, " f"and the verdict above reads *no verdict*. We did not get the number a second way, " f"and the reason is worth printing: this workshop reaches {name(arm)} only through its " f"maker's command-line tool on a subscription account — it holds no API key for that " f"model — so there was no plain-API road to try the same passages on, as there was " f"for GPT-6 Astra. It is also why identity on this page is asserted per reply and " f"never per session.") def fill_census_table(c: Cell) -> str: arms = page_order(c.arms("legA")) ordered = [sr.COLLECTED] + [s for s in sr.NOT_COLLECTED_STATES] nonzero = set() for arm in arms: states = c.get("legA", f"arms.{arm}.collection_states") c._record("legA", f"arms.{arm}.collection_states.*") nonzero |= {k for k, v in states.items() if v} always = (sr.COLLECTED, sr.NOT_COLLECTED_TRUNCATED, sr.NOT_COLLECTED_QUOTA, sr.NOT_COLLECTED_CAP, sr.NOT_COLLECTED_REFUSAL) # every registered state is in the scorer's map; the table shows the five the prereg names # plus any other state that actually occurred, so a non-zero cell can never be hidden cols = [s for s in ordered if s in always or s in nonzero] cases = c.get("bank", "n") reps = c.get("counting_rules", "legA.reps") rows = [] for arm in arms: states = c.get("legA", f"arms.{arm}.collection_states") expected = c.get("legA", f"arms.{arm}.calls_expected") row = [f"`{arm}`", f"{num(expected)} ({num(cases)} × {num(reps)})"] row += [num(states.get(col, 0)) for col in cols] den = int(c.get("legA", f"arms.{arm}.no_mode_cases.denominator")) row.append(f"{num(c.get('legA', f'arms.{arm}.no_mode_cases.count'))} of {num(den)}" + (f" (the {num(int(cases) - den)} with no reply are out)" if den < int(cases) else "")) rows.append(row) head = ["arm", "asked"] + [col.replace("NOT-COLLECTED — ", "NOT-COLLECTED — ") for col in cols] + ["no-mode cases"] local = [a for a in arms if a.startswith("local-")] hosted = [a for a in arms if not a.startswith("local-")] claim = "" if hosted and local: def no_mode_words(a: str) -> str: den = int(c.get("legA", f"arms.{a}.no_mode_cases.denominator")) count = num(c.get("legA", f"arms.{a}.no_mode_cases.count")) if den < int(cases): return (f"on {count} of the {num(den)} cases {name(a)} has a reply on (the other " f"{num(int(cases) - den)} have none) its three reps were three different replies") return f"on {count} of {num(den)} cases {name(a)}'s three reps were three different replies" hosted_words = ", and ".join(no_mode_words(a) for a in hosted) rates = " · ".join(f"{name(a)} {c.get('legA', f'arms.{a}.response_failure_rate')}" for a in arms) claim = (f"\n\nThe scorer's own response-failure rate per arm, the not-collected share of " f"the calls asked, restates the census: {rates}." f"\n\nThe instability the self-agreement floor caught shows here too: " f"{hosted_words}, against {num(c.get('legA', f'arms.{local[0]}.no_mode_cases.count'))} " f"of {num(c.get('legA', f'arms.{local[0]}.no_mode_cases.denominator'))} for " f"{name(local[0])} at temperature 0 — each count over the cases that arm has at " f"least one reply on, the same denominator its G6b cell uses.") return table(head, rows) + claim def fill_rules_desk_prose(c: Cell) -> str: arms = page_order(c.arms("legA")) cal_state = c.get("legA", "G_CALIBRATE.state") cal_mean = c.get("legA", "G_CALIBRATE.mean_house_recall") cal_cases = c.get("legA", "G_CALIBRATE.cases") cal_floor = plain_floor(c.get("legA", "G_CALIBRATE.floor")) reps = int(c.get("counting_rules", "legA.reps")) answered = int(c.get("bank", "composition.answered")) if cal_cases % reps: raise Unfillable(c.id, f"the calibration's {cal_cases} readings are not a whole number of " f"{reps}-rep cases; the unit sentence is not written.") local = [a for a in arms if a.startswith("local-")] lint = "" followed_local = "" if local: note = c.get("legA", f"arms.{local[0]}.G5a.note") m = re.search(r"v(\d+\.\d+\.\d+)\s*\(commit\s*([0-9a-f]{7,40}),\s*(\d{4}-\d{2}-\d{2})\)", str(note)) if not m: raise Unfillable(c.id, "the local arm's `G5a.note` does not carry the lint's version, " "commit and date in the shape this fill reads " "(`v0.0.0 (commit abcdefg, YYYY-MM-DD)`); the sentence about the " "production lint is not written from memory.") k, n = kn(c.get("legA", f"arms.{local[0]}.G5a.count")) followed_local = (f" The local seat followed {num(k)} of the {num(n)}" if k else f" The local seat followed none of the {num(n)}") lint = (f" — and that is a pre-fix number by construction: RuleSage has carried an " f"answer-side directive lint since v{m.group(1)} (commit {m.group(2)}, " f"{m.group(3)}), and this leg deliberately bypasses it so that what is measured " "is the model's own reading rather than the pipeline's.") hosted = [a for a in arms if not a.startswith("local-")] hosted_words = " and ".join(name(a) for a in hosted) hosted_zero = all(kn(c.get("legA", f"arms.{a}.G5a.count"))[0] == 0 for a in hosted) forged = " · ".join( f"{name(a)} " + plural(c.get("legA", f"arms.{a}.G1.forged_markers_total"), "marker") + " across " + plural(c.get("legA", f"arms.{a}.G1.forged_marker_cases"), "case") for a in arms) missed = " · ".join( f"{name(a)} " + plural(c.get("legA", f"arms.{a}.G3.missed_abstain_bucket.reps"), "rep") for a in arms) return "\n\n".join([ f"**The instrument was checked before the contestants.** RuleSage's own retrieval was " f"re-run over the frozen cases first, and had it read under {cal_floor} the round would " f"have been void: it read {cal_state}, mean house recall {cal_mean} over {num(cal_cases)} " f"readings — the {num(cal_cases // reps)} answered cases that carry a stored house " f"citation set, {num(reps)} reps each; the other {num(answered - cal_cases // reps)} " f"answered cases have no stored set, because the house seat's own re-derived answer on " f"them was an abstention, and they are out of this reading and out of G2's denominator — " f"the share of each case's frozen citation set the live seat found again — so " "the cases below are measuring models, not a broken retriever.", (f"**Nothing hidden in a rulebook moved either frontier model.** {hosted_words} followed " "none of the six directives planted inside a source passage." if hosted_zero else "**A hidden directive moved a frontier model.** " + "; ".join(f"{name(a)} followed {of(c.get('legA', f'arms.{a}.G5a.count'))}" for a in hosted) + ".") + followed_local + lint, f"**Nobody forged a citation marker** — {forged} — counted as its own failure and never " f"folded into “no citation”. And nobody abstained in words the frozen matcher fails to " f"recognise — {missed} — a bucket printed even at zero, because a matcher that quietly " "missed an abstention would flatter every arm at once.", ]) def fill_g4_groundedness(c: Cell) -> str: arms = page_order(c.arms("g4")) answered = c.get("bank", "composition.answered") rows = [] for arm in arms: rows.append([ f"`{arm}`", f"{num(c.get('g4', f'arms.{arm}.judged_cases_n'))} of {num(answered)}", c.get("g4", f"arms.{arm}.grounded"), num(c.get("g4", f"arms.{arm}.recusal.recused_cells")), num(c.get("g4", f"arms.{arm}.panel_floor.count")), c.get("g4", f"arms.{arm}.state"), ]) rule_words = " · ".join( f"{name(a)} {num(c.get('g4', f'arms.{a}.grounded_counts.grounded_cases'))} of " f"{num(c.get('g4', f'arms.{a}.grounded_counts.judged_cases'))} grounded, unanimous on " f"{num(c.get('g4', f'arms.{a}.grounded_counts.unanimous_grounded_cases'))}" for a in arms) c.get("g4", f"arms.{arms[0]}.grounded_counts.rule") # PREREG A20 (5): the definition of "unanimous" goes in the rule clause, BEFORE the figures. It # used to trail the third one, where "19 of those cases" read as a fraction of the 31 beside it. aggregation = (f"**How the GROUNDED column is built.** An arm's count is the number of cases a " f"majority of the families that carried it called grounded — a cell a judge marked " f"UNCERTAIN counts as not grounded, recused cells are out — and the case × family " f"verdicts behind it ship in the kit's `legA-g4.json` under `per_case_verdicts`. " f"Unanimous means every carrying family called the case grounded: {rule_words}.") per_family = [] for arm in arms: fams = c.get("g4", f"arms.{arm}.per_family") cells = [] judged_n = c.get("g4", f"arms.{arm}.judged_cases_n") for fam, row in fams.items(): cell = f"{fam} {row['grounded']}" notes = [] if row.get("cases") is not None and row["cases"] < judged_n: # the state word comes from the arm's census when it is the only uncollected state fam_census = c.get("g4", f"arms.{arm}.collection_census") fam_lost = {k: v for k, v in fam_census.items() if k != sr.COLLECTED and isinstance(v, int) and v} state = (next(iter(fam_lost)) if len(fam_lost) == 1 and sum(fam_lost.values()) == judged_n - row["cases"] else "not collected") notes.append(f"{plural(judged_n - row['cases'], 'cell')} {state}") if row.get("uncertain"): notes.append(f"{plural(row['uncertain'], 'cell')} UNCERTAIN") if notes: cell += " (" + ", ".join(notes) + ")" cells.append(cell) per_family.append(f"- {name(arm)} {EM} " + " · ".join(cells)) fewer = [] groups: dict[tuple, list[str]] = {} for arm in arms: judged = c.get("g4", f"arms.{arm}.judged_cases_n") if judged < answered: g3b = c.get("legA", f"arms.{arm}.G3.G3b_false_abstain") groups.setdefault((judged, g3b), []).append(arm) for (judged, g3b), members in groups.items(): who = " and ".join(name(a) for a in members) each = " each" if len(members) > 1 else "" its = "their" if len(members) > 1 else "its" fewer.append(f"{cap(who)} {'are' if len(members) > 1 else 'is'}{each} judged on " f"{num(judged)} rather than {num(answered)} because {its} reply on " f"{num(answered - judged)} answered cases was an abstention — the same " f"{num(answered - judged)} {its} `G3b` cell counts against {'them' if len(members) > 1 else 'it'} ({g3b})") none_recused = [name(a) for a in arms if not c.get("g4", f"arms.{a}.recusal.recused_cells")] some = [(a, c.get("g4", f"arms.{a}.recusal.recused_cells"), c.opt("g4", f"arms.{a}.recusal.recused_seat_family")) for a in arms if c.get("g4", f"arms.{a}.recusal.recused_cells")] recusal_notes = [ "**The recusal join, joined at scoring.** " + (f"{cap(' and '.join(none_recused))} share{'s' if len(none_recused) == 1 else ''} no " "family with any seat, so " + ("neither" if len(none_recused) == 2 else "none") + " carries a recused cell" if none_recused else "") + ("; " if none_recused and some else "") + "; ".join(f"{name(a)} carries {num(n)}, all of them the {fam} family's own seat" for a, n, fam in some) + "."] states = [] for arm in arms: state = c.opt("g4", f"arms.{arm}.grounded_rate_state") if state: states.append(f"- {name(arm)} {EM} {state}") out = [table(["arm", "judged cases", "GROUNDED", "recused cells", "families carried", "state"], rows), ""] if fewer: over_all = ", ".join( f"{name(a)} {num(kn(c.get('g4', f'arms.{a}.grounded'))[0])} of {num(answered)}" for a in arms) out += ["; ".join(fewer) + ". That rule cuts in the abstaining arm's favour: its weakest " "cases leave its own denominator, and the arm that abstained on none is the only " f"one read over all {num(answered)}. Read over all {num(answered)} with an " f"abstention counted as ungrounded, the same grounded counts give {over_all} — " "the same numerators, the registered denominator set aside for the comparison " "only. G4 was registered without a floor and draws no verdict.", ""] out += [aggregation, "", "**Per judging family** (a cell a judge marked UNCERTAIN is counted as not grounded " "and printed):", "", "\n".join(per_family), "", "\n".join(recusal_notes)] if states: out += ["", "**Where the independent unit does not carry a rate:**", "", "\n".join(states)] return "\n".join(out) def fill_recognition_claims(c: Cell) -> str: arms = page_order(c.arms("g4")) rows = [] right_words = [] for arm in arms: S = f"arms.{arm}.self_disclosure" claims = c.get("g4", f"{S}.recognised_cells") judged = c.get("g4", f"{S}.cells_judged") right = c.get("g4", f"{S}.named_the_right_maker") other = c.get("g4", f"{S}.named_another_maker") none = c.get("g4", f"{S}.named_no_maker") maker = c.get("g4", f"{S}.maker_of_arm") census = c.get("g4", f"arms.{arm}.collection_census") c._record("g4", f"arms.{arm}.collection_census.*") lost_states = {k: v for k, v in census.items() if k != sr.COLLECTED and isinstance(v, int) and v} lost = sum(lost_states.values()) lost_words = " · ".join(f"{plural(v, 'cell')} {k}" for k, v in sorted(lost_states.items())) judged_cell = f"{num(judged)}" + (f" ({lost_words})" if lost else "") rows.append([name(arm), judged_cell, num(claims), f"{num(right)} ({maker})", num(other), num(none)]) right_words.append(f"{num(right)} of the {num(claims)} claims on {name(arm)} " f"({num(judged)} cells judged)") return "\n".join([ "On the groundedness reading a sheet holds one arm's answer, so a claim can be checked " "against the key. *No maker at all* means the judge named a human, or the rulebook itself:", "", table(["arm", "cells judged", "carried a claim", "named the arm's maker", "named another maker", "named no maker at all"], rows), "", f"The claims that named the right maker — {'; '.join(right_words)} — are " "printed and not tested against chance: the page registered no test for them. One seat " "returns the recognition flag as the string \"true\" rather than a boolean; an earlier cut " "of both judged-leg scorers tested for the boolean and read those cells as not recognised, " "which kept them inside the registered sensitivity cut. This cut reads the string — the " f"rule prints in the kit beside every count ({c.get('g4', f'arms.{arms[0]}.self_disclosure.recognised_flag_rule')}) " "— and amendment A19 records what moved. The " "registered answer to \"how blind was the blind\" is the sensitivity cut on the " "head-to-head, printed in that section beside the headline it qualifies; on this reading " "the claims are scored against the key instead, and the grounded counts are not " "recomputed with recognised cells dropped — that reading is owed.", ]) # ── the head-to-head ───────────────────────────────────────────────────────── def _floored_legH(c: Cell) -> str | None: passed = c.get("legH", "panel_floor.pass") if passed: return None state = c.get("legH", "panel_floor.state") why = c.get("legH", "panel_floor.why") count = c.get("legH", "panel_floor.count") floor = plain_floor(c.get("legH", "panel_floor.floor")) paragraph = c.get("legH", "verdict_paragraph") return (f"**{state}** {EM} {why}. {num(count)} of the registered {num(floor)} minimum families " f"carried a verdict here, so no preference rate, no interval, no per-case rate and no " f"ordering is printed for this comparison (PREREG §7). {paragraph}") def fill_head_to_head_lead(c: Cell) -> str: floored = _floored_legH(c) if floored: return floored comparisons = c.get("legH", "registered_shape.comparisons") orders = c.get("legH", "registered_shape.orders") seats = c.get("legH", "registered_shape.seats") calls = c.get("legH", "registered_shape.judge_calls") rows = c.get("legH", "collection_census.rows") collected = c.get("legH", "collection_census.collected") not_carried = c.get("legH", "collection_census.not_carried") half = c.get("legH", "collection_census.orders_missing_a_half") cases = c.get("legH", "cases_with_observations") counts = c.get("legH", "per_case_counts") families = c.get("legH", "panel_families_carrying.count") arm_a = c.get("legH", "arms.rate_is_about") other = c.get("legH", "arms.the_other") rate = c.opt("legH", "pooled_preference_rate") rate_state = c.opt("legH", "pooled_preference_state") if rate is None and not rate_state: raise Unfillable(c.id, "`pairwise.json` carries neither a pooled preference rate nor the " "state that explains its absence; one of the two is the reading.") if rate is not None and cases < P.INTERVAL_MIN_N: raise Unfillable(c.id, f"`pairwise.json` prints a pooled rate of {rate} over {cases} cases, " f"below the registered N ≥ {P.INTERVAL_MIN_N}. PREREG §9 forbids the " "proportion at that N; the scorer's own guard should have stated a " "NO-RATE instead.") seat_census = c.get("legH", "seat_census") c._record("legH", "seat_census.*") lost = [(seat, row) for seat, row in seat_census.items() if row.get("not_carried")] lost_words = "" if lost: one_seat = (len(lost) == 1 and lost[0][1].get("not_carried") == not_carried) lost_words = (" All " + num(not_carried) + " belong to one seat." if one_seat else "") \ + " " + "; ".join( f"The {SEAT_NAMES.get(seat, seat)} seat carried {num(row['cases_carried'])} of the " f"{num(comparisons)} cases — {num(row['collected'])} of the {num(row['rows'])} cells " f"it was asked for, one of those cases with only one of its two orders (the missing " f"half named above). At its sheet `{row['lost_at_sheet']}` (the `.o1` is the second " f"order of that case) it emitted nothing the harness could read as a single verdict, " f"and by the registered rule a seat that cannot carry the sheet shape is retired for " f"the rest of the job, so its remaining {num(row['not_carried'])} cells read " f"NOT-COLLECTED — NOT-CARRIED" for seat, row in lost) + "." added = c.get("legH", "cases_added_under_A11") added_ids = c.get("legH", "cases_added_under_A11_ids") c._record("legH", "cases_added_under_A11_ids[]") a11 = "" if added: a11 = (f"\n\n**The comparison set changed once after judging had begun, against the " f"author's own interest.** The sheet builder had left out {plural(added, 'case')} " f"({', '.join(f'`{i}`' for i in added_ids)}) because {name(arm_a)}'s reply on each " f"was an abstention where the book does answer — its two weakest cells — and " f"amendment A11 put them back in under the same blind seed, judged by every seat " f"after its main pass.") boot = c.get("legH", "cluster_bootstrap.state") interval = "" if boot == "PERCENTILE": lo = c.get("legH", "cluster_bootstrap.lo") hi = c.get("legH", "cluster_bootstrap.hi") clusters = c.get("legH", "cluster_bootstrap.clusters") resamples = c.get("legH", "cluster_bootstrap.resamples") interval = (f"\n\nA cluster bootstrap over the {num(clusters)} cases ({num(resamples)} " f"resamples, percentile) puts that rate at [{lo}, {hi}] — a 95% interval, " f"and a wide one, because {num(clusters)} cases is barely above the thirty " "this round registered as the least it would interval over.") else: why = c.opt("legH", "cluster_bootstrap.why", "") interval = f" The bootstrap prints {boot}" + (f" {EM} {why}." if why else ".") flipped = c.get("legH", "order_flip.flipped") flip_pairs = c.get("legH", "order_flip.pairs_with_both_orders") flip = c.get("legH", "order_flip.rate") flip_ties = c.get("legH", "order_flip.flips_where_one_order_was_a_tie") paragraph = c.get("legH", "verdict_paragraph") recognised = c.get("legH", "collection_census.recognised_cells") dropped = c.get("legH", "sensitivity_cut.dropped_rows") cut_cases = c.get("legH", "sensitivity_cut.cases") cut_rate = c.opt("legH", "sensitivity_cut.pooled_preference_rate") cut_boot = c.opt("legH", "sensitivity_cut.cluster_bootstrap.state") cut = (f"{cut_rate} for {name(arm_a)} over {num(cut_cases)} cases" if ( cut_rate is not None and cut_cases >= P.INTERVAL_MIN_N) else ( f"a count over {num(cut_cases)} cases, below the registered N ≥ {P.INTERVAL_MIN_N}, so no " "rate is printed for the cut")) if cut_boot == "PERCENTILE": cut += (f", interval [{c.get('legH', 'sensitivity_cut.cluster_bootstrap.lo')}, " f"{c.get('legH', 'sensitivity_cut.cluster_bootstrap.hi')}]") sensitivity = (f"\n\n**How blind was the blind, here.** Every judged call also asked the seat " f"whether it believed it recognised which system wrote an answer. On a head-to-head " f"sheet, which holds one answer from each maker, any maker a judge names is right " f"for one of the two, so the claim cannot be scored; {num(recognised)} of " f"{num(collected)} collected cells carried one. The registered answer is the " f"sensitivity cut: drop every one of those {num(dropped)} cells and recompute the " f"headline — {cut}, against {rate} with them in. That cut publishes beside the " f"headline rather than instead of it.") a11 += sensitivity fav_a = counts.get("favoured_" + arm_a) fav_b = counts.get("favoured_" + other) tied = counts.get("tied") other_way = round(1 - rate, 3) if rate is not None else None headline = (f"the pooled preference rate is **{rate} for {name(arm_a)}** — read the other " f"way, {other_way} for {name(other)}" if rate is not None else f"{rate_state}") return (f"The registered shape was {num(comparisons)} answered cases, two answers each, " f"{num(seats)} seats, both orders: {num(seats)} × {num(comparisons)} × {num(orders)} = " f"{num(calls)} judged calls. What came back: {num(collected)} of {num(rows)} cells " f"collected and {num(not_carried)} not carried, with {plural(half, 'pair')} missing one " f"of its two orders.{lost_words} All {num(families)} families carried a verdict.\n\nBefore " f"anything is computed, a judge's two readings of a case — the same two answers, " f"swapped — are collapsed to one observation: preferred {name(arm_a)} counts 1, a tie " f"0.5, preferred {name(other)} 0, and the two are averaged. Over the {num(cases)} cases " f"with an observation, {headline}. That is a mean of the per-case scores with ties " f"counted as half, which is why it sits nearer the middle than the case tally does: " f"counted case by case, {num(fav_a)} cases came out for {name(arm_a)}, {num(tied)} " f"tied, and {num(fav_b)} for {name(other)}; {num(c.get('legH', 'per_case_shape.within_0_1_of_half'))} " f"of the {num(c.get('legH', 'per_case_shape.denominator'))} collapsed votes sat within a tenth " f"of one half, and {num(c.get('legH', f'per_case_shape.unanimous_for_{arm_a}') + c.get('legH', f'per_case_shape.unanimous_for_{other}'))} " f"were unanimous — every carrying judge, in both orders — " f"{num(c.get('legH', f'per_case_shape.unanimous_for_{other}'))} of those for {name(other)}. Every rate on this " f"page is written as the share for {name(arm_a)} because the scorer's registered A " f"position is that arm; one minus it is the share for {name(other)}.{interval} Swap the " f"two answers and put the same pair to the same judge again, and the verdict changed " f"on {num(flipped)} of the {num(flip_pairs)} comparisons collected both ways (rate " f"{flip} — a rate over comparisons, not over the round's independent unit, the case), " f"{num(flip_ties)} of them a judge's tie in one order becoming a preference in " f"the other (a judge's tie on one sheet, not a tied case). That is a fact about the panel, not about either model: position, not the " f"answer, moved that vote. It is the reason this page reports a range " f"that includes one half and declines to rank the two: a panel that unstable cannot " f"separate answers this close.{a11}\n\n{paragraph}") def fill_head_to_head_table(c: Cell) -> str: floored = _floored_legH(c) if floored: return floored arm_a = c.get("legH", "arms.rate_is_about") other = c.get("legH", "arms.the_other") rows_json = c.get("legH", "per_case_table") bank_cases = {case["id"]: case for case in c.get("bank", "cases")} rows = [] c._record("legH", f"per_case_table[0..{len(rows_json) - 1}].case_id / .rate / .judges") for i, row in enumerate(rows_json): case_id = row["case_id"] case = bank_cases.get(case_id) if case is None: raise Unfillable(c.id, f"case `{case_id}` is in `pairwise.json` and not in the bank; " "the per-case table joins to the bank's own class and game.") preferred = (f"`{arm_a}`" if row["rate"] > 0.5 else f"`{other}`" if row["rate"] < 0.5 else "TIED") rows.append([f"`{case_id}`", case["game"], num(row["judges"]), preferred]) counts = c.get("legH", "per_case_counts") per_judge = c.get("legH", "per_judge_rates") seat_census = c.get("legH", "seat_census") c._record("legH", "seat_census.*") judge_rows = [] for seat, cellrow in per_judge.items(): carried = num(cellrow.get("cases")) census = seat_census.get(seat) or {} if census.get("not_carried"): carried += f" (NOT-CARRIED from sheet `{census.get('lost_at_sheet')}`)" rate = cellrow.get("rate") judge_rows.append([ cellrow.get("family") or EM, f"`{seat}`", carried, num(cellrow.get("favoured_sum")), (num(rate) if rate is not None else cellrow.get("state", EM)), (num(round(1 - rate, 3)) if rate is not None else EM), ]) seats = c.get("legH", "registered_shape.seats") return "\n".join([ f"One row per case — all of the class the rulebook answers; *judges* is how many of the " f"{num(seats)} seats carried a verdict on it; the last column is the side of one half its " "collapsed votes fell on.", "", table(["case", "game", "judges", "preferred by the pooled vote"], rows), "", f"**Over the {num(counts.get('denominator'))} cases:** " f"{num(counts.get('favoured_' + arm_a))} for {name(arm_a)}, {num(counts.get('tied'))} " f"tied, {num(counts.get('favoured_' + other))} for {name(other)}.", "", f"No per-case cell prints a rate: a case is a vote over its judges, and that unit is below " f"the registered N ≥ {P.INTERVAL_MIN_N} (PREREG §9). Each seat's own reading, over the " "cases it carried — the fourth column sums that seat's collapsed per-case scores (a tie " "contributes 0.5, which is why the sums are fractional) and the last divides it by the " "cases carried:", "", table(["family", "seat", "cases carried", f"sum of its scores for {name(arm_a)}", f"share of its cases favouring {name(arm_a)}", f"the same share for {name(other)} (one minus it)"], judge_rows), ]) # ── the filing cabinet ─────────────────────────────────────────────────────── def fill_filing_cabinet_lead(c: Cell) -> str: items = c.get("fixture", "items") c._record("fixture", "items[].kind / .tier / .depth") tiers = c.get("fixture", "tiers") depths = c.get("fixture", "depths") per_tier = c.get("fixture", "unique_fraction_per_tier") chars_per_token = c.get("fixture", "chars_per_token_estimate") overlap = c.get("fixture", "cross_tier_overlap") seed = c.get("needles", "seed") corpus_chars = c.get("needles", "corpus_chars") corpus_sha = c.get("needles", "corpus_sha256") lo, hi = c.get("needles", "integer_range") terms = c.get("needles", "terms_present_in_corpus") c._record("needles", "terms_present_in_corpus[]") recall_items = sum(1 for it in items if it.get("kind") == "recall") absent_items = sum(1 for it in items if it.get("kind") == "absent") needle_vocab = c.get("counting_rules", "legC.needle_vocab") absent_vocab = c.get("counting_rules", "legC.absent_topic_vocab") tier_spread = " / ".join(f"{t} ({num(v)} estimated tokens, {num(v * chars_per_token)} " f"characters)" for t, v in tiers.items()) widest = max(tiers, key=lambda t: tiers[t]) tier_note = (f" — the tier named {widest} holds {num(tiers[widest])} estimated tokens, sized " "to leave the local seat's registered window some headroom" if tiers[widest] < int(widest.rstrip("k")) * 1000 else "") depth_spread = " / ".join(f"{k} = {v} of the way through" for k, v in depths.items()) unique = " · ".join(f"{t} {row['unique_fraction_within_tier']}" for t, row in per_tier.items()) overlap_line = " · ".join( f"{pair} {row['as_fraction_of_the_smaller_tier']}" for pair, row in overlap.items() if isinstance(row, dict) and "as_fraction_of_the_smaller_tier" in row) contained = " · ".join( f"the {pair.split('×')[0]} text is {row['as_fraction_of_the_smaller_tier']} contained in " f"the {pair.split('×')[1]}" for pair, row in overlap.items() if isinstance(row, dict) and "as_fraction_of_the_smaller_tier" in row) return (f"The cabinet holds {num(len(items))} items: {num(recall_items)} with a planted sentence to recall and " f"{num(absent_items)} where nothing was planted and the right answer is that the text " f"does not say — the {num(recall_items)} recall items drawn from {num(needle_vocab)} " f"needle sentences and the {num(absent_items)} absent items from {num(absent_vocab)} " f"absent topics, each spread across the three tiers, so every count below is over " f"items, not over independent questions.\n\nThey spread over three filler tiers — " f"{tier_spread}, estimated at " f"{num(chars_per_token)} characters per token{tier_note} — and three planting depths " f"({depth_spread}), two items each.\n\nNew needles in an old haystack: the filler is " f"public domain and surely memorised, and the sentences planted in it did not exist " f"before the seed was drawn: the two sentence templates and the word lists were " f"written by an agent on the author's own model and ship in the kit, and the tuples " f"were drawn by code. The filler is Foster's Complete Hoyle, Project " f"Gutenberg #53881: {num(corpus_chars)} characters, sha {short_sha(corpus_sha)}. No " f"two items inside a tier share any filler text (measured: unique fraction {unique}). " f"Across tiers the filler does overlap — {contained} — so the three tiers are not " f"independent samples of the book, and this page compares planting depths only " f"within a tier, never across them. The needles were drawn by code from seed {seed} " f"after both stated cutoffs — an invented authority, an invented person, an integer " f"from {num(lo)} to {num(hi)}, and one of {num(len(terms))} card-game terms the " f"corpus does use — and every drawn component was screened, before a prompt existed, " f"against the round's list of words that may not appear on a public page.") def _cabinet_count(c: Cell, arm: str, what: str) -> str: """`12 of 12 collected (6 of 18 items NOT-COLLECTED — CONTEXT)` or `18 of 18`.""" A = f"arms.{arm}" cell = c.get("legC", f"{A}.{what}") registered = c.get("legC", f"{A}.{what}_registered_items") missing = c.get("legC", f"{A}.{what}_not_collected") if missing: return (f"{of(cell)} collected ({num(missing)} of {num(registered)} items " f"{sr.NOT_COLLECTED_CONTEXT})") return of(cell) def fill_filing_cabinet_table(c: Cell) -> str: arms = page_order(c.arms("legC")) fixture_sha = c.get("legC", "fixture_sha256") rows = [] for arm in arms: A = f"arms.{arm}" rows.append([ f"`{arm}`", _cabinet_count(c, arm, "recall"), _cabinet_count(c, arm, "abstention"), of(c.get("legC", f"{A}.fabrications")), num(c.get("legC", f"{A}.not_classified")), num(c.get("legC", f"{A}.abstained_wrongly_on_recall")), num(c.get("legC", f"{A}.missed_on_recall")), num(c.get("legC", f"{A}.context_ratio.below_floor")), ]) main = table(["arm", "recall", "abstention", "fabrications", "NOT-CLASSIFIED", "abstained on a recall item", "missed the needle", "cells under the 0.80 context floor"], rows) tier_rows, depth_rows = [], [] for arm in arms: for tier, cellrow in c.get("legC", f"arms.{arm}.by_tier").items(): if cellrow.get("state") != sr.COLLECTED: items = cellrow.get("items") or 0 half = items // 2 tier_rows.append([f"`{arm}`", tier, f"{cellrow.get('state')} ({plural(half, 'recall item')})", f"{cellrow.get('state')} ({plural(half, 'absent item')})", f"{cellrow.get('state')} ({plural(half, 'absent item')})"]) continue tier_rows.append([f"`{arm}`", tier, f"{cellrow['recall']}/{cellrow['recall_items']}", f"{cellrow['abstention']}/{cellrow['absent_items']}", f"{cellrow['fabrications']}/{cellrow['absent_items']}"]) for depth, cellrow in c.get("legC", f"arms.{arm}.by_depth").items(): rc = f"{cellrow['recall']}/{cellrow['recall_items']}" ab = f"{cellrow['abstention']}/{cellrow['absent_items']}" if cellrow.get("recall_not_collected"): rc += f" ({num(cellrow['recall_not_collected'])} not collected)" if cellrow.get("absent_not_collected"): ab += f" ({num(cellrow['absent_not_collected'])} not collected)" depth_rows.append([f"`{arm}`", depth, rc, ab]) flat = all(len({(r[2], r[3]) for r in depth_rows if r[0] == f"`{a}`"}) == 1 for a in arms) if flat: per_arm = "; ".join( f"{name(a)} " + next(f"{r[2]} recall and {r[3]} abstention" for r in depth_rows if r[0] == f"`{a}`") + " at every depth" for a in arms) depth_block = (f"**By planting depth**, over the items collected at that depth: depth made " f"no difference to any arm — {per_arm} (d10, d50 and d90 alike).") else: depth_block = ("**By planting depth**, over the items collected at that depth:\n\n" + table(["arm", "needle depth", "recall", "abstention"], depth_rows)) by = ("**By tier** (an uncollected tier prints its state, never a zero):\n\n" + table(["arm", "tier", "recall", "abstention", "fabrications"], tier_rows) + "\n\n" + depth_block) band = c.get("legC", f"arms.{arms[0]}.tie_band") ties = [] for i, a in enumerate(arms): for b in arms[i + 1:]: for what in ("recall", "abstention"): ka = kn(c.get("legC", f"arms.{a}.{what}")) kb = kn(c.get("legC", f"arms.{b}.{what}")) if ka is None or kb is None or ka[1] != kb[1]: continue # a count over a different collected set is not comparable if abs(ka[0] - kb[0]) < band: ties.append(f"{name(a)} and {name(b)} on {what} ({ka[0]} and {kb[0]} of " f"{ka[1]})") tie_line = (f"**Differences under {num(band)} items read TIED**, and this page draws no " "ordering between them: " + "; ".join(ties) + "." if ties else f"No two arms with the same collected set are within the {num(band)}-item tie " "band on either half.") try: ctrl_arms = c.arms("legC_think_true") except Unfillable as exc: raise Unfillable(c.id, f"the polarity control is registered (PREREG A3 FOLDED (2)) and " f"prints beside the grid, never instead: {exc.why}") from exc control = [] for arm in ctrl_arms: A = f"arms.{arm}" control.append(f"{name(arm)} read recall " f"{of(c.get('legC_think_true', f'{A}.recall'))}, abstention " f"{of(c.get('legC_think_true', f'{A}.abstention'))}, fabrications " f"{of(c.get('legC_think_true', f'{A}.fabrications'))} over the items it " "could hold") primary = {arm: (c.get("legC", f"arms.{arm}.recall"), c.get("legC", f"arms.{arm}.abstention")) for arm in ctrl_arms if arm in arms} same = all( (c.get("legC_think_true", f"arms.{arm}.recall"), c.get("legC_think_true", f"arms.{arm}.abstention")) == primary[arm] for arm in primary) draws = c.opt("counting_rules", "legC.polarity_control.draws_on_disk") on_disk = c.opt("counting_rules", "legC.polarity_control.records_on_disk") twice = (f" The control ran {num(draws)} times: the first draw went out at the seat posture " f"by mistake (PREREG A10) and is set aside unpublished, {num(on_disk)} records on disk " "for the leg in all; the re-run with thinking on is what prints." if draws and draws > 1 else "") control_line = ("**The same run with the reasoning turned on.** The local seat serves with its thinking channel " "off, so we ran its whole grid a second time with thinking on, to check that " "the setting and not the model is what the counts describe: " + "; ".join(control) + ( " — exactly the same reading. Turning the reasoning on bought nothing " "here." if same else ". The second reading is printed beside the first and never " "in place of it.") + twice) queue = [] for arm in arms: q = c.get("legC", f"arms.{arm}.hand_adjudication_queue") c._record("legC", f"arms.{arm}.hand_adjudication_queue[]") states = {} for item in q: st = ((item.get("collection_state") or item.get("verdict") or "queued") if isinstance(item, dict) else "queued") states[st] = states.get(st, 0) + 1 queue.append(f"{name(arm)} {num(len(q))}" + ( " (" + ", ".join(f"{num(v)} {k}" for k, v in states.items()) + ")" if q else "")) collected_queued = [] for arm in arms: q = c.get("legC", f"arms.{arm}.hand_adjudication_queue") cells_list = c.opt("legC", f"arms.{arm}.cells") or [] cells = {(x.get("item") or x.get("id")): x for x in cells_list if isinstance(x, dict)} n_collected = sum( 1 for item in q if (cells.get(item if isinstance(item, str) else item.get("item")) or {} ).get("collection_state") == sr.COLLECTED) collected_queued.append(f"{name(arm)} {num(n_collected)}") queue_line = ("**The hand pass.** The registration owes a second reading — the orchestrator " "with the outside reader as second reader — of every reply the rules class as " "non-abstaining or fabricating. Replies so classed, per arm: " + " · ".join(collected_queued) + ". " + ("The pass had nothing to read and did not run. " if all( x.endswith(" 0") for x in collected_queued) else "") + "The scorer's queue reads " + " · ".join(queue) + "; every queued item on the " "local seat is an uncollected 32k cell — a scorer artefact, since an uncalled " "cell has no reply to adjudicate — and the count prints because the count " "publishes.") refutation = str(c.get("legC", "self_refutation.sentence")).strip() refutation = refutation[:1].upper() + refutation[1:] if not refutation.endswith("."): refutation += "." return "\n".join([ f"Fixture sha {short_sha(fixture_sha)}; every count is over the items the arm was " "actually asked, with what was not collected beside it.", "", main, "", by, "", tie_line, "", control_line, "", queue_line, "", f"**What this grid cannot tell you, said before you ask.** {refutation}", ]) def _cabinet_count_doc(c: Cell, doc: str, arm: str, what: str) -> str: A = f"arms.{arm}" cell = c.get(doc, f"{A}.{what}") registered = c.get(doc, f"{A}.{what}_registered_items") missing = c.get(doc, f"{A}.{what}_not_collected") if missing: return f"{of(cell)} collected ({num(missing)} of {num(registered)} items not collected)" return of(cell) def fill_filing_cabinet_prose(c: Cell) -> str: arms = page_order(c.arms("legC")) rows = [] for arm in arms: A = f"arms.{arm}.context_ratio" transport = c.get("legA", f"arms.{arm}.transport_class") rows.append([f"`{arm}`", PROMPT_TOKEN_RULE.get(transport, transport), str(c.get("legC", f"{A}.min")), str(c.get("legC", f"{A}.median")), num(c.get("legC", f"{A}.n")), f"{num(c.get('legC', f'{A}.below_floor'))} under {c.get('legC', f'{A}.floor')}"]) ratio_table = table(["arm", "reported prompt tokens are", "min ratio", "median ratio", "items", "cells under the floor"], rows) canaries = [] for arm in arms: doc = f"canary_{arm}" if doc not in c.inputs.docs: continue both = c.get(doc, "both_found") per = c.get(doc, "per_canary") c._record(doc, "per_canary[].found") detail = " · ".join(f"{row['position']} {'found' if row['found'] else 'not found'}" for row in per) canaries.append(f"{name(arm)} " + ("found both" if both else "did not find both") + f" ({detail})") local = [a for a in arms if a.startswith("local-")] truncation = "" if local: doc = f"effort_{local[0]}" reported = c.get(doc, "evidence.prompt_tokens_reported") estimated = c.get(doc, "evidence.prompt_tokens_estimated_chars_over_4") ratio = c.get(doc, "evidence.context_ratio_reported_over_estimated") floor = c.get("legC", f"arms.{local[0]}.context_ratio.floor") num_ctx = c.get("roster", f"arms.{local[0]}.posture.num_ctx") census = c.get("legC", f"arms.{local[0]}.collection_census") c._record("legC", f"arms.{local[0]}.collection_census.*") missing = census.get(sr.NOT_COLLECTED_CONTEXT, 0) truncation = ( f" **The local seat did not see the front of the widest tier, measured.** On that " f"same 32k canary prompt the runtime reported {num(reported)} prompt tokens against " f"our chars÷4 estimate of {num(estimated)} — a ratio of {ratio}, under the registered " f"{floor} floor — and the canary at the front was the one it did not find. The seat " f"serves with a window of {num(num_ctx)} tokens (registered, and read back from the " f"instance), so the count is not the window: the runtime reports only the tokens it " f"evaluated fresh and drops the front of a prompt that overflows, and this page does " f"not know which of the two the shortfall is, or by how much the prompt overflowed on " f"this model's tokenizer — our four-characters-per-token estimate is a guess about a " f"tokenizer that is not ours. What the two facts do say together is that the model " f"that answers our users did not read the whole of a prompt this size, which is the " f"product finding this leg exists to surface; so the seat's {num(missing)} cells in " f"that tier were never called and read {sr.NOT_COLLECTED_CONTEXT} above, rather than " f"being scored as misses, and measuring the overflow itself is owed to a later page. The registration held a remedy — a wider window for the local seat, 65,536 tokens — that would have collected all twelve cells, and it was declined (PREREG A2, A5): the wider window would have evicted a live product model from the card that serves users, and this page was not worth that.") stops = " · ".join(f"{name(a)} {num(c.get('legC', f'arms.{a}.length_stops.count'))}" for a in arms) return "\n\n".join([ "**Did the whole text arrive?** For every item, the prompt tokens the endpoint reported " "are divided by our own estimate; a cell under the floor would print CONTEXT-TRUNCATED. " "The numerator is each road's own counter, so the three ratios are not one instrument — " "the rule is printed beside each:", ratio_table, "The double canary planted at both ends of one 32k prompt, read before any scored call: " + "; ".join(canaries) + "." + truncation, f"Replies cut short by an output cap, per arm: {stops}; the local seat answers under a " "cap of the registered length and the frontier arms under none, and the cap cost " "nothing measurable here.", ]) def fill_window_open(c: Cell) -> str: return stamp_second(_window(c, "opened")) def fill_window_close(c: Cell) -> str: return stamp_second(_window(c, "closed")) # ── what this page does not say ────────────────────────────────────────────── def fill_scaffolding_delta(c: Cell) -> str: cases = c.get("scaffolding_delta", "evidence.cases") conditions = c.get("scaffolding_delta", "evidence.conditions") chars = c.get("scaffolding_delta", "evidence.preamble_chars") tokens = c.get("scaffolding_delta", "evidence.preamble_tokens_estimated") sha = c.get("scaffolding_delta", "evidence.preamble_sha256") verdict = c.get("scaffolding_delta", "verdict") pairs = c.get("scaffolding_delta", "evidence.pairs") c._record("scaffolding_delta", "evidence.pairs[].input_token_delta") c._record("scaffolding_delta", "evidence.pairs[].prompt_sha256") deltas = sorted(row["input_token_delta"] for row in pairs) if not deltas: raise Unfillable(c.id, "the scaffolding-delta receipt carries no pairs, so there is no " "measured delta to publish.") same_prompt = all( row[conditions[0]]["prompt_sha256"] == row[conditions[1]]["prompt_sha256"] for row in pairs) collected = sum(1 for row in pairs for cond in conditions if row[cond].get("collection_state") == sr.COLLECTED) c._record("scaffolding_delta", "evidence.pairs[]..collection_state") median = deltas[len(deltas) // 2] spread = (f"exactly {num(deltas[0])} input tokens on every pair" if deltas[0] == deltas[-1] else f"{num(deltas[0])} to {num(deltas[-1])} input tokens per case, median " f"{num(median)}") return (f"the preamble is {num(chars)} characters, about {num(tokens)} tokens by our " f"chars÷4 estimate (sha {short_sha(sha)}); we put {num(cases)} rules-desk cases " f"through the API arm twice, once with it and once without — {num(collected)} of " f"{num(len(pairs) * len(conditions))} cells collected, {verdict} — and the endpoint's " f"own tokenizer counted {spread}, with the user prompt's sha256 identical in both " f"conditions on {'every' if same_prompt else 'not every'} pair, so the system message " f"was the only thing that moved (the estimate and the measurement differ because one " "is a character count divided by four and the other is the vendor's tokenizer)") def fill_reasoning_tokens(c: Cell) -> str: parts = [] for arm in page_order([a for a in PAGE_ORDER if not a.startswith("local-")]): tokens = c.get("bill", f"{arm}.reasoning_tokens") records = c.get("bill", f"{arm}.records") label = "reasoning tokens" if arm.startswith("openai-") else "thinking tokens" parts.append(f"{name(arm)} {num(tokens)} {label} over its {num(records)} recorded calls " "(every scored call, warmup and probe on that road)") return "; ".join(parts) def fill_egress_matrix(c: Cell) -> str: import report_build rows = [list(row) for row in report_build.EGRESS_ROWS] c.sources.append({"file": "harness/report_build.py", "json_path": "EGRESS_ROWS (the registered matrix, PREREG §10.3)"}) matrix = table(["text", "host", "leg", "whose terms"], rows) connections = c.get("egress", "connections") c._record("egress", "connections[].ip / .rdns / .matches_vendor_host") attribution = c.get("egress", "attribution") c._record("egress", "attribution.*") sample = c.get("egress", "sample") dropped = c.get("egress", "env_receipt_from_newest_cli_attempt.dropped_count") allowed = c.get("egress", "env_receipt_from_newest_cli_attempt.allowlist") c._record("egress", "env_receipt_from_newest_cli_attempt.allowlist[]") passed = c.get("egress", "env_receipt_from_newest_cli_attempt.names_passed") c._record("egress", "env_receipt_from_newest_cli_attempt.names_passed[]") # The same block is stamped on every sealed-CLI receipt the kit SHIPS (records[0].env_receipt); # the page reads it there too, so the ten names are checkable without the withheld receipt. shipped_records = c.get("cli_tools", "records") c._record("cli_tools", "records[0].env_receipt.names_passed[], records[0].env_receipt.allowlist[], " "records[0].env_receipt.dropped_count") shipped_env = (shipped_records[0].get("env_receipt") if isinstance(shipped_records, list) and shipped_records and isinstance(shipped_records[0], dict) else None) if not isinstance(shipped_env, dict): raise Unfillable(c.id, "the shipped G-TOOLS receipt carries no records[0].env_receipt; the " "environment names print from a receipt a reader can open, or not.") if (sorted(shipped_env.get("names_passed") or []) != sorted(passed) or sorted(shipped_env.get("allowlist") or []) != sorted(allowed) or shipped_env.get("dropped_count") != dropped): raise Unfillable(c.id, "the environment receipt in the shipped G-TOOLS record differs from " "the G-EGRESS copy; the page will not print one and cite the other.") set_by_harness = sorted(set(passed) - set(allowed)) matched = sum(1 for row in connections if row.get("matches_vendor_host")) conn_rows = [] for row in connections: owners = attribution.get(row["ip"]) or [] owner = ("; ".join(sorted({o.get("process", "?") for o in owners})) if owners else ("the bench's own child, matched to a vendor host" if row.get("matches_vendor_host") else "the local loopback" if row["ip"].startswith("127.") else EM)) conn_rows.append([row["ip"], row.get("rdns") or EM, ", ".join(row.get("matches_vendor_host") or []) or "none", owner]) socket_table = table(["remote address", "reverse DNS", "matched vendor host", "whose socket"], conn_rows) counts = c.get("pen", "evidence") if not isinstance(counts, dict): raise Unfillable(c.id, "the G-PEN receipt's `evidence` is not a map of counts; the " "“personal data: none” clause is a measurement or it is not printed.") integers = {k: v for k, v in counts.items() if isinstance(v, int) and not isinstance(v, bool)} if not integers: raise Unfillable(c.id, "the G-PEN receipt's `evidence` carries no integer counts. This " "sentence publishes measured zeros, never an assurance.") c._record("pen", "evidence.") pen_verdict = c.get("pen", "verdict") scanned = c.get("pen", "scanned") tests_ran = c.get("pen", "tests_ran") pen_line = " · ".join(f"{k.replace('_', ' ')} {num(v)}" for k, v in integers.items()) clause = ("No personal data appears anywhere in this round's artifacts, measured" if all(v == 0 for v in integers.values()) else "The scan's counts are printed as measured, not summarised") scope = str(scanned).split(" — ")[0] pen_when = _receipt_when(c.get("pen", "utc")) kit_scanned = c.get("pen", "kit_scanned") c._record("pen", "kit_scanned.built_utc") if not isinstance(kit_scanned, dict) or not kit_scanned.get("built_utc"): raise Unfillable(c.id, "the G-PEN receipt does not name the kit build it scanned " "(kit_scanned.built_utc); a scan the page calls measured has to say " "which files it covered — run harness/pen_scan.py --write after the " "kit is built (PREREG A19 (6)).") kit_built = _receipt_when(kit_scanned["built_utc"]) return "\n".join([ matrix, "", f"**Measured at the socket for the two frontier roads; asserted, from the registered matrix " f"above, for the shelf and the local seat.** The sample, as the receipt records it — " f"{sample} — found {num(matched)} of {num(len(connections))} connections going to a " "vendor API host; every address is named with what owned it:", "", socket_table, "", f"{cap(plural(len(passed), 'environment name'))} reached the vendor's own binary: " f"{num(len(allowed))} inherited from a {num(dropped + len(allowed))}-name environment, of " f"which {num(dropped)} were dropped — {', '.join(f'`{a}`' for a in allowed)} — and " f"{num(len(set_by_harness))} the harness sets itself to switch off this client's " f"telemetry, error reporting, auto-update, bug command and non-essential traffic " f"({', '.join(f'`{a}`' for a in set_by_harness)}; the outside reader's disclosed finding). " "Nothing on this page claims those names did not reach it. The same environment block is " "stamped on every sealed-tool receipt the kit ships (`records[0].env_receipt`), so the ten " "names and the drop list are checkable there without the withheld socket receipt.", "", f"**{clause}** (G-PEN {pen_verdict} at {pen_when}, {num(tests_ran)} checks, each over every file " f"under {scope}): {pen_line}. That is a scan of the files that stayed here — the kit as built " f"at {kit_built}, the receipts, the fixtures, the harness — and it says nothing about the wire " "beyond the socket sample above.", ]) def fill_gate_ledger(c: Cell) -> str: arms = page_order(c.arms("legA")) lines = [] # G2 — within-arm for arm in arms: A = f"arms.{arm}.G2" g2_pass = c.get("legA", f"{A}.pass", allow_null=True) lines.append(f"- G2, {name(arm)}: **SCORED, within-arm" f"{' — ' + ('meets' if g2_pass else 'misses') + ' its floors' if g2_pass is not None else ''}** {EM} median house-set recall " f"{c.get('legA', f'{A}.median_house_recall')}, median citation-set overlap " f"(Jaccard, 0 to 1) {c.get('legA', f'{A}.median_jaccard')}, at or above the " f"per-case recall floor on {c.get('legA', f'{A}.recall_ge_floor')} — over the " f"answered cases that carry a stored house citation set, against a floor " f"written over all the answered cases; the cases with no stored set are out " f"of the reading and the floor was not re-registered for them — the floors as " f"registered: {plain_floor(c.get('legA', f'{A}.floors'))} — " + ("a drift check against the seat's own stored answers" if arm.startswith("local-") else "agreement with the local seat's own stored citation sets, a named limit " "on a hosted row and never a cross-arm score (PREREG §4 (iv))")) # G6b + the context probe first = arms[0] probe_state = c.get("legA", f"arms.{first}.G6b.ctx_probe.state") lines.append(f"- G6b's long-context probe (case 61): **{probe_state}** {EM} the case sits in the " "frozen bank and no arm was pointed at it this round; the gap prints rather than " "the bank being renumbered around it") for arm in arms: cap = c.get("legA", f"arms.{arm}.G6b.output_cap") lines.append(f"- G6b, {name(arm)}: **SCORED** {EM} " f"{of(c.get('legA', f'arms.{arm}.G6b.done_reason_length_cases'))} cases " f"stopped by a cap; output cap: {cap}") # G5c / G6c na = [] for arm in arms: for gate in ("G5c", "G6c"): state = c.opt("legA", f"arms.{arm}.{gate}.state") if state: na.append(f"{gate} on {name(arm)}") else: for path in (f"arms.{arm}.{gate}.pass",): p = c.opt("legA", path) if p is not None: lines.append(f"- {gate}, {name(arm)}: **SCORED** {EM} " f"{'cleared' if p else 'missed'}") if na: lines.append(f"- {', '.join(na)}: **NOT-APPLICABLE — transport** (PREREG §4 (vii))") for arm in page_order(c.arms("g4")): lines.append(f"- G4, {name(arm)}: **{c.get('g4', f'arms.{arm}.state')}**") fam = c.get("legH", "panel_floor.count") floor = c.get("legH", "panel_floor.floor") lines.append((f"- the head-to-head's panel floor was met {EM} {num(fam)} families carried a " f"verdict against a floor of {num(floor)}") if c.get("legH", "panel_floor.pass") else f"- the head-to-head's panel floor: **{c.get('legH', 'panel_floor.state')}** " f"{EM} {num(fam)} families carried a verdict against a floor of {num(floor)}") for arm in page_order(c.arms("legC")): census = c.get("legC", f"arms.{arm}.collection_census") c._record("legC", f"arms.{arm}.collection_census.*") lines.append(f"- the filing cabinet, {name(arm)}: " + " · ".join(f"{k} {num(v)}" for k, v in census.items())) return "\n".join(lines) # ── the round's own probes, each with its receipt ─────────────────────────── #: An amendment may read a receipt's own verdict differently afterwards; the page prints both. AMENDED_READINGS = { ("G-EFFORT", "local-gemma4-26b"): "NOT-APPLICABLE — posture (PREREG A5 (1): the seat runs " "think:false by registration, so no thinking-token field " "exists to receipt; the file's own word stands as filed)", } #: PREREG A20 (5): the descriptions the two unrowed groups may carry, and the count each is true of. #: A description prints only when the count matches; otherwise the bare count prints and the sentence #: says less rather than something it cannot support. _UNROWED_BEFORE_THE_DAY = (3, "GPT-6 Astra's smoke read and its two effort reads") _UNROWED_SAME_DAY = (1, "the shelf roster read from the morning of the round, before the roster was " "registered") #: The stamp fields a receipt of this round may carry, best provenance first: one of the round's own #: ISO stamps, then the ENDPOINT's own `created` epoch (the three pre-window Astra receipts are raw #: API responses and carry nothing else), and only then the day in the file name — which is a #: convention this round keeps (`probes.Probe.write` stamps every name) rather than a field. _RECEIPT_STAMP_FIELDS = ("utc", "started_utc", "sent_utc", "stamped_utc", "read_utc") _NAME_DAY_RE = re.compile(r"^(\d{4})-?(\d{2})-?(\d{2})") def _receipt_day(c: Cell, kit_name: str) -> str | None: """The DAY a receipt under `receipts/` is stamped. `None` when nothing in it says.""" path = c.inputs.receipts / Path(kit_name).name if path.is_file(): data = c.inputs.receipt(path, fill_id=c.id) for field in _RECEIPT_STAMP_FIELDS: if data.get(field): return _receipt_when(data[field])[:10] created = data.get("created") if isinstance(created, int) and not isinstance(created, bool): return datetime.fromtimestamp(created, timezone.utc).strftime("%Y-%m-%d") m = _NAME_DAY_RE.match(Path(kit_name).name) return f"{m.group(1)}-{m.group(2)}-{m.group(3)}" if m else None def _unrowed_rest(c: Cell, rest: list[str]) -> list[str]: """The unrowed, non-hostile, non-operator receipts, split by day against the window's own. A19 (9) stamped this clause "from before the round's co-sign", and three of the four were: the fourth — the shelf roster read at 04:37Z — is two hours and forty-three minutes AFTER the co-sign the pre-registration stamps. The split that is true of every member is the window's own DAY, which the roster already carries, so that is what the sentence is built on. """ day = stamp_day(_window(c, "opened")) before = [f for f in rest if (d := _receipt_day(c, f)) is not None and d < day] same = [f for f in rest if f not in before] out = [] if before: n, described = _UNROWED_BEFORE_THE_DAY out.append(f"{plural(len(before), 'record')} from the day before the window" + (f" ({described})" if len(before) == n else "")) if same: n, described = _UNROWED_SAME_DAY out.append(described if len(same) == n else f"{plural(len(same), 'record')} from the morning of the round, before the " "window opened") return out def fill_probe_ledger(c: Cell) -> str: rows = [] kit = c.inputs.data("kit_index", fill_id=c.id) shipped = {r["file"] for r in kit.get("files", [])} withheld = {w["file"] for w in kit.get("withheld", [])} c._record("kit_index", "files[].file, withheld[].file") rowed: set[str] = set() #: PREREG A21: the newest adversarial-read receipt ON DISK, whether or not the kit ships it. The #: tail below says "the newest the current one" about the receipts that SHIP, and in v8 that was #: false: the current pass's own receipt was withheld (the screen refused the port numbers its #: notes quoted) and the newest shipped one was two passes old. Which sentence is true is a fact #: of the index, so it is read from the index rather than assumed. Names are UTC-stamped and #: append-only, so sort order is chronological. hostile_on_disk: list[str] = [] for path in c.inputs.receipt_paths(): data = c.inputs.receipt(path, fill_id=c.id) gate = str(data.get("gate") or data.get("probe") or "").split(" (")[0].split(" /")[0] if gate.lower() == "hostile-read": hostile_on_disk.append(f"receipts/{path.name}") if not gate or gate.lower() in ("hostile-read", "operator-read"): continue rowed.add(f"receipts/{path.name}") who = data.get("arm_id") or data.get("arm") or data.get("seat") or "" verdict = data.get("verdict") filed = str(verdict) if verdict is not None else "no verdict field — a record" reading = (AMENDED_READINGS.get((gate, who), "as filed") if verdict is not None else "as filed") kit_name = f"receipts/{path.name}" if kit_name not in shipped and kit_name not in withheld: # PREREG A20 (2). A receipt the kit index lists in NEITHER place is a pour that ran out # of order — the page rendered against an index built before the receipt existed — and # the cell it would print is the worst one on the page: on the row for the receipt behind # the page's own PASS claim, "the evidence is not listed". The ordering is registered # (kit build, pen scan, kit build, fill, kit build); this refusal is what enforces it, # the way every other ordering rule in this harness enforces itself. raise Unfillable(c.id, f"the probe receipt `{kit_name}` is in neither `files` nor " f"`withheld` in `{c.inputs.docs['kit_index'].label}` — not " "listed. The kit was built before this receipt existed: rebuild " "the kit, then fill (the pour's registered order). A cell that " "told a reader the evidence is not listed would be false.") in_kit = "shipped" if kit_name in shipped else "withheld (sha in the index)" rows.append([gate, name(who) if who in DISPLAY else (f"`{who}`" if who else EM), filed, reading, in_kit]) rows.sort(key=lambda r: (r[0], r[1])) unrowed = sorted(f for f in shipped if f.startswith("receipts/") and f not in rowed) hostile = [f for f in unrowed if "hostile-read" in f] operator = [f for f in unrowed if "operator-read" in f] rest = [f for f in unrowed if f not in hostile and f not in operator] parts = [] if hostile: newest = sorted(hostile_on_disk)[-1] if hostile_on_disk else None if newest is None or newest in shipped: standing = "the newest the current one" else: standing = (f"the newest, `{newest}`, is listed in the index as withheld (its sha beside " "it)") parts.append(f"the {plural(len(hostile), 'receipt')} of the adversarial reader's passes — one per " f"pass, append-only, {standing}") if operator: parts.append("the co-signer's own read receipt") if rest: parts.extend(_unrowed_rest(c, rest)) tail = "" if unrowed: tail = (f"\n\n{cap(plural(len(unrowed), 'more record'))} ship under `receipts/` and have no row " f"here because they are not probes on the round: {'; '.join(parts)}.") return table(["probe", "arm or seat", "verdict as filed", "the registered reading", "in the kit"], rows) + tail # ── the bill ───────────────────────────────────────────────────────────────── #: The bill's own note begins with the figure it glosses (`$0.00* — …`); the page prints the #: figure once, in code, and the gloss after it. PLAN_NOTE_PREFIX = re.compile(r"^\$0\.00\*\s*[—-]\s*") def fill_bill_table(c: Cell) -> str: bill = c.inputs.data("bill", fill_id=c.id) rows_json = {k: v for k, v in bill.items() if isinstance(v, dict) and "cost_state" in v} if not rows_json: raise Unfillable(c.id, "`results/bill.json` carries no row with a `cost_state`. The four " "cost states are never collapsed (PREREG §9), so a row without one " "is not rendered as a dash — it is a refusal.") arm_ids = set(PAGE_ORDER) arm_rows = [k for k in page_order([k for k in rows_json if k in arm_ids])] seat_rows = [k for k in rows_json if k not in arm_ids] def seat_label(key: str) -> str: for seat in SEAT_NAMES: if seat.replace(".", "-") == key: return seat return key def row(name_: str) -> list[str]: if "." in name_: raise Unfillable(c.id, f"the bill row `{name_}` carries a dot in its key, which this " "file's json path syntax reads as a nesting step; rename the " "row rather than have a figure read from a path that means " "something else.") state = c.get("bill", f"{name_}.cost_state") dollars = c.opt("bill", f"{name_}.usd", EM) shown = usd(dollars) + ("\\*" if state == "plan-included" else "") # `\*`: a literal star in md if state == "no-figure-held" and dollars is not EM: shown = f"{usd(dollars)} (list-rate estimate, not a receipt)" basis = " · ".join(str(x) for x in [c.opt("bill", f"{name_}.basis") or c.opt("bill", f"{name_}.note") or EM, c.opt("bill", f"{name_}.multiplication")] if x) basis = basis.replace("LIST-RATE ESTIMATE", "list-rate estimate").replace( "UNCACHED", "uncached") if state == "plan-included": basis = f"{EM} (plan-included)" return [f"`{seat_label(name_)}`", state, num(c.opt("bill", f"{name_}.prompt_tokens", EM)), num(c.opt("bill", f"{name_}.completion_tokens", EM)), shown, basis] caps = c.get("bill", "caps") c._record("bill", "caps.*") metered = c.get("bill", "caps.metered_total_usd") cap_total = c.get("bill", "caps.registered_usd.total") cap_parts = " · ".join(f"{k} {usd(v)}" for k, v in caps["registered_usd"].items() if k != "total") bound = c.get("bill", "caps.bound") metered_rows = [k for k in rows_json if c.get("bill", f"{k}.cost_state") == "metered"] metered_parts = " + ".join(f"{name(k) if k in DISPLAY else '`' + k + '`'} " f"{usd(c.get('bill', f'{k}.usd'))}" for k in metered_rows) unrounded = [c.opt("bill", f"{k}.usd_unrounded") for k in metered_rows] if all(isinstance(u, (int, float)) for u in unrounded): metered_parts += (" — each rounded for the table; summed before rounding for the total, " + " + ".join(f"${u:.4f}" for u in unrounded) + f" = ${sum(unrounded):.4f}") estimate_rows = [k for k in rows_json if c.get("bill", f"{k}.cost_state") == "no-figure-held"] estimate_parts = "; ".join(f"{name(k)}'s row prints {usd(c.get('bill', f'{k}.usd'))}, the " "tool's own list-rate estimate of what the same tokens would cost " "on the API — labelled, and not a dollar that changed hands" for k in estimate_rows) plan_note = c.get("bill", "plan_included_note") plan_rows = sum(1 for k in rows_json if c.get("bill", f"{k}.cost_state") == "plan-included") cached_lines = [] for k in arm_rows: cached = c.opt("bill", f"{k}.cached_prompt_tokens") if cached is not None: cached_lines.append(f"{name(k)} {num(cached)}") over = [] for k in seat_rows: o = c.opt("bill", f"{k}.over_cap_usd") if o: note = str(c.get("bill", f"{k}.cap_note")) tail = note.split("; ", 1)[1] if "; " in note else note over.append(f"`{k}` metered {usd(c.get('bill', f'{k}.usd'))} against a registered " f"{usd(c.get('bill', f'{k}.cap_usd'))} seat cap — {usd(o)} over. No " f"NOT-COLLECTED — CAP cell was written, so the registered consequence " f"never fired; the runner checks a cap between calls, which is how one " f"call could carry it over, and that behaviour is the harness's and is not " f"registered") warm = c.opt("bill", "warmups") timed = c.opt("bill", "timed_out_calls") tail = [] if isinstance(warm, dict): tail.append("discarded warmups, billed and receipted: " + " · ".join(f"{k.replace('_', ' ')} {num(v)}" for k, v in warm.items() if not isinstance(v, (dict, list)))) if isinstance(timed, dict): tail.append("calls that timed out: " + " · ".join(f"{k.replace('_', ' ')} {num(v)}" for k, v in timed.items() if not isinstance(v, (dict, list)))) states = c.get("counting_rules", "vocabulary.interval_rule") return "\n".join([ f"This is what the measuring cost, in tokens and in dollars, for every model that " f"answered or judged in the window: the arms under test first, the judging seats below. " f"**The dollars that changed hands come to {usd(metered)}** ({metered_parts}), against a " f"registered cap of {usd(cap_total)} ({cap_parts}); " f"{'no cap bound and no call was cut short by one' if not bound else caps.get('sentence')}. " f"{estimate_parts}. Plan-included rows print `$0.00*` — " f"{re.sub(PLAN_NOTE_PREFIX, '', str(plan_note)).replace('not free: a flat', 'not free — a flat')}. " f"Because {num(plan_rows)} of the {num(len(rows_json))} rows below are plan-included and " f"{'one more is' if estimate_rows else 'none is'} a subscription estimate, this bill is " "not a price for reproducing the round; it is a receipt for the part of it that " "metered.", "", "**The arms:**", "", table(["arm", "cost state", "tokens in", "tokens out", "USD", "basis · the multiplication performed"], [row(k) for k in arm_rows]), "", "**The judging seats.** Every plan-included row's counters are the endpoint's own, and " "no multiplication was performed on them:", "", table(["seat", "cost state", "tokens in", "tokens out", "USD", "basis · the multiplication performed"], [row(k) for k in seat_rows]), "", ("The two dollar figures in the arms table are built by different rules and are not comparable in either direction: one is our own multiplication at a cited rate with a cache discount the endpoint reported, the other a vendor tool's estimator whose cache treatment we cannot see, over token columns that are different instruments. Cached input tokens the endpoints reported: " + " · ".join(cached_lines) + "." if cached_lines else "No endpoint reported a cached-input count."), "", ("; ".join(over) + "." if over else "No seat read over its cap."), "", (cap("; ".join(tail)) + "." if tail else ""), "", f"Every figure above is a receipt, a vendor's own list-rate estimate labelled as one, or " f"an em dash; the counting rule that governs the rest of this page is unchanged here " f"({states}).", ]) # ── the takeaways ──────────────────────────────────────────────────────────── def fill_takeaway_rules_desk(c: Cell) -> str: arms = page_order(c.arms("legA")) A = lambda arm, path: c.get("legA", f"arms.{arm}.{path}") g1 = "; ".join(f"{name(a)} {'cleared' if A(a, 'G1.pass') else 'missed'} the citation floor " f"({of(A(a, 'G1.cite_survival'))} against " f"{floor_number(A(a, 'G1.floor_source'))})" for a in arms) fa = kn(floor_number(A(arms[0], "G3.G3a_floor"))) g3a = ", ".join(f"{name(a)} {kn(A(a, 'G3.G3a_no_false_rescue'))[0]}" for a in arms) all_miss = all(kn(A(a, "G3.G3a_no_false_rescue"))[0] < fa[0] for a in arms) g5a = "; ".join(f"{name(a)} {of(A(a, 'G5a.count'))}" for a in arms) hosted_clean = all(A(a, "G1.pass") for a in arms if not a.startswith("local-")) lead = ("**Neither frontier model cleared the whole rules desk, and the gate every arm missed " "is the one a user needs most.**" if all_miss and not hosted_clean else "**The rules desk, against floors frozen before the contestants existed.**") return (f"{lead} {g1}. On the twelve questions the book does not answer, the floor asks for " f"{fa[0]} of {fa[1]} abstentions and the readings were {g3a}" + (" — no arm cleared it" if all_miss else "") + f". Hidden directives followed: {g5a}" + "; the local seat's count is a pre-fix number, caught by the product since " "2026-08-16.") def fill_takeaway_head_to_head(c: Cell) -> str: floored = _floored_legH(c) if floored: return "**Head-to-head** — " + floored arm_a = c.get("legH", "arms.rate_is_about") other = c.get("legH", "arms.the_other") counts = c.get("legH", "per_case_counts") rate = c.get("legH", "pooled_preference_rate") cases = c.get("legH", "cases_with_observations") families = c.get("legH", "panel_families_carrying.count") covers = c.get("legH", "interval_covers_half") lo = c.get("legH", "cluster_bootstrap.lo") hi = c.get("legH", "cluster_bootstrap.hi") flipped = c.get("legH", "order_flip.flipped") pairs = c.get("legH", "order_flip.pairs_with_both_orders") census = c.get("legH", "seat_census") c._record("legH", "seat_census.*") full = [s for s, v in census.items() if isinstance(v, dict) and v.get("cases_carried") == cases] partial = {s: v.get("cases_carried") for s, v in census.items() if isinstance(v, dict) and v.get("cases_carried") != cases} seat_words = (f"{num(len(full))} of them over all {num(cases)} cases" + (", and " + " and ".join(f"one over {num(n)}" for n in partial.values()) + " before it was retired for the sheet shape it could not fill" if partial else "")) lead = (f"**All {num(families)} rival families read both frontier answers blind — {seat_words} — and did not " "separate them.**" if covers else f"**All {num(families)} rival families read both frontier answers blind, and the " "panel's reading separated them.**") tail = (" — an interval that covers 0.5" if covers else " — an interval that does not cover 0.5") return (f"{lead} The pooled preference rate is {rate} for {name(arm_a)} over the {num(cases)} " f"cases, and a cluster bootstrap over those cases puts it at [{lo}, {hi}]{tail}. " f"Counted case by case the tally was {num(counts.get('favoured_' + arm_a))} to " f"{num(counts.get('favoured_' + other))} for {name(arm_a)}, and swapping the two " f"answers flipped the verdict on {num(flipped)} of {num(pairs)} judge-and-case " f"comparisons (a rate over comparisons, not over the round's independent unit, the " f"case), which is " f"why the page draws {'no ordering' if covers else 'the ordering it does'}.") def fill_takeaway_filing_cabinet(c: Cell) -> str: arms = page_order(c.arms("legC")) hosted = [a for a in arms if not a.startswith("local-")] local = [a for a in arms if a.startswith("local-")] at_ceiling = c.get("legC", "self_refutation.frontier_arms_at_ceiling") hosted_words = "; ".join(f"{name(a)} recall {of(c.get('legC', f'arms.{a}.recall'))}, " f"abstention {of(c.get('legC', f'arms.{a}.abstention'))}, " f"fabrications {of(c.get('legC', f'arms.{a}.fabrications'))}" for a in hosted) local_words = "" if local: a = local[0] census = c.get("legC", f"arms.{a}.collection_census") c._record("legC", f"arms.{a}.collection_census.*") missing = census.get(sr.NOT_COLLECTED_CONTEXT, 0) held = c.get("legC", f"arms.{a}.recall_collected_items") + c.get( "legC", f"arms.{a}.abstention_collected_items") local_words = (f"; {name(a)} recall {of(c.get('legC', f'arms.{a}.recall'))} and abstention " f"{of(c.get('legC', f'arms.{a}.abstention'))} on the {num(held)} items it " "could hold" + (f", with {num(missing)} cells in the widest tier never collected " "because the prompt did not fit its window" if missing else "")) lead = ("**On the filing cabinet both frontier models were at ceiling, and the local seat ran " "out of room.**" if at_ceiling else "**The filing cabinet.**") band = c.get("legC", f"arms.{arms[0]}.tie_band") return f"{lead} {hosted_words}{local_words}; differences under {num(band)} items read TIED." # ── the doors, and the humans ──────────────────────────────────────────────── _AMENDMENT_RE = re.compile(r"^- \*\*A\d+ ·", re.M) def fill_amendments_count(c: Cell) -> str: sealed = len(_AMENDMENT_RE.findall(c.inputs.text("prereg_sealed", fill_id=c.id))) public = len(_AMENDMENT_RE.findall(c.inputs.text("prereg_public", fill_id=c.id))) c._record("prereg_sealed", "§12 — the dated amendment bullets, counted") c._record("prereg_public", "§12 — the dated amendment bullets, counted") if sealed != public: raise Unfillable(c.id, f"the sealed pre-registration carries {sealed} dated amendments and " f"the public copy the kit ships carries {public}; the page cannot " "point a reader at a copy that is missing amendments.") if not sealed: raise Unfillable(c.id, "no dated amendment bullet was found in §12; the count is not typed.") close = re.sub(r"\D", "", str(c.get("roster", "window.closed_utc")))[:12] stamps = re.findall(r"^- \*\*A\d+ · (\S+) ·", c.inputs.text("prereg_sealed", fill_id=c.id), flags=re.M) after = sum(1 for st in stamps if re.sub(r"\D", "", st)[:12] > close) if not after: return f"{plural(sealed, 'dated amendment')}, all while the round ran" return (f"{plural(sealed, 'dated amendment')} — {num(sealed - after)} while the round ran and " f"{num(after)} after the seal, each of those changing only what the scorers print and " f"what the kit ships") def fill_operator_read(c: Cell) -> str: """The credits' human line — a projection of an operator-read receipt, never a working note: the co-sign, the egress ruling, and that a human read the page. What the read found and where it landed stays in the receipt; the page does not narrate its own drafting.""" c.get("operator_read", "version_read") when = _receipt_when(c.get("operator_read", "utc")) found = c.get("operator_read", "found") c._record("operator_read", "found[].what / .landed_in") if not isinstance(found, list) or not found or not isinstance(found[0], dict): raise Unfillable(c.id, "an operator-read receipt carries no `found[]`; the read is a " "receipt or it is not on the page.") landed = found[0].get("landed_in") if not found[0].get("what") or not isinstance(landed, int): raise Unfillable(c.id, "an operator-read receipt's `found[0]` needs `what` and an integer " "`landed_in`.") if landed > int(FILLED_VERSION): raise Unfillable(c.id, f"an operator's finding lands in version {landed}, after this one; " "the page will not say it was read and fixed before it was.") read = ("read this page before it went out" if RELEASE_DATELINE_UTC else f"read a draft of this page on the drafts shelf on {when[:10]}") return (f"An operator read the pre-registration and co-signed it before the first call, ruled " f"on what could leave our machines, and {read}.") def fill_hostile_reader(c: Cell) -> str: """The credits' disclosure of the adversarial pass — the conflict named, the reading described, and where its receipt and critique are. No counts and no drafting narrative print: the receipt carries what it raised, what landed and its verify pass, and the page points at it.""" verdict = c.get("hostile", "verdict") evidence = c.get("hostile", "evidence") if not isinstance(evidence, dict): raise Unfillable(c.id, "the hostile-reader receipt's `evidence` is not a map of counts.") integers = {k: v for k, v in evidence.items() if isinstance(v, int) and not isinstance(v, bool)} if not integers: raise Unfillable(c.id, "the hostile-reader receipt's `evidence` carries no integer counts; " "this sentence is a measurement or it is not written.") c._record("hostile", "evidence.") required = integers.get("must_fixes", integers.get("must_fix")) landed = integers.get("must_fixes_landed", integers.get("must_fix_landed")) version_read = c.opt("hostile", "version_read") c.opt("hostile", "utc") outside = (f"The pre-registration had its own outside reader, `{c.get('outside_read', 'model')}`, " f"whose raw reply (sha {short_sha(c.get('outside_read', 'reply_sha256'))}) ships in the " f"kit under `receipts/`.") if str(verdict).upper().startswith("PENDING"): return ("**No hostile reader has been through this version yet.** This draft pours to the " "drafts shelf ahead of that pass. " + outside) if required is not None and landed is not None and landed < required: raise Unfillable(c.id, f"the hostile reader required {required} changes and {landed} have " "landed; the page does not go out with a required change open.") kit = c.inputs.data("kit_index", fill_id=c.id) c._record("kit_index", "files[].file, withheld[].file") critique = f"readers/v{version_read}-hostile.md" if critique in {r["file"] for r in kit.get("files", [])}: where = f"its critique ships in the kit as `{critique}`" elif critique in {w["file"] for w in kit.get("withheld", [])}: where = "its critique is listed in the kit's index as withheld, its sha beside it" else: where = "its critique is on file with the round" verify = c.opt("hostile", "verify_pass") if isinstance(verify, dict): c._record("hostile", "verify_pass.version, verify_pass.new_must_fixes") v_new = verify.get("new_must_fixes") if not (isinstance(v_new, int) and not isinstance(v_new, bool)): raise Unfillable(c.id, "the hostile-reader receipt's `verify_pass` carries no integer " "new_must_fixes; a verify pass is a measurement or it is not written.") if str(verify.get("version")) == str(FILLED_VERSION) and v_new == 0: pass_words = "its pass on the version you are reading" elif v_new == 0: pass_words = "its later pass" else: raise Unfillable(c.id, f"the hostile reader's verify pass raised {v_new} new must-fix(es); " "the page does not go out over an open finding.") else: pass_words = "what it required" return (f"An adversarial reader running on an Anthropic model — the same company as one of the " f"two arms, because no outside model with the context to do that reading sat for it, and " f"the page names the limit rather than call the reader independent — read a late draft in " f"full against the kit, the sealed pre-registration and the scorer output; what it found, " f"what changed, and {pass_words} are in its receipt under `receipts/`, and {where}. " + outside) # ── the register: id → builder, and the documents --plan promises ──────────── @dataclass(frozen=True) class Spec: fill_id: str builder: object rule: str reads: tuple[str, ...] = field(default_factory=tuple) verbatim: bool = False # a model's own words, printed as bytes: the statistics and vocabulary # guards do not apply (they are OUR claims); the literal screen does SPECS: tuple[Spec, ...] = ( Spec("published", fill_published, "window stamp — the page is dated from the roster's own window, both ends (PLAN §2)", ("roster:window.closed_utc",)), Spec("ledger-stamp", fill_ledger_stamp, "the probe receipts' own span, the socket sample's stamp and the pen scan's, each read from " "its receipt; the scored window is not restated as if the ledger sat inside it", ("receipts:*", "egress:utc", "pen:utc")), Spec("bill-stamp", fill_bill_stamp, "the scored window from the roster, plus the earliest record the bill itself counts — read " "from the bill's own records_window, never from a receipt the bill does not carry", ("roster:window.opened_utc", "roster:window.closed_utc", "bill:records_window.earliest")), Spec("operator-read", fill_operator_read, "an operator's co-sign and the read of a draft, from an operator-read receipt: date, what " "was found, and that it was fixed — never a review with no receipt, and no draft number in " "the prose (the receipt keeps `version_read`, and the fill refuses without it)", ("operator_read:version_read", "operator_read:utc", "operator_read:found[].what / .landed_in")), Spec("dateline-date", fill_dateline_date, "window stamp — the dateline is the window's close, to the day", ("roster:window.closed_utc",)), Spec("window-stamp", fill_window_stamp, "the cold-scroll stamp under every data heading: both ends of the window, UTC " "(the one slot that may repeat)", ("roster:window.opened_utc", "roster:window.closed_utc")), Spec("limits-stamp", fill_limits_stamp, "the limits section's own stamp: the scaffolding delta's start, from its receipt, and the " "side of the window it sits on — the one measurement in that section that predates the " "window, so the section cannot carry the window stamp", ("scaffolding_delta:started_utc", "roster:window.opened_utc", "roster:window.closed_utc")), Spec("panel-shape", fill_panel_shape, "the seats, families and transports out of the registered panel; no interval, no rate", ("panel:panel.seats[]",)), Spec("outside-read-summary", fill_outside_read_summary, "receipt fields verbatim, and the findings counted out of the sealed pre-registration's " "own A3 text by disposition; no interval, no rate", ("outside_read:model", "outside_read:sent_utc", "outside_read:counters.*", "outside_read:request_sha256.prereg_bytes", "outside_read:reply_sha256", "prereg_sealed:§12 A3")), Spec("rules-desk-lead", fill_rules_desk_lead, "counts with denominators from the bank's own composition; every unit set here is under the " "registered N, so no interval", ("bank:n", "bank:frozen", "bank:composition.*", "bank:substitution.house_answers_rederived", "counting_rules:legA.reps")), Spec("users-words", fill_users_words, "counts with denominators from the queries file's own disclosure block; no interval", ("bank:n", "bank:substitution.canonical_queries", "queries:canonical_verbatim", "queries:substituted", "queries:author_model", "queries:disclosure.*")), Spec("quoted-replies", fill_quoted_replies, "three replies quoted verbatim from the rows files, each with its case, rep, length and " "sha; a model's words are not this page's claims, so the statistics and vocabulary " "guards do not run over them and the publication screen does", ("bank:cases[]", "legA:arms.*.G5a.followed", "rows::case/rep — response"), verbatim=True), Spec("gate-cards", fill_gate_cards, "one card per arm: the scorer's own cells with their denominators, the floor read out of " "the design document, and the scorer's own pass field as the verdict word", ("legA:arms.*.G1.*", "legA:arms.*.G3.*", "legA:arms.*.G5a.*", "legA:arms.*.G5b.*", "legA:arms.*.G6a.*", "legA:arms.*.G6b.*", "counting_rules:legA.arms", "bank:frozen", "bank:n")), Spec("gate-verdict", fill_gate_verdict, "the verdicts in words, from the scorers' own pass fields and the floors' own numbers; " "every count with its denominator, no interval", ("legA:arms.*.G1.*", "legA:arms.*.G3.*", "legA:arms.*.G5a.*", "legA:arms.*.G5b.*", "legA:arms.*.G6a.*", "legA:arms.*.G6b.*", "bank:frozen")), Spec("model-swap", fill_model_swap, "PREREG A13's finding, counted out of the census and the identity receipt; no interval", ("legA:arms.*.G5b.*", "legA:arms.*.collection_states.*", "legA:arms.*.calls_expected", "identity_census:served_by.*", "identity_census:the_one.block.*", "identity_census:the_one.driver_state", "legC:arms.*.collection_census.*", "bank:composition.*")), Spec("census-table", fill_census_table, "every collection state each scorer wrote, with the asked denominator in the row", ("legA:arms.*.collection_states.*", "legA:arms.*.calls_expected", "legA:arms.*.no_mode_cases.count", "legA:arms.*.no_mode_cases.denominator", "bank:n", "counting_rules:legA.reps")), Spec("rules-desk-prose", fill_rules_desk_prose, "three claims: the house control with its case count; the injection gate in words with the " "lint's version, commit and date extracted from the scorer's own note; forged markers and " "the missed-abstain bucket", ("legA:G_CALIBRATE.*", "legA:arms.*.G5a.*", "legA:arms.*.G3.missed_abstain_bucket.reps", "legA:arms.*.G1.forged_markers_total", "legA:arms.*.G1.forged_marker_cases", "counting_rules:legA.reps", "bank:composition.answered")), Spec("g4-groundedness", fill_g4_groundedness, "the judge scorer's own cells with their denominators; UNCERTAIN cells named; the " "denominator clause from the bank's answered count and the arm's G3b cell", ("g4:arms.*.judged_cases_n", "g4:arms.*.grounded", "g4:arms.*.per_family.*", "g4:arms.*.collection_census.*", "g4:arms.*.grounded_counts.*", "g4:arms.*.recusal.*", "g4:arms.*.panel_floor.count", "g4:arms.*.state", "g4:arms.*.grounded_rate_state", "bank:composition.answered", "legA:arms.*.G3.G3b_false_abstain")), Spec("recognition-claims", fill_recognition_claims, "the G4 claims checked against the key per arm, with denominators; the head-to-head " "claims and the sensitivity cut's rate only where its case count clears N ≥ 30", ("g4:arms.*.self_disclosure.*", "g4:arms.*.collection_census.*")), Spec("head-to-head-lead", fill_head_to_head_lead, "the pairwise scorer's own shape, census, seat census, pooled cell in both orientations, " "its cluster bootstrap over the cases, the order flip with its tie count, the A11 cases, " "and the paragraph it wrote; a floored panel prints only its state", ("legH:registered_shape.*", "legH:collection_census.*", "legH:seat_census.*", "legH:per_case_shape.*", "legH:cases_with_observations", "legH:pooled_preference_rate", "legH:pooled_preference_state", "legH:per_case_counts", "legH:panel_families_carrying.count", "legH:cluster_bootstrap.*", "legH:order_flip.*", "legH:cases_added_under_A11", "legH:cases_added_under_A11_ids[]", "legH:verdict_paragraph", "legH:panel_floor.*", "legH:arms.*", "legH:sensitivity_cut.*")), Spec("head-to-head-table", fill_head_to_head_table, "one row per case, joined to the bank's game; NO per-case rate — a case is a vote over " "its judges and that unit is under 30 (PREREG §9); the per-seat rows with the seat census", ("legH:per_case_table[]", "legH:per_case_counts", "legH:per_judge_rates.*", "legH:seat_census.*", "legH:registered_shape.seats", "legH:arms.*", "legH:panel_floor.*", "bank:cases[]")), Spec("filing-cabinet-lead", fill_filing_cabinet_lead, "counts and shas from the fixture and the needle file; the item split counted from the " "fixture's own items; the depths print as fractions, never as percentages", ("fixture:items[]", "fixture:tiers", "fixture:depths", "fixture:unique_fraction_per_tier.*", "fixture:chars_per_token_estimate", "fixture:cross_tier_overlap", "needles:seed", "needles:corpus_chars", "needles:corpus_sha256", "needles:integer_range", "needles:terms_present_in_corpus[]", "counting_rules:legC.needle_vocab", "counting_rules:legC.absent_topic_vocab")), Spec("filing-cabinet-table", fill_filing_cabinet_table, "the scorer's own counts over COLLECTED items with the registered and not-collected " "counts beside them (PREREG A16 (1)); an uncollected tier prints its state; the tie band " "is the scorer's own; the polarity control prints beside, never instead", ("legC:fixture_sha256", "legC:arms.*", "legC:self_refutation.*", "legC_think_true:arms.*", "counting_rules:legC.polarity_control.*")), Spec("filing-cabinet-prose", fill_filing_cabinet_prose, "ratios and token counts as measured per arm with each road's numerator rule named; the " "local seat's truncation read as PREREG A5 reads it; no percentage, no interval", ("legC:arms.*.context_ratio.*", "legC:arms.*.collection_census.*", "legC:arms.*.length_stops.count", "legA:arms.*.transport_class", "canary_cli-claude-fable-5-1:*", "canary_openai-gpt-6-astra:*", "canary_local-gemma4-26b:*", "effort_local-gemma4-26b:evidence.*", "roster:arms.*.posture.num_ctx")), Spec("window-open", fill_window_open, "window stamp from the roster, to the second (the prose beside it says UTC)", ("roster:window.opened_utc",)), Spec("window-close", fill_window_close, "window stamp from the roster, to the second (the prose beside it says UTC)", ("roster:window.closed_utc",)), Spec("scaffolding-delta", fill_scaffolding_delta, "the probe's own measured token deltas over 8 cases × 2 conditions; a range and a median " "over a unit set of 8, so no interval and no percentage", ("scaffolding_delta:verdict", "scaffolding_delta:evidence.*")), Spec("reasoning-tokens", fill_reasoning_tokens, "each frontier arm's reasoning or thinking tokens over its records, out of the bill", ("bill:*.reasoning_tokens", "bill:*.records")), Spec("egress-matrix", fill_egress_matrix, "the registered matrix, the socket sample with each address's owner from the receipt's " "own attribution, the environment names with their denominator, and the pen scan's own " "counts with its scope; the “no personal data” clause prints only when every count is zero", ("report_build:EGRESS_ROWS", "egress:sample", "egress:connections[]", "egress:attribution.*", "egress:env_receipt_from_newest_cli_attempt.*", "cli_tools:records[0].env_receipt.*", "pen:verdict", "pen:scanned", "pen:utc", "pen:kit_scanned.*", "pen:tests_ran", "pen:evidence.*")), Spec("gate-ledger", fill_gate_ledger, "every state each scorer wrote for the gates the cards do not carry: G2 within-arm, G6b " "and its probe, G5c/G6c, G4, the panel floor, the filing cabinet's collection per arm", ("legA:arms.*.G2.*", "legA:arms.*.G6b.*", "legA:arms.*.G5c.*", "legA:arms.*.G6c.*", "g4:arms.*.state", "legH:panel_floor.*", "legC:arms.*.collection_census.*")), Spec("probe-ledger", fill_probe_ledger, "one row per probe receipt on disk: its gate, its verdict as filed, the reading an " "amendment gave it, and whether the kit ships it — no number, no interval; a receipt the " "kit index lists in neither place refuses the fill, and the unrowed receipts are split by " "their stamp's day against the window's own", ("receipts:*", "kit_index:files[].file", "kit_index:withheld[].file", "roster:window.opened_utc")), Spec("bill-table", fill_bill_table, "the ledger's own rows in two tables; the caps and the metered total from the bill's own " "caps block; the plan-included asterisk; a row without a cost state is a refusal", ("bill:*.cost_state", "bill:*.prompt_tokens", "bill:*.completion_tokens", "bill:*.usd", "bill:*.basis", "bill:*.cached_prompt_tokens", "bill:*.over_cap_usd", "bill:*.cap_usd", "bill:*.cap_note", "bill:caps.*", "bill:plan_included_note", "bill:warmups", "bill:timed_out_calls", "counting_rules:vocabulary.interval_rule")), Spec("takeaway-rules-desk", fill_takeaway_rules_desk, "the same gate cells and pass fields as the cards, one claim; counts with denominators", ("legA:arms.*.G1.*", "legA:arms.*.G3.*", "legA:arms.*.G5a.count")), Spec("takeaway-head-to-head", fill_takeaway_head_to_head, "the paragraph the pairwise scorer wrote, verbatim, then the tally and the rate with " "their subject named", ("legH:verdict_paragraph", "legH:per_case_counts", "legH:pooled_preference_rate", "legH:cases_with_observations", "legH:arms.*", "legH:panel_floor.*", "legH:seat_census.*", "legH:panel_families_carrying.count", "legH:interval_covers_half", "legH:cluster_bootstrap.*", "legH:order_flip.*")), Spec("takeaway-filing-cabinet", fill_takeaway_filing_cabinet, "the scorer's own counts over collected items per arm, the tie band named, no interval", ("legC:arms.*.recall", "legC:arms.*.abstention", "legC:arms.*.fabrications", "legC:arms.*.recall_not_collected", "legC:arms.*.collection_census.*", "legC:self_refutation.frontier_arms_at_ceiling", "legC:arms.*.tie_band")), Spec("amendments-count", fill_amendments_count, "the dated amendment bullets counted in the sealed pre-registration and in the public " "copy the kit ships; a mismatch is a refusal", ("prereg_sealed:§12", "prereg_public:§12", "roster:window.closed_utc")), Spec("hostile-reader-and-outside-read", fill_hostile_reader, "the hostile reader's own receipt, in the tense its verdict field is in, and the outside " "read's sha; no receipt, no sentence", ("hostile:verdict", "hostile:evidence.*", "hostile:version_read", "hostile:utc", "hostile:verify_pass.*", "kit_index:files[].file", "kit_index:withheld[].file", "outside_read:model", "outside_read:reply_sha256")), ) SPEC_BY_ID = {s.fill_id: s for s in SPECS} #: The one slot a copy may carry more than once: the cold-scroll stamp under every data heading #: is the same two roster fields every time, and a stamp that could only print once would leave #: every section but one undated. REPEATABLE = frozenset({"window-stamp"}) # ── the copy, its placeholders, and the two files this file writes ─────────── def split_front_matter(text: str) -> tuple[str, str]: m = re.match(r"---\n(.*?\n)---\n", text, re.S) if not m: raise SystemExit("article_fill: the author copy has no YAML front matter") return m.group(1), text[m.end():] def placeholders(text: str) -> list[tuple[str, str]]: """Every `⟦id: … | …⟧` in document order. A bracket that is not one is a refusal.""" ids = [(m.group(1), m.group(2).strip()) for m in PLACEHOLDER.finditer(text)] total = len(ANY_BRACKET.findall(text)) if total != len(ids): malformed = [m.group(1)[:60] for m in ANY_BRACKET.finditer(text) if not PLACEHOLDER.fullmatch("⟦" + m.group(1) + "⟧")] raise SystemExit( f"article_fill: {len(malformed)} bracketed slot(s) in the copy are not " f"`⟦id: | ⟧` and cannot be addressed: {malformed}. An unnamed slot " "would ship as a bracket on a public page.") seen: dict[str, int] = {} for fill_id, _ in ids: seen[fill_id] = seen.get(fill_id, 0) + 1 dupes = sorted(k for k, v in seen.items() if v > 1 and k not in REPEATABLE) if dupes: raise SystemExit(f"article_fill: duplicate placeholder id(s) {dupes}; each slot is " "addressed by name and a name must mean one place.") return ids def build(*, inputs: Inputs | None = None, copy_path: Path | None = None, rules=None) -> tuple[list[dict], str]: """(`fills.json` rows, the filled copy). Raises `SystemExit` listing every refusal.""" inputs = inputs or Inputs() copy_path = Path(copy_path) if copy_path else AUTHOR_COPY if not copy_path.is_file(): raise SystemExit(f"article_fill: no author copy at {copy_path}") text = copy_path.read_text(encoding="utf-8") fm, body = split_front_matter(text) slots = placeholders(text) unknown = [i for i, _ in slots if i not in SPEC_BY_ID] orphan = [s.fill_id for s in SPECS if s.fill_id not in {i for i, _ in slots}] if unknown or orphan: raise SystemExit(f"article_fill: the copy and the register disagree — slots with no builder " f"{unknown}; builders with no slot {orphan}.") rows: list[dict] = [] refusals: list[str] = [] done: set[str] = set() for fill_id, _description in slots: if fill_id in done: continue done.add(fill_id) spec = SPEC_BY_ID[fill_id] cell = Cell(fill_id, inputs) try: built = spec.builder(cell) guard(fill_id, built, rules=rules, verbatim=spec.verbatim) except Unfillable as exc: refusals.append(f" {fill_id}: {exc.why}") continue rows.append({"id": fill_id, "text_or_table_markdown": built, "sources": cell.sources, "rule": spec.rule}) if refusals: raise SystemExit( f"article_fill: REFUSED — {len(refusals)} of {len(done)} placeholder(s) cannot be " "filled from the files on disk. Nothing was written; no figure is estimated and no " "slot ships as a bracket.\n" + "\n".join(refusals)) by_id = {r["id"]: r for r in rows} filled = _render(fm, body, by_id) return rows, filled def _render(fm: str, body: str, by_id: dict[str, dict]) -> str: """The filled copy: every slot replaced, each with an HTML comment naming its sources. The front matter gets the VALUE only — an HTML comment inside YAML is not front matter any more, and the hub's pour reads those keys. The provenance for those slots rides `fills.json`. """ def cite(row: dict) -> str: # grouped by file: a table fill reads fifty paths out of one scorer, and repeating the # filename fifty times makes the provenance harder to read, not easier. by_file: dict[str, list[str]] = {} for src in row["sources"]: by_file.setdefault(src["file"], []).append(src["json_path"]) cites = " ; ".join(f"{f}: " + ", ".join(paths) for f, paths in by_file.items()) return f"" def in_front_matter(m: re.Match) -> str: return by_id[m.group(1)]["text_or_table_markdown"] def render_line(line: str) -> list[str]: """The line with its fills in, followed by ONE comment line naming their sources. A comment never shares a BLOCK with prose: the hub's pour converts a run of non-blank lines as one markdown block, so a comment inline or directly under a paragraph is escaped into VISIBLE text and a json path's `*` becomes emphasis (v1 shipped all 21 of its comments that way). Set off by a blank line, the comment run is its own block and the converter passes it through as the HTML comment it is — provenance in the page source, invisible on the page. A block fill (a table, a multi-paragraph fill) takes its comment above, blank-line separated. """ hits = list(PLACEHOLDER.finditer(line)) if not hits: return [line] if len(hits) == 1 and "\n" in by_id[hits[0].group(1)]["text_or_table_markdown"]: row = by_id[hits[0].group(1)] lead = line[:hits[0].start()].rstrip() tail = line[hits[0].end():] block = row["text_or_table_markdown"] + tail if lead: # a block fill at the end of a lead sentence: the lead keeps its own paragraph, # and the table or list that follows is its own block (a table row glued to # prose is a pour refusal) return [lead, "", cite(row), "", block] return [cite(row), "", block] cites = [] def sub(m: re.Match) -> str: row = by_id[m.group(1)] if "\n" in row["text_or_table_markdown"]: raise SystemExit(f"article_fill: block fill `{row['id']}` shares a line with " "another slot; give it its own line.") cites.append(cite(row)) return row["text_or_table_markdown"] return [PLACEHOLDER.sub(sub, line), ""] + cites # a blank line: the comments are their own block def render_body(text: str) -> str: """Comment lines after a LIST ITEM are held until the list ends, so a dropped comment line can never split a list in two.""" out: list[str] = [] held: list[str] = [] for raw in text.split("\n"): rendered = render_line(raw) is_item = raw.lstrip().startswith(("- ", "* ")) if is_item and len(rendered) >= 3 and rendered[2].startswith("