{ "schema_version": "1.1", "instrument": { "figures_read_from": "cite-cut prose (gates.prose_of) since 2026-09-09", "warm_arm_walk": "per page, one warm immediately before that page's own questions, since 2026-09-09", "not_comparable_with": "reports whose schema_version is 1.0" }, "run_utc": "2026-09-13T03:51:59Z", "arm": "cold", "concurrency": 6, "effective_concurrency": 6, "seat": { "url_host_withheld": true, "model": "gemma-4-26b-a4b-fp8", "class": "gemma-class", "seed": "no --seed flag sent" }, "packnorm_definition_recorded_not_governing": "v3", "bank": { "path": "bank.json", "counts": { "on_page": 41, "off_page": 21, "total": 62 } }, "results": { "on_page_total": 41, "on_page_answered": 41, "on_page_partial": 0, "off_page_partial": 0, "sentences_kept": 140, "sentences_written": 142, "non_verbatim_figures": 0, "uncited": 0, "off_page_total": 21, "off_page_abstained": 21, "off_page_abstained_by_gate": 2, "off_page_abstained_by_model": 19, "off_page_flagged_but_not_verbatim": [], "on_page_abstained": 1, "off_page_cited": 19, "redaction_hits": 0, "cite_overlap_p50": 0.6795, "cite_overlap_min": 0.0, "cite_overlap_max": 1.0, "cite_overlap_rows": 60, "seat_errors": 0, "p50_seconds": 4.702, "p95_seconds": 14.539, "pages_warmed": null, "cache_hits": null, "cache_hit_of": null }, "rows": [ { "slug": "three-new-voices-at-the-narrators-chair", "q": "what did the round cost in total, and how were the three new voices judged?", "ok": true, "answer": "The metered total for the round came to $0.7652 § {#the-bill}. The three new voices were judged by a blind panel of seven seats from outside model families, six of which carried the round, reading lettered replies with no model names on them § {#seven-seats-six-carried-and-where-their-zeros-sit}. The article notes that the counts for the metered rows are upper bounds and that the subscription row's two estimates are never added into the total § {#the-bill}.", "served": "The metered total for the round came to $0.7652 § {#the-bill}. The three new voices were judged by a blind panel of seven seats from outside model families, six of which carried the round, reading lettered replies with no model names on them § {#seven-seats-six-carried-and-where-their-zeros-sit}. The article notes that the counts for the metered rows are upper bounds and that the subscription row's two estimates are never added into the total § {#the-bill}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 15.329734672035556, "ttft_ms": 12763, "tokens_in": 82794, "tokens_out": 113, "cached_tokens": 32, "cites": [ "the-bill", "seven-seats-six-carried-and-where-their-zeros-sit" ], "gates": { "cites": [ "the-bill", "seven-seats-six-carried-and-where-their-zeros-sit" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "0.7652" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-bill", "overlap": 0.8 }, { "i": 1, "cite": "seven-seats-six-carried-and-where-their-zeros-sit", "overlap": 0.684 }, { "i": 2, "cite": "the-bill", "overlap": 0.786 } ], "cite_overlap_p50": 0.786, "id": "on-01", "kind": "on-page" }, { "slug": "two-new-frontier-models-at-the-rules-desk", "q": "how many cases did the blind head-to-head use, and did it separate the two models?", "ok": true, "answer": "The blind head-to-head used 36 cases and did not separate the two models § {#head-to-head}. A seven-family panel produced a pooled preference rate of 0.544 for Claude Fable 5.1, which resulted in an interval that covers 0.5 § {#head-to-head}. The article notes that the narrator's chair job was moved out of this round before pre-registration was written § {#two-strangers-at-the-door}.", "served": "The blind head-to-head used 36 cases and did not separate the two models § {#head-to-head}. A seven-family panel produced a pooled preference rate of 0.544 for Claude Fable 5.1, which resulted in an interval that covers 0.5 § {#head-to-head}. The article notes that the narrator's chair job was moved out of this round before pre-registration was written § {#two-strangers-at-the-door}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 14.619173718965612, "ttft_ms": 3199, "tokens_in": 26913, "tokens_out": 108, "cached_tokens": 32, "cites": [ "head-to-head", "two-strangers-at-the-door" ], "gates": { "cites": [ "head-to-head", "two-strangers-at-the-door" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "36", "0.544", "5.1", "0.5" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "head-to-head", "overlap": 0.875 }, { "i": 1, "cite": "head-to-head", "overlap": 0.786 }, { "i": 2, "cite": "two-strangers-at-the-door", "overlap": 0.9 } ], "cite_overlap_p50": 0.875, "id": "on-02", "kind": "on-page" }, { "slug": "six-worlds-painted-large", "q": "how many worlds were painted, and at what size and step count?", "ok": true, "answer": "Six worlds were painted at 2048×1280 and 40 steps § {#for-the-technically-curious}. The article does not mention how many worlds were left out, moved, or not measured.", "served": "Six worlds were painted at 2048×1280 and 40 steps § {#for-the-technically-curious}. The article does not mention how many worlds were left out, moved, or not measured.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.153961843985599, "ttft_ms": 374, "tokens_in": 3145, "tokens_out": 51, "cached_tokens": 32, "cites": [ "for-the-technically-curious" ], "gates": { "cites": [ "for-the-technically-curious" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "1280", "40" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "for-the-technically-curious", "overlap": 0.857 } ], "cite_overlap_p50": 0.857, "id": "on-03", "kind": "on-page" }, { "slug": "the-beat-lab-asked-and-answered", "q": "how many answers does the page carry, and does any model run to make it?", "ok": true, "answer": "The page carries twenty-seven answers and no model is asked for any answer on this page § {#is-this-ai}. The article notes that the answers were drafted by AI agents in the byline but were read and approved word by word by humans § {#who-ran-this-and-thanks}.", "served": "The page carries twenty-seven answers and no model is asked for any answer on this page § {#is-this-ai}. The article notes that the answers were drafted by AI agents in the byline but were read and approved word by word by humans § {#who-ran-this-and-thanks}.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.31510193302529, "ttft_ms": 461, "tokens_in": 4643, "tokens_out": 64, "cached_tokens": 32, "cites": [ "is-this-ai", "who-ran-this-and-thanks" ], "gates": { "cites": [ "is-this-ai", "who-ran-this-and-thanks" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "is-this-ai", "overlap": 0.833 }, { "i": 1, "cite": "who-ran-this-and-thanks", "overlap": 0.9 } ], "cite_overlap_p50": 0.867, "id": "on-04", "kind": "on-page" }, { "slug": "ten-minutes-with-living-artists", "q": "how many tracks and releases went into the adapter, and what did the one listener prefer?", "ok": true, "answer": "The adapter was trained on 159 tracks from 41 releases § {#a-genre-not-a-composer}. The one listener preferred the untouched model both times § {#the-ear}. The article does not measure whether the untouched model is any good at this genre § {#what-we-did-not-measure}.", "served": "The adapter was trained on 159 tracks from 41 releases § {#a-genre-not-a-composer}. The one listener preferred the untouched model both times § {#the-ear}. The article does not measure whether the untouched model is any good at this genre § {#what-we-did-not-measure}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.738253566029016, "ttft_ms": 1072, "tokens_in": 13206, "tokens_out": 70, "cached_tokens": 32, "cites": [ "a-genre-not-a-composer", "the-ear", "what-we-did-not-measure" ], "gates": { "cites": [ "a-genre-not-a-composer", "the-ear", "what-we-did-not-measure" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "159", "41" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "a-genre-not-a-composer", "overlap": 0.5 }, { "i": 1, "cite": "the-ear", "overlap": 0.714 }, { "i": 2, "cite": "what-we-did-not-measure", "overlap": 0.833 } ], "cite_overlap_p50": 0.714, "id": "on-05", "kind": "on-page" }, { "slug": "listen-for-yourself", "q": "how many renders are on the page, and what was done to their loudness?", "ok": true, "answer": "There are sixty-three players on the page, each playing a listening master of a clip § {#how-to-listen}. The listening masters were created by applying one constant gain to the raw renders, with no limiter, compression, equalisation, trimming, or re-encoding § {#how-to-listen}. The gain applied was the smaller of two numbers: how far the clip sits from −16 LUFS and how far its true peak sits from a −1 dBTP ceiling § {#how-to-listen}. The article notes that the raw renders sit in `audio/` and are what every measured number on the page was read off § {#how-to-listen}.", "served": "There are sixty-three players on the page, each playing a listening master of a clip § {#how-to-listen}. The listening masters were created by applying one constant gain to the raw renders, with no limiter, compression, equalisation, trimming, or re-encoding § {#how-to-listen}. The gain applied was the smaller of two numbers: how far the clip sits from −16 LUFS and how far its true peak sits from a −1 dBTP ceiling § {#how-to-listen}. The article notes that the raw renders sit in `audio/` and are what every measured number on the page was read off § {#how-to-listen}.", "partial": false, "sentences_kept": 4, "sentences_written": 4, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 14.904880126006901, "ttft_ms": 1730, "tokens_in": 14463, "tokens_out": 141, "cached_tokens": 32, "cites": [ "how-to-listen" ], "gates": { "cites": [ "how-to-listen" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "16" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "how-to-listen", "overlap": 1.0 }, { "i": 1, "cite": "how-to-listen", "overlap": 0.786 }, { "i": 2, "cite": "how-to-listen", "overlap": 1.0 }, { "i": 3, "cite": "how-to-listen", "overlap": 0.9 } ], "cite_overlap_p50": 0.95, "id": "on-06", "kind": "on-page" }, { "slug": "half-an-hour-with-dead-composers", "q": "how long did the Sousa and Bach adapters take to train?", "ok": true, "answer": "The Sousa adapter took 14 minutes 36 seconds to train § {#sousa-a-hundred-and-six-marches}, while the Bach adapter took 8 minutes 23 seconds § {#bach-the-pianist-who-freed-her-own-recordings}. The article does not mention any training time for the Chopin adapter in this specific text, though it notes that part one's training took five minutes § {#previously-in-five-minutes}.", "served": "The Sousa adapter took 14 minutes 36 seconds to train § {#sousa-a-hundred-and-six-marches}, while the Bach adapter took 8 minutes 23 seconds § {#bach-the-pianist-who-freed-her-own-recordings}. The article does not mention any training time for the Chopin adapter in this specific text, though it notes that part one's training took five minutes § {#previously-in-five-minutes}.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 1.8942271870328113, "ttft_ms": 377, "tokens_in": 5703, "tokens_out": 103, "cached_tokens": 0, "cites": [ "sousa-a-hundred-and-six-marches", "bach-the-pianist-who-freed-her-own-recordings", "previously-in-five-minutes" ], "gates": { "cites": [ "sousa-a-hundred-and-six-marches", "bach-the-pianist-who-freed-her-own-recordings", "previously-in-five-minutes" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "14", "36", "8", "23" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "sousa-a-hundred-and-six-marches", "overlap": 0.545 }, { "i": 0, "cite": "bach-the-pianist-who-freed-her-own-recordings", "overlap": 0.455 }, { "i": 1, "cite": "previously-in-five-minutes", "overlap": 0.429 } ], "cite_overlap_p50": 0.455, "id": "on-07", "kind": "on-page" }, { "slug": "chopin-in-five-minutes", "q": "how many recordings were used, and how long did the training take by the trainer's clock?", "ok": true, "answer": "Nineteen recordings were used for training, and the process took 4 minutes 49 seconds by the trainer's own clock § {#five-minutes-of-training-honestly-accounted}. The article notes that the researchers did not measure whether a smaller GPU could have performed the training § {#what-we-did-not-measure}.", "served": "Nineteen recordings were used for training, and the process took 4 minutes 49 seconds by the trainer's own clock § {#five-minutes-of-training-honestly-accounted}. The article notes that the researchers did not measure whether a smaller GPU could have performed the training § {#what-we-did-not-measure}.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 1.4533278679591604, "ttft_ms": 323, "tokens_in": 3843, "tokens_out": 73, "cached_tokens": 32, "cites": [ "five-minutes-of-training-honestly-accounted", "what-we-did-not-measure" ], "gates": { "cites": [ "five-minutes-of-training-honestly-accounted", "what-we-did-not-measure" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "4", "49" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "five-minutes-of-training-honestly-accounted", "overlap": 0.833 }, { "i": 1, "cite": "what-we-did-not-measure", "overlap": 0.5 } ], "cite_overlap_p50": 0.666, "id": "on-08", "kind": "on-page" }, { "slug": "the-same-sixteen", "q": "what did glm-5.3-flash score on the tool bench, and what did the local models score?", "ok": true, "answer": "The glm-5.3-flash reference arm scored 16/19 — 84.2%, Wilson 95% interval [62.4%, 94.5%] on the tool bench § {#sixteen-of-nineteen}. Two local models, gemma4:12b and qwen3.6:27b, also landed on the exact same 16/19 count § {#two-local-models-sixteen-days-earlier}. Four instruments could not be honestly read on the cloud path, three never applied to a model that cannot hold a seat, and two tasks were excluded because their literal-string checkers fail correct answers § {#the-ledger-of-what-did-not-read} § {#sixteen-of-nineteen}.", "served": "The glm-5.3-flash reference arm scored 16/19 — 84.2%, Wilson 95% interval [62.4%, 94.5%] on the tool bench § {#sixteen-of-nineteen}. Two local models, gemma4:12b and qwen3.6:27b, also landed on the exact same 16/19 count § {#two-local-models-sixteen-days-earlier}. Four instruments could not be honestly read on the cloud path, three never applied to a model that cannot hold a seat, and two tasks were excluded because their literal-string checkers fail correct answers § {#the-ledger-of-what-did-not-read} § {#sixteen-of-nineteen}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.951591804041527, "ttft_ms": 411, "tokens_in": 6553, "tokens_out": 172, "cached_tokens": 32, "cites": [ "sixteen-of-nineteen", "two-local-models-sixteen-days-earlier", "the-ledger-of-what-did-not-read" ], "gates": { "cites": [ "sixteen-of-nineteen", "two-local-models-sixteen-days-earlier", "the-ledger-of-what-did-not-read" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "5.3", "16/19", "84.2", "95", "62.4", "94.5", "4:12", "3.6", "27", "16/19" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "sixteen-of-nineteen", "overlap": 0.846 }, { "i": 1, "cite": "two-local-models-sixteen-days-earlier", "overlap": 1.0 }, { "i": 2, "cite": "the-ledger-of-what-did-not-read", "overlap": 0.696 }, { "i": 2, "cite": "sixteen-of-nineteen", "overlap": 0.696 } ], "cite_overlap_p50": 0.771, "id": "on-09", "kind": "on-page" }, { "slug": "two-hours-on-battery", "q": "what speed did the mini pc hold on battery, and for how long?", "ok": true, "answer": "The mini PC held a speed of ~12 tokens/second on battery for about two hours § {#two-hours-at-12-tok-s-the-test-that-named-the-page}. The article notes that the GPU-server half of the test remains unmeasured by design § {#two-hours-at-12-tok-s-the-test-that-named-the-page}.", "served": "The mini PC held a speed of ~12 tokens/second on battery for about two hours § {#two-hours-at-12-tok-s-the-test-that-named-the-page}. The article notes that the GPU-server half of the test remains unmeasured by design § {#two-hours-at-12-tok-s-the-test-that-named-the-page}.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.359627144993283, "ttft_ms": 753, "tokens_in": 11094, "tokens_out": 91, "cached_tokens": 32, "cites": [ "two-hours-at-12-tok-s-the-test-that-named-the-page" ], "gates": { "cites": [ "two-hours-at-12-tok-s-the-test-that-named-the-page" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "12" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "two-hours-at-12-tok-s-the-test-that-named-the-page", "overlap": 1.0 }, { "i": 1, "cite": "two-hours-at-12-tok-s-the-test-that-named-the-page", "overlap": 0.714 } ], "cite_overlap_p50": 0.857, "id": "on-10", "kind": "on-page" }, { "slug": "the-instrument-travels", "q": "when the frozen exam moved to the smaller card, where did the ranked model land?", "ok": true, "answer": "The ranked model, qwen3.6:27b, cleared both judge floors and was ranked again § {#the-chair-and-who-gets-to-sit-in-it}. It achieved a kill-recall of 11 of 12 and a preservation of 8 of 9 § {#the-standings}. One preservation item was unscorable rather than lost, and the missed item moved from c1-k-bez-c1 to c1-k-bez-h1 compared to the previous run § {#what-the-battery-was-for-the-instrument-travels}.", "served": "The ranked model, qwen3.6:27b, cleared both judge floors and was ranked again § {#the-chair-and-who-gets-to-sit-in-it}. It achieved a kill-recall of 11 of 12 and a preservation of 8 of 9 § {#the-standings}. One preservation item was unscorable rather than lost, and the missed item moved from c1-k-bez-c1 to c1-k-bez-h1 compared to the previous run § {#what-the-battery-was-for-the-instrument-travels}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 5.481849998002872, "ttft_ms": 1129, "tokens_in": 8777, "tokens_out": 132, "cached_tokens": 32, "cites": [ "the-chair-and-who-gets-to-sit-in-it", "the-standings", "what-the-battery-was-for-the-instrument-travels" ], "gates": { "cites": [ "the-chair-and-who-gets-to-sit-in-it", "the-standings", "what-the-battery-was-for-the-instrument-travels" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "3.6", "27", "11", "12", "8", "9" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-chair-and-who-gets-to-sit-in-it", "overlap": 0.222 }, { "i": 1, "cite": "the-standings", "overlap": 0.8 }, { "i": 2, "cite": "what-the-battery-was-for-the-instrument-travels", "overlap": 0.692 } ], "cite_overlap_p50": 0.692, "id": "on-11", "kind": "on-page" }, { "slug": "nine-worlds-one-dog", "q": "what stays the same across the nine paintings?", "ok": true, "answer": "The founder's dog, a large black Portuguese Water Dog with one bright white chest patch, takes the same nap under the same tree in all nine painted worlds § {#top}. He maintains the same curl and the same tucked paws every time §. The article does not mention anything that was left out, moved, or not measured.", "served": "The founder's dog, a large black Portuguese Water Dog with one bright white chest patch, takes the same nap under the same tree in all nine painted worlds § {#top}. He maintains the same curl and the same tucked paws every time §. The article does not mention anything that was left out, moved, or not measured.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 3.573677868989762, "ttft_ms": 947, "tokens_in": 2058, "tokens_out": 68, "cached_tokens": 32, "cites": [ "top" ], "gates": { "cites": [ "top" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "top", "overlap": 0.526 } ], "cite_overlap_p50": 0.526, "id": "on-12", "kind": "on-page" }, { "slug": "what-150-watts-buys", "q": "what did the 450-watt cap cost the dense model, and what did it buy back?", "ok": true, "answer": "The 450 W cap cost the dense model 2.0% of throughput § {#the-answer-in-one-paragraph}. This change bought back 138 W, 11 °C, and a 28% jump in efficiency § {#the-answer-in-one-paragraph}. The article notes that it did not measure the cost for a long-context or prefill-heavy workload § {#what-this-page-does-not-say}.", "served": "The 450 W cap cost the dense model 2.0% of throughput § {#the-answer-in-one-paragraph}. This change bought back 138 W, 11 °C, and a 28% jump in efficiency § {#the-answer-in-one-paragraph}. The article notes that it did not measure the cost for a long-context or prefill-heavy workload § {#what-this-page-does-not-say}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.300551519030705, "ttft_ms": 921, "tokens_in": 7269, "tokens_out": 103, "cached_tokens": 32, "cites": [ "the-answer-in-one-paragraph", "what-this-page-does-not-say" ], "gates": { "cites": [ "the-answer-in-one-paragraph", "what-this-page-does-not-say" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "450", "2.0", "138", "11", "28" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-answer-in-one-paragraph", "overlap": 0.714 }, { "i": 1, "cite": "the-answer-in-one-paragraph", "overlap": 0.75 }, { "i": 2, "cite": "what-this-page-does-not-say", "overlap": 0.667 } ], "cite_overlap_p50": 0.714, "id": "on-13", "kind": "on-page" }, { "slug": "how-the-beat-lab-works", "q": "where is the sound made, and how long does a typical wish take?", "ok": true, "answer": "Every sound in the Beat Lab is synthesised by the user's own browser § {#the-sound-is-made-on-your-device}. A typical wish takes about 4 s § {#what-happens-when-you-ask-the-genie}. The article does not provide a measurement for how long the genie's voice takes to arrive after a wish is made § {#what-happens-when-you-ask-the-genie}.", "served": "Every sound in the Beat Lab is synthesised by the user's own browser § {#the-sound-is-made-on-your-device}. A typical wish takes about 4 s § {#what-happens-when-you-ask-the-genie}. The article does not provide a measurement for how long the genie's voice takes to arrive after a wish is made § {#what-happens-when-you-ask-the-genie}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.213663403003011, "ttft_ms": 2522, "tokens_in": 31469, "tokens_out": 100, "cached_tokens": 32, "cites": [ "the-sound-is-made-on-your-device", "what-happens-when-you-ask-the-genie" ], "gates": { "cites": [ "the-sound-is-made-on-your-device", "what-happens-when-you-ask-the-genie" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "4" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-sound-is-made-on-your-device", "overlap": 0.875 }, { "i": 1, "cite": "what-happens-when-you-ask-the-genie", "overlap": 1.0 }, { "i": 2, "cite": "what-happens-when-you-ask-the-genie", "overlap": 0.6 } ], "cite_overlap_p50": 0.875, "id": "on-14", "kind": "on-page" }, { "slug": "the-typist-and-the-developer", "q": "how long did the developer take to develop one picture?", "ok": true, "answer": "The developer developed a picture in 1.54 seconds § {#the-developer}. When the typist was streaming tokens at the same time, the developer took 1.82 seconds per sketch § {#one-card-both-jobs-at-once}. The article notes that it did not measure how much traffic was present during the run § {#what-we-did-not-measure-said-out-loud}.", "served": "The developer developed a picture in 1.54 seconds § {#the-developer}. When the typist was streaming tokens at the same time, the developer took 1.82 seconds per sketch § {#one-card-both-jobs-at-once}. The article notes that it did not measure how much traffic was present during the run § {#what-we-did-not-measure-said-out-loud}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.2762534620123915, "ttft_ms": 360, "tokens_in": 5536, "tokens_out": 90, "cached_tokens": 32, "cites": [ "the-developer", "one-card-both-jobs-at-once", "what-we-did-not-measure-said-out-loud" ], "gates": { "cites": [ "the-developer", "one-card-both-jobs-at-once", "what-we-did-not-measure-said-out-loud" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "1.54", "1.82" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-developer", "overlap": 1.0 }, { "i": 1, "cite": "one-card-both-jobs-at-once", "overlap": 0.909 }, { "i": 2, "cite": "what-we-did-not-measure-said-out-loud", "overlap": 0.571 } ], "cite_overlap_p50": 0.909, "id": "on-15", "kind": "on-page" }, { "slug": "a-rig-your-friend-already-owns", "q": "how much faster was the workstation card than the 3090 on the game's own art?", "ok": true, "answer": "The article does not provide a single speed comparison for the workstation card against the 3090 on the game's own art, instead providing model-specific medians § {#the-table-that-answers-the-question}. For the game's sketch artist, the 96G card was 3.4× faster than the 3090 § {#the-sketch-ladder-the-number-the-game-lives-on}. The text notes that the 8–12 GB question remains untested § {#what-this-means-for-the-game}.", "served": "This page doesn't say. The nearest it gets is § {#the-table-that-answers-the-question}. “Fifty-Seven Milliseconds: What a Diffusion Step Actually Buys” covers it more directly.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": true, "abstain_source": "model", "abstain_section": "the-table-that-answers-the-question", "seconds": 2.8597708760062233, "ttft_ms": 353, "tokens_in": 4777, "tokens_out": 126, "cached_tokens": 32, "cites": [ "the-table-that-answers-the-question", "the-sketch-ladder-the-number-the-game-lives-on", "what-this-means-for-the-game" ], "gates": { "cites": [ "the-table-that-answers-the-question", "the-sketch-ladder-the-number-the-game-lives-on", "what-this-means-for-the-game" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "3090", "96", "3.4", "3090", "8-12" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-table-that-answers-the-question", "overlap": 0.333 }, { "i": 1, "cite": "the-sketch-ladder-the-number-the-game-lives-on", "overlap": 0.75 }, { "i": 2, "cite": "what-this-means-for-the-game", "overlap": 0.571 } ], "cite_overlap_p50": 0.571, "id": "on-16", "kind": "on-page" }, { "slug": "how-a-vision-model-sees", "q": "how many tokens does an 896 px picture become, and how long does its prefill take?", "ok": true, "answer": "An 896 px picture becomes 199 tokens and its prefill takes 184 ms § {#finding-one-a-picture-is-199-tokens-at-896-px-and-the}. This measurement was recorded on the seat the cove ships, which is the minicpm-v4.5 model § {#finding-one-a-picture-is-199-tokens-at-896-px-and-the}.", "served": "An 896 px picture becomes 199 tokens and its prefill takes 184 ms § {#finding-one-a-picture-is-199-tokens-at-896-px-and-the}. This measurement was recorded on the seat the cove ships, which is the minicpm-v4.5 model § {#finding-one-a-picture-is-199-tokens-at-896-px-and-the}.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.718999456963502, "ttft_ms": 655, "tokens_in": 10311, "tokens_out": 107, "cached_tokens": 32, "cites": [ "finding-one-a-picture-is-199-tokens-at-896-px-and-the" ], "gates": { "cites": [ "finding-one-a-picture-is-199-tokens-at-896-px-and-the" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "896", "199", "184", "4.5" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "finding-one-a-picture-is-199-tokens-at-896-px-and-the", "overlap": 0.8 }, { "i": 1, "cite": "finding-one-a-picture-is-199-tokens-at-896-px-and-the", "overlap": 0.714 } ], "cite_overlap_p50": 0.757, "id": "on-17", "kind": "on-page" }, { "slug": "fifty-seven-milliseconds", "q": "which painter is the fastest cell, and at what time?", "ok": true, "answer": "The distilled painter is the fastest cell in the second run, recording a time of 0.62 s at 4 steps § {#finding-four-the-argument-with-our-own-settings-and-what-it}. This measurement was taken on a 96G VRAM workstation § {#finding-four-the-argument-with-our-own-settings-and-what-it}. The article does not mention what was left out, moved, or not measured in relation to this specific cell.", "served": "The distilled painter is the fastest cell in the second run, recording a time of 0.62 s at 4 steps § {#finding-four-the-argument-with-our-own-settings-and-what-it}. This measurement was taken on a 96G VRAM workstation § {#finding-four-the-argument-with-our-own-settings-and-what-it}. The article does not mention what was left out, moved, or not measured in relation to this specific cell.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.304640636953991, "ttft_ms": 454, "tokens_in": 5434, "tokens_out": 110, "cached_tokens": 32, "cites": [ "finding-four-the-argument-with-our-own-settings-and-what-it" ], "gates": { "cites": [ "finding-four-the-argument-with-our-own-settings-and-what-it" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "0.62", "4", "96" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "finding-four-the-argument-with-our-own-settings-and-what-it", "overlap": 0.9 }, { "i": 1, "cite": "finding-four-the-argument-with-our-own-settings-and-what-it", "overlap": 0.6 } ], "cite_overlap_p50": 0.75, "id": "on-18", "kind": "on-page" }, { "slug": "the-dog-the-dice-and-the-painter", "q": "how big is the file the whole world is written in?", "ok": true, "answer": "The world is written in a single YAML file containing 16,705 lines § {#the-world-is-a-file}. This file holds six regions, forty NPCs, fifty-nine quests, and three kinds of adoptable pet § {#the-world-is-a-file}. No scores or instrumentation were used for this walk § {#the-walk-is-a-walk}.", "served": "The world is written in a single YAML file containing 16,705 lines § {#the-world-is-a-file}. This file holds six regions, forty NPCs, fifty-nine quests, and three kinds of adoptable pet § {#the-world-is-a-file}. No scores or instrumentation were used for this walk § {#the-walk-is-a-walk}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 1.4023211580351926, "ttft_ms": 414, "tokens_in": 6116, "tokens_out": 85, "cached_tokens": 32, "cites": [ "the-world-is-a-file" ], "gates": { "cites": [ "the-world-is-a-file" ], "cites_dropped": [ "the-walk-is-a-walk" ], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "cite: 1 cite(s) not in this page's anchors: the-walk-is-a-walk" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [ "the-walk-is-a-walk" ] }, "figures": [ "16,705" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-world-is-a-file", "overlap": 0.778 }, { "i": 1, "cite": "the-world-is-a-file", "overlap": 0.917 } ], "cite_overlap_p50": 0.848, "id": "on-19", "kind": "on-page" }, { "slug": "where-new-knowledge-comes-from", "q": "what are the two engines the page names for where new knowledge comes from?", "ok": true, "answer": "The two engines are present in the data, unmined, and implied by the rules, unvisited § {#two-kinds-of-there-all-along}. The first engine involves reading patterns that were already in archives, such as co-evolution fingerprints or molecular shapes in a library § {#the-first-engine-the-archive-nobody-had-read}. The second engine involves walking regions of possibility space that are implied by rules, such as a planet's position or a specific move in a game § {#the-second-engine-the-rooms-the-rules-imply}. Structural biologists noted that these engines left out how proteins pair up, how they move, and what one mutation does § {#the-first-engine-the-archive-nobody-had-read}.", "served": "The two engines are present in the data, unmined, and implied by the rules, unvisited § {#two-kinds-of-there-all-along}. The first engine involves reading patterns that were already in archives, such as co-evolution fingerprints or molecular shapes in a library § {#the-first-engine-the-archive-nobody-had-read}. The second engine involves walking regions of possibility space that are implied by rules, such as a planet's position or a specific move in a game § {#the-second-engine-the-rooms-the-rules-imply}. Structural biologists noted that these engines left out how proteins pair up, how they move, and what one mutation does § {#the-first-engine-the-archive-nobody-had-read}.", "partial": false, "sentences_kept": 4, "sentences_written": 4, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 3.2190957440179773, "ttft_ms": 360, "tokens_in": 5472, "tokens_out": 166, "cached_tokens": 32, "cites": [ "two-kinds-of-there-all-along", "the-first-engine-the-archive-nobody-had-read", "the-second-engine-the-rooms-the-rules-imply" ], "gates": { "cites": [ "two-kinds-of-there-all-along", "the-first-engine-the-archive-nobody-had-read", "the-second-engine-the-rooms-the-rules-imply" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "two-kinds-of-there-all-along", "overlap": 1.0 }, { "i": 1, "cite": "the-first-engine-the-archive-nobody-had-read", "overlap": 0.385 }, { "i": 2, "cite": "the-second-engine-the-rooms-the-rules-imply", "overlap": 0.533 }, { "i": 3, "cite": "the-first-engine-the-archive-nobody-had-read", "overlap": 0.231 } ], "cite_overlap_p50": 0.459, "id": "on-20", "kind": "on-page" }, { "slug": "compression-is-the-objective", "q": "what does the page say a language model is, and what prize does it point to?", "ok": true, "answer": "A language model is a compression of its corpus § {#the-punchline}. This is described as the training objective rather than a figure of speech § {#the-short-version}. The article points to the Hutter Prize, which charges for everything including self-extracting entries and every decompressor byte counted § {#the-punchline}.", "served": "A language model is a compression of its corpus § {#the-punchline}. This is described as the training objective rather than a figure of speech § {#the-short-version}. The article points to the Hutter Prize, which charges for everything including self-extracting entries and every decompressor byte counted § {#the-punchline}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 1.994717275025323, "ttft_ms": 358, "tokens_in": 5448, "tokens_out": 71, "cached_tokens": 32, "cites": [ "the-punchline", "top" ], "gates": { "cites": [ "the-punchline", "top" ], "cites_dropped": [], "cites_resolved": [ "top" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-punchline", "overlap": 1.0 }, { "i": 1, "cite": "top", "overlap": 0.667 }, { "i": 2, "cite": "the-punchline", "overlap": 0.333 } ], "cite_overlap_p50": 0.667, "id": "on-21", "kind": "on-page" }, { "slug": "the-free-speed-wasnt-free", "q": "what was the production median before the change, and what caused the speed-up?", "ok": true, "answer": "The production median before the change was 138.6 tok/s over 1,982 live rulings § {#the-day-the-box-got-faster}. The speed-up was caused by an ollama commit that changed the default repeat_penalty from 1.1 to 1.0, which stopped a penalties stage from running on every token § {#the-two-line-commit}. The article notes that the quality half of this change remains unmeasured § {#the-part-that-isnt-about-speed}.", "served": "The production median before the change was 138.6 tok/s over 1,982 live rulings § {#the-day-the-box-got-faster}. The speed-up was caused by an ollama commit that changed the default repeat_penalty from 1.1 to 1.0, which stopped a penalties stage from running on every token § {#the-two-line-commit}. The article notes that the quality half of this change remains unmeasured § {#the-part-that-isnt-about-speed}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.5043272000038996, "ttft_ms": 461, "tokens_in": 7529, "tokens_out": 117, "cached_tokens": 32, "cites": [ "the-day-the-box-got-faster", "the-two-line-commit", "the-part-that-isnt-about-speed" ], "gates": { "cites": [ "the-day-the-box-got-faster", "the-two-line-commit", "the-part-that-isnt-about-speed" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "138.6", "1,982", "1.1", "1.0" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-day-the-box-got-faster", "overlap": 0.9 }, { "i": 1, "cite": "the-two-line-commit", "overlap": 0.562 }, { "i": 2, "cite": "the-part-that-isnt-about-speed", "overlap": 0.667 } ], "cite_overlap_p50": 0.667, "id": "on-22", "kind": "on-page" }, { "slug": "the-compressed-photograph", "q": "how much slower was the uncompressed original than the compressed build?", "ok": true, "answer": "The uncompressed original was 3.4× the speed of the trained-4-bit build § {#the-same-question-down-the-whole-ladder}. The 8-bit copy was 28% slower in tokens per second than the q4 build § {#the-same-brain-at-two-compressions}. The article does not explicitly state the exact speed difference between the uncompressed original and the q4 build, though it notes the original's speed was 58.1 tok/s compared to the q4 build's 129.4 tok/s § {#the-same-question-down-the-whole-ladder}. The text notes that it does not measure perplexity § {#what-this-page-does-not-say}.", "served": "The uncompressed original was 3.4× the speed of the trained-4-bit build § {#the-same-question-down-the-whole-ladder}. The 8-bit copy was 28% slower in tokens per second than the q4 build § {#the-same-brain-at-two-compressions}. The article does not explicitly state the exact speed difference between the uncompressed original and the q4 build, though it notes the original's speed was 58.1 tok/s compared to the q4 build's 129.4 tok/s § {#the-same-question-down-the-whole-ladder}. The text notes that it does not measure perplexity § {#what-this-page-does-not-say}.", "partial": false, "sentences_kept": 4, "sentences_written": 4, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 9.88981123798294, "ttft_ms": 560, "tokens_in": 8637, "tokens_out": 168, "cached_tokens": 32, "cites": [ "the-same-question-down-the-whole-ladder", "the-same-brain-at-two-compressions", "what-this-page-does-not-say" ], "gates": { "cites": [ "the-same-question-down-the-whole-ladder", "the-same-brain-at-two-compressions", "what-this-page-does-not-say" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "3.4", "4", "8", "28", "4", "4", "58.1", "4", "129.4" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-same-question-down-the-whole-ladder", "overlap": 1.0 }, { "i": 1, "cite": "the-same-brain-at-two-compressions", "overlap": 1.0 }, { "i": 2, "cite": "the-same-question-down-the-whole-ladder", "overlap": 0.667 }, { "i": 3, "cite": "what-this-page-does-not-say", "overlap": 0.75 } ], "cite_overlap_p50": 0.875, "id": "on-23", "kind": "on-page" }, { "slug": "three-at-the-table", "q": "what is the true active share for the live seat, and what does the page call the two numbers in a model name?", "ok": true, "answer": "The true active share for the live seat is 14.4% of the model's bytes § {#the-payroll-and-the-meeting}. The two numbers in a model name are the payroll and the meeting § {#the-tag-with-two-numbers}. The article notes that it does not measure the latency trade-off of streaming experts from system RAM § {#the-part-nobody-warns-you-about-the-whole-payroll-needs-an}.", "served": "The true active share for the live seat is 14.4% of the model's bytes § {#the-payroll-and-the-meeting}. The two numbers in a model name are the payroll and the meeting § {#the-tag-with-two-numbers}. The article notes that it does not measure the latency trade-off of streaming experts from system RAM § {#the-part-nobody-warns-you-about-the-whole-payroll-needs-an}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.007939080009237, "ttft_ms": 699, "tokens_in": 7685, "tokens_out": 103, "cached_tokens": 32, "cites": [ "the-payroll-and-the-meeting", "the-tag-with-two-numbers", "the-part-nobody-warns-you-about-the-whole-payroll-needs-an" ], "gates": { "cites": [ "the-payroll-and-the-meeting", "the-tag-with-two-numbers", "the-part-nobody-warns-you-about-the-whole-payroll-needs-an" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "14.4" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-payroll-and-the-meeting", "overlap": 1.0 }, { "i": 1, "cite": "the-tag-with-two-numbers", "overlap": 0.833 }, { "i": 2, "cite": "the-part-nobody-warns-you-about-the-whole-payroll-needs-an", "overlap": 0.5 } ], "cite_overlap_p50": 0.833, "id": "on-24", "kind": "on-page" }, { "slug": "reading-is-fast", "q": "what was the median reading speed, and why is reading faster than writing?", "ok": true, "answer": "The median reading speed was 5,415 tokens/s § {#a-measurement-from-our-own-card}. Reading is faster because it is parallel, meaning every token's math can happen at the same time since the text already exists § {#reading-is-a-group-photo}. Writing is slower because it is serial, as the model must choose tokens one at a time because the future of the sentence does not yet exist § {#writing-is-one-at-a-time}.", "served": "The median reading speed was 5,415 tokens/s § {#a-measurement-from-our-own-card}. Reading is faster because it is parallel, meaning every token's math can happen at the same time since the text already exists § {#reading-is-a-group-photo}. Writing is slower because it is serial, as the model must choose tokens one at a time because the future of the sentence does not yet exist § {#writing-is-one-at-a-time}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.0724026190000586, "ttft_ms": 834, "tokens_in": 5557, "tokens_out": 108, "cached_tokens": 32, "cites": [ "a-measurement-from-our-own-card", "reading-is-a-group-photo", "writing-is-one-at-a-time" ], "gates": { "cites": [ "a-measurement-from-our-own-card", "reading-is-a-group-photo", "writing-is-one-at-a-time" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "5,415" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "a-measurement-from-our-own-card", "overlap": 1.0 }, { "i": 1, "cite": "reading-is-a-group-photo", "overlap": 0.667 }, { "i": 2, "cite": "writing-is-one-at-a-time", "overlap": 0.786 } ], "cite_overlap_p50": 0.786, "id": "on-25", "kind": "on-page" }, { "slug": "three-librarians", "q": "how many dimensions does the embedder's map have, and what does the third librarian do?", "ok": true, "answer": "The embedder's map has 768 directions § {#the-second-librarian-searches-by-meaning}. The third librarian works only within the aisle of the specific game being asked about, picking up the edition's rulebook and any matched correction documents to rank passages by meaning § {#the-third-librarian-never-left-your-aisle}. The article does not state what was left out, moved, or not measured regarding the embedder or the third librarian § {#the-nerd-shelf-the-numbers-from-the-shipped-configuration}.", "served": "The embedder's map has 768 directions § {#the-second-librarian-searches-by-meaning}. The third librarian works only within the aisle of the specific game being asked about, picking up the edition's rulebook and any matched correction documents to rank passages by meaning § {#the-third-librarian-never-left-your-aisle}. The article does not state what was left out, moved, or not measured regarding the embedder or the third librarian § {#the-nerd-shelf-the-numbers-from-the-shipped-configuration}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 8.72151337802643, "ttft_ms": 366, "tokens_in": 5913, "tokens_out": 126, "cached_tokens": 32, "cites": [ "the-second-librarian-searches-by-meaning", "the-third-librarian-never-left-your-aisle", "the-nerd-shelf-the-numbers-from-the-shipped-configuration" ], "gates": { "cites": [ "the-second-librarian-searches-by-meaning", "the-third-librarian-never-left-your-aisle", "the-nerd-shelf-the-numbers-from-the-shipped-configuration" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "768" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-second-librarian-searches-by-meaning", "overlap": 0.75 }, { "i": 1, "cite": "the-third-librarian-never-left-your-aisle", "overlap": 0.7 }, { "i": 2, "cite": "the-nerd-shelf-the-numbers-from-the-shipped-configuration", "overlap": 0.111 } ], "cite_overlap_p50": 0.7, "id": "on-26", "kind": "on-page" }, { "slug": "the-new-kid", "q": "which seat did the new model pass, and which floor did it miss?", "ok": true, "answer": "The qwen3.8:27b model passed the classifier seat, clearing all five pre-registered gates § {#the-classifier-seat-five-gates-all-pass-faster}, but it failed the judge seat by achieving preservation of 11 of 16 against a floor of 13 § {#the-judge-seat-and-the-floor-it-missed}. The article notes that the classifier seat's G1 result of 0/20 was based on about half the power the pre-registration intended § {#the-classifier-seat-five-gates-all-pass-faster}.", "served": "The qwen3.8:27b model passed the classifier seat, clearing all five pre-registered gates § {#the-classifier-seat-five-gates-all-pass-faster}, but it failed the judge seat by achieving preservation of 11 of 16 against a floor of 13 § {#the-judge-seat-and-the-floor-it-missed}. The article notes that the classifier seat's G1 result of 0/20 was based on about half the power the pre-registration intended § {#the-classifier-seat-five-gates-all-pass-faster}.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 9.463259060983546, "ttft_ms": 1009, "tokens_in": 15091, "tokens_out": 133, "cached_tokens": 32, "cites": [ "the-classifier-seat-five-gates-all-pass-faster", "the-judge-seat-and-the-floor-it-missed" ], "gates": { "cites": [ "the-classifier-seat-five-gates-all-pass-faster", "the-judge-seat-and-the-floor-it-missed" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "3.8", "27", "11", "16", "13", "0/20" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-classifier-seat-five-gates-all-pass-faster", "overlap": 0.632 }, { "i": 0, "cite": "the-judge-seat-and-the-floor-it-missed", "overlap": 0.737 }, { "i": 1, "cite": "the-classifier-seat-five-gates-all-pass-faster", "overlap": 0.636 } ], "cite_overlap_p50": 0.636, "id": "on-27", "kind": "on-page" }, { "slug": "llms-txt", "q": "did any ai crawler fetch the file in the first thirty days, and how big is the largest measured llms-full file?", "ok": true, "answer": "No AI crawler asked for the file in thirty days of logs § {#does-anyone-fetch-it}. The largest measured llms-full file weighs 30.7 MiB § {#where-the-file-came-from}. The article notes that the measurement of the largest file is an estimate rather than a measured count every other token figure on the page carries § {#where-the-file-came-from}.", "served": "No AI crawler asked for the file in thirty days of logs § {#does-anyone-fetch-it}. The largest measured llms-full file weighs 30.7 MiB § {#where-the-file-came-from}. The article notes that the measurement of the largest file is an estimate rather than a measured count every other token figure on the page carries § {#where-the-file-came-from}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 8.834598341025412, "ttft_ms": 1571, "tokens_in": 12541, "tokens_out": 89, "cached_tokens": 32, "cites": [ "does-anyone-fetch-it", "where-the-file-came-from" ], "gates": { "cites": [ "does-anyone-fetch-it", "where-the-file-came-from" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "30.7" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "does-anyone-fetch-it", "overlap": 0.571 }, { "i": 1, "cite": "where-the-file-came-from", "overlap": 1.0 }, { "i": 2, "cite": "where-the-file-came-from", "overlap": 0.846 } ], "cite_overlap_p50": 0.846, "id": "on-28", "kind": "on-page" }, { "slug": "the-open-call", "q": "how many models were handed the frozen moment, and what did the round cost?", "ok": true, "answer": "Twenty language models were handed the frozen moment § {#top}. The round's metered spend was $2.2826 § {#2-2826-over-56-metered-calls-against-38-49-of-caps}. The agent transport holds no cost figure and no token counters § {#the-author-of-this-page-is-a-contestant-in-it}.", "served": "Twenty language models were handed the frozen moment § {#top}. The round's metered spend was $2.2826 § {#2-2826-over-56-metered-calls-against-38-49-of-caps}. The agent transport holds no cost figure and no token counters § {#the-author-of-this-page-is-a-contestant-in-it}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 9.348230316012632, "ttft_ms": 7308, "tokens_in": 63492, "tokens_out": 93, "cached_tokens": 32, "cites": [ "top", "2-2826-over-56-metered-calls-against-38-49-of-caps", "the-author-of-this-page-is-a-contestant-in-it" ], "gates": { "cites": [ "top", "2-2826-over-56-metered-calls-against-38-49-of-caps", "the-author-of-this-page-is-a-contestant-in-it" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "2.2826" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "top", "overlap": 1.0 }, { "i": 1, "cite": "2-2826-over-56-metered-calls-against-38-49-of-caps", "overlap": 0.5 }, { "i": 2, "cite": "the-author-of-this-page-is-a-contestant-in-it", "overlap": 1.0 } ], "cite_overlap_p50": 1.0, "id": "on-29", "kind": "on-page" }, { "slug": "where-your-question-goes", "q": "how long does every notebook on the server keep its entries?", "ok": true, "answer": "Every notebook on the server keeps thirty days of entries § {#the-first-promise-your-visit-stays-between-us}. This rule applies to all notebooks, including the tour guide's, and is maintained by a nightly sweep § {#the-first-promise-your-visit-stays-between-us}. Previously, the notebooks were set to keep their lines for up to a hundred years § {#the-first-promise-your-visit-stays-between-us}.", "served": "Every notebook on the server keeps thirty days of entries § {#the-first-promise-your-visit-stays-between-us}. This rule applies to all notebooks, including the tour guide's, and is maintained by a nightly sweep § {#the-first-promise-your-visit-stays-between-us}. Previously, the notebooks were set to keep their lines for up to a hundred years § {#the-first-promise-your-visit-stays-between-us}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 8.729885520995595, "ttft_ms": 1313, "tokens_in": 4985, "tokens_out": 102, "cached_tokens": 32, "cites": [ "the-first-promise-your-visit-stays-between-us" ], "gates": { "cites": [ "the-first-promise-your-visit-stays-between-us" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-first-promise-your-visit-stays-between-us", "overlap": 0.857 }, { "i": 1, "cite": "the-first-promise-your-visit-stays-between-us", "overlap": 0.778 }, { "i": 2, "cite": "the-first-promise-your-visit-stays-between-us", "overlap": 0.889 } ], "cite_overlap_p50": 0.857, "id": "on-30", "kind": "on-page" }, { "slug": "how-we-work", "q": "what does the page say about the humans and agents who make each exhibit?", "ok": true, "answer": "The human generates and judges while the agent amplifies and verifies § {#the-plain-english-version}. Agents perform tasks such as recon, planning, and auditing, while the human operator provides the spark of noticing, makes binding rulings, and provides the final sign-off for release § {#the-thorough-walk}. The article notes that the voice trial's original five questions were never persisted, so a rebuilt probe was used instead § {#the-plain-english-version}.", "served": "The human generates and judges while the agent amplifies and verifies § {#the-plain-english-version}. Agents perform tasks such as recon, planning, and auditing, while the human operator provides the spark of noticing, makes binding rulings, and provides the final sign-off for release § {#the-thorough-walk}. The article notes that the voice trial's original five questions were never persisted, so a rebuilt probe was used instead § {#the-plain-english-version}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.82501988200238, "ttft_ms": 365, "tokens_in": 5210, "tokens_out": 100, "cached_tokens": 32, "cites": [ "the-plain-english-version", "the-thorough-walk" ], "gates": { "cites": [ "the-plain-english-version", "the-thorough-walk" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-plain-english-version", "overlap": 0.429 }, { "i": 1, "cite": "the-thorough-walk", "overlap": 0.579 }, { "i": 2, "cite": "the-plain-english-version", "overlap": 0.667 } ], "cite_overlap_p50": 0.579, "id": "on-31", "kind": "on-page" }, { "slug": "the-move", "q": "what did the median search time go from and to after the move?", "ok": true, "answer": "The median search time went from 242 ms in the laptop era to 77.5 ms in the new era § {#instrument-two · the-receipts}. This change reflects the retrieval leg that the move actually touched § {#instrument-two · the-receipts}. The article notes that the move changed where the wires run but not whose machines think, as generation stays on the original GPU box § {#what-moved}.", "served": "The median search time went from 242 ms in the laptop era to 77.5 ms in the new era § {#instrument-two · the-receipts}. This change reflects the retrieval leg that the move actually touched § {#instrument-two · the-receipts}. The article notes that the move changed where the wires run but not whose machines think, as generation stays on the original GPU box § {#what-moved}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.579826882982161, "ttft_ms": 480, "tokens_in": 6101, "tokens_out": 91, "cached_tokens": 32, "cites": [ "receipts", "what-moved" ], "gates": { "cites": [ "receipts", "what-moved" ], "cites_dropped": [], "cites_resolved": [ "receipts" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "242", "77.5" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "receipts", "overlap": 0.9 }, { "i": 1, "cite": "receipts", "overlap": 0.857 }, { "i": 2, "cite": "what-moved", "overlap": 0.846 } ], "cite_overlap_p50": 0.857, "id": "on-32", "kind": "on-page" }, { "slug": "chair-trials", "q": "how many models and arms sat the five exams, and did the box stop serving?", "ok": true, "answer": "Nine models and eleven arms sat the five exams on a 96G workstation that stayed in service throughout § {#the-short-version}. The article notes that no on-disk blob size was registered § {#roster} and that the cold probe measured a cold queue rather than a cold seat § {#limits-stated-plainly}.", "served": "Nine models and eleven arms sat the five exams on a 96G workstation that stayed in service throughout § {#the-short-version}. The article notes that no on-disk blob size was registered § {#roster} and that the cold probe measured a cold queue rather than a cold seat § {#limits-stated-plainly}.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 5.274401688016951, "ttft_ms": 2278, "tokens_in": 30322, "tokens_out": 70, "cached_tokens": 32, "cites": [ "top", "roster", "limits-stated-plainly" ], "gates": { "cites": [ "top", "roster", "limits-stated-plainly" ], "cites_dropped": [], "cites_resolved": [ "top" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "96" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "top", "overlap": 1.0 }, { "i": 1, "cite": "roster", "overlap": 0.909 }, { "i": 1, "cite": "limits-stated-plainly", "overlap": 0.545 } ], "cite_overlap_p50": 0.909, "id": "on-33", "kind": "on-page" }, { "slug": "cove-voice-head-to-head", "q": "what happened to the August class's best voice at the cove's gate?", "ok": true, "answer": "The August class's best voice, the candidate qwen3.6:27b, was refused at the gate § {#the-verdict}. This refusal occurred because the model failed the canon floor by inventing facts twice in thirteen prompts and failed the latency floor § {#the-verdict}. The article notes that a refused candidate receives no seat proposal, no live flip, no deployment change, and no threshold table § {#the-verdict}.", "served": "The August class's best voice, the candidate qwen3.6:27b, was refused at the gate § {#the-verdict}. This refusal occurred because the model failed the canon floor by inventing facts twice in thirteen prompts and failed the latency floor § {#the-verdict}. The article notes that a refused candidate receives no seat proposal, no live flip, no deployment change, and no threshold table § {#the-verdict}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 5.411122156016063, "ttft_ms": 2843, "tokens_in": 16668, "tokens_out": 93, "cached_tokens": 32, "cites": [ "the-verdict" ], "gates": { "cites": [ "the-verdict" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "3.6", "27" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-verdict", "overlap": 0.444 }, { "i": 1, "cite": "the-verdict", "overlap": 0.692 }, { "i": 2, "cite": "the-verdict", "overlap": 0.75 } ], "cite_overlap_p50": 0.692, "id": "on-34", "kind": "on-page" }, { "slug": "outside-judges", "q": "how many outside labs re-judged, and what was the widest cross-vendor gap?", "ok": true, "answer": "Four rival frontier labs re-judged every sealed round § {#vendors}. The widest cross-vendor gap on any arm was 0.178 § {#vendors}. This gap occurred on muse-glimmer:30b-q8_0-dflash, where mistral read it at 0.450 against nvidia at 0.628 § {#vendors}.", "served": "Four rival frontier labs re-judged every sealed round § {#vendors}. The widest cross-vendor gap on any arm was 0.178 § {#vendors}. This gap occurred on muse-glimmer:30b-q8_0-dflash, where mistral read it at 0.450 against nvidia at 0.628 § {#vendors}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.831789096991997, "ttft_ms": 2736, "tokens_in": 10362, "tokens_out": 82, "cached_tokens": 32, "cites": [ "vendors" ], "gates": { "cites": [ "vendors" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "0.178", "30", "80", "0.450", "0.628" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "vendors", "overlap": 0.625 }, { "i": 1, "cite": "vendors", "overlap": 1.0 }, { "i": 2, "cite": "vendors", "overlap": 0.909 } ], "cite_overlap_p50": 0.909, "id": "on-35", "kind": "on-page" }, { "slug": "august-arrivals", "q": "how many models landed that week, and what share broke the response contract?", "ok": true, "answer": "Four open-weight models landed in one week of August 2026 § {#the-story}, and all four broke the response contract on more than 10% of calls § {#exam-one-the-judge-chair}. Specifically, the failure rates for the four candidates were 16.3%, 51.2%, 55.8%, and 62.8% § {#exam-one-the-judge-chair}. The article notes that olmo-3.1:32b-think-q4_K_M was on the residency census but never sat the voice exam § {#exam-two-the-narrators-chair}.", "served": "Four open-weight models landed in one week of August 2026 § {#the-story}, and all four broke the response contract on more than 10% of calls § {#exam-one-the-judge-chair}. Specifically, the failure rates for the four candidates were 16.3%, 51.2%, 55.8%, and 62.8% § {#exam-one-the-judge-chair}. The article notes that olmo-3.1:32b-think-q4_K_M was on the residency census but never sat the voice exam § {#exam-two-the-narrators-chair}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 6.9106033850112, "ttft_ms": 2996, "tokens_in": 11270, "tokens_out": 145, "cached_tokens": 32, "cites": [ "the-story", "exam-one-the-judge-chair", "exam-two-the-narrators-chair" ], "gates": { "cites": [ "the-story", "exam-one-the-judge-chair", "exam-two-the-narrators-chair" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "10", "16.3", "51.2", "55.8", "62.8", "3.1", "32", "4" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-story", "overlap": 0.429 }, { "i": 0, "cite": "exam-one-the-judge-chair", "overlap": 0.857 }, { "i": 1, "cite": "exam-one-the-judge-chair", "overlap": 0.778 }, { "i": 2, "cite": "exam-two-the-narrators-chair", "overlap": 0.778 } ], "cite_overlap_p50": 0.778, "id": "on-36", "kind": "on-page" }, { "slug": "seat-trials", "q": "what are the kill-recall and preservation floors a model must clear?", "ok": true, "answer": "A model must achieve a kill-recall of at least 23/27 and a preservation of at least 13/16 to clear the floors § {#the-instrument}. These two floors are written down before any model runs and each binds on its own § {#the-instrument}. The article notes that no model in the trial actually failed the kill-recall floor, as every measurable seat passed it § {#the-finding-we-did-not-expect}.", "served": "A model must achieve a kill-recall of at least 23/27 and a preservation of at least 13/16 to clear the floors § {#the-instrument}. These two floors are written down before any model runs and each binds on its own § {#the-instrument}. The article notes that no model in the trial actually failed the kill-recall floor, as every measurable seat passed it § {#the-finding-we-did-not-expect}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 1.8637114890152588, "ttft_ms": 451, "tokens_in": 7238, "tokens_out": 99, "cached_tokens": 32, "cites": [ "the-instrument", "the-finding-we-did-not-expect" ], "gates": { "cites": [ "the-instrument", "the-finding-we-did-not-expect" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "23/27", "13/16" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-instrument", "overlap": 0.6 }, { "i": 1, "cite": "the-instrument", "overlap": 1.0 }, { "i": 2, "cite": "the-finding-we-did-not-expect", "overlap": 0.545 } ], "cite_overlap_p50": 0.6, "id": "on-37", "kind": "on-page" }, { "slug": "voice-trials", "q": "how many model tags were tried, and what does the anchor model read?", "ok": true, "answer": "The article states that there were 24 distinct model tags across 40 scored runs § {#provenance-4cae43}. The anchor model reads 9.0 on the first scale, 7.0 on the second, and 6.2 on the third § {#the-method-limits-first}. No delta across record sets is ever published § {#the-method-limits-first}.", "served": "The article states that there were 24 distinct model tags across 40 scored runs § {#provenance-4cae43}. The anchor model reads 9.0 on the first scale, 7.0 on the second, and 6.2 on the third § {#the-method-limits-first}. No delta across record sets is ever published § {#the-method-limits-first}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 1.7733465190394782, "ttft_ms": 713, "tokens_in": 10384, "tokens_out": 87, "cached_tokens": 32, "cites": [ "provenance-4cae43", "the-method-limits-first" ], "gates": { "cites": [ "provenance-4cae43", "the-method-limits-first" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "24", "40", "9.0", "7.0", "6.2" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "provenance-4cae43", "overlap": 0.889 }, { "i": 1, "cite": "the-method-limits-first", "overlap": 0.8 }, { "i": 2, "cite": "the-method-limits-first", "overlap": 1.0 } ], "cite_overlap_p50": 0.889, "id": "on-38", "kind": "on-page" }, { "slug": "licences", "q": "what does the page say it does with each model's licence?", "ok": true, "answer": "The page states that every model's licence is read first-hand and stamped with the date it was last read in full § {#how-to-read-this-page}. This process involves reading the licence file packaged with the weights, the platform's raw permission record, or the vendor's current text § {#top}. The article notes that unread models are withheld and that some models, such as gemma2:9b, are not currently installed on the bench box so their shipped licence cannot be re-read § {#the-brains-chat-vision-memory-all-read-2026-08-11} § {#provenance-063a00}.", "served": "The page states that every model's licence is read first-hand and stamped with the date it was last read in full § {#how-to-read-this-page}. This process involves reading the licence file packaged with the weights, the platform's raw permission record, or the vendor's current text § {#top}. The article notes that unread models are withheld and that some models, such as gemma2:9b, are not currently installed on the bench box so their shipped licence cannot be re-read § {#the-brains-chat-vision-memory-all-read-2026-08-11} § {#provenance-063a00}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 14.086038831970654, "ttft_ms": 572, "tokens_in": 8595, "tokens_out": 148, "cached_tokens": 32, "cites": [ "how-to-read-this-page", "top", "the-brains-chat-vision-memory-all-read-2026-08-11", "provenance-063a00" ], "gates": { "cites": [ "how-to-read-this-page", "top", "the-brains-chat-vision-memory-all-read-2026-08-11", "provenance-063a00" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "2", "9" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "how-to-read-this-page", "overlap": 0.7 }, { "i": 1, "cite": "top", "overlap": 0.714 }, { "i": 2, "cite": "the-brains-chat-vision-memory-all-read-2026-08-11", "overlap": 0.529 }, { "i": 2, "cite": "provenance-063a00", "overlap": 0.235 } ], "cite_overlap_p50": 0.615, "id": "on-39", "kind": "on-page" }, { "slug": "hear-it-for-yourself", "q": "how many clips does the page carry, and how many presentations made up the sitting?", "ok": true, "answer": "The page carries eighty-four clips § {#the-twelve-requests}, and the sitting was made up of thirty-six presentations § {#the-blind-sheet-unsealed}. These presentations consisted of twenty-four scoring pairs and twelve sentinels § {#the-blind-sheet-unsealed}. The article notes that the run actually rendered eighty-five clips § {#how-to-check-these-clips}.", "served": "The page carries eighty-four clips § {#the-twelve-requests}, and the sitting was made up of thirty-six presentations § {#the-blind-sheet-unsealed}. These presentations consisted of twenty-four scoring pairs and twelve sentinels § {#the-blind-sheet-unsealed}. The article notes that the run actually rendered eighty-five clips § {#how-to-check-these-clips}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.617595033021644, "ttft_ms": 1565, "tokens_in": 19071, "tokens_out": 88, "cached_tokens": 32, "cites": [ "the-twelve-requests", "the-blind-sheet-unsealed", "how-to-check-these-clips" ], "gates": { "cites": [ "the-twelve-requests", "the-blind-sheet-unsealed", "how-to-check-these-clips" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-twelve-requests", "overlap": 0.375 }, { "i": 0, "cite": "the-blind-sheet-unsealed", "overlap": 0.625 }, { "i": 1, "cite": "the-blind-sheet-unsealed", "overlap": 0.857 }, { "i": 2, "cite": "how-to-check-these-clips", "overlap": 0.667 } ], "cite_overlap_p50": 0.646, "id": "on-40", "kind": "on-page" }, { "slug": "the-ceiling-is-not-the-corpus", "q": "how did the adapter score against the base model in the listening, and on how many house prompts did it win?", "ok": true, "answer": "The adapter was preferred on four pairs and the untouched model was preferred on eleven of the fifteen pairs where a preference was expressed § {#the-blind-sitting}. On the four requests that asked for house, the untouched model won every pair the listener could separate § {#what-the-adapter-actually-did}. The article notes that the listener abstained on nine of the twenty-four pairs § {#the-blind-sitting}.", "served": "The adapter was preferred on four pairs and the untouched model was preferred on eleven of the fifteen pairs where a preference was expressed § {#the-blind-sitting}. On the four requests that asked for house, the untouched model won every pair the listener could separate § {#what-the-adapter-actually-did}. The article notes that the listener abstained on nine of the twenty-four pairs § {#the-blind-sitting}.", "partial": false, "sentences_kept": 3, "sentences_written": 3, "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.801733147003688, "ttft_ms": 2069, "tokens_in": 9267, "tokens_out": 88, "cached_tokens": 32, "cites": [ "the-blind-sitting", "what-the-adapter-actually-did" ], "gates": { "cites": [ "the-blind-sitting", "what-the-adapter-actually-did" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-blind-sitting", "overlap": 0.9 }, { "i": 1, "cite": "what-the-adapter-actually-did", "overlap": 0.818 }, { "i": 2, "cite": "the-blind-sitting", "overlap": 0.833 } ], "cite_overlap_p50": 0.833, "id": "on-41", "kind": "on-page" }, { "slug": "what-150-watts-buys", "q": "what does a large cloud provider charge for the same work?", "ok": true, "answer": "The article does not say what a large cloud provider charges for the same work § {#the-moment-this-page-is-about}.", "served": "This page doesn't say. The nearest it gets is § {#the-moment-this-page-is-about}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "the-moment-this-page-is-about", "seconds": 2.2686058080289513, "ttft_ms": 1617, "tokens_in": 7259, "tokens_out": 30, "cached_tokens": 32, "cites": [ "the-moment-this-page-is-about" ], "gates": { "cites": [ "the-moment-this-page-is-about" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-moment-this-page-is-about", "overlap": 0.143 } ], "cite_overlap_p50": 0.143, "id": "off-01", "kind": "off-page" }, { "slug": "how-we-work", "q": "what is the postal address of the workshop that ran this?", "ok": true, "answer": "The article does not provide the postal address of the workshop § {#the-plain-english-version}.", "served": "This page doesn't say. The nearest it gets is § {#the-plain-english-version}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "the-plain-english-version", "seconds": 2.142180830007419, "ttft_ms": 1368, "tokens_in": 5207, "tokens_out": 22, "cached_tokens": 32, "cites": [ "the-plain-english-version" ], "gates": { "cites": [ "the-plain-english-version" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-plain-english-version", "overlap": 0.25 } ], "cite_overlap_p50": 0.25, "id": "off-02", "kind": "off-page" }, { "slug": "llms-txt", "q": "how many people subscribe to this site's feed?", "ok": true, "answer": "The article does not provide information regarding how many people subscribe to the site's feed. The section § {#does-anyone-fetch-it} comes nearest by discussing web server request logs and the number of requests made by various agents.", "served": "This page doesn't say. The nearest it gets is § {#does-anyone-fetch-it}.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": true, "abstain_source": "model", "abstain_section": "does-anyone-fetch-it", "seconds": 11.727773635007907, "ttft_ms": 769, "tokens_in": 12526, "tokens_out": 49, "cached_tokens": 32, "cites": [ "does-anyone-fetch-it" ], "gates": { "cites": [ "does-anyone-fetch-it" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 1, "cite": "does-anyone-fetch-it", "overlap": 0.538 } ], "cite_overlap_p50": 0.538, "id": "off-03", "kind": "off-page" }, { "slug": "three-new-voices-at-the-narrators-chair", "q": "what will the next article on this site be about?", "ok": true, "answer": "The article does not state what the next article on the site will be about § {#the-short-version}.", "served": "This page doesn't say. The nearest it gets is § {#top}. “qwen3.8:27b across four house benches: one seat filled, one floor missed” covers it more directly.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "top", "seconds": 11.83235738199437, "ttft_ms": 10628, "tokens_in": 82788, "tokens_out": 24, "cached_tokens": 32, "cites": [ "top" ], "gates": { "cites": [ "top" ], "cites_dropped": [], "cites_resolved": [ "top" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "top", "overlap": 0.0 } ], "cite_overlap_p50": 0.0, "id": "off-04", "kind": "off-page" }, { "slug": "two-hours-on-battery", "q": "what did this cost in electricity at my own utility's rate?", "ok": true, "answer": "The article does not provide information regarding the cost of electricity at a utility's rate § {#two-hours-at-12-tok-s-the-test-that-named-the-page}.", "served": "This page doesn't say. The nearest it gets is § {#two-hours-at-12-tok-s-the-test-that-named-the-page}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "two-hours-at-12-tok-s-the-test-that-named-the-page", "seconds": 11.184571510006208, "ttft_ms": 1135, "tokens_in": 11093, "tokens_out": 45, "cached_tokens": 32, "cites": [ "two-hours-at-12-tok-s-the-test-that-named-the-page" ], "gates": { "cites": [ "two-hours-at-12-tok-s-the-test-that-named-the-page" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "two-hours-at-12-tok-s-the-test-that-named-the-page", "overlap": 0.0 } ], "cite_overlap_p50": 0.0, "id": "off-05", "kind": "off-page" }, { "slug": "reading-is-fast", "q": "who won the football world cup in 2022?", "ok": true, "answer": "The article does not answer this question § {#the-mystery-stated-honestly}.", "served": "This page doesn't say. The nearest it gets is § {#the-mystery-stated-honestly}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "the-mystery-stated-honestly", "seconds": 1.6907988900202326, "ttft_ms": 698, "tokens_in": 5555, "tokens_out": 19, "cached_tokens": 0, "cites": [ "the-mystery-stated-honestly" ], "gates": { "cites": [ "the-mystery-stated-honestly" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-mystery-stated-honestly", "overlap": 1.0 } ], "cite_overlap_p50": 1.0, "id": "off-06", "kind": "off-page" }, { "slug": "a-rig-your-friend-already-owns", "q": "what does one of these cards cost in my country today?", "ok": true, "answer": "The article does not answer what any of these cards cost in the reader's country today § {#the-moment-this-page-is-about}.", "served": "This page doesn't say. The nearest it gets is § {#the-moment-this-page-is-about}. “Three New Voices at the Narrator's Chair” covers it more directly.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "the-moment-this-page-is-about", "seconds": 1.8897495880373754, "ttft_ms": 643, "tokens_in": 4767, "tokens_out": 33, "cached_tokens": 32, "cites": [ "the-moment-this-page-is-about" ], "gates": { "cites": [ "the-moment-this-page-is-about" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-moment-this-page-is-about", "overlap": 0.167 } ], "cite_overlap_p50": 0.167, "id": "off-07", "kind": "off-page" }, { "slug": "the-move", "q": "can you write a python script that reproduces this for me?", "ok": true, "answer": "The article does not provide a python script or instructions for writing one. The section § {#how-to-check-our-work-and-see-it-live} discusses how to check the work by asking a real question or taking the provided data files, but it does not offer code for reproduction.", "served": "This page doesn't say. The nearest it gets is § {#how-to-check-our-work-and-see-it-live}.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": true, "abstain_source": "model", "abstain_section": "how-to-check-our-work-and-see-it-live", "seconds": 2.2540833260281943, "ttft_ms": 571, "tokens_in": 6099, "tokens_out": 64, "cached_tokens": 32, "cites": [ "how-to-check-our-work-and-see-it-live" ], "gates": { "cites": [ "how-to-check-our-work-and-see-it-live" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 1, "cite": "how-to-check-our-work-and-see-it-live", "overlap": 0.214 } ], "cite_overlap_p50": 0.214, "id": "off-08", "kind": "off-page" }, { "slug": "fifty-seven-milliseconds", "q": "what was the weather like on the day this was measured?", "ok": true, "answer": "The article does not say what the weather was like on the day of measurement § {#how-to-check-our-work-and-see-it-live}.", "served": "This page doesn't say. The nearest it gets is § {#how-to-check-our-work-and-see-it-live}. “What 150 Watts Buys” covers it more directly.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "how-to-check-our-work-and-see-it-live", "seconds": 1.5458378240000457, "ttft_ms": 744, "tokens_in": 5434, "tokens_out": 36, "cached_tokens": 32, "cites": [ "how-to-check-our-work-and-see-it-live" ], "gates": { "cites": [ "how-to-check-our-work-and-see-it-live" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "how-to-check-our-work-and-see-it-live", "overlap": 0.0 } ], "cite_overlap_p50": 0.0, "id": "off-09", "kind": "off-page" }, { "slug": "six-worlds-painted-large", "q": "how many hours a week do the people who ran this work?", "ok": true, "answer": "The article does not state how many hours a week the people who ran this work § {#who-ran-this-and-thanks}.", "served": "This page doesn't say. The nearest it gets is § {#who-ran-this-and-thanks}. “How the print lab works” covers it more directly.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "who-ran-this-and-thanks", "seconds": 1.3415143070160411, "ttft_ms": 692, "tokens_in": 3144, "tokens_out": 29, "cached_tokens": 32, "cites": [ "who-ran-this-and-thanks" ], "gates": { "cites": [ "who-ran-this-and-thanks" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "who-ran-this-and-thanks", "overlap": 0.143 } ], "cite_overlap_p50": 0.143, "id": "off-10", "kind": "off-page" }, { "slug": "outside-judges", "q": "what is this workshop's hourly rate for consulting work?", "ok": true, "answer": "The article does not state the workshop's hourly rate for consulting work § {#boundary}.", "served": "This page doesn't say. The nearest it gets is § {#boundary}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "boundary", "seconds": 1.2063351329998113, "ttft_ms": 909, "tokens_in": 10355, "tokens_out": 19, "cached_tokens": 32, "cites": [ "boundary" ], "gates": { "cites": [ "boundary" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "boundary", "overlap": 0.0 } ], "cite_overlap_p50": 0.0, "id": "off-11", "kind": "off-page" }, { "slug": "the-typist-and-the-developer", "q": "what is this workshop's annual revenue?", "ok": false, "answer": "The article does not state the workshop's annual revenue § {#the-typist-and-the-developer}.", "served": "This page doesn't say. The nearest it gets is § {#top}.", "partial": false, "sentences_kept": 0, "sentences_written": 1, "abstained": true, "abstain_source": "gate", "abstain_section": "top", "seconds": 7.561411835951731, "ttft_ms": 375, "tokens_in": 5534, "tokens_out": 25, "cached_tokens": 32, "cites": [], "gates": { "cites": [], "cites_dropped": [ "the-typist-and-the-developer" ], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": true, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "cite: 1 cite(s) not in this page's anchors: the-typist-and-the-developer", "cite: no valid cite" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [ "the-typist-and-the-developer" ] }, "figures": [], "figures_verbatim": true, "cites_valid": false, "redaction_hits": 0, "cite_overlap": [], "cite_overlap_p50": null, "id": "off-12", "kind": "off-page" }, { "slug": "compression-is-the-objective", "q": "how would these numbers change on windows instead?", "ok": true, "answer": "The article does not state how these numbers would change on windows instead § {#what-we-can-show-you}.", "served": "This page doesn't say. The nearest it gets is § {#what-we-can-show-you}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "what-we-can-show-you", "seconds": 7.540606366004795, "ttft_ms": 462, "tokens_in": 5439, "tokens_out": 26, "cached_tokens": 32, "cites": [ "what-we-can-show-you" ], "gates": { "cites": [ "what-we-can-show-you" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "what-we-can-show-you", "overlap": 0.4 } ], "cite_overlap_p50": 0.4, "id": "off-13", "kind": "off-page" }, { "slug": "the-open-call", "q": "what did other reviewers say about this piece after it was published?", "ok": true, "answer": "The article does not state what other reviewers said about this piece after it was published § {#the-short-version}. The section that comes nearest to discussing external feedback is the provenance section § {#provenance}, which describes how the page is authored and audited.", "served": "This page doesn't say. The nearest it gets is § {#top}. “Three New Voices at the Narrator's Chair” covers it more directly.", "partial": false, "sentences_kept": 2, "sentences_written": 2, "abstained": true, "abstain_source": "model", "abstain_section": "top", "seconds": 8.428588390001096, "ttft_ms": 7182, "tokens_in": 63489, "tokens_out": 53, "cached_tokens": 32, "cites": [ "top", "provenance" ], "gates": { "cites": [ "top", "provenance" ], "cites_dropped": [], "cites_resolved": [ "top" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "top", "overlap": 0.0 }, { "i": 1, "cite": "provenance", "overlap": 0.2 } ], "cite_overlap_p50": 0.1, "id": "off-14", "kind": "off-page" }, { "slug": "the-new-kid", "q": "how does this compare to a result published in 2028?", "ok": false, "answer": "The article does not provide information regarding any results published in 2028 § {#the-limits}.", "served": "This page doesn't say. The nearest it gets is § {#the-arrival}. “The outside judges — four rival labs re-check our work.” covers it more directly.", "partial": false, "sentences_kept": 0, "sentences_written": 1, "abstained": true, "abstain_source": "gate", "abstain_section": "the-arrival", "seconds": 7.623324447951745, "ttft_ms": 1255, "tokens_in": 15091, "tokens_out": 23, "cached_tokens": 32, "cites": [], "gates": { "cites": [], "cites_dropped": [ "the-limits" ], "cites_resolved": [], "new_noun_tokens": [ "2028" ], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": true, "new_noun": true, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "cite: 1 cite(s) not in this page's anchors: the-limits", "cite: no valid cite", "new noun: 1 token(s) the article does not carry" ], "new_noun_tokens": [ "2028" ], "new_noun_tokens_withheld": 0, "cites_dropped": [ "the-limits" ] }, "figures": [], "figures_verbatim": true, "cites_valid": false, "redaction_hits": 0, "cite_overlap": [], "cite_overlap_p50": null, "id": "off-15", "kind": "off-page" }, { "slug": "where-your-question-goes", "q": "which country is the workshop's business registered in?", "ok": true, "answer": "The article does not state which country the workshop's business is registered in § {#the-first-promise-your-visit-stays-between-us}.", "served": "This page doesn't say. The nearest it gets is § {#the-first-promise-your-visit-stays-between-us}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "the-first-promise-your-visit-stays-between-us", "seconds": 7.479893285955768, "ttft_ms": 542, "tokens_in": 4984, "tokens_out": 34, "cached_tokens": 32, "cites": [ "the-first-promise-your-visit-stays-between-us" ], "gates": { "cites": [ "the-first-promise-your-visit-stays-between-us" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-first-promise-your-visit-stays-between-us", "overlap": 0.2 } ], "cite_overlap_p50": 0.2, "id": "off-16", "kind": "off-page" }, { "slug": "nine-worlds-one-dog", "q": "how many other readers have asked this same question?", "ok": true, "answer": "The article does not answer how many other readers have asked this same question § {#for-the-technically-curious}.", "served": "This page doesn't say. The nearest it gets is § {#for-the-technically-curious}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "for-the-technically-curious", "seconds": 6.916126455995254, "ttft_ms": 6606, "tokens_in": 2059, "tokens_out": 27, "cached_tokens": 32, "cites": [ "for-the-technically-curious" ], "gates": { "cites": [ "for-the-technically-curious" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "for-the-technically-curious", "overlap": 0.286 } ], "cite_overlap_p50": 0.286, "id": "off-17", "kind": "off-page" }, { "slug": "how-the-beat-lab-works", "q": "what phone number reaches support?", "ok": true, "answer": "The article does not provide a phone number to reach support § {#what-it-is}.", "served": "This page doesn't say. The nearest it gets is § {#what-it-is}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "what-it-is", "seconds": 3.6364194829948246, "ttft_ms": 2572, "tokens_in": 31460, "tokens_out": 20, "cached_tokens": 32, "cites": [ "what-it-is" ], "gates": { "cites": [ "what-it-is" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "what-it-is", "overlap": 0.0 } ], "cite_overlap_p50": 0.0, "id": "off-18", "kind": "off-page" }, { "slug": "three-at-the-table", "q": "what would this measurement look like at twice the batch size?", "ok": true, "answer": "The article does not say what this measurement would look like at twice the batch size § {#the-part-nobody-warns-you-about-the-whole-payroll-needs-an}.", "served": "This page doesn't say. The nearest it gets is § {#the-part-nobody-warns-you-about-the-whole-payroll-needs-an}.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "the-part-nobody-warns-you-about-the-whole-payroll-needs-an", "seconds": 3.3875002319691703, "ttft_ms": 493, "tokens_in": 7672, "tokens_out": 42, "cached_tokens": 32, "cites": [ "the-part-nobody-warns-you-about-the-whole-payroll-needs-an" ], "gates": { "cites": [ "the-part-nobody-warns-you-about-the-whole-payroll-needs-an" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-part-nobody-warns-you-about-the-whole-payroll-needs-an", "overlap": 0.143 } ], "cite_overlap_p50": 0.143, "id": "off-19", "kind": "off-page" }, { "slug": "the-same-sixteen", "q": "which of these models is the best one to use for my own project?", "ok": true, "answer": "The article does not answer which model is best to use for a project § {#a-different-kind-of-row}.", "served": "This page doesn't say. The nearest it gets is § {#a-different-kind-of-row}. “Three New Voices at the Narrator's Chair” covers it more directly.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "a-different-kind-of-row", "seconds": 3.16276158898836, "ttft_ms": 2588, "tokens_in": 6545, "tokens_out": 27, "cached_tokens": 32, "cites": [ "a-different-kind-of-row" ], "gates": { "cites": [ "a-different-kind-of-row" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "a-different-kind-of-row", "overlap": 0.4 } ], "cite_overlap_p50": 0.4, "id": "off-20", "kind": "off-page" }, { "slug": "seat-trials", "q": "how many models will sit this exam next year?", "ok": true, "answer": "The article does not say how many models will sit this exam next year § {#the-instrument}.", "served": "This page doesn't say. The nearest it gets is § {#the-instrument}. “The Instrument Travels” covers it more directly.", "partial": false, "sentences_kept": 1, "sentences_written": 1, "abstained": true, "abstain_source": "model", "abstain_section": "the-instrument", "seconds": 3.0463393969694152, "ttft_ms": 2723, "tokens_in": 7234, "tokens_out": 21, "cached_tokens": 32, "cites": [ "the-instrument" ], "gates": { "cites": [ "the-instrument" ], "cites_dropped": [], "cites_resolved": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "spelled_figure": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-instrument", "overlap": 0.143 } ], "cite_overlap_p50": 0.143, "id": "off-21", "kind": "off-page" } ], "verdicts": { "on_page_answered": true, "non_verbatim_figures": true, "off_page_abstained": true, "redaction_hits": true }, "passed": true }