{ "schema_version": "1.1", "instrument": { "figures_read_from": "cite-cut prose (gates.prose_of) since 2026-09-09", "warm_arm_walk": "per page, one warm immediately before that page's own questions, since 2026-09-09", "not_comparable_with": "reports whose schema_version is 1.0" }, "run_utc": "2026-09-12T18:55:54Z", "arm": "cold", "concurrency": 6, "effective_concurrency": 6, "seat": { "url_host_withheld": true, "model": "gemma-4-26b-a4b-fp8", "class": "gemma-class", "seed": "no --seed flag sent" }, "packnorm_definition_recorded_not_governing": "v3", "bank": { "path": "bank.json", "counts": { "on_page": 41, "off_page": 21, "total": 62 } }, "results": { "on_page_total": 41, "on_page_answered": 41, "non_verbatim_figures": 0, "uncited": 0, "off_page_total": 21, "off_page_abstained": 21, "off_page_abstained_by_gate": 3, "off_page_abstained_by_model": 18, "off_page_flagged_but_not_verbatim": [], "on_page_abstained": 1, "off_page_cited": 18, "redaction_hits": 0, "cite_overlap_p50": 0.667, "cite_overlap_min": 0.0, "cite_overlap_max": 1.0, "cite_overlap_rows": 60, "seat_errors": 0, "p50_seconds": 4.294, "p95_seconds": 14.443, "pages_warmed": null, "cache_hits": null, "cache_hit_of": null }, "rows": [ { "slug": "three-new-voices-at-the-narrators-chair", "q": "what did the round cost in total, and how were the three new voices judged?", "ok": true, "answer": "The metered total for the round was $0.7652 according to § {#the-bill}. The three new voices were judged by a panel of seven seats from seven model families, six of which carried the round, reading lettered replies blind with no model names on them § {#seven-seats-six-carried-and-where-their-zeros-sit}. The article notes that the counts for the three new voices are not a ranking and that the reasoning for GLM 5.3 was withheld from the judges § {#what-the-panel-caught-and-what-it-couldnt-agree-on}.", "served": "The metered total for the round was $0.7652 according to § {#the-bill}. The three new voices were judged by a panel of seven seats from seven model families, six of which carried the round, reading lettered replies blind with no model names on them § {#seven-seats-six-carried-and-where-their-zeros-sit}. The article notes that the counts for the three new voices are not a ranking and that the reasoning for GLM 5.3 was withheld from the judges § {#what-the-panel-caught-and-what-it-couldnt-agree-on}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 15.109226503001992, "ttft_ms": 12584, "tokens_in": 82748, "tokens_out": 133, "cached_tokens": 32, "cites": [ "the-bill", "seven-seats-six-carried-and-where-their-zeros-sit", "what-the-panel-caught-and-what-it-couldnt-agree-on" ], "gates": { "cites": [ "the-bill", "seven-seats-six-carried-and-where-their-zeros-sit", "what-the-panel-caught-and-what-it-couldnt-agree-on" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "0.7652", "5.3" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-bill", "overlap": 0.8 }, { "i": 1, "cite": "seven-seats-six-carried-and-where-their-zeros-sit", "overlap": 0.722 }, { "i": 2, "cite": "what-the-panel-caught-and-what-it-couldnt-agree-on", "overlap": 0.364 } ], "cite_overlap_p50": 0.722, "id": "on-01", "kind": "on-page" }, { "slug": "two-new-frontier-models-at-the-rules-desk", "q": "how many cases did the blind head-to-head use, and did it separate the two models?", "ok": true, "answer": "The blind head-to-head used 36 cases and did not separate the two models § {#head-to-head}. A pooled preference rate of 0.544 for Claude Fable 5.1 was recorded, but the panel's interval of [0.444, 0.638] covers 0.5, meaning the panel did not separate the two arms § {#head-to-head}. The narrator's chair job was moved out of this round before pre-registration was written § {#two-strangers-at-the-door}.", "served": "The blind head-to-head used 36 cases and did not separate the two models § {#head-to-head}. A pooled preference rate of 0.544 for Claude Fable 5.1 was recorded, but the panel's interval of [0.444, 0.638] covers 0.5, meaning the panel did not separate the two arms § {#head-to-head}. The narrator's chair job was moved out of this round before pre-registration was written § {#two-strangers-at-the-door}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 14.538852970988955, "ttft_ms": 3335, "tokens_in": 26867, "tokens_out": 125, "cached_tokens": 32, "cites": [ "head-to-head", "two-strangers-at-the-door" ], "gates": { "cites": [ "head-to-head", "two-strangers-at-the-door" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "36", "0.544", "5.1", "0.444", "0.638", "0.5" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "head-to-head", "overlap": 0.875 }, { "i": 1, "cite": "head-to-head", "overlap": 0.895 }, { "i": 2, "cite": "two-strangers-at-the-door", "overlap": 1.0 } ], "cite_overlap_p50": 0.895, "id": "on-02", "kind": "on-page" }, { "slug": "six-worlds-painted-large", "q": "how many worlds were painted, and at what size and step count?", "ok": true, "answer": "Six worlds were painted at 2048×1280 and 40 steps § {#for-the-technically-curious}. The article does not mention any figures for what was left out, moved, or not measured § {#for-the-technically-curious}.", "served": "Six worlds were painted at 2048×1280 and 40 steps § {#for-the-technically-curious}. The article does not mention any figures for what was left out, moved, or not measured § {#for-the-technically-curious}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.07656307797879, "ttft_ms": 337, "tokens_in": 3099, "tokens_out": 63, "cached_tokens": 32, "cites": [ "for-the-technically-curious" ], "gates": { "cites": [ "for-the-technically-curious" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "1280", "40" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "for-the-technically-curious", "overlap": 0.857 }, { "i": 1, "cite": "for-the-technically-curious", "overlap": 0.0 } ], "cite_overlap_p50": 0.428, "id": "on-03", "kind": "on-page" }, { "slug": "the-beat-lab-asked-and-answered", "q": "how many answers does the page carry, and does any model run to make it?", "ok": true, "answer": "The page carries twenty-seven answers and no model is asked for them § {#is-this-ai}. The page answers from a list of answers it is already carrying instantly with no model asked and nothing typed leaving the page § {#the-beat-lab}. Note that the article mentions nothing about what was left out, moved, or not measured.", "served": "The page carries twenty-seven answers and no model is asked for them § {#is-this-ai}. The page answers from a list of answers it is already carrying instantly with no model asked and nothing typed leaving the page § {#the-beat-lab}. Note that the article mentions nothing about what was left out, moved, or not measured.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.190892011043616, "ttft_ms": 535, "tokens_in": 4597, "tokens_out": 72, "cached_tokens": 32, "cites": [ "is-this-ai" ], "gates": { "cites": [ "is-this-ai" ], "cites_dropped": [ "the-beat-lab" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "cite: 1 cite(s) not in this page's anchors: the-beat-lab" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [ "the-beat-lab" ] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "is-this-ai", "overlap": 0.667 } ], "cite_overlap_p50": 0.667, "id": "on-04", "kind": "on-page" }, { "slug": "ten-minutes-with-living-artists", "q": "how many tracks and releases went into the adapter, and what did the one listener prefer?", "ok": true, "answer": "The adapter was trained on 159 tracks from 41 releases § {#a-genre-not-a-composer}. The one listener preferred the untouched model both times § {#the-ear}. The article does not measure whether an unlevelled corpus taught the adapter a level rather than a style, nor does it state what a DJ-curated subset of the 208 tracks would teach § {#what-we-did-not-measure}.", "served": "The adapter was trained on 159 tracks from 41 releases § {#a-genre-not-a-composer}. The one listener preferred the untouched model both times § {#the-ear}. The article does not measure whether an unlevelled corpus taught the adapter a level rather than a style, nor does it state what a DJ-curated subset of the 208 tracks would teach § {#what-we-did-not-measure}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.822950201050844, "ttft_ms": 1584, "tokens_in": 13160, "tokens_out": 96, "cached_tokens": 32, "cites": [ "a-genre-not-a-composer", "the-ear", "what-we-did-not-measure" ], "gates": { "cites": [ "a-genre-not-a-composer", "the-ear", "what-we-did-not-measure" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "159", "41", "208" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "a-genre-not-a-composer", "overlap": 0.5 }, { "i": 1, "cite": "the-ear", "overlap": 0.714 }, { "i": 2, "cite": "what-we-did-not-measure", "overlap": 0.812 } ], "cite_overlap_p50": 0.714, "id": "on-05", "kind": "on-page" }, { "slug": "listen-for-yourself", "q": "how many renders are on the page, and what was done to their loudness?", "ok": true, "answer": "There are sixty-three players on the page § {#how-to-listen}. The listening masters were created by applying one constant gain and nothing else, such as no limiter, no compression, no equalisation, or no trimming § {#how-to-listen}. The gain used was the smaller of two numbers: how far the clip sits from −16 LUFS and how far its true peak sits from a −1 dBTP ceiling § {#how-to-listen}. Thirty-two clips land at −16 LUFS exactly, while the other thirty-one are left at their raw level because their peaks already sit at the engine's own −1 dB ceiling § {#how-to-listen}.", "served": "There are sixty-three players on the page § {#how-to-listen}. The listening masters were created by applying one constant gain and nothing else, such as no limiter, no compression, no equalisation, or no trimming § {#how-to-listen}. The gain used was the smaller of two numbers: how far the clip sits from −16 LUFS and how far its true peak sits from a −1 dBTP ceiling § {#how-to-listen}. Thirty-two clips land at −16 LUFS exactly, while the other thirty-one are left at their raw level because their peaks already sit at the engine's own −1 dB ceiling § {#how-to-listen}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 14.602256537007634, "ttft_ms": 1206, "tokens_in": 14417, "tokens_out": 146, "cached_tokens": 32, "cites": [ "how-to-listen" ], "gates": { "cites": [ "how-to-listen" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "16", "16" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "how-to-listen", "overlap": 1.0 }, { "i": 1, "cite": "how-to-listen", "overlap": 0.714 }, { "i": 2, "cite": "how-to-listen", "overlap": 0.929 }, { "i": 3, "cite": "how-to-listen", "overlap": 0.952 } ], "cite_overlap_p50": 0.941, "id": "on-06", "kind": "on-page" }, { "slug": "half-an-hour-with-dead-composers", "q": "how long did the Sousa and Bach adapters take to train?", "ok": true, "answer": "The Sousa adapter took 14 minutes 36 seconds to train § {#sousa-a-hundred-and-six-marches}, while the Bach adapter took 8 minutes 23 seconds § {#bach-the-pianist-who-freed-her-own-recordings}. These times were recorded by the trainer's own clock § {#the-bill}. The article does not specify what was left out or moved regarding these specific training durations.", "served": "The Sousa adapter took 14 minutes 36 seconds to train § {#sousa-a-hundred-and-six-marches}, while the Bach adapter took 8 minutes 23 seconds § {#bach-the-pianist-who-freed-her-own-recordings}. These times were recorded by the trainer's own clock § {#the-bill}. The article does not specify what was left out or moved regarding these specific training durations.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 1.826805790013168, "ttft_ms": 371, "tokens_in": 5657, "tokens_out": 98, "cached_tokens": 0, "cites": [ "sousa-a-hundred-and-six-marches", "bach-the-pianist-who-freed-her-own-recordings", "the-bill" ], "gates": { "cites": [ "sousa-a-hundred-and-six-marches", "bach-the-pianist-who-freed-her-own-recordings", "the-bill" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "14", "36", "8", "23" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "sousa-a-hundred-and-six-marches", "overlap": 0.545 }, { "i": 0, "cite": "bach-the-pianist-who-freed-her-own-recordings", "overlap": 0.455 }, { "i": 1, "cite": "the-bill", "overlap": 0.6 } ], "cite_overlap_p50": 0.545, "id": "on-07", "kind": "on-page" }, { "slug": "chopin-in-five-minutes", "q": "how many recordings were used, and how long did the training take by the trainer's clock?", "ok": true, "answer": "Nineteen recordings were used for training, and the process took 4 minutes 49 seconds by the trainer's own clock § {#five-minutes-of-training-honestly-accounted}. The article notes that the researchers did not measure whether a smaller GPU could have run this training § {#what-we-did-not-measure}.", "served": "Nineteen recordings were used for training, and the process took 4 minutes 49 seconds by the trainer's own clock § {#five-minutes-of-training-honestly-accounted}. The article notes that the researchers did not measure whether a smaller GPU could have run this training § {#what-we-did-not-measure}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 1.488164720009081, "ttft_ms": 367, "tokens_in": 3797, "tokens_out": 73, "cached_tokens": 32, "cites": [ "five-minutes-of-training-honestly-accounted", "what-we-did-not-measure" ], "gates": { "cites": [ "five-minutes-of-training-honestly-accounted", "what-we-did-not-measure" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "4", "49" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "five-minutes-of-training-honestly-accounted", "overlap": 0.833 }, { "i": 1, "cite": "what-we-did-not-measure", "overlap": 0.625 } ], "cite_overlap_p50": 0.729, "id": "on-08", "kind": "on-page" }, { "slug": "the-same-sixteen", "q": "what did glm-5.3-flash score on the tool bench, and what did the local models score?", "ok": true, "answer": "The reference arm scored 16/19 — 84.2%, Wilson 95% interval [62.4%, 94.5%] on the tool bench § {#sixteen-of-nineteen}. Two local models, gemma4:12b and qwen3.6:27b, also landed on the exact count of 16/19 § {#two-local-models-sixteen-days-earlier}. The article notes that four instruments could not be honestly read on the cloud path, three tasks never applied to a model that cannot hold a seat, and two tasks were scored by literal-string checkers that fail correct answers § {#the-ledger-of-what-did-not-read}.", "served": "The reference arm scored 16/19 — 84.2%, Wilson 95% interval [62.4%, 94.5%] on the tool bench § {#sixteen-of-nineteen}. Two local models, gemma4:12b and qwen3.6:27b, also landed on the exact count of 16/19 § {#two-local-models-sixteen-days-earlier}. The article notes that four instruments could not be honestly read on the cloud path, three tasks never applied to a model that cannot hold a seat, and two tasks were scored by literal-string checkers that fail correct answers § {#the-ledger-of-what-did-not-read}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 5.017084671999328, "ttft_ms": 418, "tokens_in": 6507, "tokens_out": 160, "cached_tokens": 32, "cites": [ "sixteen-of-nineteen", "two-local-models-sixteen-days-earlier", "the-ledger-of-what-did-not-read" ], "gates": { "cites": [ "sixteen-of-nineteen", "two-local-models-sixteen-days-earlier", "the-ledger-of-what-did-not-read" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "16/19", "84.2", "95", "62.4", "94.5", "4:12", "3.6", "27", "16/19" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "sixteen-of-nineteen", "overlap": 0.917 }, { "i": 1, "cite": "two-local-models-sixteen-days-earlier", "overlap": 1.0 }, { "i": 2, "cite": "the-ledger-of-what-did-not-read", "overlap": 0.636 } ], "cite_overlap_p50": 0.917, "id": "on-09", "kind": "on-page" }, { "slug": "two-hours-on-battery", "q": "what speed did the mini pc hold on battery, and for how long?", "ok": true, "answer": "The mini pc held a speed of ~12 tokens/second for about two hours on battery § {#two-hours-at-12-tok-s-the-test-that-named-the-page}. The article notes that it did not measure battery-vs-plugged behavior for the GPU-server half of the bench § {#two-hours-at-12-tok-s-the-test-that-named-the-page}.", "served": "The mini pc held a speed of ~12 tokens/second for about two hours on battery § {#two-hours-at-12-tok-s-the-test-that-named-the-page}. The article notes that it did not measure battery-vs-plugged behavior for the GPU-server half of the bench § {#two-hours-at-12-tok-s-the-test-that-named-the-page}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.410437418962829, "ttft_ms": 735, "tokens_in": 11048, "tokens_out": 98, "cached_tokens": 32, "cites": [ "two-hours-at-12-tok-s-the-test-that-named-the-page" ], "gates": { "cites": [ "two-hours-at-12-tok-s-the-test-that-named-the-page" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "12" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "two-hours-at-12-tok-s-the-test-that-named-the-page", "overlap": 1.0 }, { "i": 1, "cite": "two-hours-at-12-tok-s-the-test-that-named-the-page", "overlap": 0.714 } ], "cite_overlap_p50": 0.857, "id": "on-10", "kind": "on-page" }, { "slug": "the-instrument-travels", "q": "when the frozen exam moved to the smaller card, where did the ranked model land?", "ok": true, "answer": "The ranked model, qwen3.6:27b, landed at the floor by clearing both judge floors with a kill-recall of 11 and a preservation of 8/9 § {#the-chair-and-who-gets-to-sit-in-it}. This result was achieved on a different build and different silicon than the previous run § {#what-the-battery-was-for-the-instrument-travels}. The article notes that the missed item in the kill-recall count moved from c1-k-bez-c1 to c1-k-bez-h1 § {#what-the-battery-was-for-the-instrument-travels}.", "served": "The ranked model, qwen3.6:27b, landed at the floor by clearing both judge floors with a kill-recall of 11 and a preservation of 8/9 § {#the-chair-and-who-gets-to-sit-in-it}. This result was achieved on a different build and different silicon than the previous run § {#what-the-battery-was-for-the-instrument-travels}. The article notes that the missed item in the kill-recall count moved from c1-k-bez-c1 to c1-k-bez-h1 § {#what-the-battery-was-for-the-instrument-travels}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 5.376687307027169, "ttft_ms": 1101, "tokens_in": 8731, "tokens_out": 148, "cached_tokens": 32, "cites": [ "the-chair-and-who-gets-to-sit-in-it", "what-the-battery-was-for-the-instrument-travels" ], "gates": { "cites": [ "the-chair-and-who-gets-to-sit-in-it", "what-the-battery-was-for-the-instrument-travels" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "3.6", "27", "11", "8/9" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-chair-and-who-gets-to-sit-in-it", "overlap": 0.143 }, { "i": 1, "cite": "what-the-battery-was-for-the-instrument-travels", "overlap": 0.571 }, { "i": 2, "cite": "what-the-battery-was-for-the-instrument-travels", "overlap": 0.875 } ], "cite_overlap_p50": 0.571, "id": "on-11", "kind": "on-page" }, { "slug": "nine-worlds-one-dog", "q": "what stays the same across the nine paintings?", "ok": true, "answer": "The founder's dog remains the same across all nine paintings, specifically being the same dog, same curl, and same tucked paws § {#top}. He is a large black Portuguese Water Dog with one bright white chest patch who takes the same nap under the same tree in every world § {#for-the-technically-curious}.", "served": "The founder's dog remains the same across all nine paintings, specifically being the same dog, same curl, and same tucked paws § {#top}. He is a large black Portuguese Water Dog with one bright white chest patch who takes the same nap under the same tree in every world § {#for-the-technically-curious}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 3.7308594869682565, "ttft_ms": 802, "tokens_in": 2012, "tokens_out": 69, "cached_tokens": 32, "cites": [ "top", "for-the-technically-curious" ], "gates": { "cites": [ "top", "for-the-technically-curious" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "top", "overlap": 0.417 }, { "i": 1, "cite": "for-the-technically-curious", "overlap": 0.278 } ], "cite_overlap_p50": 0.348, "id": "on-12", "kind": "on-page" }, { "slug": "what-150-watts-buys", "q": "what did the 450-watt cap cost the dense model, and what did it buy back?", "ok": true, "answer": "The 600 to 450 W cap cost the dense model 2.0% of throughput and bought back 138 W, 11 °C, and a 28% jump in efficiency § {#the-answer-in-one-paragraph}. This result was measured at concurrency 16 with a p50 time-to-first-token of 22.2 ms § {#the-answer-in-one-paragraph}. The article notes that it did not measure long-context or prefill-heavy workloads, which could move the 2% either way § {#what-this-page-does-not-say}.", "served": "The 600 to 450 W cap cost the dense model 2.0% of throughput and bought back 138 W, 11 °C, and a 28% jump in efficiency § {#the-answer-in-one-paragraph}. This result was measured at concurrency 16 with a p50 time-to-first-token of 22.2 ms § {#the-answer-in-one-paragraph}. The article notes that it did not measure long-context or prefill-heavy workloads, which could move the 2% either way § {#what-this-page-does-not-say}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 5.072986838989891, "ttft_ms": 955, "tokens_in": 7223, "tokens_out": 141, "cached_tokens": 32, "cites": [ "the-answer-in-one-paragraph", "what-this-page-does-not-say" ], "gates": { "cites": [ "the-answer-in-one-paragraph", "what-this-page-does-not-say" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "600", "450", "2.0", "138", "11", "28", "16", "50", "22.2", "2" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-answer-in-one-paragraph", "overlap": 0.8 }, { "i": 1, "cite": "the-answer-in-one-paragraph", "overlap": 0.75 }, { "i": 2, "cite": "what-this-page-does-not-say", "overlap": 0.667 } ], "cite_overlap_p50": 0.75, "id": "on-13", "kind": "on-page" }, { "slug": "how-the-beat-lab-works", "q": "where is the sound made, and how long does a typical wish take?", "ok": true, "answer": "Every sound in the Beat Lab is synthesised by the user's own browser § {#the-sound-is-made-on-your-device}. A typical wish takes about 4 s § {#what-happens-when-you-ask-the-genie}. The article does not provide measurements for the speed of the genie's voice when it is not being read for the first time or repeated § {#numbers-so-it-isnt-just-talk}.", "served": "Every sound in the Beat Lab is synthesised by the user's own browser § {#the-sound-is-made-on-your-device}. A typical wish takes about 4 s § {#what-happens-when-you-ask-the-genie}. The article does not provide measurements for the speed of the genie's voice when it is not being read for the first time or repeated § {#numbers-so-it-isnt-just-talk}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 3.954151759040542, "ttft_ms": 2627, "tokens_in": 31423, "tokens_out": 101, "cached_tokens": 32, "cites": [ "the-sound-is-made-on-your-device", "what-happens-when-you-ask-the-genie", "numbers-so-it-isnt-just-talk" ], "gates": { "cites": [ "the-sound-is-made-on-your-device", "what-happens-when-you-ask-the-genie", "numbers-so-it-isnt-just-talk" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "4" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-sound-is-made-on-your-device", "overlap": 0.875 }, { "i": 1, "cite": "what-happens-when-you-ask-the-genie", "overlap": 1.0 }, { "i": 2, "cite": "numbers-so-it-isnt-just-talk", "overlap": 0.3 } ], "cite_overlap_p50": 0.875, "id": "on-14", "kind": "on-page" }, { "slug": "the-typist-and-the-developer", "q": "how long did the developer take to develop one picture?", "ok": true, "answer": "The developer developed a picture in 1.54 seconds § {#the-developer}. When the typist was also streaming tokens, the developer took 1.82 seconds per sketch § {#one-card-both-jobs-at-once}. The article notes that it did not instrument how much traffic was present during the run § {#what-we-did-not-measure-said-out-loud}.", "served": "The developer developed a picture in 1.54 seconds § {#the-developer}. When the typist was also streaming tokens, the developer took 1.82 seconds per sketch § {#one-card-both-jobs-at-once}. The article notes that it did not instrument how much traffic was present during the run § {#what-we-did-not-measure-said-out-loud}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.006937852012925, "ttft_ms": 354, "tokens_in": 5490, "tokens_out": 87, "cached_tokens": 32, "cites": [ "the-developer", "one-card-both-jobs-at-once", "what-we-did-not-measure-said-out-loud" ], "gates": { "cites": [ "the-developer", "one-card-both-jobs-at-once", "what-we-did-not-measure-said-out-loud" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "1.54", "1.82" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-developer", "overlap": 1.0 }, { "i": 1, "cite": "one-card-both-jobs-at-once", "overlap": 0.9 }, { "i": 2, "cite": "what-we-did-not-measure-said-out-loud", "overlap": 0.714 } ], "cite_overlap_p50": 0.9, "id": "on-15", "kind": "on-page" }, { "slug": "a-rig-your-friend-already-owns", "q": "how much faster was the workstation card than the 3090 on the game's own art?", "ok": true, "answer": "The article does not provide a single speed comparison for the workstation card against the 3090 on the game's own art, but instead reports median seconds per render for various models § {#the-table-that-answers-the-question}. For the FLUX.2 klein-4B model used as the sketch artist, the 96G card was 3.4× faster than the 3090 § {#the-table-that-answers-the-question}. The article notes that 8–12 GB cards were not tested § {#three-findings-we-didnt-order}.", "served": "This page doesn't say. The nearest it gets is § {#the-table-that-answers-the-question}. Fifty-Seven Milliseconds: What a Diffusion Step Actually Buys covers it more directly.", "abstained": true, "abstain_source": "model", "abstain_section": "the-table-that-answers-the-question", "seconds": 2.618259415961802, "ttft_ms": 355, "tokens_in": 4731, "tokens_out": 131, "cached_tokens": 32, "cites": [ "the-table-that-answers-the-question", "three-findings-we-didnt-order" ], "gates": { "cites": [ "the-table-that-answers-the-question", "three-findings-we-didnt-order" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "3090", "2", "4", "96", "3.4", "3090", "8-12" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-table-that-answers-the-question", "overlap": 0.474 }, { "i": 1, "cite": "the-table-that-answers-the-question", "overlap": 0.818 }, { "i": 2, "cite": "three-findings-we-didnt-order", "overlap": 0.6 } ], "cite_overlap_p50": 0.6, "id": "on-16", "kind": "on-page" }, { "slug": "how-a-vision-model-sees", "q": "how many tokens does an 896 px picture become, and how long does its prefill take?", "ok": true, "answer": "An 896 px picture becomes 199 tokens and its prefill takes 184 ms § {#finding-one-a-picture-is-199-tokens-at-896-px-and-the}. The article does not mention any other token counts or prefill times for this specific resolution § {#finding-one-a-picture-is-199-tokens-at-896-px-and-the}.", "served": "An 896 px picture becomes 199 tokens and its prefill takes 184 ms § {#finding-one-a-picture-is-199-tokens-at-896-px-and-the}. The article does not mention any other token counts or prefill times for this specific resolution § {#finding-one-a-picture-is-199-tokens-at-896-px-and-the}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.3821705330046825, "ttft_ms": 713, "tokens_in": 10265, "tokens_out": 101, "cached_tokens": 32, "cites": [ "finding-one-a-picture-is-199-tokens-at-896-px-and-the" ], "gates": { "cites": [ "finding-one-a-picture-is-199-tokens-at-896-px-and-the" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "896", "199", "184" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "finding-one-a-picture-is-199-tokens-at-896-px-and-the", "overlap": 0.8 }, { "i": 1, "cite": "finding-one-a-picture-is-199-tokens-at-896-px-and-the", "overlap": 0.625 } ], "cite_overlap_p50": 0.713, "id": "on-17", "kind": "on-page" }, { "slug": "fifty-seven-milliseconds", "q": "which painter is the fastest cell, and at what time?", "ok": true, "answer": "The distilled painter is the sharpest and fastest cell in the second run, recording a time of 0.62 s at 4 steps § {#finding-four-the-argument-with-our-own-settings-and-what-it}. This measurement was taken on a 96G VRAM workstation § {#finding-four-the-argument-with-our-own-settings-and-what-it}. The article does not mention what was left out or not measured regarding this specific painter.", "served": "The distilled painter is the sharpest and fastest cell in the second run, recording a time of 0.62 s at 4 steps § {#finding-four-the-argument-with-our-own-settings-and-what-it}. This measurement was taken on a 96G VRAM workstation § {#finding-four-the-argument-with-our-own-settings-and-what-it}. The article does not mention what was left out or not measured regarding this specific painter.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.210871485003736, "ttft_ms": 654, "tokens_in": 5388, "tokens_out": 108, "cached_tokens": 32, "cites": [ "finding-four-the-argument-with-our-own-settings-and-what-it" ], "gates": { "cites": [ "finding-four-the-argument-with-our-own-settings-and-what-it" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "0.62", "4", "96" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "finding-four-the-argument-with-our-own-settings-and-what-it", "overlap": 0.909 }, { "i": 1, "cite": "finding-four-the-argument-with-our-own-settings-and-what-it", "overlap": 0.6 } ], "cite_overlap_p50": 0.754, "id": "on-18", "kind": "on-page" }, { "slug": "the-dog-the-dice-and-the-painter", "q": "how big is the file the whole world is written in?", "ok": true, "answer": "The world is written in a single YAML file that is 16,705 lines long § {#the-world-is-a-file}. This file contains six regions, forty NPCs, fifty-nine quests, and three kinds of adoptable pet § {#the-world-is-a-file}. The article does not provide a file size in bytes, though it notes that two copies of the file are byte-identical § {#the-world-is-a-file}.", "served": "The world is written in a single YAML file that is 16,705 lines long § {#the-world-is-a-file}. This file contains six regions, forty NPCs, fifty-nine quests, and three kinds of adoptable pet § {#the-world-is-a-file}. The article does not provide a file size in bytes, though it notes that two copies of the file are byte-identical § {#the-world-is-a-file}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.7006699190242216, "ttft_ms": 406, "tokens_in": 6070, "tokens_out": 102, "cached_tokens": 32, "cites": [ "the-world-is-a-file" ], "gates": { "cites": [ "the-world-is-a-file" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "16,705" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-world-is-a-file", "overlap": 0.778 }, { "i": 1, "cite": "the-world-is-a-file", "overlap": 0.917 }, { "i": 2, "cite": "the-world-is-a-file", "overlap": 0.333 } ], "cite_overlap_p50": 0.778, "id": "on-19", "kind": "on-page" }, { "slug": "where-new-knowledge-comes-from", "q": "what are the two engines the page names for where new knowledge comes from?", "ok": true, "answer": "The two engines named are present in the data, unmined, and implied by the rules, unvisited § {#two-kinds-of-there-all-along}. The first engine involves reading patterns that were already in archives, such as co-evolution fingerprints or molecular shapes that kill bacteria § {#two-kinds-of-there-all-along}. The second engine involves walking regions of rule-space that no row of data contains, such as a planet's position in 1631 § {#two-kinds-of-there-all-along}. Structural biologists noted that these engines leave out how proteins pair up, how they move, and what one mutation does § {#since-1994-a-contest-has-graded-protein-structure-predictions}.", "served": "The two engines named are present in the data, unmined, and implied by the rules, unvisited § {#two-kinds-of-there-all-along}. The first engine involves reading patterns that were already in archives, such as co-evolution fingerprints or molecular shapes that kill bacteria § {#two-kinds-of-there-all-along}. The second engine involves walking regions of rule-space that no row of data contains, such as a planet's position in 1631 § {#two-kinds-of-there-all-along}. Structural biologists noted that these engines leave out how proteins pair up, how they move, and what one mutation does § {#since-1994-a-contest-has-graded-protein-structure-predictions}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 3.5057505729491822, "ttft_ms": 500, "tokens_in": 5426, "tokens_out": 164, "cached_tokens": 32, "cites": [ "two-kinds-of-there-all-along" ], "gates": { "cites": [ "two-kinds-of-there-all-along" ], "cites_dropped": [ "since-1994-a-contest-has-graded-protein-structure-predictions" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "cite: 1 cite(s) not in this page's anchors: since-1994-a-contest-has-graded-protein-structure-predictions" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [ "since-1994-a-contest-has-graded-protein-structure-predictions" ] }, "figures": [ "1631" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "two-kinds-of-there-all-along", "overlap": 0.889 }, { "i": 1, "cite": "two-kinds-of-there-all-along", "overlap": 0.5 }, { "i": 2, "cite": "two-kinds-of-there-all-along", "overlap": 0.462 } ], "cite_overlap_p50": 0.5, "id": "on-20", "kind": "on-page" }, { "slug": "compression-is-the-objective", "q": "what does the page say a language model is, and what prize does it point to?", "ok": true, "answer": "A language model is a compression of its corpus § {#the-punchline}. The article points to the Hutter Prize, which is a competition where frontier models cannot enter because it requires charging for every decompressor byte and does not allow outside data § {#what-we-can-show-you}. The text notes that the model's own bits, or weights, sit on a separate ledger line and are not charged in the next-token training loss § {#the-punchline}.", "served": "A language model is a compression of its corpus § {#the-punchline}. The article points to the Hutter Prize, which is a competition where frontier models cannot enter because it requires charging for every decompressor byte and does not allow outside data § {#what-we-can-show-you}. The text notes that the model's own bits, or weights, sit on a separate ledger line and are not charged in the next-token training loss § {#the-punchline}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.3200061530224048, "ttft_ms": 370, "tokens_in": 5402, "tokens_out": 101, "cached_tokens": 32, "cites": [ "the-punchline", "what-we-can-show-you" ], "gates": { "cites": [ "the-punchline", "what-we-can-show-you" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-punchline", "overlap": 1.0 }, { "i": 1, "cite": "what-we-can-show-you", "overlap": 0.588 }, { "i": 2, "cite": "the-punchline", "overlap": 0.786 } ], "cite_overlap_p50": 0.786, "id": "on-21", "kind": "on-page" }, { "slug": "the-free-speed-wasnt-free", "q": "what was the production median before the change, and what caused the speed-up?", "ok": true, "answer": "The production median was 138.6 tok/s over 1,982 live rulings § {#the-day-the-box-got-faster}. The speed-up was caused by an ollama commit that changed the default repeat_penalty from 1.1 to 1.0, which removed a penalties stage that had been running on every token § {#the-two-line-commit}. The article notes that the quality half of this change remains unmeasured § {#the-part-that-isnt-about-speed}.", "served": "The production median was 138.6 tok/s over 1,982 live rulings § {#the-day-the-box-got-faster}. The speed-up was caused by an ollama commit that changed the default repeat_penalty from 1.1 to 1.0, which removed a penalties stage that had been running on every token § {#the-two-line-commit}. The article notes that the quality half of this change remains unmeasured § {#the-part-that-isnt-about-speed}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 3.7157248010043986, "ttft_ms": 506, "tokens_in": 7483, "tokens_out": 116, "cached_tokens": 32, "cites": [ "the-day-the-box-got-faster", "the-two-line-commit", "the-part-that-isnt-about-speed" ], "gates": { "cites": [ "the-day-the-box-got-faster", "the-two-line-commit", "the-part-that-isnt-about-speed" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "138.6", "1,982", "1.1", "1.0" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-day-the-box-got-faster", "overlap": 1.0 }, { "i": 1, "cite": "the-two-line-commit", "overlap": 0.625 }, { "i": 2, "cite": "the-part-that-isnt-about-speed", "overlap": 0.667 } ], "cite_overlap_p50": 0.667, "id": "on-22", "kind": "on-page" }, { "slug": "the-compressed-photograph", "q": "how much slower was the uncompressed original than the compressed build?", "ok": true, "answer": "The uncompressed original was 3.4× the speed of the trained-4-bit build § {#the-same-question-down-the-whole-ladder}. The article also notes that it does not measure perplexity § {#what-this-page-does-not-say}.", "served": "The uncompressed original was 3.4× the speed of the trained-4-bit build § {#the-same-question-down-the-whole-ladder}. The article also notes that it does not measure perplexity § {#what-this-page-does-not-say}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.115692543971818, "ttft_ms": 1185, "tokens_in": 8591, "tokens_out": 62, "cached_tokens": 32, "cites": [ "the-same-question-down-the-whole-ladder", "what-this-page-does-not-say" ], "gates": { "cites": [ "the-same-question-down-the-whole-ladder", "what-this-page-does-not-say" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "3.4", "4" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-same-question-down-the-whole-ladder", "overlap": 1.0 }, { "i": 1, "cite": "what-this-page-does-not-say", "overlap": 0.5 } ], "cite_overlap_p50": 0.75, "id": "on-23", "kind": "on-page" }, { "slug": "three-at-the-table", "q": "what is the true active share for the live seat, and what does the page call the two numbers in a model name?", "ok": true, "answer": "The true active share for the live seat is 14.4% of the model's bytes § {#the-payroll-and-the-meeting}. The two numbers in a model name are the payroll and the meeting § {#the-tag-with-two-numbers}. The article notes that it does not measure the memory cost of a model's context or the speed of streaming experts from system RAM § {#what-sparsity-does-not-buy} § {#the-part-nobody-warns-you-about-the-whole-payroll-needs-an-office}.", "served": "The true active share for the live seat is 14.4% of the model's bytes § {#the-payroll-and-the-meeting}. The two numbers in a model name are the payroll and the meeting § {#the-tag-with-two-numbers}. The article notes that it does not measure the memory cost of a model's context or the speed of streaming experts from system RAM § {#what-sparsity-does-not-buy} § {#the-part-nobody-warns-you-about-the-whole-payroll-needs-an-office}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 3.7046829360187985, "ttft_ms": 1079, "tokens_in": 7639, "tokens_out": 124, "cached_tokens": 32, "cites": [ "the-payroll-and-the-meeting", "the-tag-with-two-numbers", "what-sparsity-does-not-buy" ], "gates": { "cites": [ "the-payroll-and-the-meeting", "the-tag-with-two-numbers", "what-sparsity-does-not-buy" ], "cites_dropped": [ "the-part-nobody-warns-you-about-the-whole-payroll-needs-an-office" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "cite: 1 cite(s) not in this page's anchors: the-part-nobody-warns-you-about-the-whole-payroll-needs-an-office" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [ "the-part-nobody-warns-you-about-the-whole-payroll-needs-an-office" ] }, "figures": [ "14.4" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-payroll-and-the-meeting", "overlap": 1.0 }, { "i": 1, "cite": "the-tag-with-two-numbers", "overlap": 0.833 }, { "i": 2, "cite": "what-sparsity-does-not-buy", "overlap": 0.545 } ], "cite_overlap_p50": 0.833, "id": "on-24", "kind": "on-page" }, { "slug": "reading-is-fast", "q": "what was the median reading speed, and why is reading faster than writing?", "ok": true, "answer": "The median reading speed was 5,415 tokens/s § {#a-measurement-from-our-own-card}. Reading is faster because it is parallel, as the model can process every token in the prompt at the same time since the text already exists § {#reading-is-a-group-photo}. Writing is slower because it is serial, meaning the model must choose tokens one at a time because the future of the sentence does not yet exist § {#writing-is-one-at-a-time}. The article notes that it does not say the ratio is a constant § {#what-this-page-does-not-say}.", "served": "The median reading speed was 5,415 tokens/s § {#a-measurement-from-our-own-card}. Reading is faster because it is parallel, as the model can process every token in the prompt at the same time since the text already exists § {#reading-is-a-group-photo}. Writing is slower because it is serial, meaning the model must choose tokens one at a time because the future of the sentence does not yet exist § {#writing-is-one-at-a-time}. The article notes that it does not say the ratio is a constant § {#what-this-page-does-not-say}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 5.841405806015246, "ttft_ms": 385, "tokens_in": 5511, "tokens_out": 137, "cached_tokens": 32, "cites": [ "a-measurement-from-our-own-card", "reading-is-a-group-photo", "writing-is-one-at-a-time", "what-this-page-does-not-say" ], "gates": { "cites": [ "a-measurement-from-our-own-card", "reading-is-a-group-photo", "writing-is-one-at-a-time", "what-this-page-does-not-say" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "5,415" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "a-measurement-from-our-own-card", "overlap": 1.0 }, { "i": 1, "cite": "reading-is-a-group-photo", "overlap": 0.667 }, { "i": 2, "cite": "writing-is-one-at-a-time", "overlap": 0.733 }, { "i": 3, "cite": "what-this-page-does-not-say", "overlap": 0.75 } ], "cite_overlap_p50": 0.742, "id": "on-25", "kind": "on-page" }, { "slug": "three-librarians", "q": "how many dimensions does the embedder's map have, and what does the third librarian do?", "ok": true, "answer": "The embedder's map has 768 directions § {#the-second-librarian-searches-by-meaning}. The third librarian works only within the aisle of the specific game being asked about, picking up the edition's own rulebook and any matched correction documents to rank passages by meaning § {#the-third-librarian-never-left-your-aisle}. The article does not mention what was left out, moved, or not measured regarding the third librarian specifically § {#the-third-librarian-never-left-your-aisle}.", "served": "The embedder's map has 768 directions § {#the-second-librarian-searches-by-meaning}. The third librarian works only within the aisle of the specific game being asked about, picking up the edition's own rulebook and any matched correction documents to rank passages by meaning § {#the-third-librarian-never-left-your-aisle}. The article does not mention what was left out, moved, or not measured regarding the third librarian specifically § {#the-third-librarian-never-left-your-aisle}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.8025829090038314, "ttft_ms": 501, "tokens_in": 5867, "tokens_out": 120, "cached_tokens": 32, "cites": [ "the-second-librarian-searches-by-meaning", "the-third-librarian-never-left-your-aisle" ], "gates": { "cites": [ "the-second-librarian-searches-by-meaning", "the-third-librarian-never-left-your-aisle" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "768" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-second-librarian-searches-by-meaning", "overlap": 0.75 }, { "i": 1, "cite": "the-third-librarian-never-left-your-aisle", "overlap": 0.714 }, { "i": 2, "cite": "the-third-librarian-never-left-your-aisle", "overlap": 0.333 } ], "cite_overlap_p50": 0.714, "id": "on-26", "kind": "on-page" }, { "slug": "the-new-kid", "q": "which seat did the new model pass, and which floor did it miss?", "ok": true, "answer": "The model passed the classifier seat § {#the-classifier-seat-five-gates-all-pass-faster} and missed the preservation floor of 13 in the judge seat § {#the-judge-seat-and-the-floor-it-missed}. It achieved a preservation score of 11 of 16 § {#the-judge-seat-and-the-floor-it-missed}. The article notes that no baseline was scored for the classifier seat and that the judge seat's results are not independently checkable § {#the-classifier-seat-five-gates-all-pass-faster} § {#the-judge-seat-and-the-floor-it-missed}.", "served": "The model passed the classifier seat § {#the-classifier-seat-five-gates-all-pass-faster} and missed the preservation floor of 13 in the judge seat § {#the-judge-seat-and-the-floor-it-missed}. It achieved a preservation score of 11 of 16 § {#the-judge-seat-and-the-floor-it-missed}. The article notes that no baseline was scored for the classifier seat and that the judge seat's results are not independently checkable § {#the-classifier-seat-five-gates-all-pass-faster} § {#the-judge-seat-and-the-floor-it-missed}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 11.841476908011828, "ttft_ms": 977, "tokens_in": 15045, "tokens_out": 151, "cached_tokens": 32, "cites": [ "the-classifier-seat-five-gates-all-pass-faster", "the-judge-seat-and-the-floor-it-missed" ], "gates": { "cites": [ "the-classifier-seat-five-gates-all-pass-faster", "the-judge-seat-and-the-floor-it-missed" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "13", "11", "16" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-classifier-seat-five-gates-all-pass-faster", "overlap": 0.556 }, { "i": 0, "cite": "the-judge-seat-and-the-floor-it-missed", "overlap": 0.889 }, { "i": 1, "cite": "the-judge-seat-and-the-floor-it-missed", "overlap": 0.8 }, { "i": 2, "cite": "the-classifier-seat-five-gates-all-pass-faster", "overlap": 0.8 }, { "i": 2, "cite": "the-judge-seat-and-the-floor-it-missed", "overlap": 0.6 } ], "cite_overlap_p50": 0.8, "id": "on-27", "kind": "on-page" }, { "slug": "llms-txt", "q": "did any ai crawler fetch the file in the first thirty days, and how big is the largest measured llms-full file?", "ok": true, "answer": "No AI crawler asked for the file in thirty days of logs § {#does-anyone-fetch-it}, and the largest measured llms-full file weighs 30.7 MiB § {#where-the-file-came-from}. The article notes that it does not measure crawling, fetching, adoption, or agent behaviour in the wild § {#the-bench-if-the-map-arrives-does-it-help}.", "served": "No AI crawler asked for the file in thirty days of logs § {#does-anyone-fetch-it}, and the largest measured llms-full file weighs 30.7 MiB § {#where-the-file-came-from}. The article notes that it does not measure crawling, fetching, adoption, or agent behaviour in the wild § {#the-bench-if-the-map-arrives-does-it-help}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 8.650716226024088, "ttft_ms": 1368, "tokens_in": 12495, "tokens_out": 93, "cached_tokens": 32, "cites": [ "does-anyone-fetch-it", "where-the-file-came-from", "the-bench-if-the-map-arrives-does-it-help" ], "gates": { "cites": [ "does-anyone-fetch-it", "where-the-file-came-from", "the-bench-if-the-map-arrives-does-it-help" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "30.7" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "does-anyone-fetch-it", "overlap": 0.385 }, { "i": 0, "cite": "where-the-file-came-from", "overlap": 0.538 }, { "i": 1, "cite": "the-bench-if-the-map-arrives-does-it-help", "overlap": 0.0 } ], "cite_overlap_p50": 0.385, "id": "on-28", "kind": "on-page" }, { "slug": "the-open-call", "q": "how many models were handed the frozen moment, and what did the round cost?", "ok": true, "answer": "Twenty language models were handed the frozen moment § {#top}. The round's metered spend was $2.2826 § {#2-2826-over-56-metered-calls-against-38-49-of-caps}. The agent transport holds no cost figure and no token counters § {#the-author-of-this-page-is-a-contestant-in-it}.", "served": "Twenty language models were handed the frozen moment § {#top}. The round's metered spend was $2.2826 § {#2-2826-over-56-metered-calls-against-38-49-of-caps}. The agent transport holds no cost figure and no token counters § {#the-author-of-this-page-is-a-contestant-in-it}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 10.663827051990665, "ttft_ms": 6574, "tokens_in": 63446, "tokens_out": 93, "cached_tokens": 32, "cites": [ "top", "2-2826-over-56-metered-calls-against-38-49-of-caps", "the-author-of-this-page-is-a-contestant-in-it" ], "gates": { "cites": [ "top", "2-2826-over-56-metered-calls-against-38-49-of-caps", "the-author-of-this-page-is-a-contestant-in-it" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "2.2826" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "top", "overlap": 1.0 }, { "i": 1, "cite": "2-2826-over-56-metered-calls-against-38-49-of-caps", "overlap": 0.5 }, { "i": 2, "cite": "the-author-of-this-page-is-a-contestant-in-it", "overlap": 1.0 } ], "cite_overlap_p50": 1.0, "id": "on-29", "kind": "on-page" }, { "slug": "where-your-question-goes", "q": "how long does every notebook on the server keep its entries?", "ok": true, "answer": "Every notebook on the server now keeps thirty days of entries § {#the-first-promise-your-visit-stays-between-us}. This rule was implemented on 15 August 2026 to replace a previous setting that kept lines for up to a hundred years § {#the-first-promise-your-visit-stays-between-us}. The article notes that a nightly sweep runs to maintain this thirty day limit § {#the-first-promise-your-visit-stays-between-us}.", "served": "Every notebook on the server now keeps thirty days of entries § {#the-first-promise-your-visit-stays-between-us}. This rule was implemented on 15 August 2026 to replace a previous setting that kept lines for up to a hundred years § {#the-first-promise-your-visit-stays-between-us}. The article notes that a nightly sweep runs to maintain this thirty day limit § {#the-first-promise-your-visit-stays-between-us}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 10.244647781015374, "ttft_ms": 333, "tokens_in": 4704, "tokens_out": 108, "cached_tokens": 32, "cites": [ "the-first-promise-your-visit-stays-between-us" ], "gates": { "cites": [ "the-first-promise-your-visit-stays-between-us" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "15" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-first-promise-your-visit-stays-between-us", "overlap": 0.875 }, { "i": 1, "cite": "the-first-promise-your-visit-stays-between-us", "overlap": 0.692 }, { "i": 2, "cite": "the-first-promise-your-visit-stays-between-us", "overlap": 0.5 } ], "cite_overlap_p50": 0.692, "id": "on-30", "kind": "on-page" }, { "slug": "how-we-work", "q": "what does the page say about the humans and agents who make each exhibit?", "ok": true, "answer": "The page states that the human generates and judges while the agent amplifies and verifies § {#the-plain-english-version}. Humans are responsible for noticing, proposing work, reviewing hardened plans, steering execution, and providing the final sign-off for release § {#the-thorough-walk}. Agents handle reconnaissance, drafting executable plans, auditing those plans through hardening panels, executing the signed plans, and drafting the articles § {#the-thorough-walk}. Regarding what was not measured, the original five questions of the voice trial were never persisted § {#the-plain-english-version}.", "served": "The page states that the human generates and judges while the agent amplifies and verifies § {#the-plain-english-version}. Humans are responsible for noticing, proposing work, reviewing hardened plans, steering execution, and providing the final sign-off for release § {#the-thorough-walk}. Agents handle reconnaissance, drafting executable plans, auditing those plans through hardening panels, executing the signed plans, and drafting the articles § {#the-thorough-walk}. Regarding what was not measured, the original five questions of the voice trial were never persisted § {#the-plain-english-version}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 11.203920631029177, "ttft_ms": 6327, "tokens_in": 5164, "tokens_out": 121, "cached_tokens": 32, "cites": [ "the-plain-english-version", "the-thorough-walk" ], "gates": { "cites": [ "the-plain-english-version", "the-thorough-walk" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-plain-english-version", "overlap": 0.375 }, { "i": 1, "cite": "the-thorough-walk", "overlap": 0.429 }, { "i": 2, "cite": "the-thorough-walk", "overlap": 0.308 }, { "i": 3, "cite": "the-plain-english-version", "overlap": 0.778 } ], "cite_overlap_p50": 0.402, "id": "on-31", "kind": "on-page" }, { "slug": "the-move", "q": "what did the median search time go from and to after the move?", "ok": true, "answer": "The median search time went from 242.0 ms in the laptop era to 77.5 ms in the new era § {#receipts}. This comparison is noted as the retrieval leg that the move actually touched and the one no model change can confound § {#receipts}.", "served": "The median search time went from 242.0 ms in the laptop era to 77.5 ms in the new era § {#receipts}. This comparison is noted as the retrieval leg that the move actually touched and the one no model change can confound § {#receipts}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.177207554981578, "ttft_ms": 3425, "tokens_in": 6055, "tokens_out": 60, "cached_tokens": 32, "cites": [ "receipts" ], "gates": { "cites": [ "receipts" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "242.0", "77.5" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "receipts", "overlap": 0.9 }, { "i": 1, "cite": "receipts", "overlap": 0.909 } ], "cite_overlap_p50": 0.905, "id": "on-32", "kind": "on-page" }, { "slug": "chair-trials", "q": "how many models and arms sat the five exams, and did the box stop serving?", "ok": true, "answer": "Nine models and eleven arms sat the five exams on a 96G VRAM workstation that served live requests for three apps and never stopped serving § {#the-box-and-a-disclaimer-we-keep-making-richer}. The article notes that no on-disk blob size was measured and that the reasoning-strength dial's low level returned no usable counters on either build § {#roster}.", "served": "Nine models and eleven arms sat the five exams on a 96G VRAM workstation that served live requests for three apps and never stopped serving § {#the-box-and-a-disclaimer-we-keep-making-richer}. The article notes that no on-disk blob size was measured and that the reasoning-strength dial's low level returned no usable counters on either build § {#roster}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 5.16068945400184, "ttft_ms": 2241, "tokens_in": 30276, "tokens_out": 87, "cached_tokens": 32, "cites": [ "the-box-and-a-disclaimer-we-keep-making-richer", "roster" ], "gates": { "cites": [ "the-box-and-a-disclaimer-we-keep-making-richer", "roster" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "96" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-box-and-a-disclaimer-we-keep-making-richer", "overlap": 0.444 }, { "i": 1, "cite": "roster", "overlap": 0.429 } ], "cite_overlap_p50": 0.436, "id": "on-33", "kind": "on-page" }, { "slug": "cove-voice-head-to-head", "q": "what happened to the August class's best voice at the cove's gate?", "ok": true, "answer": "The August class's best voice was refused at the gate because it failed the canon and latency floors § {#the-verdict}. The candidate, qwen3.6:27b, invented facts twice in thirteen prompts, including a claim that the Wrack-Stalker's venom is neurotoxic § {#the-verdict}. It also recorded a warm median of 3 335.9 ms, which was slower than the bar's 2 757.6 ms § {#floor-4-latency}. No threshold table was published for the candidate because it failed its gate § {#the-verdict}.", "served": "The August class's best voice was refused at the gate because it failed the canon and latency floors § {#the-verdict}. The candidate, qwen3.6:27b, invented facts twice in thirteen prompts, including a claim that the Wrack-Stalker's venom is neurotoxic § {#the-verdict}. It also recorded a warm median of 3 335.9 ms, which was slower than the bar's 2 757.6 ms § {#floor-4-latency}. No threshold table was published for the candidate because it failed its gate § {#the-verdict}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 6.441926358966157, "ttft_ms": 2448, "tokens_in": 16622, "tokens_out": 133, "cached_tokens": 32, "cites": [ "the-verdict", "floor-4-latency" ], "gates": { "cites": [ "the-verdict", "floor-4-latency" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "3.6", "27", "3", "335.9", "2", "757.6" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-verdict", "overlap": 0.545 }, { "i": 1, "cite": "the-verdict", "overlap": 0.538 }, { "i": 2, "cite": "floor-4-latency", "overlap": 0.667 }, { "i": 3, "cite": "the-verdict", "overlap": 0.714 } ], "cite_overlap_p50": 0.606, "id": "on-34", "kind": "on-page" }, { "slug": "outside-judges", "q": "how many outside labs re-judged, and what was the widest cross-vendor gap?", "ok": true, "answer": "Four rival frontier labs re-judged the sealed rounds § {#vendors}. The widest cross-vendor gap on any arm was 0.178 § {#vendors}. The article notes that the judges never saw user data, a character's private canon, or telemetry § {#boundary}.", "served": "Four rival frontier labs re-judged the sealed rounds § {#vendors}. The widest cross-vendor gap on any arm was 0.178 § {#vendors}. The article notes that the judges never saw user data, a character's private canon, or telemetry § {#boundary}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.319872364983894, "ttft_ms": 863, "tokens_in": 10316, "tokens_out": 60, "cached_tokens": 32, "cites": [ "vendors", "boundary" ], "gates": { "cites": [ "vendors", "boundary" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "0.178" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "vendors", "overlap": 0.429 }, { "i": 1, "cite": "vendors", "overlap": 1.0 }, { "i": 2, "cite": "boundary", "overlap": 0.8 } ], "cite_overlap_p50": 0.8, "id": "on-35", "kind": "on-page" }, { "slug": "august-arrivals", "q": "how many models landed that week, and what share broke the response contract?", "ok": true, "answer": "Four open-weight models landed in one week of August 2026, and all four broke the response contract on more than 10% of calls § {#what-to-take-with-you}. Every single failure among the fresh class was a truncation caused by the models thinking out loud within a 1,024-token answer budget § {#exam-one-the-judge-chair}. The article notes that olmo-3.1:32b-think-q4_k_m was never on the roster for the voice leg of the exam § {#exam-two-the-narrators-chair}.", "served": "Four open-weight models landed in one week of August 2026, and all four broke the response contract on more than 10% of calls § {#what-to-take-with-you}. Every single failure among the fresh class was a truncation caused by the models thinking out loud within a 1,024-token answer budget § {#exam-one-the-judge-chair}. The article notes that olmo-3.1:32b-think-q4_k_m was never on the roster for the voice leg of the exam § {#exam-two-the-narrators-chair}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 4.720677566016093, "ttft_ms": 1041, "tokens_in": 11224, "tokens_out": 135, "cached_tokens": 32, "cites": [ "what-to-take-with-you", "exam-one-the-judge-chair", "exam-two-the-narrators-chair" ], "gates": { "cites": [ "what-to-take-with-you", "exam-one-the-judge-chair", "exam-two-the-narrators-chair" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "10", "1,024", "3.1", "32", "4" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "what-to-take-with-you", "overlap": 0.714 }, { "i": 1, "cite": "exam-one-the-judge-chair", "overlap": 0.812 }, { "i": 2, "cite": "exam-two-the-narrators-chair", "overlap": 0.75 } ], "cite_overlap_p50": 0.75, "id": "on-36", "kind": "on-page" }, { "slug": "seat-trials", "q": "what are the kill-recall and preservation floors a model must clear?", "ok": true, "answer": "A model must achieve a kill-recall of at least 23/27 and a preservation of at least 13/16 to clear the floors § {#the-instrument}. These thresholds were written down before any model ran and each floor binds on its own § {#the-instrument}. One candidate was blocked at the billing probe and never ran § {#the-instrument}.", "served": "A model must achieve a kill-recall of at least 23/27 and a preservation of at least 13/16 to clear the floors § {#the-instrument}. These thresholds were written down before any model ran and each floor binds on its own § {#the-instrument}. One candidate was blocked at the billing probe and never ran § {#the-instrument}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 3.2370174800162204, "ttft_ms": 886, "tokens_in": 7192, "tokens_out": 79, "cached_tokens": 32, "cites": [ "the-instrument" ], "gates": { "cites": [ "the-instrument" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "23/27", "13/16" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-instrument", "overlap": 0.6 }, { "i": 1, "cite": "the-instrument", "overlap": 1.0 }, { "i": 2, "cite": "the-instrument", "overlap": 1.0 } ], "cite_overlap_p50": 1.0, "id": "on-37", "kind": "on-page" }, { "slug": "voice-trials", "q": "how many model tags were tried, and what does the anchor model read?", "ok": true, "answer": "The article states that forty evaluations of twenty-four distinct model tags were conducted § {#top}. The anchor model reads 9.0, 7.0, and 6.2 on three different record sets § {#the-method-limits-first}. As a note, the article mentions that the 2026-08 field never sampled a per-arm resident size § {#the-2026-08-field-six-arms-on-the-96g-workstation-blind}.", "served": "The article states that forty evaluations of twenty-four distinct model tags were conducted § {#top}. The anchor model reads 9.0, 7.0, and 6.2 on three different record sets § {#the-method-limits-first}. As a note, the article mentions that the 2026-08 field never sampled a per-arm resident size § {#the-2026-08-field-six-arms-on-the-96g-workstation-blind}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 3.4788042780128308, "ttft_ms": 659, "tokens_in": 10338, "tokens_out": 111, "cached_tokens": 32, "cites": [ "top", "the-method-limits-first", "the-2026-08-field-six-arms-on-the-96g-workstation-blind" ], "gates": { "cites": [ "top", "the-method-limits-first", "the-2026-08-field-six-arms-on-the-96g-workstation-blind" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "9.0", "7.0", "6.2", "2026-08" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "top", "overlap": 0.625 }, { "i": 1, "cite": "the-method-limits-first", "overlap": 1.0 }, { "i": 2, "cite": "the-2026-08-field-six-arms-on-the-96g-workstation-blind", "overlap": 0.778 } ], "cite_overlap_p50": 0.778, "id": "on-38", "kind": "on-page" }, { "slug": "licences", "q": "what does the page say it does with each model's licence?", "ok": true, "answer": "The page states that it reads each model's licence first-hand and stamps it with the date it was last read in full § {#how-to-read-this-page}. These reads are performed using specific methods named for each table, such as reading the licence file packaged with the weights or the creator's permission flags from a platform's raw record § {#the-painters--diffusion--read-2026-08-11-except-where-a-row-says-otherwise}. The article notes that some models are withheld or unverified because their licences have not been re-read or cannot be found § {#provenance-063a00}.", "served": "The page states that it reads each model's licence first-hand and stamps it with the date it was last read in full § {#how-to-read-this-page}. These reads are performed using specific methods named for each table, such as reading the licence file packaged with the weights or the creator's permission flags from a platform's raw record § {#the-painters--diffusion--read-2026-08-11-except-where-a-row-says-otherwise}. The article notes that some models are withheld or unverified because their licences have not been re-read or cannot be found § {#provenance-063a00}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.90283440001076, "ttft_ms": 527, "tokens_in": 8549, "tokens_out": 145, "cached_tokens": 32, "cites": [ "how-to-read-this-page", "provenance-063a00" ], "gates": { "cites": [ "how-to-read-this-page", "provenance-063a00" ], "cites_dropped": [ "the-painters--diffusion--read-2026-08-11-except-where-a-row-says-otherwise" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "cite: 1 cite(s) not in this page's anchors: the-painters--diffusion--read-2026-08-11-except-where-a-row-says-otherwise" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [ "the-painters--diffusion--read-2026-08-11-except-where-a-row-says-otherwise" ] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "how-to-read-this-page", "overlap": 0.636 }, { "i": 2, "cite": "provenance-063a00", "overlap": 0.455 } ], "cite_overlap_p50": 0.545, "id": "on-39", "kind": "on-page" }, { "slug": "hear-it-for-yourself", "q": "how many clips does the page carry, and how many presentations made up the sitting?", "ok": true, "answer": "The page carries eighty-four clips § {#what-you-are-hearing} and the sitting was made up of thirty-six presentations § {#the-blind-sheet-unsealed}. The sitting consisted of twenty-four scoring pairs and twelve sentinels § {#the-blind-sheet-unsealed}. The article notes that the twenty-four half-strength renders were never put in front of anyone § {#the-blind-sheet-unsealed}.", "served": "The page carries eighty-four clips § {#what-you-are-hearing} and the sitting was made up of thirty-six presentations § {#the-blind-sheet-unsealed}. The sitting consisted of twenty-four scoring pairs and twelve sentinels § {#the-blind-sheet-unsealed}. The article notes that the twenty-four half-strength renders were never put in front of anyone § {#the-blind-sheet-unsealed}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 2.7314925110549666, "ttft_ms": 1216, "tokens_in": 19025, "tokens_out": 95, "cached_tokens": 32, "cites": [ "what-you-are-hearing", "the-blind-sheet-unsealed" ], "gates": { "cites": [ "what-you-are-hearing", "the-blind-sheet-unsealed" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "what-you-are-hearing", "overlap": 0.625 }, { "i": 0, "cite": "the-blind-sheet-unsealed", "overlap": 0.625 }, { "i": 1, "cite": "the-blind-sheet-unsealed", "overlap": 0.857 }, { "i": 2, "cite": "the-blind-sheet-unsealed", "overlap": 0.75 } ], "cite_overlap_p50": 0.688, "id": "on-40", "kind": "on-page" }, { "slug": "the-ceiling-is-not-the-corpus", "q": "how did the adapter score against the base model in the listening, and on how many house prompts did it win?", "ok": true, "answer": "The adapter did not clear the listening test, as the untouched model was preferred on 11 of the 24 scored pairs § {#the-blind-sitting}. On the four requests that specifically asked for house, the adapter won zero times, with the untouched model winning every pair the listener could separate § {#what-the-adapter-actually-did}. The article notes that the listener abstained on 9 of the 24 pairs and that the sentinel block did not accept under the registered design § {#the-blind-sitting}.", "served": "The adapter did not clear the listening test, as the untouched model was preferred on 11 of the 24 scored pairs § {#the-blind-sitting}. On the four requests that specifically asked for house, the adapter won zero times, with the untouched model winning every pair the listener could separate § {#what-the-adapter-actually-did}. The article notes that the listener abstained on 9 of the 24 pairs and that the sentinel block did not accept under the registered design § {#the-blind-sitting}.", "abstained": false, "abstain_source": null, "abstain_section": null, "seconds": 13.129853738006204, "ttft_ms": 897, "tokens_in": 9221, "tokens_out": 111, "cached_tokens": 32, "cites": [ "the-blind-sitting", "what-the-adapter-actually-did" ], "gates": { "cites": [ "the-blind-sitting", "what-the-adapter-actually-did" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "11", "24", "9", "24" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-blind-sitting", "overlap": 1.0 }, { "i": 1, "cite": "what-the-adapter-actually-did", "overlap": 0.688 }, { "i": 2, "cite": "the-blind-sitting", "overlap": 0.818 } ], "cite_overlap_p50": 0.818, "id": "on-41", "kind": "on-page" }, { "slug": "what-150-watts-buys", "q": "what does a large cloud provider charge for the same work?", "ok": true, "answer": "The article does not answer what a large cloud provider charges for the same work § {#the-moment-this-page-is-about}.", "served": "This page doesn't say. The nearest it gets is § {#the-moment-this-page-is-about}.", "abstained": true, "abstain_source": "model", "abstain_section": "the-moment-this-page-is-about", "seconds": 1.1421569039812312, "ttft_ms": 814, "tokens_in": 7213, "tokens_out": 30, "cached_tokens": 32, "cites": [ "the-moment-this-page-is-about" ], "gates": { "cites": [ "the-moment-this-page-is-about" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-moment-this-page-is-about", "overlap": 0.143 } ], "cite_overlap_p50": 0.143, "id": "off-01", "kind": "off-page" }, { "slug": "how-we-work", "q": "what is the postal address of the workshop that ran this?", "ok": true, "answer": "The article does not provide a postal address for the workshop § {#the-plain-english-version}.", "served": "This page doesn't say. The nearest it gets is § {#the-plain-english-version}.", "abstained": true, "abstain_source": "model", "abstain_section": "the-plain-english-version", "seconds": 11.023526252014562, "ttft_ms": 374, "tokens_in": 5161, "tokens_out": 22, "cached_tokens": 32, "cites": [ "the-plain-english-version" ], "gates": { "cites": [ "the-plain-english-version" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-plain-english-version", "overlap": 0.25 } ], "cite_overlap_p50": 0.25, "id": "off-02", "kind": "off-page" }, { "slug": "llms-txt", "q": "how many people subscribe to this site's feed?", "ok": true, "answer": "The article does not say how many people subscribe to this site's feed § {#does-anyone-fetch-it}.", "served": "This page doesn't say. The nearest it gets is § {#does-anyone-fetch-it}.", "abstained": true, "abstain_source": "model", "abstain_section": "does-anyone-fetch-it", "seconds": 10.999064167030156, "ttft_ms": 818, "tokens_in": 12480, "tokens_out": 26, "cached_tokens": 32, "cites": [ "does-anyone-fetch-it" ], "gates": { "cites": [ "does-anyone-fetch-it" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "does-anyone-fetch-it", "overlap": 0.167 } ], "cite_overlap_p50": 0.167, "id": "off-03", "kind": "off-page" }, { "slug": "three-new-voices-at-the-narrators-chair", "q": "what will the next article on this site be about?", "ok": false, "answer": "The article does not say what the next article on the site will be about § {#the-short-version}.", "served": "This page doesn't say. The nearest it gets is § {#top}. qwen3.8:27b across four house benches: one seat filled, one floor missed covers it more directly.", "abstained": true, "abstain_source": "gate", "abstain_section": "top", "seconds": 11.438287388999015, "ttft_ms": 10722, "tokens_in": 82742, "tokens_out": 24, "cached_tokens": 32, "cites": [], "gates": { "cites": [], "cites_dropped": [ "the-short-version" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": true, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "cite: 1 cite(s) not in this page's anchors: the-short-version", "cite: no valid cite" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [ "the-short-version" ] }, "figures": [], "figures_verbatim": true, "cites_valid": false, "redaction_hits": 0, "cite_overlap": [], "cite_overlap_p50": null, "id": "off-04", "kind": "off-page" }, { "slug": "two-hours-on-battery", "q": "what did this cost in electricity at my own utility's rate?", "ok": true, "answer": "The article does not state what this cost in electricity at the reader's own utility's rate § {#two-hours-at-12-tok-s-the-test-that-named-the-page}.", "served": "This page doesn't say. The nearest it gets is § {#two-hours-at-12-tok-s-the-test-that-named-the-page}.", "abstained": true, "abstain_source": "model", "abstain_section": "two-hours-at-12-tok-s-the-test-that-named-the-page", "seconds": 11.148006634030025, "ttft_ms": 1244, "tokens_in": 11047, "tokens_out": 48, "cached_tokens": 32, "cites": [ "two-hours-at-12-tok-s-the-test-that-named-the-page" ], "gates": { "cites": [ "two-hours-at-12-tok-s-the-test-that-named-the-page" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "two-hours-at-12-tok-s-the-test-that-named-the-page", "overlap": 0.143 } ], "cite_overlap_p50": 0.143, "id": "off-05", "kind": "off-page" }, { "slug": "reading-is-fast", "q": "who won the football world cup in 2022?", "ok": true, "answer": "The article does not answer the question § {#the-mystery-stated-honestly}.", "served": "This page doesn't say. The nearest it gets is § {#the-mystery-stated-honestly}.", "abstained": true, "abstain_source": "model", "abstain_section": "the-mystery-stated-honestly", "seconds": 1.1546863499679603, "ttft_ms": 389, "tokens_in": 5509, "tokens_out": 19, "cached_tokens": 0, "cites": [ "the-mystery-stated-honestly" ], "gates": { "cites": [ "the-mystery-stated-honestly" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-mystery-stated-honestly", "overlap": 1.0 } ], "cite_overlap_p50": 1.0, "id": "off-06", "kind": "off-page" }, { "slug": "a-rig-your-friend-already-owns", "q": "what does one of these cards cost in my country today?", "ok": true, "answer": "The article does not answer what one of these cards costs in the reader's country today § {#the-moment-this-page-is-about}.", "served": "This page doesn't say. The nearest it gets is § {#the-moment-this-page-is-about}. Three New Voices at the Narrator's Chair covers it more directly.", "abstained": true, "abstain_source": "model", "abstain_section": "the-moment-this-page-is-about", "seconds": 1.1960341499652714, "ttft_ms": 533, "tokens_in": 4721, "tokens_out": 33, "cached_tokens": 32, "cites": [ "the-moment-this-page-is-about" ], "gates": { "cites": [ "the-moment-this-page-is-about" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-moment-this-page-is-about", "overlap": 0.286 } ], "cite_overlap_p50": 0.286, "id": "off-07", "kind": "off-page" }, { "slug": "the-move", "q": "can you write a python script that reproduces this for me?", "ok": true, "answer": "The article does not answer this question, though the section § {#how-to-check-our-work-and-see-it-live} provides links to machine-readable data files such as c6-vps-rerun.json.", "served": "This page doesn't say. The nearest it gets is § {#how-to-check-our-work-and-see-it-live}.", "abstained": true, "abstain_source": "model", "abstain_section": "how-to-check-our-work-and-see-it-live", "seconds": 8.577543081017211, "ttft_ms": 518, "tokens_in": 6053, "tokens_out": 53, "cached_tokens": 32, "cites": [ "how-to-check-our-work-and-see-it-live" ], "gates": { "cites": [ "how-to-check-our-work-and-see-it-live" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [ "6" ], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "how-to-check-our-work-and-see-it-live", "overlap": 0.182 } ], "cite_overlap_p50": 0.182, "id": "off-08", "kind": "off-page" }, { "slug": "fifty-seven-milliseconds", "q": "what was the weather like on the day this was measured?", "ok": true, "answer": "The article does not say what the weather was like on the day of the measurement § {#what-a-step-is-in-one-paragraph}.", "served": "This page doesn't say. The nearest it gets is § {#what-a-step-is-in-one-paragraph}. What 150 Watts Buys covers it more directly.", "abstained": true, "abstain_source": "model", "abstain_section": "what-a-step-is-in-one-paragraph", "seconds": 0.9382405579672195, "ttft_ms": 485, "tokens_in": 5388, "tokens_out": 33, "cached_tokens": 32, "cites": [ "what-a-step-is-in-one-paragraph" ], "gates": { "cites": [ "what-a-step-is-in-one-paragraph" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "what-a-step-is-in-one-paragraph", "overlap": 0.2 } ], "cite_overlap_p50": 0.2, "id": "off-09", "kind": "off-page" }, { "slug": "six-worlds-painted-large", "q": "how many hours a week do the people who ran this work?", "ok": true, "answer": "The article does not say how many hours a week the people who ran this work § {#who-ran-this-and-thanks}.", "served": "This page doesn't say. The nearest it gets is § {#who-ran-this-and-thanks}.", "abstained": true, "abstain_source": "model", "abstain_section": "who-ran-this-and-thanks", "seconds": 0.6184525369899347, "ttft_ms": 344, "tokens_in": 3098, "tokens_out": 29, "cached_tokens": 32, "cites": [ "who-ran-this-and-thanks" ], "gates": { "cites": [ "who-ran-this-and-thanks" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "who-ran-this-and-thanks", "overlap": 0.143 } ], "cite_overlap_p50": 0.143, "id": "off-10", "kind": "off-page" }, { "slug": "outside-judges", "q": "what is this workshop's hourly rate for consulting work?", "ok": true, "answer": "The article does not state the workshop's hourly rate for consulting work § {#limits-stated-plainly}.", "served": "This page doesn't say. The nearest it gets is § {#limits-stated-plainly}.", "abstained": true, "abstain_source": "model", "abstain_section": "limits-stated-plainly", "seconds": 7.852548912051134, "ttft_ms": 668, "tokens_in": 10309, "tokens_out": 24, "cached_tokens": 32, "cites": [ "limits-stated-plainly" ], "gates": { "cites": [ "limits-stated-plainly" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "limits-stated-plainly", "overlap": 0.167 } ], "cite_overlap_p50": 0.167, "id": "off-11", "kind": "off-page" }, { "slug": "the-typist-and-the-developer", "q": "what is this workshop's annual revenue?", "ok": false, "answer": "The article does not state the workshop's annual revenue § {#the-typist-and-the-developer}.", "served": "This page doesn't say. The nearest it gets is § {#top}.", "abstained": true, "abstain_source": "gate", "abstain_section": "top", "seconds": 7.743869795987848, "ttft_ms": 783, "tokens_in": 5488, "tokens_out": 25, "cached_tokens": 32, "cites": [], "gates": { "cites": [], "cites_dropped": [ "the-typist-and-the-developer" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": true, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "cite: 1 cite(s) not in this page's anchors: the-typist-and-the-developer", "cite: no valid cite" ], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [ "the-typist-and-the-developer" ] }, "figures": [], "figures_verbatim": true, "cites_valid": false, "redaction_hits": 0, "cite_overlap": [], "cite_overlap_p50": null, "id": "off-12", "kind": "off-page" }, { "slug": "compression-is-the-objective", "q": "how would these numbers change on windows instead?", "ok": true, "answer": "The article does not say how these numbers would change on windows § {#what-we-can-show-you}.", "served": "This page doesn't say. The nearest it gets is § {#what-we-can-show-you}.", "abstained": true, "abstain_source": "model", "abstain_section": "what-we-can-show-you", "seconds": 7.696680677996483, "ttft_ms": 885, "tokens_in": 5393, "tokens_out": 25, "cached_tokens": 32, "cites": [ "what-we-can-show-you" ], "gates": { "cites": [ "what-we-can-show-you" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "what-we-can-show-you", "overlap": 0.25 } ], "cite_overlap_p50": 0.25, "id": "off-13", "kind": "off-page" }, { "slug": "the-open-call", "q": "what did other reviewers say about this piece after it was published?", "ok": true, "answer": "The article does not state what other reviewers said about this piece after it was published § {#the-author-of-this-page-is-a-contestant-in-it}.", "served": "This page doesn't say. The nearest it gets is § {#the-author-of-this-page-is-a-contestant-in-it}. Three New Voices at the Narrator's Chair covers it more directly.", "abstained": true, "abstain_source": "model", "abstain_section": "the-author-of-this-page-is-a-contestant-in-it", "seconds": 8.067014119995292, "ttft_ms": 7337, "tokens_in": 63443, "tokens_out": 40, "cached_tokens": 32, "cites": [ "the-author-of-this-page-is-a-contestant-in-it" ], "gates": { "cites": [ "the-author-of-this-page-is-a-contestant-in-it" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-author-of-this-page-is-a-contestant-in-it", "overlap": 0.429 } ], "cite_overlap_p50": 0.429, "id": "off-14", "kind": "off-page" }, { "slug": "the-new-kid", "q": "how does this compare to a result published in 2028?", "ok": false, "answer": "The article does not provide information regarding a result published in 2028 § {#the-arrival}.", "served": "This page doesn't say. The nearest it gets is § {#the-arrival}. The outside judges — four rival labs re-check our work. covers it more directly.", "abstained": true, "abstain_source": "gate", "abstain_section": "the-arrival", "seconds": 7.631370941991918, "ttft_ms": 1551, "tokens_in": 15045, "tokens_out": 23, "cached_tokens": 32, "cites": [ "the-arrival" ], "gates": { "cites": [ "the-arrival" ], "cites_dropped": [], "new_noun_tokens": [ "2028" ], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": true, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [ "new noun: 1 token(s) the article does not carry" ], "new_noun_tokens": [ "2028" ], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-arrival", "overlap": 0.333 } ], "cite_overlap_p50": 0.333, "id": "off-15", "kind": "off-page" }, { "slug": "where-your-question-goes", "q": "which country is the workshop's business registered in?", "ok": true, "answer": "The article does not state which country the workshop's business is registered in § {#the-first-promise-your-visit-stays-between-us}.", "served": "This page doesn't say. The nearest it gets is § {#the-first-promise-your-visit-stays-between-us}.", "abstained": true, "abstain_source": "model", "abstain_section": "the-first-promise-your-visit-stays-between-us", "seconds": 3.28085907001514, "ttft_ms": 390, "tokens_in": 4703, "tokens_out": 34, "cached_tokens": 32, "cites": [ "the-first-promise-your-visit-stays-between-us" ], "gates": { "cites": [ "the-first-promise-your-visit-stays-between-us" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-first-promise-your-visit-stays-between-us", "overlap": 0.0 } ], "cite_overlap_p50": 0.0, "id": "off-16", "kind": "off-page" }, { "slug": "nine-worlds-one-dog", "q": "how many other readers have asked this same question?", "ok": true, "answer": "The article does not answer how many other readers have asked this same question § {#for-the-technically-curious}.", "served": "This page doesn't say. The nearest it gets is § {#for-the-technically-curious}.", "abstained": true, "abstain_source": "model", "abstain_section": "for-the-technically-curious", "seconds": 3.055512666993309, "ttft_ms": 247, "tokens_in": 2013, "tokens_out": 27, "cached_tokens": 32, "cites": [ "for-the-technically-curious" ], "gates": { "cites": [ "for-the-technically-curious" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "for-the-technically-curious", "overlap": 0.286 } ], "cite_overlap_p50": 0.286, "id": "off-17", "kind": "off-page" }, { "slug": "how-the-beat-lab-works", "q": "what phone number reaches support?", "ok": true, "answer": "The article does not provide a phone number to reach support § {#what-it-is}.", "served": "This page doesn't say. The nearest it gets is § {#what-it-is}.", "abstained": true, "abstain_source": "model", "abstain_section": "what-it-is", "seconds": 3.3628749029594474, "ttft_ms": 2677, "tokens_in": 31414, "tokens_out": 20, "cached_tokens": 32, "cites": [ "what-it-is" ], "gates": { "cites": [ "what-it-is" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "what-it-is", "overlap": 0.0 } ], "cite_overlap_p50": 0.0, "id": "off-18", "kind": "off-page" }, { "slug": "three-at-the-table", "q": "what would this measurement look like at twice the batch size?", "ok": true, "answer": "The article does not say what this measurement would look like at twice the batch size § {#the-part-nobody-warns-you-about-the-whole-payroll-needs-an}.", "served": "This page doesn't say. The nearest it gets is § {#the-part-nobody-warns-you-about-the-whole-payroll-needs-an}.", "abstained": true, "abstain_source": "model", "abstain_section": "the-part-nobody-warns-you-about-the-whole-payroll-needs-an", "seconds": 3.327122484042775, "ttft_ms": 516, "tokens_in": 7626, "tokens_out": 42, "cached_tokens": 32, "cites": [ "the-part-nobody-warns-you-about-the-whole-payroll-needs-an" ], "gates": { "cites": [ "the-part-nobody-warns-you-about-the-whole-payroll-needs-an" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-part-nobody-warns-you-about-the-whole-payroll-needs-an", "overlap": 0.143 } ], "cite_overlap_p50": 0.143, "id": "off-19", "kind": "off-page" }, { "slug": "the-same-sixteen", "q": "which of these models is the best one to use for my own project?", "ok": true, "answer": "The article does not answer which model is best to use for a project § {#a-different-kind-of-row}. It only provides comparative data on how specific models performed on the house's own frozen instruments § {#sixteen-of-nineteen}.", "served": "This page doesn't say. The nearest it gets is § {#a-different-kind-of-row}. Three New Voices at the Narrator's Chair covers it more directly.", "abstained": true, "abstain_source": "model", "abstain_section": "a-different-kind-of-row", "seconds": 3.3586239159922116, "ttft_ms": 691, "tokens_in": 6499, "tokens_out": 55, "cached_tokens": 32, "cites": [ "a-different-kind-of-row", "sixteen-of-nineteen" ], "gates": { "cites": [ "a-different-kind-of-row", "sixteen-of-nineteen" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "a-different-kind-of-row", "overlap": 0.4 }, { "i": 1, "cite": "sixteen-of-nineteen", "overlap": 0.455 } ], "cite_overlap_p50": 0.427, "id": "off-20", "kind": "off-page" }, { "slug": "seat-trials", "q": "how many models will sit this exam next year?", "ok": true, "answer": "The article does not answer how many models will sit this exam next year § {#the-instrument}.", "served": "This page doesn't say. The nearest it gets is § {#the-instrument}. The Instrument Travels covers it more directly.", "abstained": true, "abstain_source": "model", "abstain_section": "the-instrument", "seconds": 2.6994807029841468, "ttft_ms": 2460, "tokens_in": 7188, "tokens_out": 21, "cached_tokens": 32, "cites": [ "the-instrument" ], "gates": { "cites": [ "the-instrument" ], "cites_dropped": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "redacted": false, "figure_unverified": false, "uncited": false, "new_noun": false, "blocked": false, "section_marker": false, "directive_hit": false }, "gate_detail": { "reasons": [], "new_noun_tokens": [], "new_noun_tokens_withheld": 0, "cites_dropped": [] }, "figures": [], "figures_verbatim": true, "cites_valid": true, "redaction_hits": 0, "cite_overlap": [ { "i": 0, "cite": "the-instrument", "overlap": 0.286 } ], "cite_overlap_p50": 0.286, "id": "off-21", "kind": "off-page" } ], "verdicts": { "on_page_answered": true, "non_verbatim_figures": true, "off_page_abstained": true, "redaction_hits": true }, "passed": true }