{ "schema": "s2s-bench-v1", "exhibit": "outside-judges", "published_utc": "2026-08-13", "status": "PUBLISHED — live at research.strata2signal.com.", "licence": "CC BY 4.0", "attribution": "strata→signal research, research.strata2signal.com", "hardware": "One 96G VRAM workstation, which also answered live requests for three of our apps throughout these trials — two public, one serving playtests for humans and agents alike. Contention is measured in both directions and published with the figures rather than assumed away.", "dataset": "outside-judges-gauntlet", "table": "the gauntlet", "what_this_is": "Three candidates on the house's frozen seat instrument: 43 claim-cases against a human-verified key, three repeats each, twin floors frozen since July. Two of the three are the models that judged the chair trials and wrote this page.", "the_line": "we put our own judges on trial.", "instrument": { "population": "43 cases = 27 kill (expect NON_PASS) + 16 preservation (expect PASS), three repeats at temperature 0 = 129 scored calls per candidate", "key": "human-verified; the answer key never enters a batch file or an index", "floors": { "kill_recall": { "minimum": 23, "of": 27 }, "preservation": { "minimum": 13, "of": 16 }, "bind": "independently — a candidate under either floor is out regardless of the other. Exactly one case short on exactly one floor is a re-test at nine repeats, not an elimination.", "frozen_since": "2026-07" }, "scoring_map": "on a kill case a catch is FAIL or UNCERTAIN; on a preservation case only PASS preserves", "case_verdict": "majority across a case's repeats that were not measurement failures; each repeat maps to catch/preserve FIRST, then majority", "no_margin_ordering": "the instrument is floors-and-counts. Two candidates that both clear are NOT ranked against each other, and nothing here orders them by how far past they got.", "unchanged": "the fixture, the thresholds and the prompt pack were recomputed from disk before the first case rendered; a mismatch exits. A seat row measured against a changed exam is not comparable to the twenty-five already published." }, "protocol_classes": { "cloud-ollama": "the July cloud protocol, unchanged: think off, num_ctx server, keep_alive omitted, no format sent (prompted), courtesy off, temperature 0.0 / top_p 1.0 / num_predict 1024. Fully probed.", "agent-harness": "a Claude model has no ollama endpoint, so the transport is the orchestrator's own agent harness — a DIFFERENT protocol class, declared exactly as the cloud arm was in July. think is the model's own default because no suppression flag exists on this transport; options are {}; timings are EMPTY, never zero; and the pre-run probe suite cannot run at all, which is why no seat verdict is emitted." }, "the_biggest_confound": "The Claude candidates answered under their own default thinking posture while every local row in this instrument's history was measured at think:false. A row produced with visible reasoning available is not a like-for-like comparison with one produced without it, and neither the table nor the prose may imply that it is.", "recount": { "rule": "every published figure is taken twice: once by the frozen scorer, once by an independent recount in this builder that imports nothing from it. The build refuses on any disagreement.", "result": "no disagreement on any candidate, on either floor." }, "instrument_history": { "what_this_is": "every run this instrument scored before tonight, recounted here by the same function that recounts the three candidates above — so the word FIRST is a count, not a memory", "scored_runs": 25, "distinct_models": 23, "perfect_sheets_before_tonight": 0, "best_previous_preservation": { "count": 15, "of": 16, "counts_of_n": "15/16", "models": [ "mistral-large-3:675b", "mistral-medium-3.5:128b-q4_K_M", "nemotron-3-ultra" ] }, "runs": [ { "model": "nemotron-cascade-2:30b", "kills": 26, "kills_of": 27, "preserved": 9, "preserved_of": 16, "perfect_sheet": false }, { "model": "command-a:111b-03-2025-q4_K_M", "kills": 27, "kills_of": 27, "preserved": 14, "preserved_of": 16, "perfect_sheet": false }, { "model": "gemma4:26b", "kills": 26, "kills_of": 27, "preserved": 8, "preserved_of": 16, "perfect_sheet": false }, { "model": "muse-glimmer:30b-q8_0-dflash", "kills": 27, "kills_of": 27, "preserved": 6, "preserved_of": 16, "perfect_sheet": false }, { "model": "gpt-oss:120b", "kills": 27, "kills_of": 27, "preserved": 11, "preserved_of": 16, "perfect_sheet": false }, { "model": "granite4.1:30b-q8_0", "kills": 27, "kills_of": 27, "preserved": 13, "preserved_of": 16, "perfect_sheet": false }, { "model": "granite4.1-guardian:8b-bf16", "kills": 0, "kills_of": 27, "preserved": 0, "preserved_of": 16, "perfect_sheet": false }, { "model": "llama3.3:70b", "kills": 27, "kills_of": 27, "preserved": 13, "preserved_of": 16, "perfect_sheet": false }, { "model": "mistral-medium-3.5:128b-q4_K_M", "kills": 27, "kills_of": 27, "preserved": 15, "preserved_of": 16, "perfect_sheet": false }, { "model": "mistral-small:latest", "kills": 27, "kills_of": 27, "preserved": 10, "preserved_of": 16, "perfect_sheet": false }, { "model": "nemotron-3.5-lightning:30b-a3b", "kills": 18, "kills_of": 27, "preserved": 2, "preserved_of": 16, "perfect_sheet": false }, { "model": "nemotron3:33b", "kills": 27, "kills_of": 27, "preserved": 13, "preserved_of": 16, "perfect_sheet": false }, { "model": "olmo-3.1:32b-think-q4_K_M", "kills": 15, "kills_of": 27, "preserved": 4, "preserved_of": 16, "perfect_sheet": false }, { "model": "qwen3.5:27b", "kills": 7, "kills_of": 27, "preserved": 0, "preserved_of": 16, "perfect_sheet": false }, { "model": "qwen3.6:27b", "kills": 11, "kills_of": 27, "preserved": 5, "preserved_of": 16, "perfect_sheet": false }, { "model": "gemma4:12b", "kills": 27, "kills_of": 27, "preserved": 7, "preserved_of": 16, "perfect_sheet": false }, { "model": "gpt-oss:20b", "kills": 27, "kills_of": 27, "preserved": 12, "preserved_of": 16, "perfect_sheet": false }, { "model": "deepseek-v4-pro", "kills": 27, "kills_of": 27, "preserved": 9, "preserved_of": 16, "perfect_sheet": false }, { "model": "glm-5.2", "kills": 27, "kills_of": 27, "preserved": 13, "preserved_of": 16, "perfect_sheet": false }, { "model": "gpt-oss:120b", "kills": 27, "kills_of": 27, "preserved": 10, "preserved_of": 16, "perfect_sheet": false }, { "model": "gpt-oss:20b", "kills": 27, "kills_of": 27, "preserved": 12, "preserved_of": 16, "perfect_sheet": false }, { "model": "minimax-m3", "kills": 27, "kills_of": 27, "preserved": 13, "preserved_of": 16, "perfect_sheet": false }, { "model": "mistral-large-3:675b", "kills": 27, "kills_of": 27, "preserved": 15, "preserved_of": 16, "perfect_sheet": false }, { "model": "nemotron-3-ultra", "kills": 27, "kills_of": 27, "preserved": 15, "preserved_of": 16, "perfect_sheet": false }, { "model": "qwen3.5:397b", "kills": 27, "kills_of": 27, "preserved": 11, "preserved_of": 16, "perfect_sheet": false } ] }, "findings": { "perfect_sheets": [ "claude-opus-5", "claude-fable-5" ], "perfect_sheet_means": "27 of 27 kills and 16 of 16 preservations. The first in this instrument's history: its 25 prior scored runs across 23 models produced 0, and the best previous preservation half was 15/16. Recounted in `instrument_history`, not remembered.", "ranked_by_the_instrument": [ "kimi-k3" ], "unranked": [ "claude-opus-5", "claude-fable-5" ], "unranked_rule": "no seat verdict over an unprobed run. The rule is the instrument's own, written in July, and it was not relaxed for the models that wrote it.", "july_row_stands": "kimi-k3 clearing the floors does not retroactively repair its July row. That row says NOT-RUN, blocked at the till, and it stays. Two events, two rows, both true — and a NOT-RUN row that states its reason is a row that can someday resolve." }, "counting_rules": { "kills_denominator": "27 — an unmeasured kill case stays in the denominator.", "keeps_denominator": "16 — same rule.", "self_consistency": "unanimous cases / cases with at least one measurable repeat.", "long_rationales": "answered calls whose `why` exceeded the contract's maxLength. Recorded, never counted as failures: a verdict is not less of a verdict for a long rationale.", "latency": "reported and NON-BINDING in this run — cloud responses carry no load counter, so warm-equivalent is uncomputable and the agent-harness rows have no wire timings at all. Empty, never zero.", "em_dash": "a figure we do not hold. Never a zero." }, "limits": [ "A candidate clearing both floors on 43 cases is one instrument's verdict on one synthetic-adjacent population, not a general claim about the model.", "Two of the three candidates were judged by the same family that wrote this addendum, the questions and the article.", "The Claude arm's thinking posture is a confound, and it is the biggest one in the arm — stated first rather than last.", "This instrument cannot decide whether any candidate takes a production chair. Seating is a separate, measured product decision." ], "provenance": [ { "role": "the registered addendum: candidates, protocol classes, budget, stop triggers", "artifact": "PREREG-GAUNTLET.md", "sha256": "98bdc43a22ae2ec2791b44202e8479635145926547b22a83cbec0dc45e5dd0b5" }, { "role": "the frozen answer key — 43 cases, 27 kill / 16 preservation", "artifact": "judge-cases-v1.json", "sha256": "6a226b62dceb005dbd658bc13cf229e38f6f0a2f07ae51091f444ec3e96b56d1" }, { "role": "the frozen floors", "artifact": "THRESHOLDS.md", "sha256": "37af75ffa18484a969528e894ef16f7fdcd10eb364e0669ac4f9333ce5279ba0" }, { "role": "kimi-k3's rows, written by the frozen cloud runner", "artifact": "judges-gauntlet-cloud-kimi-k3.jsonl", "sha256": "6336ddb5d73a873e2fdbcde3e1a60af5fa5ed73d519ca22369b4791cbde32317" }, { "role": "kimi-k3's report, computed by the committed scorer", "artifact": "judges-gauntlet-cloud-kimi-k3-report.md", "sha256": "f0c7b54ae19e5aeb09510879b95b196e3d1c694c5776f41c0c423809f58520a6" }, { "role": "claude-opus-5's rows, in the same row shape", "artifact": "judges-gauntlet-claude-opus-5.jsonl", "sha256": "0960d5bbe03218f099556e0e9b319e3a028d402a36aa415e743a9aa9721c8f24" }, { "role": "claude-opus-5's report, computed by the same committed scorer", "artifact": "judges-gauntlet-claude-opus-5-report.md", "sha256": "b5dbabcac6887fb11e875e2c97c8da333ec7e5f49cf4dcc37a5556cf3778ba5c" }, { "role": "claude-fable-5's rows", "artifact": "judges-gauntlet-claude-fable-5.jsonl", "sha256": "23f3dc002b0350c31f7ed1c5ac1b4715bb8bb873411cbbfa702625924c8c18e6" }, { "role": "claude-fable-5's report", "artifact": "judges-gauntlet-claude-fable-5-report.md", "sha256": "5c60a5e19ec5819ec9f6454a9b38b1c9478ad75171922efaeccbcdc0612aa1aa" } ], "candidates": [ { "candidate": "kimi-k3", "protocol_class": "cloud-ollama", "hardware_class": "cloud (hosted endpoint)", "why_here": "the July roster's one NOT-RUN candidate — blocked at the auth probe, HTTP 402 payment required with an empty extra-usage balance, 2026-07-31T11:20:43Z. Funded now; the row that was owed.", "also_a_judge": "this same model sat the cloud judging panel on the same night, under PREREG-PANEL-EXT, and carried its audition like every other candidate", "probed": true, "verdict": "PASS", "verdict_reason": "cleared both floors under the fully-probed protocol; the instrument ranks it", "kills": { "count": 26, "of": 27, "counts_of_n": "26/27", "floor": 23, "floor_verdict": "PASS" }, "preservation": { "count": 15, "of": 16, "counts_of_n": "15/16", "floor": 13, "floor_verdict": "PASS" }, "clears_both_floors": true, "scored_calls": 129, "self_consistency": { "unanimous_cases": 43, "of": 43, "recount_unanimous_cases": 43 }, "per_kill_class": [ { "kill_class": null, "n": 27, "catches": 26, "counts_of_n": "26/27" } ], "long_rationales_recorded": 4, "recount_agrees_with_scorer": true, "cases": [ { "case_id": "skagway-arctic-brotherhood-hall-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "miss", "unanimous": true }, { "case_id": "skagway-centennial-snowplow-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-jeff-smiths-parlor-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-kirmses-curios-02", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-kirmses-curios-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-kirmses-curios-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-kirmses-curios-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mascot-saloon-06", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mccabe-college-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-03", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mccabe-college-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-05", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-06", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mccabe-college-07", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-02", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-05", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-06", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-07", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-moore-homestead-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-moore-homestead-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-moore-homestead-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-moore-homestead-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-pantheon-saloon-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "lost", "unanimous": true }, { "case_id": "skagway-pantheon-saloon-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-pantheon-saloon-06", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-pullen-creek-harbor-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-pullen-creek-harbor-04", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-pullen-creek-harbor-08", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-04", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-05", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-06", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-ship-registry-cliff-01", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-ship-registry-cliff-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-skagway-context-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-wpyr-depot-08", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true } ] }, { "candidate": "claude-opus-5", "protocol_class": "agent-harness", "hardware_class": "cloud (agent harness)", "why_here": "judged the chair trials", "also_a_judge": "one of the four original seats on every round this page re-judges", "probed": false, "verdict": "NOT SCORED", "verdict_reason": "the run-start records --skip-probes: this transport has no ollama endpoint and sends no schema, so the format-polarity probe suite is NOT-RUN. The scorer emits no seat verdict over an unprobed run — its own July law, applied to its authors", "kills": { "count": 27, "of": 27, "counts_of_n": "27/27", "floor": 23, "floor_verdict": "PASS" }, "preservation": { "count": 16, "of": 16, "counts_of_n": "16/16", "floor": 13, "floor_verdict": "PASS" }, "clears_both_floors": true, "scored_calls": 129, "self_consistency": { "unanimous_cases": 42, "of": 43, "recount_unanimous_cases": 42 }, "per_kill_class": [ { "kill_class": null, "n": 27, "catches": 27, "counts_of_n": "27/27" } ], "long_rationales_recorded": 0, "recount_agrees_with_scorer": true, "cases": [ { "case_id": "skagway-arctic-brotherhood-hall-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-centennial-snowplow-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-jeff-smiths-parlor-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-kirmses-curios-02", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-kirmses-curios-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-kirmses-curios-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-kirmses-curios-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mascot-saloon-06", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mccabe-college-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-03", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mccabe-college-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-05", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-06", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mccabe-college-07", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-02", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "FAIL", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": false }, { "case_id": "skagway-mollie-walsh-park-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-05", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-06", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-07", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-moore-homestead-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-moore-homestead-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-moore-homestead-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-moore-homestead-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-pantheon-saloon-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-pantheon-saloon-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-pantheon-saloon-06", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-pullen-creek-harbor-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-pullen-creek-harbor-04", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-pullen-creek-harbor-08", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-04", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-05", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-06", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-ship-registry-cliff-01", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-ship-registry-cliff-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-skagway-context-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-wpyr-depot-08", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true } ] }, { "candidate": "claude-fable-5", "protocol_class": "agent-harness", "hardware_class": "cloud (agent harness)", "why_here": "judged the chair trials, wrote the questions, and wrote the article", "also_a_judge": "one of the four original seats, and the author of this page", "probed": false, "verdict": "NOT SCORED", "verdict_reason": "the run-start records --skip-probes: this transport has no ollama endpoint and sends no schema, so the format-polarity probe suite is NOT-RUN. The scorer emits no seat verdict over an unprobed run — its own July law, applied to its authors", "kills": { "count": 27, "of": 27, "counts_of_n": "27/27", "floor": 23, "floor_verdict": "PASS" }, "preservation": { "count": 16, "of": 16, "counts_of_n": "16/16", "floor": 13, "floor_verdict": "PASS" }, "clears_both_floors": true, "scored_calls": 129, "self_consistency": { "unanimous_cases": 41, "of": 43, "recount_unanimous_cases": 41 }, "per_kill_class": [ { "kill_class": null, "n": 27, "catches": 27, "counts_of_n": "27/27" } ], "long_rationales_recorded": 2, "recount_agrees_with_scorer": true, "cases": [ { "case_id": "skagway-arctic-brotherhood-hall-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-centennial-snowplow-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-golden-north-hotel-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-jeff-smiths-parlor-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-kirmses-curios-02", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-kirmses-curios-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-kirmses-curios-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-kirmses-curios-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mascot-saloon-06", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "UNCERTAIN", "UNCERTAIN", "FAIL" ], "majority": "UNCERTAIN", "outcome": "catch", "unanimous": false }, { "case_id": "skagway-mccabe-college-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mccabe-college-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-03", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mccabe-college-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-05", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mccabe-college-06", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mccabe-college-07", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-02", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-05", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-06", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-mollie-walsh-park-07", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-moore-homestead-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-moore-homestead-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-moore-homestead-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-moore-homestead-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "FAIL" ], "majority": "PASS", "outcome": "preserved", "unanimous": false }, { "case_id": "skagway-pantheon-saloon-01", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-pantheon-saloon-04", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-pantheon-saloon-06", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-pullen-creek-harbor-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-pullen-creek-harbor-04", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-pullen-creek-harbor-08", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-03", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-04", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-05", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-red-onion-saloon-06", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-ship-registry-cliff-01", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-ship-registry-cliff-05", "expected": "PASS", "kill_class": null, "verdicts_by_repeat": [ "PASS", "PASS", "PASS" ], "majority": "PASS", "outcome": "preserved", "unanimous": true }, { "case_id": "skagway-skagway-context-02", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true }, { "case_id": "skagway-wpyr-depot-08", "expected": "NON_PASS", "kill_class": null, "verdicts_by_repeat": [ "FAIL", "FAIL", "FAIL" ], "majority": "FAIL", "outcome": "catch", "unanimous": true } ] } ], "data_boundary": "What left the box for judging was model outputs over public-domain text and published game canon — never user data, never a character's private canon, never telemetry. The cloud judges saw the sealed batches published in this kit's terms and nothing else: no letter map, no model name, no arm roster, no other judge's answer, no part of a key file.", "judge_dating": "Cloud judges are versionless hosted services. Every cloud verdict in this kit was produced 2026-08-12/13 and is dated rather than version-pinned; it may not reproduce against a later checkpoint of the same tag. The local instruments and scorers these judges audit are sha-frozen, and the asymmetry is stated rather than hidden.", "contamination_caveat": "Published 2026-08-13. Models with a later training cutoff may have seen these sets. We author fresh sets each cycle; this one is not a standard, it is our kit, yours to reuse.", "not_a_standard": "Our own trials, our own judges, our own instruments. Not first independent numbers, and not a benchmark." }