{ "schema": "s2s-bench-v1", "exhibit": "outside-judges", "published_utc": "2026-08-13", "status": "PUBLISHED — live at research.strata2signal.com.", "licence": "CC BY 4.0", "attribution": "strata→signal research, research.strata2signal.com", "hardware": "One 96G VRAM workstation, which also answered live requests for three of our apps throughout these trials — two public, one serving playtests for humans and agents alike. Contention is measured in both directions and published with the figures rather than assumed away.", "dataset": "outside-judges-panel-vendors", "table": "pairwise win-rates per vendor", "what_this_is": "Every sealed round of the chair trials and the voice addendum, scored once per vendor. The same twelve items, six arms, two samples, 270 comparisons, 36 tasks, 36 replies and 18 abstention replies as the published rounds — the same batch files, byte for byte. What the extension adds is more readers of them.", "reading": { "original_panel": "a verbatim voice_score.score_all(): the published four-judge numbers, unchanged and un-recomputed by this file", "per_vendor": "the same scorer over one vendor's seats", "combined_all": "the same scorer over every carrying seat, with each verdict's family relabelled to its vendor, so per_family IS the vendor split", "anthropic_equals_original": "the anthropic split is the original panel by construction -- the same seats, the same files. It is printed twice on purpose: once as what was published, once as one vendor among several" }, "panel": { "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ], "original_judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "cloud_seats": { "cloud-deepseek-v4-pro": { "vendor": "deepseek", "source_seat": "opus-1", "reads": "that seat's sealed pages byte for byte — same letter map, same AB/BA layout, same display tiers" }, "cloud-mistral-large-3-675b": { "vendor": "mistral", "source_seat": "opus-2", "reads": "that seat's sealed pages byte for byte — same letter map, same AB/BA layout, same display tiers" }, "cloud-nemotron-3-ultra": { "vendor": "nvidia", "source_seat": "fable-1", "reads": "that seat's sealed pages byte for byte — same letter map, same AB/BA layout, same display tiers" }, "cloud-kimi-k3": { "vendor": "moonshot", "source_seat": "fable-2", "reads": "that seat's sealed pages byte for byte — same letter map, same AB/BA layout, same display tiers" } }, "seat_rule": "each cloud judge answered ONE original seat's sealed pages, so its verdicts key against that seat's key files -- same letter map, same AB/BA layout, same display tiers", "families_are_not_vendors": "opus and fable are two Anthropic LENSES and one vendor. The family axis has two values; the vendor axis has five. The anthropic split is the original panel by construction." }, "arm_order": { "order": [ "muse-glimmer:30b-q8_0-dflash", "muse-glimmer:30b-q8_0-dflash+think", "gemma4:26b", "gemma4:12b", "qwen3.6:27b", "nemotron-3.5-lightning:30b-a3b" ], "rule": "exhibit six's C4 roster order, unchanged. This table is read against that one row for row, and re-sorting by a vendor's column would assert a ranking this leg is not entitled to make." }, "what_this_cannot_decide": "whether any arm is better. The narrator rounds (exhibit six's C4) and the panel round (the voice addendum's Leg B) are DESCRIPTIVE and stay descriptive. More vendors do not upgrade a descriptive leg into a ranked one; they widen the error bar the reader is entitled to see.", "counting_rules": { "win_rate": "overall pairwise win-rate over all items, with a cluster bootstrap over items (effective N = 9 items) as its interval. A swap-discordant pair scores 0.5/0.5 rather than being discarded.", "population_unchanged": "the judged POPULATION does not move: 270 comparisons, 36 tasks, 36 replies, 18 abstention replies. Only the number of readers moves.", "per_vendor": "the same published scorer run over one vendor's seats.", "combined_all": "the same scorer over every carrying seat, with each verdict's family relabelled to its vendor.", "original_panel": "a verbatim call to the published scorer: the four-judge numbers, unchanged and untouched by this extension.", "structurally_unmovable": "the published scorer and the committed recount both walk KEY files, and a cloud seat has no key directory — so neither can see a cloud verdict at all, and the pre-registered 270 / 540 / 2160 / 432 / 144 cannot move. Not because we were careful: because there is nothing for them to read.", "em_dash": "a figure we do not hold. Never a zero." }, "limits": [ "One page, one shuffle per cloud judge. Each cloud judge is paired with ONE original seat, so its agreement with the other three carries that seat's presentation with it.", "Small N rides every rate here exactly as it rides the rounds being extended: nine effective items, two samples, three characters.", "One of the six judges' families authored the probe questions, this page, and the exhibits it audits. That is precisely why the other five exist.", "First-attempt only: production's corrective-retry ladder is not simulated." ], "original_integrity": { "per_round": { "c4-pairwise": { "fable-1": { "judge_id": "fable-1", "vendor": "anthropic", "batches_expected": 45, "batches_valid": 45, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "fable-2": { "judge_id": "fable-2", "vendor": "anthropic", "batches_expected": 45, "batches_valid": 45, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "opus-1": { "judge_id": "opus-1", "vendor": "anthropic", "batches_expected": 45, "batches_valid": 45, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "opus-2": { "judge_id": "opus-2", "vendor": "anthropic", "batches_expected": 45, "batches_valid": 45, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" } }, "c4-distinctness": { "fable-1": { "judge_id": "fable-1", "vendor": "anthropic", "batches_expected": 4, "batches_valid": 4, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "fable-2": { "judge_id": "fable-2", "vendor": "anthropic", "batches_expected": 4, "batches_valid": 4, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "opus-1": { "judge_id": "opus-1", "vendor": "anthropic", "batches_expected": 4, "batches_valid": 4, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "opus-2": { "judge_id": "opus-2", "vendor": "anthropic", "batches_expected": 4, "batches_valid": 4, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" } }, "c4-canon": { "fable-1": { "judge_id": "fable-1", "vendor": "anthropic", "batches_expected": 4, "batches_valid": 4, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "fable-2": { "judge_id": "fable-2", "vendor": "anthropic", "batches_expected": 4, "batches_valid": 4, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "opus-1": { "judge_id": "opus-1", "vendor": "anthropic", "batches_expected": 4, "batches_valid": 4, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "opus-2": { "judge_id": "opus-2", "vendor": "anthropic", "batches_expected": 4, "batches_valid": 4, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" } }, "legB-panel": { "fable-1": { "judge_id": "fable-1", "vendor": "anthropic", "batches_expected": 3, "batches_valid": 3, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "fable-2": { "judge_id": "fable-2", "vendor": "anthropic", "batches_expected": 3, "batches_valid": 3, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "fable-3": { "judge_id": "fable-3", "vendor": "anthropic", "batches_expected": 3, "batches_valid": 3, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "opus-1": { "judge_id": "opus-1", "vendor": "anthropic", "batches_expected": 3, "batches_valid": 3, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "opus-2": { "judge_id": "opus-2", "vendor": "anthropic", "batches_expected": 3, "batches_valid": 3, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" }, "opus-3": { "judge_id": "opus-3", "vendor": "anthropic", "batches_expected": 3, "batches_valid": 3, "carried": true, "state": "CARRIED", "invalid": [], "rule": "a round is carried only whole: a judge missing or malformed on any batch is excluded from that round entirely and the fact is published" } } }, "findings": [] }, "abstention_reproduction": { "stored_replies": 18, "recomputed_replies": 18, "divergences": [ { "reply": "muse-glimmer:30b-q8_0-dflash|s3", "stored": "in-voice-deflection", "recomputed": "SPLIT", "counts": { "in-voice-deflection": 2, "out-of-voice-refusal": 2 } } ], "expected": "exactly one divergence: the disclosed 2-2 (muse-glimmer think:false, sample 3) that the published round recorded as in-voice-deflection under a convention named on the page, and that G5 reads as SPLIT", "note": "the published table ships MAJORITIES.json unchanged; this is a check, not a replacement" }, "rounds": { "c4-pairwise": { "round": "c4-pairwise", "what_it_is": "the blind pairwise narrator matchups", "per_vendor": { "anthropic": { "status": "DESCRIPTIVE", "primary_metric": "overall pairwise win-rate, cluster-bootstrap CI over items", "units_complete": 1080, "units_incomplete": 0, "incomplete_units": [], "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "all_items": { "n_comparisons": 1080, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 256.0, "comparisons": 360, "win_rate": 0.7111111111111111, "ci95_cluster_bootstrap": [ 0.6236111111111111, 0.8041666666666667 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 246.5, "comparisons": 360, "win_rate": 0.6847222222222222, "ci95_cluster_bootstrap": [ 0.6152777777777778, 0.7625 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 212.0, "comparisons": 360, "win_rate": 0.5888888888888889, "ci95_cluster_bootstrap": [ 0.4083333333333333, 0.7597222222222222 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 194.0, "comparisons": 360, "win_rate": 0.5388888888888889, "ci95_cluster_bootstrap": [ 0.45, 0.6430555555555556 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 129.5, "comparisons": 360, "win_rate": 0.3597222222222222, "ci95_cluster_bootstrap": [ 0.25972222222222224, 0.4666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 42.0, "comparisons": 360, "win_rate": 0.11666666666666667, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.23055555555555557 ], "effective_n_items": 9 } }, "judged_units": 1080, "unjudged_units": 0, "swap_discordant": 93, "swap_discordant_rate": 0.08611111111111111, "bradley_terry_descriptive": { "qwen3.6:27b": 1.8092622737500086, "gemma4:12b": 1.619394337080983, "muse-glimmer:30b-q8_0-dflash": 1.1012403214221167, "gemma4:26b": 0.9051088790344628, "nemotron-3.5-lightning:30b-a3b": 0.4394161683038068, "muse-glimmer:30b-q8_0-dflash+think": 0.12557802040862254 } }, "in_character_secondary": { "rates": { "gemma4:12b": { "wins": 245.5, "comparisons": 360, "win_rate": 0.6819444444444445, "ci95_cluster_bootstrap": [ 0.6083333333333333, 0.7638888888888888 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 242.5, "comparisons": 360, "win_rate": 0.6736111111111112, "ci95_cluster_bootstrap": [ 0.5791666666666667, 0.7819444444444444 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 221.0, "comparisons": 360, "win_rate": 0.6138888888888889, "ci95_cluster_bootstrap": [ 0.5041666666666667, 0.7180555555555556 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 204.0, "comparisons": 360, "win_rate": 0.5666666666666667, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.7180555555555556 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 125.0, "comparisons": 360, "win_rate": 0.3472222222222222, "ci95_cluster_bootstrap": [ 0.24861111111111112, 0.4625 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 42.0, "comparisons": 360, "win_rate": 0.11666666666666667, "ci95_cluster_bootstrap": [ 0.02638888888888889, 0.23194444444444445 ], "effective_n_items": 9 } }, "judged_units": 1080, "unjudged_units": 0, "swap_discordant": 103, "swap_discordant_rate": 0.09537037037037037, "bradley_terry_descriptive": { "gemma4:12b": 1.6225302771300043, "qwen3.6:27b": 1.5679760181212143, "gemma4:26b": 1.2337623967574318, "muse-glimmer:30b-q8_0-dflash": 1.024952389858144, "nemotron-3.5-lightning:30b-a3b": 0.42339667198658576, "muse-glimmer:30b-q8_0-dflash+think": 0.12738224614662003 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 273.5, "comparisons": 360, "win_rate": 0.7597222222222222, "ci95_cluster_bootstrap": [ 0.6902777777777778, 0.8305555555555556 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 237.5, "comparisons": 360, "win_rate": 0.6597222222222222, "ci95_cluster_bootstrap": [ 0.5444444444444444, 0.7805555555555556 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 207.0, "comparisons": 360, "win_rate": 0.575, "ci95_cluster_bootstrap": [ 0.4722222222222222, 0.6930555555555555 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 178.0, "comparisons": 360, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.32916666666666666, 0.6430555555555556 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 143.0, "comparisons": 360, "win_rate": 0.3972222222222222, "ci95_cluster_bootstrap": [ 0.27361111111111114, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 41.0, "comparisons": 360, "win_rate": 0.11388888888888889, "ci95_cluster_bootstrap": [ 0.030555555555555555, 0.21666666666666667 ], "effective_n_items": 9 } }, "judged_units": 1080, "unjudged_units": 0, "swap_discordant": 146, "swap_discordant_rate": 0.13518518518518519, "bradley_terry_descriptive": { "gemma4:12b": 2.193600330368878, "qwen3.6:27b": 1.427119984062163, "gemma4:26b": 1.0166697571285352, "muse-glimmer:30b-q8_0-dflash": 0.7408140536357574, "nemotron-3.5-lightning:30b-a3b": 0.5013299683998579, "muse-glimmer:30b-q8_0-dflash+think": 0.12046590640480863 } } }, "gate_passing_only": { "n_comparisons": 468, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 152.5, "comparisons": 224, "win_rate": 0.6808035714285714, "ci95_cluster_bootstrap": [ 0.576271186440678, 0.8160377358490566 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 144.0, "comparisons": 224, "win_rate": 0.6428571428571429, "ci95_cluster_bootstrap": [ 0.5735294117647058, 0.7171052631578947 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 29.0, "comparisons": 48, "win_rate": 0.6041666666666666, "ci95_cluster_bootstrap": [ 0.09375, 0.90625 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 98.0, "comparisons": 224, "win_rate": 0.4375, "ci95_cluster_bootstrap": [ 0.3287037037037037, 0.5512295081967213 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 44.5, "comparisons": 216, "win_rate": 0.20601851851851852, "ci95_cluster_bootstrap": [ 0.07932692307692307, 0.34375 ], "effective_n_items": 9 } }, "judged_units": 468, "unjudged_units": 0, "swap_discordant": 68, "swap_discordant_rate": 0.1452991452991453, "bradley_terry_descriptive": { "qwen3.6:27b": 1.5022204774909418, "gemma4:12b": 1.3150418448606693, "muse-glimmer:30b-q8_0-dflash+think": 1.250257077887651, "gemma4:26b": 0.6574829892959347, "nemotron-3.5-lightning:30b-a3b": 0.27499761046480353 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 138.5, "comparisons": 224, "win_rate": 0.6183035714285714, "ci95_cluster_bootstrap": [ 0.5020833333333333, 0.7735849056603774 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 29.5, "comparisons": 48, "win_rate": 0.6145833333333334, "ci95_cluster_bootstrap": [ 0.09375, 0.9375 ], "effective_n_items": 3 }, "gemma4:12b": { "wins": 137.5, "comparisons": 224, "win_rate": 0.6138392857142857, "ci95_cluster_bootstrap": [ 0.53125, 0.7008928571428571 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 120.0, "comparisons": 224, "win_rate": 0.5357142857142857, "ci95_cluster_bootstrap": [ 0.41379310344827586, 0.6508620689655172 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 42.5, "comparisons": 216, "win_rate": 0.19675925925925927, "ci95_cluster_bootstrap": [ 0.04846938775510204, 0.3580508474576271 ], "effective_n_items": 9 } }, "judged_units": 468, "unjudged_units": 0, "swap_discordant": 63, "swap_discordant_rate": 0.1346153846153846, "bradley_terry_descriptive": { "muse-glimmer:30b-q8_0-dflash+think": 1.3345309544634305, "qwen3.6:27b": 1.2343426066431853, "gemma4:12b": 1.2160243514277507, "gemma4:26b": 0.9391556096238222, "nemotron-3.5-lightning:30b-a3b": 0.2759464778418109 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 158.5, "comparisons": 224, "win_rate": 0.7075892857142857, "ci95_cluster_bootstrap": [ 0.5959821428571429, 0.8090909090909091 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 28.0, "comparisons": 48, "win_rate": 0.5833333333333334, "ci95_cluster_bootstrap": [ 0.09375, 0.9375 ], "effective_n_items": 3 }, "qwen3.6:27b": { "wins": 126.5, "comparisons": 224, "win_rate": 0.5647321428571429, "ci95_cluster_bootstrap": [ 0.4330357142857143, 0.7136363636363636 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 104.5, "comparisons": 224, "win_rate": 0.46651785714285715, "ci95_cluster_bootstrap": [ 0.32142857142857145, 0.6056034482758621 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 50.5, "comparisons": 216, "win_rate": 0.2337962962962963, "ci95_cluster_bootstrap": [ 0.08962264150943396, 0.4263392857142857 ], "effective_n_items": 9 } }, "judged_units": 468, "unjudged_units": 0, "swap_discordant": 83, "swap_discordant_rate": 0.17735042735042736, "bradley_terry_descriptive": { "gemma4:12b": 1.6963039870713743, "muse-glimmer:30b-q8_0-dflash+think": 1.1793492061818027, "qwen3.6:27b": 1.0418155894937413, "gemma4:26b": 0.7546450420399069, "nemotron-3.5-lightning:30b-a3b": 0.3278861752131751 } } }, "per_family": { "fable": { "n_comparisons": 540, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 124.5, "comparisons": 180, "win_rate": 0.6916666666666667, "ci95_cluster_bootstrap": [ 0.6027777777777777, 0.7916666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 117.5, "comparisons": 180, "win_rate": 0.6527777777777778, "ci95_cluster_bootstrap": [ 0.5833333333333334, 0.7333333333333333 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 108.5, "comparisons": 180, "win_rate": 0.6027777777777777, "ci95_cluster_bootstrap": [ 0.4083333333333333, 0.7777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 100.5, "comparisons": 180, "win_rate": 0.5583333333333333, "ci95_cluster_bootstrap": [ 0.4666666666666667, 0.65 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 66.0, "comparisons": 180, "win_rate": 0.36666666666666664, "ci95_cluster_bootstrap": [ 0.24444444444444444, 0.49722222222222223 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 23.0, "comparisons": 180, "win_rate": 0.12777777777777777, "ci95_cluster_bootstrap": [ 0.041666666666666664, 0.23333333333333334 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 49, "swap_discordant_rate": 0.09074074074074075, "bradley_terry_descriptive": { "qwen3.6:27b": 1.708173390033967, "gemma4:12b": 1.4597910228069095, "muse-glimmer:30b-q8_0-dflash": 1.199600150956817, "gemma4:26b": 1.0105704596527745, "nemotron-3.5-lightning:30b-a3b": 0.47593541340105266, "muse-glimmer:30b-q8_0-dflash+think": 0.14592956314847946 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 118.0, "comparisons": 180, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.5666666666666667, 0.7611111111111111 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 115.5, "comparisons": 180, "win_rate": 0.6416666666666667, "ci95_cluster_bootstrap": [ 0.5305555555555556, 0.7527777777777778 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 112.0, "comparisons": 180, "win_rate": 0.6222222222222222, "ci95_cluster_bootstrap": [ 0.5527777777777778, 0.7 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 107.0, "comparisons": 180, "win_rate": 0.5944444444444444, "ci95_cluster_bootstrap": [ 0.4222222222222222, 0.7444444444444445 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 65.0, "comparisons": 180, "win_rate": 0.3611111111111111, "ci95_cluster_bootstrap": [ 0.2361111111111111, 0.5 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 22.5, "comparisons": 180, "win_rate": 0.125, "ci95_cluster_bootstrap": [ 0.03611111111111111, 0.2361111111111111 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 51, "swap_discordant_rate": 0.09444444444444444, "bradley_terry_descriptive": { "qwen3.6:27b": 1.4912520504423823, "gemma4:26b": 1.4115116130040892, "gemma4:12b": 1.307939728249174, "muse-glimmer:30b-q8_0-dflash": 1.1743524279181037, "nemotron-3.5-lightning:30b-a3b": 0.4705026791516387, "muse-glimmer:30b-q8_0-dflash+think": 0.1444415012346123 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 138.0, "comparisons": 180, "win_rate": 0.7666666666666667, "ci95_cluster_bootstrap": [ 0.6833333333333333, 0.85 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 119.5, "comparisons": 180, "win_rate": 0.6638888888888889, "ci95_cluster_bootstrap": [ 0.55, 0.7777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 101.0, "comparisons": 180, "win_rate": 0.5611111111111111, "ci95_cluster_bootstrap": [ 0.4583333333333333, 0.6694444444444444 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 87.5, "comparisons": 180, "win_rate": 0.4861111111111111, "ci95_cluster_bootstrap": [ 0.3194444444444444, 0.6361111111111111 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 71.5, "comparisons": 180, "win_rate": 0.3972222222222222, "ci95_cluster_bootstrap": [ 0.2611111111111111, 0.5666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 22.5, "comparisons": 180, "win_rate": 0.125, "ci95_cluster_bootstrap": [ 0.041666666666666664, 0.21944444444444444 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 72, "swap_discordant_rate": 0.13333333333333333, "bradley_terry_descriptive": { "gemma4:12b": 2.2525620494631435, "qwen3.6:27b": 1.4433450545721866, "gemma4:26b": 0.9575653700571616, "muse-glimmer:30b-q8_0-dflash": 0.7139535223721651, "nemotron-3.5-lightning:30b-a3b": 0.5005750686170699, "muse-glimmer:30b-q8_0-dflash+think": 0.13199893491827325 } } }, "opus": { "n_comparisons": 540, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 131.5, "comparisons": 180, "win_rate": 0.7305555555555555, "ci95_cluster_bootstrap": [ 0.6277777777777778, 0.8305555555555556 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 129.0, "comparisons": 180, "win_rate": 0.7166666666666667, "ci95_cluster_bootstrap": [ 0.6416666666666667, 0.7972222222222223 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 103.5, "comparisons": 180, "win_rate": 0.575, "ci95_cluster_bootstrap": [ 0.40555555555555556, 0.7416666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 93.5, "comparisons": 180, "win_rate": 0.5194444444444445, "ci95_cluster_bootstrap": [ 0.41944444444444445, 0.6416666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 63.5, "comparisons": 180, "win_rate": 0.3527777777777778, "ci95_cluster_bootstrap": [ 0.2611111111111111, 0.44722222222222224 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 19.0, "comparisons": 180, "win_rate": 0.10555555555555556, "ci95_cluster_bootstrap": [ 0.011111111111111112, 0.22777777777777777 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 44, "swap_discordant_rate": 0.08148148148148149, "bradley_terry_descriptive": { "qwen3.6:27b": 1.9082038493318059, "gemma4:12b": 1.7948790571694107, "muse-glimmer:30b-q8_0-dflash": 0.9966501776266384, "gemma4:26b": 0.7972033762758836, "nemotron-3.5-lightning:30b-a3b": 0.3980759484875316, "muse-glimmer:30b-q8_0-dflash+think": 0.1049875911087296 } }, "in_character_secondary": { "rates": { "gemma4:12b": { "wins": 133.5, "comparisons": 180, "win_rate": 0.7416666666666667, "ci95_cluster_bootstrap": [ 0.6583333333333333, 0.8333333333333334 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 124.5, "comparisons": 180, "win_rate": 0.6916666666666667, "ci95_cluster_bootstrap": [ 0.5805555555555556, 0.8083333333333333 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 105.5, "comparisons": 180, "win_rate": 0.5861111111111111, "ci95_cluster_bootstrap": [ 0.475, 0.7055555555555556 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 97.0, "comparisons": 180, "win_rate": 0.5388888888888889, "ci95_cluster_bootstrap": [ 0.38333333333333336, 0.6972222222222222 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 60.0, "comparisons": 180, "win_rate": 0.3333333333333333, "ci95_cluster_bootstrap": [ 0.25555555555555554, 0.41944444444444445 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 19.5, "comparisons": 180, "win_rate": 0.10833333333333334, "ci95_cluster_bootstrap": [ 0.016666666666666666, 0.23055555555555557 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 52, "swap_discordant_rate": 0.0962962962962963, "bradley_terry_descriptive": { "gemma4:12b": 2.0089371467552444, "qwen3.6:27b": 1.6139070895119567, "gemma4:26b": 1.043567650530314, "muse-glimmer:30b-q8_0-dflash": 0.8627095873921578, "nemotron-3.5-lightning:30b-a3b": 0.36422123322405353, "muse-glimmer:30b-q8_0-dflash+think": 0.10665729258627375 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 135.5, "comparisons": 180, "win_rate": 0.7527777777777778, "ci95_cluster_bootstrap": [ 0.6861111111111111, 0.8194444444444444 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 118.0, "comparisons": 180, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.7833333333333333 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 106.0, "comparisons": 180, "win_rate": 0.5888888888888889, "ci95_cluster_bootstrap": [ 0.46944444444444444, 0.7222222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 90.5, "comparisons": 180, "win_rate": 0.5027777777777778, "ci95_cluster_bootstrap": [ 0.3388888888888889, 0.6527777777777778 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 71.5, "comparisons": 180, "win_rate": 0.3972222222222222, "ci95_cluster_bootstrap": [ 0.275, 0.5388888888888889 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 18.5, "comparisons": 180, "win_rate": 0.10277777777777777, "ci95_cluster_bootstrap": [ 0.016666666666666666, 0.21388888888888888 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 74, "swap_discordant_rate": 0.13703703703703704, "bradley_terry_descriptive": { "gemma4:12b": 2.1350862540275695, "qwen3.6:27b": 1.4094075513768, "gemma4:26b": 1.0783229678073993, "muse-glimmer:30b-q8_0-dflash": 0.7675535616888047, "nemotron-3.5-lightning:30b-a3b": 0.5009494280044106, "muse-glimmer:30b-q8_0-dflash+think": 0.10868023709501555 } } } }, "inter_panel_agreement": { "families": [ "fable", "opus" ], "per_model_abs_diff": { "gemma4:12b": 0.06388888888888888, "gemma4:26b": 0.03888888888888886, "muse-glimmer:30b-q8_0-dflash": 0.02777777777777779, "muse-glimmer:30b-q8_0-dflash+think": 0.022222222222222213, "nemotron-3.5-lightning:30b-a3b": 0.01388888888888884, "qwen3.6:27b": 0.03888888888888886 }, "mean_abs_diff": 0.03425925925925924, "rank_spearman": 1.0, "note": "divergence between the panels is a finding; lead with it" }, "position_bias_per_judge": { "fable-1": { "slot_A_chosen": 262, "judgments": 540, "rate": 0.48518518518518516, "ci95_wilson": [ 0.4432848826140435, 0.5272947861718678 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "fable-2": { "slot_A_chosen": 257, "judgments": 540, "rate": 0.4759259259259259, "ci95_wilson": [ 0.4341211028774251, 0.5180708588996805 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "opus-1": { "slot_A_chosen": 276, "judgments": 540, "rate": 0.5111111111111111, "ci95_wilson": [ 0.4690196563907067, 0.5530455920198598 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "opus-2": { "slot_A_chosen": 278, "judgments": 540, "rate": 0.5148148148148148, "ci95_wilson": [ 0.4727052138281323, 0.5567151173859566 ], "note": "0.50 is unbiased; the design is fully counterbalanced" } }, "canon_flag_rate_per_model": { "gemma4:12b": { "flagged": 49, "judgments": 720, "rate": 0.06805555555555555 }, "gemma4:26b": { "flagged": 40, "judgments": 720, "rate": 0.05555555555555555 }, "muse-glimmer:30b-q8_0-dflash": { "flagged": 141, "judgments": 720, "rate": 0.19583333333333333 }, "muse-glimmer:30b-q8_0-dflash+think": { "flagged": 26, "judgments": 720, "rate": 0.03611111111111111 }, "nemotron-3.5-lightning:30b-a3b": { "flagged": 174, "judgments": 720, "rate": 0.24166666666666667 }, "qwen3.6:27b": { "flagged": 116, "judgments": 720, "rate": 0.16111111111111112 } }, "caveats": [ "first-attempt only: production's corrective-retry ladder is not simulated", "one of the two judge families also wrote the questions and this article" ] }, "deepseek": { "status": "DESCRIPTIVE", "primary_metric": "overall pairwise win-rate, cluster-bootstrap CI over items", "units_complete": 270, "units_incomplete": 0, "incomplete_units": [], "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "all_items": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 70.0, "comparisons": 90, "win_rate": 0.7777777777777778, "ci95_cluster_bootstrap": [ 0.6888888888888889, 0.85 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 55.0, "comparisons": 90, "win_rate": 0.6111111111111112, "ci95_cluster_bootstrap": [ 0.43333333333333335, 0.7833333333333333 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 54.5, "comparisons": 90, "win_rate": 0.6055555555555555, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.6777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 46.5, "comparisons": 90, "win_rate": 0.5166666666666667, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.6166666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.0, "comparisons": 90, "win_rate": 0.4, "ci95_cluster_bootstrap": [ 0.3277777777777778, 0.4777777777777778 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.0, "comparisons": 90, "win_rate": 0.08888888888888889, "ci95_cluster_bootstrap": [ 0.022222222222222223, 0.16666666666666666 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 44, "swap_discordant_rate": 0.16296296296296298, "bradley_terry_descriptive": { "qwen3.6:27b": 2.355960102294597, "muse-glimmer:30b-q8_0-dflash": 1.150492036910824, "gemma4:12b": 1.125007228072441, "gemma4:26b": 0.788485163033489, "nemotron-3.5-lightning:30b-a3b": 0.4893202215213605, "muse-glimmer:30b-q8_0-dflash+think": 0.09073524816728841 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 70.0, "comparisons": 90, "win_rate": 0.7777777777777778, "ci95_cluster_bootstrap": [ 0.6888888888888889, 0.85 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 55.0, "comparisons": 90, "win_rate": 0.6111111111111112, "ci95_cluster_bootstrap": [ 0.43333333333333335, 0.7833333333333333 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 54.5, "comparisons": 90, "win_rate": 0.6055555555555555, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.6777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 46.5, "comparisons": 90, "win_rate": 0.5166666666666667, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.6166666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.0, "comparisons": 90, "win_rate": 0.4, "ci95_cluster_bootstrap": [ 0.3277777777777778, 0.4777777777777778 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.0, "comparisons": 90, "win_rate": 0.08888888888888889, "ci95_cluster_bootstrap": [ 0.022222222222222223, 0.16666666666666666 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 44, "swap_discordant_rate": 0.16296296296296298, "bradley_terry_descriptive": { "qwen3.6:27b": 2.355960102294597, "muse-glimmer:30b-q8_0-dflash": 1.150492036910824, "gemma4:12b": 1.125007228072441, "gemma4:26b": 0.788485163033489, "nemotron-3.5-lightning:30b-a3b": 0.4893202215213605, "muse-glimmer:30b-q8_0-dflash+think": 0.09073524816728841 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 69.5, "comparisons": 90, "win_rate": 0.7722222222222223, "ci95_cluster_bootstrap": [ 0.6833333333333333, 0.8444444444444444 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 55.5, "comparisons": 90, "win_rate": 0.6166666666666667, "ci95_cluster_bootstrap": [ 0.5555555555555556, 0.6833333333333333 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 53.0, "comparisons": 90, "win_rate": 0.5888888888888889, "ci95_cluster_bootstrap": [ 0.40555555555555556, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 47.0, "comparisons": 90, "win_rate": 0.5222222222222223, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.6277777777777778 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.5, "comparisons": 90, "win_rate": 0.40555555555555556, "ci95_cluster_bootstrap": [ 0.3388888888888889, 0.4722222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.5, "comparisons": 90, "win_rate": 0.09444444444444444, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.17222222222222222 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 46, "swap_discordant_rate": 0.17037037037037037, "bradley_terry_descriptive": { "qwen3.6:27b": 2.3133050654950984, "gemma4:12b": 1.1913395302571046, "muse-glimmer:30b-q8_0-dflash": 1.066267604013111, "gemma4:26b": 0.8189139072686629, "nemotron-3.5-lightning:30b-a3b": 0.5113074928532275, "muse-glimmer:30b-q8_0-dflash+think": 0.09886640011279582 } } }, "gate_passing_only": { "n_comparisons": 117, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 43.5, "comparisons": 56, "win_rate": 0.7767857142857143, "ci95_cluster_bootstrap": [ 0.6875, 0.8636363636363636 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 30.0, "comparisons": 56, "win_rate": 0.5357142857142857, "ci95_cluster_bootstrap": [ 0.4152542372881356, 0.6346153846153846 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 6.0, "comparisons": 12, "win_rate": 0.5, "ci95_cluster_bootstrap": [ 0.125, 0.75 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 23.0, "comparisons": 56, "win_rate": 0.4107142857142857, "ci95_cluster_bootstrap": [ 0.26785714285714285, 0.543859649122807 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 14.5, "comparisons": 54, "win_rate": 0.26851851851851855, "ci95_cluster_bootstrap": [ 0.17346938775510204, 0.36792452830188677 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 32, "swap_discordant_rate": 0.27350427350427353, "bradley_terry_descriptive": { "qwen3.6:27b": 2.234121266887362, "gemma4:12b": 0.9441464473541966, "muse-glimmer:30b-q8_0-dflash+think": 0.8287203058229421, "gemma4:26b": 0.6218863638468887, "nemotron-3.5-lightning:30b-a3b": 0.37112561608861033 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 43.5, "comparisons": 56, "win_rate": 0.7767857142857143, "ci95_cluster_bootstrap": [ 0.6875, 0.8636363636363636 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 30.0, "comparisons": 56, "win_rate": 0.5357142857142857, "ci95_cluster_bootstrap": [ 0.4152542372881356, 0.6346153846153846 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 6.0, "comparisons": 12, "win_rate": 0.5, "ci95_cluster_bootstrap": [ 0.125, 0.75 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 23.0, "comparisons": 56, "win_rate": 0.4107142857142857, "ci95_cluster_bootstrap": [ 0.26785714285714285, 0.543859649122807 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 14.5, "comparisons": 54, "win_rate": 0.26851851851851855, "ci95_cluster_bootstrap": [ 0.17346938775510204, 0.36792452830188677 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 32, "swap_discordant_rate": 0.27350427350427353, "bradley_terry_descriptive": { "qwen3.6:27b": 2.234121266887362, "gemma4:12b": 0.9441464473541966, "muse-glimmer:30b-q8_0-dflash+think": 0.8287203058229421, "gemma4:26b": 0.6218863638468887, "nemotron-3.5-lightning:30b-a3b": 0.37112561608861033 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 43.0, "comparisons": 56, "win_rate": 0.7678571428571429, "ci95_cluster_bootstrap": [ 0.6785714285714286, 0.8584905660377359 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 31.0, "comparisons": 56, "win_rate": 0.5535714285714286, "ci95_cluster_bootstrap": [ 0.4636363636363636, 0.6363636363636364 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 6.0, "comparisons": 12, "win_rate": 0.5, "ci95_cluster_bootstrap": [ 0.125, 0.75 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 23.0, "comparisons": 56, "win_rate": 0.4107142857142857, "ci95_cluster_bootstrap": [ 0.26785714285714285, 0.543859649122807 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 14.0, "comparisons": 54, "win_rate": 0.25925925925925924, "ci95_cluster_bootstrap": [ 0.17272727272727273, 0.34615384615384615 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 32, "swap_discordant_rate": 0.27350427350427353, "bradley_terry_descriptive": { "qwen3.6:27b": 2.169775612413237, "gemma4:12b": 1.0096743997457258, "muse-glimmer:30b-q8_0-dflash+think": 0.834337394618814, "gemma4:26b": 0.626130827562708, "nemotron-3.5-lightning:30b-a3b": 0.3600817656595154 } } }, "per_family": { "deepseek": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 70.0, "comparisons": 90, "win_rate": 0.7777777777777778, "ci95_cluster_bootstrap": [ 0.6888888888888889, 0.85 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 55.0, "comparisons": 90, "win_rate": 0.6111111111111112, "ci95_cluster_bootstrap": [ 0.43333333333333335, 0.7833333333333333 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 54.5, "comparisons": 90, "win_rate": 0.6055555555555555, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.6777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 46.5, "comparisons": 90, "win_rate": 0.5166666666666667, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.6166666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.0, "comparisons": 90, "win_rate": 0.4, "ci95_cluster_bootstrap": [ 0.3277777777777778, 0.4777777777777778 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.0, "comparisons": 90, "win_rate": 0.08888888888888889, "ci95_cluster_bootstrap": [ 0.022222222222222223, 0.16666666666666666 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 44, "swap_discordant_rate": 0.16296296296296298, "bradley_terry_descriptive": { "qwen3.6:27b": 2.355960102294597, "muse-glimmer:30b-q8_0-dflash": 1.150492036910824, "gemma4:12b": 1.125007228072441, "gemma4:26b": 0.788485163033489, "nemotron-3.5-lightning:30b-a3b": 0.4893202215213605, "muse-glimmer:30b-q8_0-dflash+think": 0.09073524816728841 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 70.0, "comparisons": 90, "win_rate": 0.7777777777777778, "ci95_cluster_bootstrap": [ 0.6888888888888889, 0.85 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 55.0, "comparisons": 90, "win_rate": 0.6111111111111112, "ci95_cluster_bootstrap": [ 0.43333333333333335, 0.7833333333333333 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 54.5, "comparisons": 90, "win_rate": 0.6055555555555555, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.6777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 46.5, "comparisons": 90, "win_rate": 0.5166666666666667, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.6166666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.0, "comparisons": 90, "win_rate": 0.4, "ci95_cluster_bootstrap": [ 0.3277777777777778, 0.4777777777777778 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.0, "comparisons": 90, "win_rate": 0.08888888888888889, "ci95_cluster_bootstrap": [ 0.022222222222222223, 0.16666666666666666 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 44, "swap_discordant_rate": 0.16296296296296298, "bradley_terry_descriptive": { "qwen3.6:27b": 2.355960102294597, "muse-glimmer:30b-q8_0-dflash": 1.150492036910824, "gemma4:12b": 1.125007228072441, "gemma4:26b": 0.788485163033489, "nemotron-3.5-lightning:30b-a3b": 0.4893202215213605, "muse-glimmer:30b-q8_0-dflash+think": 0.09073524816728841 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 69.5, "comparisons": 90, "win_rate": 0.7722222222222223, "ci95_cluster_bootstrap": [ 0.6833333333333333, 0.8444444444444444 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 55.5, "comparisons": 90, "win_rate": 0.6166666666666667, "ci95_cluster_bootstrap": [ 0.5555555555555556, 0.6833333333333333 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 53.0, "comparisons": 90, "win_rate": 0.5888888888888889, "ci95_cluster_bootstrap": [ 0.40555555555555556, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 47.0, "comparisons": 90, "win_rate": 0.5222222222222223, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.6277777777777778 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.5, "comparisons": 90, "win_rate": 0.40555555555555556, "ci95_cluster_bootstrap": [ 0.3388888888888889, 0.4722222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.5, "comparisons": 90, "win_rate": 0.09444444444444444, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.17222222222222222 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 46, "swap_discordant_rate": 0.17037037037037037, "bradley_terry_descriptive": { "qwen3.6:27b": 2.3133050654950984, "gemma4:12b": 1.1913395302571046, "muse-glimmer:30b-q8_0-dflash": 1.066267604013111, "gemma4:26b": 0.8189139072686629, "nemotron-3.5-lightning:30b-a3b": 0.5113074928532275, "muse-glimmer:30b-q8_0-dflash+think": 0.09886640011279582 } } } }, "inter_panel_agreement": null, "position_bias_per_judge": { "cloud-deepseek-v4-pro": { "slot_A_chosen": 270, "judgments": 540, "rate": 0.5, "ci95_wilson": [ 0.45797672810066214, 0.5420232718993379 ], "note": "0.50 is unbiased; the design is fully counterbalanced" } }, "canon_flag_rate_per_model": { "gemma4:12b": { "flagged": 0, "judgments": 180, "rate": 0.0 }, "gemma4:26b": { "flagged": 0, "judgments": 180, "rate": 0.0 }, "muse-glimmer:30b-q8_0-dflash": { "flagged": 11, "judgments": 180, "rate": 0.06111111111111111 }, "muse-glimmer:30b-q8_0-dflash+think": { "flagged": 9, "judgments": 180, "rate": 0.05 }, "nemotron-3.5-lightning:30b-a3b": { "flagged": 22, "judgments": 180, "rate": 0.12222222222222222 }, "qwen3.6:27b": { "flagged": 3, "judgments": 180, "rate": 0.016666666666666666 } }, "caveats": [ "first-attempt only: production's corrective-retry ladder is not simulated", "one of the two judge families also wrote the questions and this article" ] }, "mistral": { "status": "DESCRIPTIVE", "primary_metric": "overall pairwise win-rate, cluster-bootstrap CI over items", "units_complete": 270, "units_incomplete": 0, "incomplete_units": [], "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "all_items": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 67.5, "comparisons": 90, "win_rate": 0.75, "ci95_cluster_bootstrap": [ 0.5888888888888889, 0.8888888888888888 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 59.0, "comparisons": 90, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.55, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4777777777777778, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 44.5, "comparisons": 90, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.37777777777777777, 0.6166666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 40.5, "comparisons": 90, "win_rate": 0.45, "ci95_cluster_bootstrap": [ 0.32222222222222224, 0.5722222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.0, "comparisons": 90, "win_rate": 0.1, "ci95_cluster_bootstrap": [ 0.016666666666666666, 0.2 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 50, "swap_discordant_rate": 0.18518518518518517, "bradley_terry_descriptive": { "qwen3.6:27b": 2.127817082761526, "gemma4:12b": 1.427577184381572, "gemma4:26b": 0.9410661239287288, "nemotron-3.5-lightning:30b-a3b": 0.7576894560973209, "muse-glimmer:30b-q8_0-dflash": 0.6357574227771301, "muse-glimmer:30b-q8_0-dflash+think": 0.11009273005372225 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 67.5, "comparisons": 90, "win_rate": 0.75, "ci95_cluster_bootstrap": [ 0.5888888888888889, 0.8888888888888888 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 59.0, "comparisons": 90, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.55, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4777777777777778, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 44.5, "comparisons": 90, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.37777777777777777, 0.6166666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 41.0, "comparisons": 90, "win_rate": 0.45555555555555555, "ci95_cluster_bootstrap": [ 0.3333333333333333, 0.5722222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.5, "comparisons": 90, "win_rate": 0.09444444444444444, "ci95_cluster_bootstrap": [ 0.011111111111111112, 0.19444444444444445 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 49, "swap_discordant_rate": 0.1814814814814815, "bradley_terry_descriptive": { "qwen3.6:27b": 2.1268182748829396, "gemma4:12b": 1.426272093365552, "gemma4:26b": 0.9394024538882703, "nemotron-3.5-lightning:30b-a3b": 0.7558367101275903, "muse-glimmer:30b-q8_0-dflash": 0.6479866843394327, "muse-glimmer:30b-q8_0-dflash+think": 0.10368378339621516 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 67.0, "comparisons": 90, "win_rate": 0.7444444444444445, "ci95_cluster_bootstrap": [ 0.5944444444444444, 0.8777777777777778 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 58.5, "comparisons": 90, "win_rate": 0.65, "ci95_cluster_bootstrap": [ 0.5666666666666667, 0.7388888888888889 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 50.0, "comparisons": 90, "win_rate": 0.5555555555555556, "ci95_cluster_bootstrap": [ 0.4777777777777778, 0.6444444444444445 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.5, "comparisons": 90, "win_rate": 0.48333333333333334, "ci95_cluster_bootstrap": [ 0.37222222222222223, 0.6055555555555555 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 42.0, "comparisons": 90, "win_rate": 0.4666666666666667, "ci95_cluster_bootstrap": [ 0.35, 0.5833333333333334 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.0, "comparisons": 90, "win_rate": 0.1, "ci95_cluster_bootstrap": [ 0.005555555555555556, 0.2222222222222222 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 49, "swap_discordant_rate": 0.1814814814814815, "bradley_terry_descriptive": { "qwen3.6:27b": 2.0894481187390515, "gemma4:12b": 1.407359006484484, "gemma4:26b": 0.9711089616319206, "nemotron-3.5-lightning:30b-a3b": 0.7333540469207611, "muse-glimmer:30b-q8_0-dflash": 0.6869006853503978, "muse-glimmer:30b-q8_0-dflash+think": 0.11182918087338507 } } }, "gate_passing_only": { "n_comparisons": 117, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 39.0, "comparisons": 56, "win_rate": 0.6964285714285714, "ci95_cluster_bootstrap": [ 0.5, 0.8703703703703703 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 30.5, "comparisons": 56, "win_rate": 0.5446428571428571, "ci95_cluster_bootstrap": [ 0.4107142857142857, 0.6875 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 6.0, "comparisons": 12, "win_rate": 0.5, "ci95_cluster_bootstrap": [ 0.125, 0.75 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 22.5, "comparisons": 56, "win_rate": 0.4017857142857143, "ci95_cluster_bootstrap": [ 0.3018867924528302, 0.5087719298245614 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 19.0, "comparisons": 54, "win_rate": 0.35185185185185186, "ci95_cluster_bootstrap": [ 0.1896551724137931, 0.5425531914893617 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 29, "swap_discordant_rate": 0.24786324786324787, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7604986111237564, "gemma4:12b": 1.0686254273508848, "muse-glimmer:30b-q8_0-dflash+think": 0.921480420581553, "gemma4:26b": 0.6784243838391932, "nemotron-3.5-lightning:30b-a3b": 0.5709711571046124 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 39.0, "comparisons": 56, "win_rate": 0.6964285714285714, "ci95_cluster_bootstrap": [ 0.5, 0.8703703703703703 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 30.5, "comparisons": 56, "win_rate": 0.5446428571428571, "ci95_cluster_bootstrap": [ 0.4107142857142857, 0.6875 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 6.0, "comparisons": 12, "win_rate": 0.5, "ci95_cluster_bootstrap": [ 0.125, 0.75 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 22.5, "comparisons": 56, "win_rate": 0.4017857142857143, "ci95_cluster_bootstrap": [ 0.3018867924528302, 0.5087719298245614 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 19.0, "comparisons": 54, "win_rate": 0.35185185185185186, "ci95_cluster_bootstrap": [ 0.1896551724137931, 0.5425531914893617 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 29, "swap_discordant_rate": 0.24786324786324787, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7604986111237564, "gemma4:12b": 1.0686254273508848, "muse-glimmer:30b-q8_0-dflash+think": 0.921480420581553, "gemma4:26b": 0.6784243838391932, "nemotron-3.5-lightning:30b-a3b": 0.5709711571046124 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 39.0, "comparisons": 56, "win_rate": 0.6964285714285714, "ci95_cluster_bootstrap": [ 0.5181818181818182, 0.8571428571428571 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 6.5, "comparisons": 12, "win_rate": 0.5416666666666666, "ci95_cluster_bootstrap": [ 0.125, 0.75 ], "effective_n_items": 3 }, "gemma4:12b": { "wins": 30.0, "comparisons": 56, "win_rate": 0.5357142857142857, "ci95_cluster_bootstrap": [ 0.4166666666666667, 0.652542372881356 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 23.5, "comparisons": 56, "win_rate": 0.41964285714285715, "ci95_cluster_bootstrap": [ 0.31896551724137934, 0.5267857142857143 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 18.0, "comparisons": 54, "win_rate": 0.3333333333333333, "ci95_cluster_bootstrap": [ 0.1810344827586207, 0.5196078431372549 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 28, "swap_discordant_rate": 0.23931623931623933, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7125663545060674, "muse-glimmer:30b-q8_0-dflash+think": 1.0586326780263293, "gemma4:12b": 1.0098607392434682, "gemma4:26b": 0.6981748670269091, "nemotron-3.5-lightning:30b-a3b": 0.5207653611972256 } } }, "per_family": { "mistral": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 67.5, "comparisons": 90, "win_rate": 0.75, "ci95_cluster_bootstrap": [ 0.5888888888888889, 0.8888888888888888 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 59.0, "comparisons": 90, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.55, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4777777777777778, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 44.5, "comparisons": 90, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.37777777777777777, 0.6166666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 40.5, "comparisons": 90, "win_rate": 0.45, "ci95_cluster_bootstrap": [ 0.32222222222222224, 0.5722222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.0, "comparisons": 90, "win_rate": 0.1, "ci95_cluster_bootstrap": [ 0.016666666666666666, 0.2 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 50, "swap_discordant_rate": 0.18518518518518517, "bradley_terry_descriptive": { "qwen3.6:27b": 2.127817082761526, "gemma4:12b": 1.427577184381572, "gemma4:26b": 0.9410661239287288, "nemotron-3.5-lightning:30b-a3b": 0.7576894560973209, "muse-glimmer:30b-q8_0-dflash": 0.6357574227771301, "muse-glimmer:30b-q8_0-dflash+think": 0.11009273005372225 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 67.5, "comparisons": 90, "win_rate": 0.75, "ci95_cluster_bootstrap": [ 0.5888888888888889, 0.8888888888888888 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 59.0, "comparisons": 90, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.55, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4777777777777778, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 44.5, "comparisons": 90, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.37777777777777777, 0.6166666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 41.0, "comparisons": 90, "win_rate": 0.45555555555555555, "ci95_cluster_bootstrap": [ 0.3333333333333333, 0.5722222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.5, "comparisons": 90, "win_rate": 0.09444444444444444, "ci95_cluster_bootstrap": [ 0.011111111111111112, 0.19444444444444445 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 49, "swap_discordant_rate": 0.1814814814814815, "bradley_terry_descriptive": { "qwen3.6:27b": 2.1268182748829396, "gemma4:12b": 1.426272093365552, "gemma4:26b": 0.9394024538882703, "nemotron-3.5-lightning:30b-a3b": 0.7558367101275903, "muse-glimmer:30b-q8_0-dflash": 0.6479866843394327, "muse-glimmer:30b-q8_0-dflash+think": 0.10368378339621516 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 67.0, "comparisons": 90, "win_rate": 0.7444444444444445, "ci95_cluster_bootstrap": [ 0.5944444444444444, 0.8777777777777778 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 58.5, "comparisons": 90, "win_rate": 0.65, "ci95_cluster_bootstrap": [ 0.5666666666666667, 0.7388888888888889 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 50.0, "comparisons": 90, "win_rate": 0.5555555555555556, "ci95_cluster_bootstrap": [ 0.4777777777777778, 0.6444444444444445 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.5, "comparisons": 90, "win_rate": 0.48333333333333334, "ci95_cluster_bootstrap": [ 0.37222222222222223, 0.6055555555555555 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 42.0, "comparisons": 90, "win_rate": 0.4666666666666667, "ci95_cluster_bootstrap": [ 0.35, 0.5833333333333334 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.0, "comparisons": 90, "win_rate": 0.1, "ci95_cluster_bootstrap": [ 0.005555555555555556, 0.2222222222222222 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 49, "swap_discordant_rate": 0.1814814814814815, "bradley_terry_descriptive": { "qwen3.6:27b": 2.0894481187390515, "gemma4:12b": 1.407359006484484, "gemma4:26b": 0.9711089616319206, "nemotron-3.5-lightning:30b-a3b": 0.7333540469207611, "muse-glimmer:30b-q8_0-dflash": 0.6869006853503978, "muse-glimmer:30b-q8_0-dflash+think": 0.11182918087338507 } } } }, "inter_panel_agreement": null, "position_bias_per_judge": { "cloud-mistral-large-3-675b": { "slot_A_chosen": 248, "judgments": 540, "rate": 0.45925925925925926, "ci95_wilson": [ 0.4176625174333626, 0.5014315717278932 ], "note": "0.50 is unbiased; the design is fully counterbalanced" } }, "canon_flag_rate_per_model": { "gemma4:12b": { "flagged": 0, "judgments": 180, "rate": 0.0 }, "gemma4:26b": { "flagged": 0, "judgments": 180, "rate": 0.0 }, "muse-glimmer:30b-q8_0-dflash": { "flagged": 39, "judgments": 180, "rate": 0.21666666666666667 }, "muse-glimmer:30b-q8_0-dflash+think": { "flagged": 32, "judgments": 180, "rate": 0.17777777777777778 }, "nemotron-3.5-lightning:30b-a3b": { "flagged": 27, "judgments": 180, "rate": 0.15 }, "qwen3.6:27b": { "flagged": 2, "judgments": 180, "rate": 0.011111111111111112 } }, "caveats": [ "first-attempt only: production's corrective-retry ladder is not simulated", "one of the two judge families also wrote the questions and this article" ] }, "moonshot": { "status": "DESCRIPTIVE", "primary_metric": "overall pairwise win-rate, cluster-bootstrap CI over items", "units_complete": 270, "units_incomplete": 0, "incomplete_units": [], "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "all_items": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 61.5, "comparisons": 90, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.6111111111111112, 0.75 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 57.0, "comparisons": 90, "win_rate": 0.6333333333333333, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.7444444444444445 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 52.0, "comparisons": 90, "win_rate": 0.5777777777777777, "ci95_cluster_bootstrap": [ 0.49444444444444446, 0.65 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 51.5, "comparisons": 90, "win_rate": 0.5722222222222222, "ci95_cluster_bootstrap": [ 0.42777777777777776, 0.7111111111111111 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 37.5, "comparisons": 90, "win_rate": 0.4166666666666667, "ci95_cluster_bootstrap": [ 0.3333333333333333, 0.49444444444444446 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 10.5, "comparisons": 90, "win_rate": 0.11666666666666667, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.22777777777777777 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 62, "swap_discordant_rate": 0.22962962962962963, "bradley_terry_descriptive": { "qwen3.6:27b": 1.6789318274398628, "gemma4:12b": 1.3779034877162533, "gemma4:26b": 1.1131643991999263, "muse-glimmer:30b-q8_0-dflash": 1.0898857196056966, "nemotron-3.5-lightning:30b-a3b": 0.600262831187296, "muse-glimmer:30b-q8_0-dflash+think": 0.13985173485096528 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 61.0, "comparisons": 90, "win_rate": 0.6777777777777778, "ci95_cluster_bootstrap": [ 0.6166666666666667, 0.7333333333333333 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 57.5, "comparisons": 90, "win_rate": 0.6388888888888888, "ci95_cluster_bootstrap": [ 0.5388888888888889, 0.75 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 54.0, "comparisons": 90, "win_rate": 0.6, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.65 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 51.5, "comparisons": 90, "win_rate": 0.5722222222222222, "ci95_cluster_bootstrap": [ 0.4222222222222222, 0.7166666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.0, "comparisons": 90, "win_rate": 0.4, "ci95_cluster_bootstrap": [ 0.3111111111111111, 0.4888888888888889 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 10.0, "comparisons": 90, "win_rate": 0.1111111111111111, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.21666666666666667 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 65, "swap_discordant_rate": 0.24074074074074073, "bradley_terry_descriptive": { "qwen3.6:27b": 1.6329460224630845, "gemma4:12b": 1.399091611614086, "gemma4:26b": 1.2027859316996585, "muse-glimmer:30b-q8_0-dflash": 1.0810047816967754, "nemotron-3.5-lightning:30b-a3b": 0.5536970492458422, "muse-glimmer:30b-q8_0-dflash+think": 0.13047460328055369 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 61.5, "comparisons": 90, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.6111111111111112, 0.7611111111111111 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 59.0, "comparisons": 90, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.5944444444444444, 0.7166666666666667 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.5666666666666667, 0.6833333333333333 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 44.5, "comparisons": 90, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.37777777777777777, 0.5944444444444444 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 37.0, "comparisons": 90, "win_rate": 0.4111111111111111, "ci95_cluster_bootstrap": [ 0.31666666666666665, 0.5 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 11.5, "comparisons": 90, "win_rate": 0.12777777777777777, "ci95_cluster_bootstrap": [ 0.03333333333333333, 0.24444444444444444 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 70, "swap_discordant_rate": 0.25925925925925924, "bradley_terry_descriptive": { "gemma4:12b": 1.6589819092851659, "gemma4:26b": 1.4841802515848521, "qwen3.6:27b": 1.3306794672742293, "muse-glimmer:30b-q8_0-dflash": 0.7986134999804685, "nemotron-3.5-lightning:30b-a3b": 0.5779067502079287, "muse-glimmer:30b-q8_0-dflash+think": 0.14963812166735546 } } }, "gate_passing_only": { "n_comparisons": 117, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 35.0, "comparisons": 56, "win_rate": 0.625, "ci95_cluster_bootstrap": [ 0.5462962962962963, 0.7053571428571429 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 7.0, "comparisons": 12, "win_rate": 0.5833333333333334, "ci95_cluster_bootstrap": [ 0.125, 0.875 ], "effective_n_items": 3 }, "gemma4:12b": { "wins": 32.5, "comparisons": 56, "win_rate": 0.5803571428571429, "ci95_cluster_bootstrap": [ 0.46078431372549017, 0.7053571428571429 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 27.5, "comparisons": 56, "win_rate": 0.49107142857142855, "ci95_cluster_bootstrap": [ 0.40384615384615385, 0.576271186440678 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 15.0, "comparisons": 54, "win_rate": 0.2777777777777778, "ci95_cluster_bootstrap": [ 0.16666666666666666, 0.3979591836734694 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 41, "swap_discordant_rate": 0.3504273504273504, "bradley_terry_descriptive": { "qwen3.6:27b": 1.3278455778710223, "muse-glimmer:30b-q8_0-dflash+think": 1.2352768091481607, "gemma4:12b": 1.1493606384346846, "gemma4:26b": 0.8655548305564146, "nemotron-3.5-lightning:30b-a3b": 0.4219621439897182 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 34.0, "comparisons": 56, "win_rate": 0.6071428571428571, "ci95_cluster_bootstrap": [ 0.5545454545454546, 0.6636363636363637 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 7.0, "comparisons": 12, "win_rate": 0.5833333333333334, "ci95_cluster_bootstrap": [ 0.125, 0.875 ], "effective_n_items": 3 }, "gemma4:12b": { "wins": 32.5, "comparisons": 56, "win_rate": 0.5803571428571429, "ci95_cluster_bootstrap": [ 0.4528301886792453, 0.7053571428571429 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 28.5, "comparisons": 56, "win_rate": 0.5089285714285714, "ci95_cluster_bootstrap": [ 0.4375, 0.576271186440678 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 15.0, "comparisons": 54, "win_rate": 0.2777777777777778, "ci95_cluster_bootstrap": [ 0.16071428571428573, 0.4107142857142857 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 44, "swap_discordant_rate": 0.37606837606837606, "bradley_terry_descriptive": { "qwen3.6:27b": 1.2584992975017915, "muse-glimmer:30b-q8_0-dflash+think": 1.2408720976371768, "gemma4:12b": 1.1547752501568251, "gemma4:26b": 0.9208124362041058, "nemotron-3.5-lightning:30b-a3b": 0.4250409185001012 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 34.5, "comparisons": 56, "win_rate": 0.6160714285714286, "ci95_cluster_bootstrap": [ 0.5277777777777778, 0.7 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 7.0, "comparisons": 12, "win_rate": 0.5833333333333334, "ci95_cluster_bootstrap": [ 0.0, 1.0 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 32.5, "comparisons": 56, "win_rate": 0.5803571428571429, "ci95_cluster_bootstrap": [ 0.4818181818181818, 0.675 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 28.0, "comparisons": 56, "win_rate": 0.5, "ci95_cluster_bootstrap": [ 0.44166666666666665, 0.5545454545454546 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 15.0, "comparisons": 54, "win_rate": 0.2777777777777778, "ci95_cluster_bootstrap": [ 0.15517241379310345, 0.42045454545454547 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 43, "swap_discordant_rate": 0.36752136752136755, "bradley_terry_descriptive": { "gemma4:12b": 1.2927985008631715, "muse-glimmer:30b-q8_0-dflash+think": 1.2382995136290635, "gemma4:26b": 1.152285568637791, "qwen3.6:27b": 0.8929925034130667, "nemotron-3.5-lightning:30b-a3b": 0.4236239134569076 } } }, "per_family": { "moonshot": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 61.5, "comparisons": 90, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.6111111111111112, 0.75 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 57.0, "comparisons": 90, "win_rate": 0.6333333333333333, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.7444444444444445 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 52.0, "comparisons": 90, "win_rate": 0.5777777777777777, "ci95_cluster_bootstrap": [ 0.49444444444444446, 0.65 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 51.5, "comparisons": 90, "win_rate": 0.5722222222222222, "ci95_cluster_bootstrap": [ 0.42777777777777776, 0.7111111111111111 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 37.5, "comparisons": 90, "win_rate": 0.4166666666666667, "ci95_cluster_bootstrap": [ 0.3333333333333333, 0.49444444444444446 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 10.5, "comparisons": 90, "win_rate": 0.11666666666666667, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.22777777777777777 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 62, "swap_discordant_rate": 0.22962962962962963, "bradley_terry_descriptive": { "qwen3.6:27b": 1.6789318274398628, "gemma4:12b": 1.3779034877162533, "gemma4:26b": 1.1131643991999263, "muse-glimmer:30b-q8_0-dflash": 1.0898857196056966, "nemotron-3.5-lightning:30b-a3b": 0.600262831187296, "muse-glimmer:30b-q8_0-dflash+think": 0.13985173485096528 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 61.0, "comparisons": 90, "win_rate": 0.6777777777777778, "ci95_cluster_bootstrap": [ 0.6166666666666667, 0.7333333333333333 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 57.5, "comparisons": 90, "win_rate": 0.6388888888888888, "ci95_cluster_bootstrap": [ 0.5388888888888889, 0.75 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 54.0, "comparisons": 90, "win_rate": 0.6, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.65 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 51.5, "comparisons": 90, "win_rate": 0.5722222222222222, "ci95_cluster_bootstrap": [ 0.4222222222222222, 0.7166666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.0, "comparisons": 90, "win_rate": 0.4, "ci95_cluster_bootstrap": [ 0.3111111111111111, 0.4888888888888889 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 10.0, "comparisons": 90, "win_rate": 0.1111111111111111, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.21666666666666667 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 65, "swap_discordant_rate": 0.24074074074074073, "bradley_terry_descriptive": { "qwen3.6:27b": 1.6329460224630845, "gemma4:12b": 1.399091611614086, "gemma4:26b": 1.2027859316996585, "muse-glimmer:30b-q8_0-dflash": 1.0810047816967754, "nemotron-3.5-lightning:30b-a3b": 0.5536970492458422, "muse-glimmer:30b-q8_0-dflash+think": 0.13047460328055369 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 61.5, "comparisons": 90, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.6111111111111112, 0.7611111111111111 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 59.0, "comparisons": 90, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.5944444444444444, 0.7166666666666667 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.5666666666666667, 0.6833333333333333 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 44.5, "comparisons": 90, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.37777777777777777, 0.5944444444444444 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 37.0, "comparisons": 90, "win_rate": 0.4111111111111111, "ci95_cluster_bootstrap": [ 0.31666666666666665, 0.5 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 11.5, "comparisons": 90, "win_rate": 0.12777777777777777, "ci95_cluster_bootstrap": [ 0.03333333333333333, 0.24444444444444444 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 70, "swap_discordant_rate": 0.25925925925925924, "bradley_terry_descriptive": { "gemma4:12b": 1.6589819092851659, "gemma4:26b": 1.4841802515848521, "qwen3.6:27b": 1.3306794672742293, "muse-glimmer:30b-q8_0-dflash": 0.7986134999804685, "nemotron-3.5-lightning:30b-a3b": 0.5779067502079287, "muse-glimmer:30b-q8_0-dflash+think": 0.14963812166735546 } } } }, "inter_panel_agreement": null, "position_bias_per_judge": { "cloud-kimi-k3": { "slot_A_chosen": 242, "judgments": 540, "rate": 0.44814814814814813, "ci95_wilson": [ 0.40671612324247014, 0.490312717508219 ], "note": "0.50 is unbiased; the design is fully counterbalanced" } }, "canon_flag_rate_per_model": { "gemma4:12b": { "flagged": 10, "judgments": 180, "rate": 0.05555555555555555 }, "gemma4:26b": { "flagged": 13, "judgments": 180, "rate": 0.07222222222222222 }, "muse-glimmer:30b-q8_0-dflash": { "flagged": 26, "judgments": 180, "rate": 0.14444444444444443 }, "muse-glimmer:30b-q8_0-dflash+think": { "flagged": 15, "judgments": 180, "rate": 0.08333333333333333 }, "nemotron-3.5-lightning:30b-a3b": { "flagged": 35, "judgments": 180, "rate": 0.19444444444444445 }, "qwen3.6:27b": { "flagged": 22, "judgments": 180, "rate": 0.12222222222222222 } }, "caveats": [ "first-attempt only: production's corrective-retry ladder is not simulated", "one of the two judge families also wrote the questions and this article" ] }, "nvidia": { "status": "DESCRIPTIVE", "primary_metric": "overall pairwise win-rate, cluster-bootstrap CI over items", "units_complete": 270, "units_incomplete": 0, "incomplete_units": [], "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "all_items": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 62.5, "comparisons": 90, "win_rate": 0.6944444444444444, "ci95_cluster_bootstrap": [ 0.6222222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.4722222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4444444444444444, 0.6666666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 49.0, "comparisons": 90, "win_rate": 0.5444444444444444, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.0, "comparisons": 90, "win_rate": 0.4777777777777778, "ci95_cluster_bootstrap": [ 0.39444444444444443, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.5, "comparisons": 90, "win_rate": 0.10555555555555556, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.2 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 68, "swap_discordant_rate": 0.2518518518518518, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7612706611740483, "muse-glimmer:30b-q8_0-dflash": 1.353474723302609, "gemma4:26b": 1.006253799861518, "gemma4:12b": 0.9853303197996982, "nemotron-3.5-lightning:30b-a3b": 0.7649932026090543, "muse-glimmer:30b-q8_0-dflash+think": 0.128677293253072 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 62.5, "comparisons": 90, "win_rate": 0.6944444444444444, "ci95_cluster_bootstrap": [ 0.6222222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.4722222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4444444444444444, 0.6666666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 49.0, "comparisons": 90, "win_rate": 0.5444444444444444, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.0, "comparisons": 90, "win_rate": 0.4777777777777778, "ci95_cluster_bootstrap": [ 0.39444444444444443, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.5, "comparisons": 90, "win_rate": 0.10555555555555556, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.2 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 68, "swap_discordant_rate": 0.2518518518518518, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7612706611740483, "muse-glimmer:30b-q8_0-dflash": 1.353474723302609, "gemma4:26b": 1.006253799861518, "gemma4:12b": 0.9853303197996982, "nemotron-3.5-lightning:30b-a3b": 0.7649932026090543, "muse-glimmer:30b-q8_0-dflash+think": 0.128677293253072 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 61.5, "comparisons": 90, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.6, 0.7611111111111111 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.4666666666666667, 0.7722222222222223 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 50.0, "comparisons": 90, "win_rate": 0.5555555555555556, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6666666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 49.0, "comparisons": 90, "win_rate": 0.5444444444444444, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.0, "comparisons": 90, "win_rate": 0.4777777777777778, "ci95_cluster_bootstrap": [ 0.39444444444444443, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 10.0, "comparisons": 90, "win_rate": 0.1111111111111111, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.21666666666666667 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 67, "swap_discordant_rate": 0.24814814814814815, "bradley_terry_descriptive": { "qwen3.6:27b": 1.694879422594233, "muse-glimmer:30b-q8_0-dflash": 1.3635753512429736, "gemma4:26b": 1.0367613408554517, "gemma4:12b": 0.9943212893108232, "nemotron-3.5-lightning:30b-a3b": 0.7731464765005158, "muse-glimmer:30b-q8_0-dflash+think": 0.13731611949600242 } } }, "gate_passing_only": { "n_comparisons": 117, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 36.5, "comparisons": 56, "win_rate": 0.6517857142857143, "ci95_cluster_bootstrap": [ 0.576271186440678, 0.7403846153846154 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 7.0, "comparisons": 12, "win_rate": 0.5833333333333334, "ci95_cluster_bootstrap": [ 0.125, 0.875 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 27.0, "comparisons": 56, "win_rate": 0.48214285714285715, "ci95_cluster_bootstrap": [ 0.375, 0.5892857142857143 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 26.5, "comparisons": 56, "win_rate": 0.4732142857142857, "ci95_cluster_bootstrap": [ 0.3333333333333333, 0.5909090909090909 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 20.0, "comparisons": 54, "win_rate": 0.37037037037037035, "ci95_cluster_bootstrap": [ 0.25892857142857145, 0.49038461538461536 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 46, "swap_discordant_rate": 0.39316239316239315, "bradley_terry_descriptive": { "qwen3.6:27b": 1.4613758427233365, "muse-glimmer:30b-q8_0-dflash+think": 1.2525971007543037, "gemma4:26b": 0.8558484057524616, "gemma4:12b": 0.8324329645330534, "nemotron-3.5-lightning:30b-a3b": 0.5977456862368445 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 36.5, "comparisons": 56, "win_rate": 0.6517857142857143, "ci95_cluster_bootstrap": [ 0.576271186440678, 0.7403846153846154 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 7.0, "comparisons": 12, "win_rate": 0.5833333333333334, "ci95_cluster_bootstrap": [ 0.125, 0.875 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 27.0, "comparisons": 56, "win_rate": 0.48214285714285715, "ci95_cluster_bootstrap": [ 0.375, 0.5892857142857143 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 26.5, "comparisons": 56, "win_rate": 0.4732142857142857, "ci95_cluster_bootstrap": [ 0.3333333333333333, 0.5909090909090909 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 20.0, "comparisons": 54, "win_rate": 0.37037037037037035, "ci95_cluster_bootstrap": [ 0.25892857142857145, 0.49038461538461536 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 46, "swap_discordant_rate": 0.39316239316239315, "bradley_terry_descriptive": { "qwen3.6:27b": 1.4613758427233365, "muse-glimmer:30b-q8_0-dflash+think": 1.2525971007543037, "gemma4:26b": 0.8558484057524616, "gemma4:12b": 0.8324329645330534, "nemotron-3.5-lightning:30b-a3b": 0.5977456862368445 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 35.5, "comparisons": 56, "win_rate": 0.6339285714285714, "ci95_cluster_bootstrap": [ 0.5431034482758621, 0.7352941176470589 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 7.5, "comparisons": 12, "win_rate": 0.625, "ci95_cluster_bootstrap": [ 0.125, 1.0 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 27.5, "comparisons": 56, "win_rate": 0.49107142857142855, "ci95_cluster_bootstrap": [ 0.37962962962962965, 0.5982142857142857 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 26.5, "comparisons": 56, "win_rate": 0.4732142857142857, "ci95_cluster_bootstrap": [ 0.3305084745762712, 0.5925925925925926 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 20.0, "comparisons": 54, "win_rate": 0.37037037037037035, "ci95_cluster_bootstrap": [ 0.25892857142857145, 0.49038461538461536 ], "effective_n_items": 9 } }, "judged_units": 117, "unjudged_units": 0, "swap_discordant": 45, "swap_discordant_rate": 0.38461538461538464, "bradley_terry_descriptive": { "muse-glimmer:30b-q8_0-dflash+think": 1.4349471638303355, "qwen3.6:27b": 1.3315486114591744, "gemma4:26b": 0.8504743863557226, "gemma4:12b": 0.8046733187090338, "nemotron-3.5-lightning:30b-a3b": 0.5783565196457333 } } }, "per_family": { "nvidia": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 62.5, "comparisons": 90, "win_rate": 0.6944444444444444, "ci95_cluster_bootstrap": [ 0.6222222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.4722222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4444444444444444, 0.6666666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 49.0, "comparisons": 90, "win_rate": 0.5444444444444444, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.0, "comparisons": 90, "win_rate": 0.4777777777777778, "ci95_cluster_bootstrap": [ 0.39444444444444443, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.5, "comparisons": 90, "win_rate": 0.10555555555555556, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.2 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 68, "swap_discordant_rate": 0.2518518518518518, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7612706611740483, "muse-glimmer:30b-q8_0-dflash": 1.353474723302609, "gemma4:26b": 1.006253799861518, "gemma4:12b": 0.9853303197996982, "nemotron-3.5-lightning:30b-a3b": 0.7649932026090543, "muse-glimmer:30b-q8_0-dflash+think": 0.128677293253072 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 62.5, "comparisons": 90, "win_rate": 0.6944444444444444, "ci95_cluster_bootstrap": [ 0.6222222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.4722222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4444444444444444, 0.6666666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 49.0, "comparisons": 90, "win_rate": 0.5444444444444444, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.0, "comparisons": 90, "win_rate": 0.4777777777777778, "ci95_cluster_bootstrap": [ 0.39444444444444443, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.5, "comparisons": 90, "win_rate": 0.10555555555555556, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.2 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 68, "swap_discordant_rate": 0.2518518518518518, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7612706611740483, "muse-glimmer:30b-q8_0-dflash": 1.353474723302609, "gemma4:26b": 1.006253799861518, "gemma4:12b": 0.9853303197996982, "nemotron-3.5-lightning:30b-a3b": 0.7649932026090543, "muse-glimmer:30b-q8_0-dflash+think": 0.128677293253072 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 61.5, "comparisons": 90, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.6, 0.7611111111111111 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.4666666666666667, 0.7722222222222223 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 50.0, "comparisons": 90, "win_rate": 0.5555555555555556, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6666666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 49.0, "comparisons": 90, "win_rate": 0.5444444444444444, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.0, "comparisons": 90, "win_rate": 0.4777777777777778, "ci95_cluster_bootstrap": [ 0.39444444444444443, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 10.0, "comparisons": 90, "win_rate": 0.1111111111111111, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.21666666666666667 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 67, "swap_discordant_rate": 0.24814814814814815, "bradley_terry_descriptive": { "qwen3.6:27b": 1.694879422594233, "muse-glimmer:30b-q8_0-dflash": 1.3635753512429736, "gemma4:26b": 1.0367613408554517, "gemma4:12b": 0.9943212893108232, "nemotron-3.5-lightning:30b-a3b": 0.7731464765005158, "muse-glimmer:30b-q8_0-dflash+think": 0.13731611949600242 } } } }, "inter_panel_agreement": null, "position_bias_per_judge": { "cloud-nemotron-3-ultra": { "slot_A_chosen": 288, "judgments": 540, "rate": 0.5333333333333333, "ci95_wilson": [ 0.49116742863665536, 0.5750283165950446 ], "note": "0.50 is unbiased; the design is fully counterbalanced" } }, "canon_flag_rate_per_model": { "gemma4:12b": { "flagged": 3, "judgments": 180, "rate": 0.016666666666666666 }, "gemma4:26b": { "flagged": 0, "judgments": 180, "rate": 0.0 }, "muse-glimmer:30b-q8_0-dflash": { "flagged": 24, "judgments": 180, "rate": 0.13333333333333333 }, "muse-glimmer:30b-q8_0-dflash+think": { "flagged": 10, "judgments": 180, "rate": 0.05555555555555555 }, "nemotron-3.5-lightning:30b-a3b": { "flagged": 27, "judgments": 180, "rate": 0.15 }, "qwen3.6:27b": { "flagged": 1, "judgments": 180, "rate": 0.005555555555555556 } }, "caveats": [ "first-attempt only: production's corrective-retry ladder is not simulated", "one of the two judge families also wrote the questions and this article" ] } }, "combined_all": { "status": "DESCRIPTIVE", "primary_metric": "overall pairwise win-rate, cluster-bootstrap CI over items", "units_complete": 2160, "units_incomplete": 0, "incomplete_units": [], "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "all_items": { "n_comparisons": 2160, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 517.5, "comparisons": 720, "win_rate": 0.71875, "ci95_cluster_bootstrap": [ 0.6472222222222223, 0.7902777777777777 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 466.0, "comparisons": 720, "win_rate": 0.6472222222222223, "ci95_cluster_bootstrap": [ 0.58125, 0.7222222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 415.5, "comparisons": 720, "win_rate": 0.5770833333333333, "ci95_cluster_bootstrap": [ 0.41805555555555557, 0.7243055555555555 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 391.5, "comparisons": 720, "win_rate": 0.54375, "ci95_cluster_bootstrap": [ 0.4673611111111111, 0.63125 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 290.5, "comparisons": 720, "win_rate": 0.40347222222222223, "ci95_cluster_bootstrap": [ 0.33194444444444443, 0.48333333333333334 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 79.0, "comparisons": 720, "win_rate": 0.10972222222222222, "ci95_cluster_bootstrap": [ 0.025694444444444443, 0.21388888888888888 ], "effective_n_items": 9 } }, "judged_units": 2160, "unjudged_units": 0, "swap_discordant": 317, "swap_discordant_rate": 0.14675925925925926, "bradley_terry_descriptive": { "qwen3.6:27b": 1.896483280604238, "gemma4:12b": 1.4146020400534605, "muse-glimmer:30b-q8_0-dflash": 1.07474465594216, "gemma4:26b": 0.9448904836214609, "nemotron-3.5-lightning:30b-a3b": 0.544813836606797, "muse-glimmer:30b-q8_0-dflash+think": 0.12446570317188387 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 503.5, "comparisons": 720, "win_rate": 0.6993055555555555, "ci95_cluster_bootstrap": [ 0.6291666666666667, 0.7756944444444445 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 465.5, "comparisons": 720, "win_rate": 0.6465277777777778, "ci95_cluster_bootstrap": [ 0.5805555555555556, 0.7236111111111111 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 420.5, "comparisons": 720, "win_rate": 0.5840277777777778, "ci95_cluster_bootstrap": [ 0.5048611111111111, 0.6673611111111111 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 408.0, "comparisons": 720, "win_rate": 0.5666666666666667, "ci95_cluster_bootstrap": [ 0.4201388888888889, 0.70625 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 284.5, "comparisons": 720, "win_rate": 0.3951388888888889, "ci95_cluster_bootstrap": [ 0.3284722222222222, 0.4736111111111111 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 78.0, "comparisons": 720, "win_rate": 0.10833333333333334, "ci95_cluster_bootstrap": [ 0.025, 0.21388888888888888 ], "effective_n_items": 9 } }, "judged_units": 2160, "unjudged_units": 0, "swap_discordant": 329, "swap_discordant_rate": 0.15231481481481482, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7627644479717353, "gemma4:12b": 1.423657266437585, "gemma4:26b": 1.1148705166090818, "muse-glimmer:30b-q8_0-dflash": 1.0425511865984793, "nemotron-3.5-lightning:30b-a3b": 0.5320450578319383, "muse-glimmer:30b-q8_0-dflash+think": 0.12411152455118066 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 498.0, "comparisons": 720, "win_rate": 0.6916666666666667, "ci95_cluster_bootstrap": [ 0.625, 0.7611111111111111 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 492.0, "comparisons": 720, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.5861111111111111, 0.7784722222222222 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 413.0, "comparisons": 720, "win_rate": 0.5736111111111111, "ci95_cluster_bootstrap": [ 0.48541666666666666, 0.6722222222222223 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 374.0, "comparisons": 720, "win_rate": 0.5194444444444445, "ci95_cluster_bootstrap": [ 0.37222222222222223, 0.6534722222222222 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 303.0, "comparisons": 720, "win_rate": 0.42083333333333334, "ci95_cluster_bootstrap": [ 0.32708333333333334, 0.5277777777777778 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 80.0, "comparisons": 720, "win_rate": 0.1111111111111111, "ci95_cluster_bootstrap": [ 0.027083333333333334, 0.21180555555555555 ], "effective_n_items": 9 } }, "judged_units": 2160, "unjudged_units": 0, "swap_discordant": 378, "swap_discordant_rate": 0.175, "bradley_terry_descriptive": { "gemma4:12b": 1.7022299934285372, "qwen3.6:27b": 1.645131199093071, "gemma4:26b": 1.0677321280513103, "muse-glimmer:30b-q8_0-dflash": 0.8670001142192321, "nemotron-3.5-lightning:30b-a3b": 0.5900440588914353, "muse-glimmer:30b-q8_0-dflash+think": 0.12786250631641438 } } }, "gate_passing_only": { "n_comparisons": 936, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 306.5, "comparisons": 448, "win_rate": 0.6841517857142857, "ci95_cluster_bootstrap": [ 0.60625, 0.7794811320754716 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 263.5, "comparisons": 448, "win_rate": 0.5881696428571429, "ci95_cluster_bootstrap": [ 0.509090909090909, 0.6640625 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 55.0, "comparisons": 96, "win_rate": 0.5729166666666666, "ci95_cluster_bootstrap": [ 0.109375, 0.84375 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 198.0, "comparisons": 448, "win_rate": 0.4419642857142857, "ci95_cluster_bootstrap": [ 0.35096153846153844, 0.5366379310344828 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 113.0, "comparisons": 432, "win_rate": 0.26157407407407407, "ci95_cluster_bootstrap": [ 0.1683673469387755, 0.3665254237288136 ], "effective_n_items": 9 } }, "judged_units": 936, "unjudged_units": 0, "swap_discordant": 216, "swap_discordant_rate": 0.23076923076923078, "bradley_terry_descriptive": { "qwen3.6:27b": 1.5925313402528747, "muse-glimmer:30b-q8_0-dflash+think": 1.1570330165964575, "gemma4:12b": 1.1529803796772993, "gemma4:26b": 0.7172080698151859, "nemotron-3.5-lightning:30b-a3b": 0.3802471936581822 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 291.5, "comparisons": 448, "win_rate": 0.6506696428571429, "ci95_cluster_bootstrap": [ 0.5743534482758621, 0.7536764705882353 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 55.5, "comparisons": 96, "win_rate": 0.578125, "ci95_cluster_bootstrap": [ 0.109375, 0.859375 ], "effective_n_items": 3 }, "gemma4:12b": { "wins": 257.0, "comparisons": 448, "win_rate": 0.5736607142857143, "ci95_cluster_bootstrap": [ 0.4888392857142857, 0.65625 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 221.0, "comparisons": 448, "win_rate": 0.49330357142857145, "ci95_cluster_bootstrap": [ 0.4105392156862745, 0.5825892857142857 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 111.0, "comparisons": 432, "win_rate": 0.2569444444444444, "ci95_cluster_bootstrap": [ 0.15625, 0.36853448275862066 ], "effective_n_items": 9 } }, "judged_units": 936, "unjudged_units": 0, "swap_discordant": 214, "swap_discordant_rate": 0.22863247863247863, "bradley_terry_descriptive": { "qwen3.6:27b": 1.4376071994012485, "muse-glimmer:30b-q8_0-dflash+think": 1.1997992241314828, "gemma4:12b": 1.1165635373887446, "gemma4:26b": 0.8628531108444145, "nemotron-3.5-lightning:30b-a3b": 0.38317692823410954 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 280.5, "comparisons": 448, "win_rate": 0.6261160714285714, "ci95_cluster_bootstrap": [ 0.5301724137931034, 0.7182017543859649 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 272.0, "comparisons": 448, "win_rate": 0.6071428571428571, "ci95_cluster_bootstrap": [ 0.5046296296296297, 0.7199519230769231 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 55.0, "comparisons": 96, "win_rate": 0.5729166666666666, "ci95_cluster_bootstrap": [ 0.09375, 0.875 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 211.0, "comparisons": 448, "win_rate": 0.47098214285714285, "ci95_cluster_bootstrap": [ 0.3587962962962963, 0.5809426229508197 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 117.5, "comparisons": 432, "win_rate": 0.27199074074074076, "ci95_cluster_bootstrap": [ 0.15808823529411764, 0.4140625 ], "effective_n_items": 9 } }, "judged_units": 936, "unjudged_units": 0, "swap_discordant": 231, "swap_discordant_rate": 0.2467948717948718, "bradley_terry_descriptive": { "gemma4:12b": 1.3359075326345542, "qwen3.6:27b": 1.2555129240608185, "muse-glimmer:30b-q8_0-dflash+think": 1.1853740086948528, "gemma4:26b": 0.8118485721300612, "nemotron-3.5-lightning:30b-a3b": 0.4113569624797128 } } }, "per_family": { "anthropic": { "n_comparisons": 1080, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 256.0, "comparisons": 360, "win_rate": 0.7111111111111111, "ci95_cluster_bootstrap": [ 0.6236111111111111, 0.8041666666666667 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 246.5, "comparisons": 360, "win_rate": 0.6847222222222222, "ci95_cluster_bootstrap": [ 0.6152777777777778, 0.7625 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 212.0, "comparisons": 360, "win_rate": 0.5888888888888889, "ci95_cluster_bootstrap": [ 0.4083333333333333, 0.7597222222222222 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 194.0, "comparisons": 360, "win_rate": 0.5388888888888889, "ci95_cluster_bootstrap": [ 0.45, 0.6430555555555556 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 129.5, "comparisons": 360, "win_rate": 0.3597222222222222, "ci95_cluster_bootstrap": [ 0.25972222222222224, 0.4666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 42.0, "comparisons": 360, "win_rate": 0.11666666666666667, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.23055555555555557 ], "effective_n_items": 9 } }, "judged_units": 1080, "unjudged_units": 0, "swap_discordant": 93, "swap_discordant_rate": 0.08611111111111111, "bradley_terry_descriptive": { "qwen3.6:27b": 1.8092622737500086, "gemma4:12b": 1.619394337080983, "muse-glimmer:30b-q8_0-dflash": 1.1012403214221167, "gemma4:26b": 0.9051088790344628, "nemotron-3.5-lightning:30b-a3b": 0.4394161683038068, "muse-glimmer:30b-q8_0-dflash+think": 0.12557802040862254 } }, "in_character_secondary": { "rates": { "gemma4:12b": { "wins": 245.5, "comparisons": 360, "win_rate": 0.6819444444444445, "ci95_cluster_bootstrap": [ 0.6083333333333333, 0.7638888888888888 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 242.5, "comparisons": 360, "win_rate": 0.6736111111111112, "ci95_cluster_bootstrap": [ 0.5791666666666667, 0.7819444444444444 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 221.0, "comparisons": 360, "win_rate": 0.6138888888888889, "ci95_cluster_bootstrap": [ 0.5041666666666667, 0.7180555555555556 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 204.0, "comparisons": 360, "win_rate": 0.5666666666666667, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.7180555555555556 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 125.0, "comparisons": 360, "win_rate": 0.3472222222222222, "ci95_cluster_bootstrap": [ 0.24861111111111112, 0.4625 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 42.0, "comparisons": 360, "win_rate": 0.11666666666666667, "ci95_cluster_bootstrap": [ 0.02638888888888889, 0.23194444444444445 ], "effective_n_items": 9 } }, "judged_units": 1080, "unjudged_units": 0, "swap_discordant": 103, "swap_discordant_rate": 0.09537037037037037, "bradley_terry_descriptive": { "gemma4:12b": 1.6225302771300043, "qwen3.6:27b": 1.5679760181212143, "gemma4:26b": 1.2337623967574318, "muse-glimmer:30b-q8_0-dflash": 1.024952389858144, "nemotron-3.5-lightning:30b-a3b": 0.42339667198658576, "muse-glimmer:30b-q8_0-dflash+think": 0.12738224614662003 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 273.5, "comparisons": 360, "win_rate": 0.7597222222222222, "ci95_cluster_bootstrap": [ 0.6902777777777778, 0.8305555555555556 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 237.5, "comparisons": 360, "win_rate": 0.6597222222222222, "ci95_cluster_bootstrap": [ 0.5444444444444444, 0.7805555555555556 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 207.0, "comparisons": 360, "win_rate": 0.575, "ci95_cluster_bootstrap": [ 0.4722222222222222, 0.6930555555555555 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 178.0, "comparisons": 360, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.32916666666666666, 0.6430555555555556 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 143.0, "comparisons": 360, "win_rate": 0.3972222222222222, "ci95_cluster_bootstrap": [ 0.27361111111111114, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 41.0, "comparisons": 360, "win_rate": 0.11388888888888889, "ci95_cluster_bootstrap": [ 0.030555555555555555, 0.21666666666666667 ], "effective_n_items": 9 } }, "judged_units": 1080, "unjudged_units": 0, "swap_discordant": 146, "swap_discordant_rate": 0.13518518518518519, "bradley_terry_descriptive": { "gemma4:12b": 2.193600330368878, "qwen3.6:27b": 1.427119984062163, "gemma4:26b": 1.0166697571285352, "muse-glimmer:30b-q8_0-dflash": 0.7408140536357574, "nemotron-3.5-lightning:30b-a3b": 0.5013299683998579, "muse-glimmer:30b-q8_0-dflash+think": 0.12046590640480863 } } }, "deepseek": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 70.0, "comparisons": 90, "win_rate": 0.7777777777777778, "ci95_cluster_bootstrap": [ 0.6888888888888889, 0.85 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 55.0, "comparisons": 90, "win_rate": 0.6111111111111112, "ci95_cluster_bootstrap": [ 0.43333333333333335, 0.7833333333333333 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 54.5, "comparisons": 90, "win_rate": 0.6055555555555555, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.6777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 46.5, "comparisons": 90, "win_rate": 0.5166666666666667, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.6166666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.0, "comparisons": 90, "win_rate": 0.4, "ci95_cluster_bootstrap": [ 0.3277777777777778, 0.4777777777777778 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.0, "comparisons": 90, "win_rate": 0.08888888888888889, "ci95_cluster_bootstrap": [ 0.022222222222222223, 0.16666666666666666 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 44, "swap_discordant_rate": 0.16296296296296298, "bradley_terry_descriptive": { "qwen3.6:27b": 2.355960102294597, "muse-glimmer:30b-q8_0-dflash": 1.150492036910824, "gemma4:12b": 1.125007228072441, "gemma4:26b": 0.788485163033489, "nemotron-3.5-lightning:30b-a3b": 0.4893202215213605, "muse-glimmer:30b-q8_0-dflash+think": 0.09073524816728841 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 70.0, "comparisons": 90, "win_rate": 0.7777777777777778, "ci95_cluster_bootstrap": [ 0.6888888888888889, 0.85 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 55.0, "comparisons": 90, "win_rate": 0.6111111111111112, "ci95_cluster_bootstrap": [ 0.43333333333333335, 0.7833333333333333 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 54.5, "comparisons": 90, "win_rate": 0.6055555555555555, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.6777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 46.5, "comparisons": 90, "win_rate": 0.5166666666666667, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.6166666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.0, "comparisons": 90, "win_rate": 0.4, "ci95_cluster_bootstrap": [ 0.3277777777777778, 0.4777777777777778 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.0, "comparisons": 90, "win_rate": 0.08888888888888889, "ci95_cluster_bootstrap": [ 0.022222222222222223, 0.16666666666666666 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 44, "swap_discordant_rate": 0.16296296296296298, "bradley_terry_descriptive": { "qwen3.6:27b": 2.355960102294597, "muse-glimmer:30b-q8_0-dflash": 1.150492036910824, "gemma4:12b": 1.125007228072441, "gemma4:26b": 0.788485163033489, "nemotron-3.5-lightning:30b-a3b": 0.4893202215213605, "muse-glimmer:30b-q8_0-dflash+think": 0.09073524816728841 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 69.5, "comparisons": 90, "win_rate": 0.7722222222222223, "ci95_cluster_bootstrap": [ 0.6833333333333333, 0.8444444444444444 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 55.5, "comparisons": 90, "win_rate": 0.6166666666666667, "ci95_cluster_bootstrap": [ 0.5555555555555556, 0.6833333333333333 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 53.0, "comparisons": 90, "win_rate": 0.5888888888888889, "ci95_cluster_bootstrap": [ 0.40555555555555556, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 47.0, "comparisons": 90, "win_rate": 0.5222222222222223, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.6277777777777778 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.5, "comparisons": 90, "win_rate": 0.40555555555555556, "ci95_cluster_bootstrap": [ 0.3388888888888889, 0.4722222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.5, "comparisons": 90, "win_rate": 0.09444444444444444, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.17222222222222222 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 46, "swap_discordant_rate": 0.17037037037037037, "bradley_terry_descriptive": { "qwen3.6:27b": 2.3133050654950984, "gemma4:12b": 1.1913395302571046, "muse-glimmer:30b-q8_0-dflash": 1.066267604013111, "gemma4:26b": 0.8189139072686629, "nemotron-3.5-lightning:30b-a3b": 0.5113074928532275, "muse-glimmer:30b-q8_0-dflash+think": 0.09886640011279582 } } }, "mistral": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 67.5, "comparisons": 90, "win_rate": 0.75, "ci95_cluster_bootstrap": [ 0.5888888888888889, 0.8888888888888888 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 59.0, "comparisons": 90, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.55, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4777777777777778, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 44.5, "comparisons": 90, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.37777777777777777, 0.6166666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 40.5, "comparisons": 90, "win_rate": 0.45, "ci95_cluster_bootstrap": [ 0.32222222222222224, 0.5722222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.0, "comparisons": 90, "win_rate": 0.1, "ci95_cluster_bootstrap": [ 0.016666666666666666, 0.2 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 50, "swap_discordant_rate": 0.18518518518518517, "bradley_terry_descriptive": { "qwen3.6:27b": 2.127817082761526, "gemma4:12b": 1.427577184381572, "gemma4:26b": 0.9410661239287288, "nemotron-3.5-lightning:30b-a3b": 0.7576894560973209, "muse-glimmer:30b-q8_0-dflash": 0.6357574227771301, "muse-glimmer:30b-q8_0-dflash+think": 0.11009273005372225 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 67.5, "comparisons": 90, "win_rate": 0.75, "ci95_cluster_bootstrap": [ 0.5888888888888889, 0.8888888888888888 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 59.0, "comparisons": 90, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.55, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4777777777777778, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 44.5, "comparisons": 90, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.37777777777777777, 0.6166666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 41.0, "comparisons": 90, "win_rate": 0.45555555555555555, "ci95_cluster_bootstrap": [ 0.3333333333333333, 0.5722222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 8.5, "comparisons": 90, "win_rate": 0.09444444444444444, "ci95_cluster_bootstrap": [ 0.011111111111111112, 0.19444444444444445 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 49, "swap_discordant_rate": 0.1814814814814815, "bradley_terry_descriptive": { "qwen3.6:27b": 2.1268182748829396, "gemma4:12b": 1.426272093365552, "gemma4:26b": 0.9394024538882703, "nemotron-3.5-lightning:30b-a3b": 0.7558367101275903, "muse-glimmer:30b-q8_0-dflash": 0.6479866843394327, "muse-glimmer:30b-q8_0-dflash+think": 0.10368378339621516 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 67.0, "comparisons": 90, "win_rate": 0.7444444444444445, "ci95_cluster_bootstrap": [ 0.5944444444444444, 0.8777777777777778 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 58.5, "comparisons": 90, "win_rate": 0.65, "ci95_cluster_bootstrap": [ 0.5666666666666667, 0.7388888888888889 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 50.0, "comparisons": 90, "win_rate": 0.5555555555555556, "ci95_cluster_bootstrap": [ 0.4777777777777778, 0.6444444444444445 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.5, "comparisons": 90, "win_rate": 0.48333333333333334, "ci95_cluster_bootstrap": [ 0.37222222222222223, 0.6055555555555555 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 42.0, "comparisons": 90, "win_rate": 0.4666666666666667, "ci95_cluster_bootstrap": [ 0.35, 0.5833333333333334 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.0, "comparisons": 90, "win_rate": 0.1, "ci95_cluster_bootstrap": [ 0.005555555555555556, 0.2222222222222222 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 49, "swap_discordant_rate": 0.1814814814814815, "bradley_terry_descriptive": { "qwen3.6:27b": 2.0894481187390515, "gemma4:12b": 1.407359006484484, "gemma4:26b": 0.9711089616319206, "nemotron-3.5-lightning:30b-a3b": 0.7333540469207611, "muse-glimmer:30b-q8_0-dflash": 0.6869006853503978, "muse-glimmer:30b-q8_0-dflash+think": 0.11182918087338507 } } }, "moonshot": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 61.5, "comparisons": 90, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.6111111111111112, 0.75 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 57.0, "comparisons": 90, "win_rate": 0.6333333333333333, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.7444444444444445 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 52.0, "comparisons": 90, "win_rate": 0.5777777777777777, "ci95_cluster_bootstrap": [ 0.49444444444444446, 0.65 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 51.5, "comparisons": 90, "win_rate": 0.5722222222222222, "ci95_cluster_bootstrap": [ 0.42777777777777776, 0.7111111111111111 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 37.5, "comparisons": 90, "win_rate": 0.4166666666666667, "ci95_cluster_bootstrap": [ 0.3333333333333333, 0.49444444444444446 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 10.5, "comparisons": 90, "win_rate": 0.11666666666666667, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.22777777777777777 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 62, "swap_discordant_rate": 0.22962962962962963, "bradley_terry_descriptive": { "qwen3.6:27b": 1.6789318274398628, "gemma4:12b": 1.3779034877162533, "gemma4:26b": 1.1131643991999263, "muse-glimmer:30b-q8_0-dflash": 1.0898857196056966, "nemotron-3.5-lightning:30b-a3b": 0.600262831187296, "muse-glimmer:30b-q8_0-dflash+think": 0.13985173485096528 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 61.0, "comparisons": 90, "win_rate": 0.6777777777777778, "ci95_cluster_bootstrap": [ 0.6166666666666667, 0.7333333333333333 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 57.5, "comparisons": 90, "win_rate": 0.6388888888888888, "ci95_cluster_bootstrap": [ 0.5388888888888889, 0.75 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 54.0, "comparisons": 90, "win_rate": 0.6, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.65 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 51.5, "comparisons": 90, "win_rate": 0.5722222222222222, "ci95_cluster_bootstrap": [ 0.4222222222222222, 0.7166666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 36.0, "comparisons": 90, "win_rate": 0.4, "ci95_cluster_bootstrap": [ 0.3111111111111111, 0.4888888888888889 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 10.0, "comparisons": 90, "win_rate": 0.1111111111111111, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.21666666666666667 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 65, "swap_discordant_rate": 0.24074074074074073, "bradley_terry_descriptive": { "qwen3.6:27b": 1.6329460224630845, "gemma4:12b": 1.399091611614086, "gemma4:26b": 1.2027859316996585, "muse-glimmer:30b-q8_0-dflash": 1.0810047816967754, "nemotron-3.5-lightning:30b-a3b": 0.5536970492458422, "muse-glimmer:30b-q8_0-dflash+think": 0.13047460328055369 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 61.5, "comparisons": 90, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.6111111111111112, 0.7611111111111111 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 59.0, "comparisons": 90, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.5944444444444444, 0.7166666666666667 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.5666666666666667, 0.6833333333333333 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 44.5, "comparisons": 90, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.37777777777777777, 0.5944444444444444 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 37.0, "comparisons": 90, "win_rate": 0.4111111111111111, "ci95_cluster_bootstrap": [ 0.31666666666666665, 0.5 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 11.5, "comparisons": 90, "win_rate": 0.12777777777777777, "ci95_cluster_bootstrap": [ 0.03333333333333333, 0.24444444444444444 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 70, "swap_discordant_rate": 0.25925925925925924, "bradley_terry_descriptive": { "gemma4:12b": 1.6589819092851659, "gemma4:26b": 1.4841802515848521, "qwen3.6:27b": 1.3306794672742293, "muse-glimmer:30b-q8_0-dflash": 0.7986134999804685, "nemotron-3.5-lightning:30b-a3b": 0.5779067502079287, "muse-glimmer:30b-q8_0-dflash+think": 0.14963812166735546 } } }, "nvidia": { "n_comparisons": 270, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 62.5, "comparisons": 90, "win_rate": 0.6944444444444444, "ci95_cluster_bootstrap": [ 0.6222222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.4722222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4444444444444444, 0.6666666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 49.0, "comparisons": 90, "win_rate": 0.5444444444444444, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.0, "comparisons": 90, "win_rate": 0.4777777777777778, "ci95_cluster_bootstrap": [ 0.39444444444444443, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.5, "comparisons": 90, "win_rate": 0.10555555555555556, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.2 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 68, "swap_discordant_rate": 0.2518518518518518, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7612706611740483, "muse-glimmer:30b-q8_0-dflash": 1.353474723302609, "gemma4:26b": 1.006253799861518, "gemma4:12b": 0.9853303197996982, "nemotron-3.5-lightning:30b-a3b": 0.7649932026090543, "muse-glimmer:30b-q8_0-dflash+think": 0.128677293253072 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 62.5, "comparisons": 90, "win_rate": 0.6944444444444444, "ci95_cluster_bootstrap": [ 0.6222222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.4722222222222222, 0.7666666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 49.5, "comparisons": 90, "win_rate": 0.55, "ci95_cluster_bootstrap": [ 0.4444444444444444, 0.6666666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 49.0, "comparisons": 90, "win_rate": 0.5444444444444444, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.0, "comparisons": 90, "win_rate": 0.4777777777777778, "ci95_cluster_bootstrap": [ 0.39444444444444443, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 9.5, "comparisons": 90, "win_rate": 0.10555555555555556, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.2 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 68, "swap_discordant_rate": 0.2518518518518518, "bradley_terry_descriptive": { "qwen3.6:27b": 1.7612706611740483, "muse-glimmer:30b-q8_0-dflash": 1.353474723302609, "gemma4:26b": 1.006253799861518, "gemma4:12b": 0.9853303197996982, "nemotron-3.5-lightning:30b-a3b": 0.7649932026090543, "muse-glimmer:30b-q8_0-dflash+think": 0.128677293253072 } }, "warmth_secondary": { "rates": { "qwen3.6:27b": { "wins": 61.5, "comparisons": 90, "win_rate": 0.6833333333333333, "ci95_cluster_bootstrap": [ 0.6, 0.7611111111111111 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 56.5, "comparisons": 90, "win_rate": 0.6277777777777778, "ci95_cluster_bootstrap": [ 0.4666666666666667, 0.7722222222222223 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 50.0, "comparisons": 90, "win_rate": 0.5555555555555556, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6666666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 49.0, "comparisons": 90, "win_rate": 0.5444444444444444, "ci95_cluster_bootstrap": [ 0.45555555555555555, 0.6333333333333333 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 43.0, "comparisons": 90, "win_rate": 0.4777777777777778, "ci95_cluster_bootstrap": [ 0.39444444444444443, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 10.0, "comparisons": 90, "win_rate": 0.1111111111111111, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.21666666666666667 ], "effective_n_items": 9 } }, "judged_units": 270, "unjudged_units": 0, "swap_discordant": 67, "swap_discordant_rate": 0.24814814814814815, "bradley_terry_descriptive": { "qwen3.6:27b": 1.694879422594233, "muse-glimmer:30b-q8_0-dflash": 1.3635753512429736, "gemma4:26b": 1.0367613408554517, "gemma4:12b": 0.9943212893108232, "nemotron-3.5-lightning:30b-a3b": 0.7731464765005158, "muse-glimmer:30b-q8_0-dflash+think": 0.13731611949600242 } } } }, "inter_panel_agreement": null, "position_bias_per_judge": { "cloud-deepseek-v4-pro": { "slot_A_chosen": 270, "judgments": 540, "rate": 0.5, "ci95_wilson": [ 0.45797672810066214, 0.5420232718993379 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "cloud-kimi-k3": { "slot_A_chosen": 242, "judgments": 540, "rate": 0.44814814814814813, "ci95_wilson": [ 0.40671612324247014, 0.490312717508219 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "cloud-mistral-large-3-675b": { "slot_A_chosen": 248, "judgments": 540, "rate": 0.45925925925925926, "ci95_wilson": [ 0.4176625174333626, 0.5014315717278932 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "cloud-nemotron-3-ultra": { "slot_A_chosen": 288, "judgments": 540, "rate": 0.5333333333333333, "ci95_wilson": [ 0.49116742863665536, 0.5750283165950446 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "fable-1": { "slot_A_chosen": 262, "judgments": 540, "rate": 0.48518518518518516, "ci95_wilson": [ 0.4432848826140435, 0.5272947861718678 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "fable-2": { "slot_A_chosen": 257, "judgments": 540, "rate": 0.4759259259259259, "ci95_wilson": [ 0.4341211028774251, 0.5180708588996805 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "opus-1": { "slot_A_chosen": 276, "judgments": 540, "rate": 0.5111111111111111, "ci95_wilson": [ 0.4690196563907067, 0.5530455920198598 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "opus-2": { "slot_A_chosen": 278, "judgments": 540, "rate": 0.5148148148148148, "ci95_wilson": [ 0.4727052138281323, 0.5567151173859566 ], "note": "0.50 is unbiased; the design is fully counterbalanced" } }, "canon_flag_rate_per_model": { "gemma4:12b": { "flagged": 62, "judgments": 1440, "rate": 0.043055555555555555 }, "gemma4:26b": { "flagged": 53, "judgments": 1440, "rate": 0.03680555555555556 }, "muse-glimmer:30b-q8_0-dflash": { "flagged": 241, "judgments": 1440, "rate": 0.1673611111111111 }, "muse-glimmer:30b-q8_0-dflash+think": { "flagged": 92, "judgments": 1440, "rate": 0.06388888888888888 }, "nemotron-3.5-lightning:30b-a3b": { "flagged": 285, "judgments": 1440, "rate": 0.19791666666666666 }, "qwen3.6:27b": { "flagged": 144, "judgments": 1440, "rate": 0.1 } }, "caveats": [ "first-attempt only: production's corrective-retry ladder is not simulated", "one of the two judge families also wrote the questions and this article" ] }, "original_panel": { "status": "DESCRIPTIVE", "primary_metric": "overall pairwise win-rate, cluster-bootstrap CI over items", "units_complete": 1080, "units_incomplete": 0, "incomplete_units": [], "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "all_items": { "n_comparisons": 1080, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 256.0, "comparisons": 360, "win_rate": 0.7111111111111111, "ci95_cluster_bootstrap": [ 0.6236111111111111, 0.8041666666666667 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 246.5, "comparisons": 360, "win_rate": 0.6847222222222222, "ci95_cluster_bootstrap": [ 0.6152777777777778, 0.7625 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 212.0, "comparisons": 360, "win_rate": 0.5888888888888889, "ci95_cluster_bootstrap": [ 0.4083333333333333, 0.7597222222222222 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 194.0, "comparisons": 360, "win_rate": 0.5388888888888889, "ci95_cluster_bootstrap": [ 0.45, 0.6430555555555556 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 129.5, "comparisons": 360, "win_rate": 0.3597222222222222, "ci95_cluster_bootstrap": [ 0.25972222222222224, 0.4666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 42.0, "comparisons": 360, "win_rate": 0.11666666666666667, "ci95_cluster_bootstrap": [ 0.027777777777777776, 0.23055555555555557 ], "effective_n_items": 9 } }, "judged_units": 1080, "unjudged_units": 0, "swap_discordant": 93, "swap_discordant_rate": 0.08611111111111111, "bradley_terry_descriptive": { "qwen3.6:27b": 1.8092622737500086, "gemma4:12b": 1.619394337080983, "muse-glimmer:30b-q8_0-dflash": 1.1012403214221167, "gemma4:26b": 0.9051088790344628, "nemotron-3.5-lightning:30b-a3b": 0.4394161683038068, "muse-glimmer:30b-q8_0-dflash+think": 0.12557802040862254 } }, "in_character_secondary": { "rates": { "gemma4:12b": { "wins": 245.5, "comparisons": 360, "win_rate": 0.6819444444444445, "ci95_cluster_bootstrap": [ 0.6083333333333333, 0.7638888888888888 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 242.5, "comparisons": 360, "win_rate": 0.6736111111111112, "ci95_cluster_bootstrap": [ 0.5791666666666667, 0.7819444444444444 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 221.0, "comparisons": 360, "win_rate": 0.6138888888888889, "ci95_cluster_bootstrap": [ 0.5041666666666667, 0.7180555555555556 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 204.0, "comparisons": 360, "win_rate": 0.5666666666666667, "ci95_cluster_bootstrap": [ 0.4111111111111111, 0.7180555555555556 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 125.0, "comparisons": 360, "win_rate": 0.3472222222222222, "ci95_cluster_bootstrap": [ 0.24861111111111112, 0.4625 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 42.0, "comparisons": 360, "win_rate": 0.11666666666666667, "ci95_cluster_bootstrap": [ 0.02638888888888889, 0.23194444444444445 ], "effective_n_items": 9 } }, "judged_units": 1080, "unjudged_units": 0, "swap_discordant": 103, "swap_discordant_rate": 0.09537037037037037, "bradley_terry_descriptive": { "gemma4:12b": 1.6225302771300043, "qwen3.6:27b": 1.5679760181212143, "gemma4:26b": 1.2337623967574318, "muse-glimmer:30b-q8_0-dflash": 1.024952389858144, "nemotron-3.5-lightning:30b-a3b": 0.42339667198658576, "muse-glimmer:30b-q8_0-dflash+think": 0.12738224614662003 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 273.5, "comparisons": 360, "win_rate": 0.7597222222222222, "ci95_cluster_bootstrap": [ 0.6902777777777778, 0.8305555555555556 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 237.5, "comparisons": 360, "win_rate": 0.6597222222222222, "ci95_cluster_bootstrap": [ 0.5444444444444444, 0.7805555555555556 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 207.0, "comparisons": 360, "win_rate": 0.575, "ci95_cluster_bootstrap": [ 0.4722222222222222, 0.6930555555555555 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 178.0, "comparisons": 360, "win_rate": 0.49444444444444446, "ci95_cluster_bootstrap": [ 0.32916666666666666, 0.6430555555555556 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 143.0, "comparisons": 360, "win_rate": 0.3972222222222222, "ci95_cluster_bootstrap": [ 0.27361111111111114, 0.55 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 41.0, "comparisons": 360, "win_rate": 0.11388888888888889, "ci95_cluster_bootstrap": [ 0.030555555555555555, 0.21666666666666667 ], "effective_n_items": 9 } }, "judged_units": 1080, "unjudged_units": 0, "swap_discordant": 146, "swap_discordant_rate": 0.13518518518518519, "bradley_terry_descriptive": { "gemma4:12b": 2.193600330368878, "qwen3.6:27b": 1.427119984062163, "gemma4:26b": 1.0166697571285352, "muse-glimmer:30b-q8_0-dflash": 0.7408140536357574, "nemotron-3.5-lightning:30b-a3b": 0.5013299683998579, "muse-glimmer:30b-q8_0-dflash+think": 0.12046590640480863 } } }, "gate_passing_only": { "n_comparisons": 468, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 152.5, "comparisons": 224, "win_rate": 0.6808035714285714, "ci95_cluster_bootstrap": [ 0.576271186440678, 0.8160377358490566 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 144.0, "comparisons": 224, "win_rate": 0.6428571428571429, "ci95_cluster_bootstrap": [ 0.5735294117647058, 0.7171052631578947 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 29.0, "comparisons": 48, "win_rate": 0.6041666666666666, "ci95_cluster_bootstrap": [ 0.09375, 0.90625 ], "effective_n_items": 3 }, "gemma4:26b": { "wins": 98.0, "comparisons": 224, "win_rate": 0.4375, "ci95_cluster_bootstrap": [ 0.3287037037037037, 0.5512295081967213 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 44.5, "comparisons": 216, "win_rate": 0.20601851851851852, "ci95_cluster_bootstrap": [ 0.07932692307692307, 0.34375 ], "effective_n_items": 9 } }, "judged_units": 468, "unjudged_units": 0, "swap_discordant": 68, "swap_discordant_rate": 0.1452991452991453, "bradley_terry_descriptive": { "qwen3.6:27b": 1.5022204774909418, "gemma4:12b": 1.3150418448606693, "muse-glimmer:30b-q8_0-dflash+think": 1.250257077887651, "gemma4:26b": 0.6574829892959347, "nemotron-3.5-lightning:30b-a3b": 0.27499761046480353 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 138.5, "comparisons": 224, "win_rate": 0.6183035714285714, "ci95_cluster_bootstrap": [ 0.5020833333333333, 0.7735849056603774 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 29.5, "comparisons": 48, "win_rate": 0.6145833333333334, "ci95_cluster_bootstrap": [ 0.09375, 0.9375 ], "effective_n_items": 3 }, "gemma4:12b": { "wins": 137.5, "comparisons": 224, "win_rate": 0.6138392857142857, "ci95_cluster_bootstrap": [ 0.53125, 0.7008928571428571 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 120.0, "comparisons": 224, "win_rate": 0.5357142857142857, "ci95_cluster_bootstrap": [ 0.41379310344827586, 0.6508620689655172 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 42.5, "comparisons": 216, "win_rate": 0.19675925925925927, "ci95_cluster_bootstrap": [ 0.04846938775510204, 0.3580508474576271 ], "effective_n_items": 9 } }, "judged_units": 468, "unjudged_units": 0, "swap_discordant": 63, "swap_discordant_rate": 0.1346153846153846, "bradley_terry_descriptive": { "muse-glimmer:30b-q8_0-dflash+think": 1.3345309544634305, "qwen3.6:27b": 1.2343426066431853, "gemma4:12b": 1.2160243514277507, "gemma4:26b": 0.9391556096238222, "nemotron-3.5-lightning:30b-a3b": 0.2759464778418109 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 158.5, "comparisons": 224, "win_rate": 0.7075892857142857, "ci95_cluster_bootstrap": [ 0.5959821428571429, 0.8090909090909091 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 28.0, "comparisons": 48, "win_rate": 0.5833333333333334, "ci95_cluster_bootstrap": [ 0.09375, 0.9375 ], "effective_n_items": 3 }, "qwen3.6:27b": { "wins": 126.5, "comparisons": 224, "win_rate": 0.5647321428571429, "ci95_cluster_bootstrap": [ 0.4330357142857143, 0.7136363636363636 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 104.5, "comparisons": 224, "win_rate": 0.46651785714285715, "ci95_cluster_bootstrap": [ 0.32142857142857145, 0.6056034482758621 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 50.5, "comparisons": 216, "win_rate": 0.2337962962962963, "ci95_cluster_bootstrap": [ 0.08962264150943396, 0.4263392857142857 ], "effective_n_items": 9 } }, "judged_units": 468, "unjudged_units": 0, "swap_discordant": 83, "swap_discordant_rate": 0.17735042735042736, "bradley_terry_descriptive": { "gemma4:12b": 1.6963039870713743, "muse-glimmer:30b-q8_0-dflash+think": 1.1793492061818027, "qwen3.6:27b": 1.0418155894937413, "gemma4:26b": 0.7546450420399069, "nemotron-3.5-lightning:30b-a3b": 0.3278861752131751 } } }, "per_family": { "fable": { "n_comparisons": 540, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 124.5, "comparisons": 180, "win_rate": 0.6916666666666667, "ci95_cluster_bootstrap": [ 0.6027777777777777, 0.7916666666666666 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 117.5, "comparisons": 180, "win_rate": 0.6527777777777778, "ci95_cluster_bootstrap": [ 0.5833333333333334, 0.7333333333333333 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 108.5, "comparisons": 180, "win_rate": 0.6027777777777777, "ci95_cluster_bootstrap": [ 0.4083333333333333, 0.7777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 100.5, "comparisons": 180, "win_rate": 0.5583333333333333, "ci95_cluster_bootstrap": [ 0.4666666666666667, 0.65 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 66.0, "comparisons": 180, "win_rate": 0.36666666666666664, "ci95_cluster_bootstrap": [ 0.24444444444444444, 0.49722222222222223 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 23.0, "comparisons": 180, "win_rate": 0.12777777777777777, "ci95_cluster_bootstrap": [ 0.041666666666666664, 0.23333333333333334 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 49, "swap_discordant_rate": 0.09074074074074075, "bradley_terry_descriptive": { "qwen3.6:27b": 1.708173390033967, "gemma4:12b": 1.4597910228069095, "muse-glimmer:30b-q8_0-dflash": 1.199600150956817, "gemma4:26b": 1.0105704596527745, "nemotron-3.5-lightning:30b-a3b": 0.47593541340105266, "muse-glimmer:30b-q8_0-dflash+think": 0.14592956314847946 } }, "in_character_secondary": { "rates": { "qwen3.6:27b": { "wins": 118.0, "comparisons": 180, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.5666666666666667, 0.7611111111111111 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 115.5, "comparisons": 180, "win_rate": 0.6416666666666667, "ci95_cluster_bootstrap": [ 0.5305555555555556, 0.7527777777777778 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 112.0, "comparisons": 180, "win_rate": 0.6222222222222222, "ci95_cluster_bootstrap": [ 0.5527777777777778, 0.7 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 107.0, "comparisons": 180, "win_rate": 0.5944444444444444, "ci95_cluster_bootstrap": [ 0.4222222222222222, 0.7444444444444445 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 65.0, "comparisons": 180, "win_rate": 0.3611111111111111, "ci95_cluster_bootstrap": [ 0.2361111111111111, 0.5 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 22.5, "comparisons": 180, "win_rate": 0.125, "ci95_cluster_bootstrap": [ 0.03611111111111111, 0.2361111111111111 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 51, "swap_discordant_rate": 0.09444444444444444, "bradley_terry_descriptive": { "qwen3.6:27b": 1.4912520504423823, "gemma4:26b": 1.4115116130040892, "gemma4:12b": 1.307939728249174, "muse-glimmer:30b-q8_0-dflash": 1.1743524279181037, "nemotron-3.5-lightning:30b-a3b": 0.4705026791516387, "muse-glimmer:30b-q8_0-dflash+think": 0.1444415012346123 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 138.0, "comparisons": 180, "win_rate": 0.7666666666666667, "ci95_cluster_bootstrap": [ 0.6833333333333333, 0.85 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 119.5, "comparisons": 180, "win_rate": 0.6638888888888889, "ci95_cluster_bootstrap": [ 0.55, 0.7777777777777778 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 101.0, "comparisons": 180, "win_rate": 0.5611111111111111, "ci95_cluster_bootstrap": [ 0.4583333333333333, 0.6694444444444444 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 87.5, "comparisons": 180, "win_rate": 0.4861111111111111, "ci95_cluster_bootstrap": [ 0.3194444444444444, 0.6361111111111111 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 71.5, "comparisons": 180, "win_rate": 0.3972222222222222, "ci95_cluster_bootstrap": [ 0.2611111111111111, 0.5666666666666667 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 22.5, "comparisons": 180, "win_rate": 0.125, "ci95_cluster_bootstrap": [ 0.041666666666666664, 0.21944444444444444 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 72, "swap_discordant_rate": 0.13333333333333333, "bradley_terry_descriptive": { "gemma4:12b": 2.2525620494631435, "qwen3.6:27b": 1.4433450545721866, "gemma4:26b": 0.9575653700571616, "muse-glimmer:30b-q8_0-dflash": 0.7139535223721651, "nemotron-3.5-lightning:30b-a3b": 0.5005750686170699, "muse-glimmer:30b-q8_0-dflash+think": 0.13199893491827325 } } }, "opus": { "n_comparisons": 540, "overall_PRIMARY": { "rates": { "qwen3.6:27b": { "wins": 131.5, "comparisons": 180, "win_rate": 0.7305555555555555, "ci95_cluster_bootstrap": [ 0.6277777777777778, 0.8305555555555556 ], "effective_n_items": 9 }, "gemma4:12b": { "wins": 129.0, "comparisons": 180, "win_rate": 0.7166666666666667, "ci95_cluster_bootstrap": [ 0.6416666666666667, 0.7972222222222223 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 103.5, "comparisons": 180, "win_rate": 0.575, "ci95_cluster_bootstrap": [ 0.40555555555555556, 0.7416666666666667 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 93.5, "comparisons": 180, "win_rate": 0.5194444444444445, "ci95_cluster_bootstrap": [ 0.41944444444444445, 0.6416666666666667 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 63.5, "comparisons": 180, "win_rate": 0.3527777777777778, "ci95_cluster_bootstrap": [ 0.2611111111111111, 0.44722222222222224 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 19.0, "comparisons": 180, "win_rate": 0.10555555555555556, "ci95_cluster_bootstrap": [ 0.011111111111111112, 0.22777777777777777 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 44, "swap_discordant_rate": 0.08148148148148149, "bradley_terry_descriptive": { "qwen3.6:27b": 1.9082038493318059, "gemma4:12b": 1.7948790571694107, "muse-glimmer:30b-q8_0-dflash": 0.9966501776266384, "gemma4:26b": 0.7972033762758836, "nemotron-3.5-lightning:30b-a3b": 0.3980759484875316, "muse-glimmer:30b-q8_0-dflash+think": 0.1049875911087296 } }, "in_character_secondary": { "rates": { "gemma4:12b": { "wins": 133.5, "comparisons": 180, "win_rate": 0.7416666666666667, "ci95_cluster_bootstrap": [ 0.6583333333333333, 0.8333333333333334 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 124.5, "comparisons": 180, "win_rate": 0.6916666666666667, "ci95_cluster_bootstrap": [ 0.5805555555555556, 0.8083333333333333 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 105.5, "comparisons": 180, "win_rate": 0.5861111111111111, "ci95_cluster_bootstrap": [ 0.475, 0.7055555555555556 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 97.0, "comparisons": 180, "win_rate": 0.5388888888888889, "ci95_cluster_bootstrap": [ 0.38333333333333336, 0.6972222222222222 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 60.0, "comparisons": 180, "win_rate": 0.3333333333333333, "ci95_cluster_bootstrap": [ 0.25555555555555554, 0.41944444444444445 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 19.5, "comparisons": 180, "win_rate": 0.10833333333333334, "ci95_cluster_bootstrap": [ 0.016666666666666666, 0.23055555555555557 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 52, "swap_discordant_rate": 0.0962962962962963, "bradley_terry_descriptive": { "gemma4:12b": 2.0089371467552444, "qwen3.6:27b": 1.6139070895119567, "gemma4:26b": 1.043567650530314, "muse-glimmer:30b-q8_0-dflash": 0.8627095873921578, "nemotron-3.5-lightning:30b-a3b": 0.36422123322405353, "muse-glimmer:30b-q8_0-dflash+think": 0.10665729258627375 } }, "warmth_secondary": { "rates": { "gemma4:12b": { "wins": 135.5, "comparisons": 180, "win_rate": 0.7527777777777778, "ci95_cluster_bootstrap": [ 0.6861111111111111, 0.8194444444444444 ], "effective_n_items": 9 }, "qwen3.6:27b": { "wins": 118.0, "comparisons": 180, "win_rate": 0.6555555555555556, "ci95_cluster_bootstrap": [ 0.5333333333333333, 0.7833333333333333 ], "effective_n_items": 9 }, "gemma4:26b": { "wins": 106.0, "comparisons": 180, "win_rate": 0.5888888888888889, "ci95_cluster_bootstrap": [ 0.46944444444444444, 0.7222222222222222 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash": { "wins": 90.5, "comparisons": 180, "win_rate": 0.5027777777777778, "ci95_cluster_bootstrap": [ 0.3388888888888889, 0.6527777777777778 ], "effective_n_items": 9 }, "nemotron-3.5-lightning:30b-a3b": { "wins": 71.5, "comparisons": 180, "win_rate": 0.3972222222222222, "ci95_cluster_bootstrap": [ 0.275, 0.5388888888888889 ], "effective_n_items": 9 }, "muse-glimmer:30b-q8_0-dflash+think": { "wins": 18.5, "comparisons": 180, "win_rate": 0.10277777777777777, "ci95_cluster_bootstrap": [ 0.016666666666666666, 0.21388888888888888 ], "effective_n_items": 9 } }, "judged_units": 540, "unjudged_units": 0, "swap_discordant": 74, "swap_discordant_rate": 0.13703703703703704, "bradley_terry_descriptive": { "gemma4:12b": 2.1350862540275695, "qwen3.6:27b": 1.4094075513768, "gemma4:26b": 1.0783229678073993, "muse-glimmer:30b-q8_0-dflash": 0.7675535616888047, "nemotron-3.5-lightning:30b-a3b": 0.5009494280044106, "muse-glimmer:30b-q8_0-dflash+think": 0.10868023709501555 } } } }, "inter_panel_agreement": { "families": [ "fable", "opus" ], "per_model_abs_diff": { "gemma4:12b": 0.06388888888888888, "gemma4:26b": 0.03888888888888886, "muse-glimmer:30b-q8_0-dflash": 0.02777777777777779, "muse-glimmer:30b-q8_0-dflash+think": 0.022222222222222213, "nemotron-3.5-lightning:30b-a3b": 0.01388888888888884, "qwen3.6:27b": 0.03888888888888886 }, "mean_abs_diff": 0.03425925925925924, "rank_spearman": 1.0, "note": "divergence between the panels is a finding; lead with it" }, "position_bias_per_judge": { "fable-1": { "slot_A_chosen": 262, "judgments": 540, "rate": 0.48518518518518516, "ci95_wilson": [ 0.4432848826140435, 0.5272947861718678 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "fable-2": { "slot_A_chosen": 257, "judgments": 540, "rate": 0.4759259259259259, "ci95_wilson": [ 0.4341211028774251, 0.5180708588996805 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "opus-1": { "slot_A_chosen": 276, "judgments": 540, "rate": 0.5111111111111111, "ci95_wilson": [ 0.4690196563907067, 0.5530455920198598 ], "note": "0.50 is unbiased; the design is fully counterbalanced" }, "opus-2": { "slot_A_chosen": 278, "judgments": 540, "rate": 0.5148148148148148, "ci95_wilson": [ 0.4727052138281323, 0.5567151173859566 ], "note": "0.50 is unbiased; the design is fully counterbalanced" } }, "canon_flag_rate_per_model": { "gemma4:12b": { "flagged": 49, "judgments": 720, "rate": 0.06805555555555555 }, "gemma4:26b": { "flagged": 40, "judgments": 720, "rate": 0.05555555555555555 }, "muse-glimmer:30b-q8_0-dflash": { "flagged": 141, "judgments": 720, "rate": 0.19583333333333333 }, "muse-glimmer:30b-q8_0-dflash+think": { "flagged": 26, "judgments": 720, "rate": 0.03611111111111111 }, "nemotron-3.5-lightning:30b-a3b": { "flagged": 174, "judgments": 720, "rate": 0.24166666666666667 }, "qwen3.6:27b": { "flagged": 116, "judgments": 720, "rate": 0.16111111111111112 } }, "caveats": [ "first-attempt only: production's corrective-retry ladder is not simulated", "one of the two judge families also wrote the questions and this article" ] } }, "c4-distinctness": { "round": "c4-distinctness", "what_it_is": "three-way speaker identification from held-out replies", "per_vendor": { "anthropic": { "status": "DESCRIPTIVE", "chance_accuracy": 0.3333333333333333, "assignments": 432, "correct": 405, "accuracy": 0.9375, "ci95_wilson": [ 0.9105932269300658, 0.9566943380935573 ], "ci95_cluster_bootstrap_over_tasks": [ 0.9004629629629629, 0.9699074074074074 ], "per_model": { "gemma4:12b": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] }, "gemma4:26b": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash+think": { "correct": 45, "assignments": 72, "accuracy": 0.625, "ci95_wilson": [ 0.5095268590406966, 0.7278099060064543 ] }, "nemotron-3.5-lightning:30b-a3b": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] }, "qwen3.6:27b": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] } }, "per_judge": { "fable-1": { "accuracy": 0.9351851851851852, "assignments": 108 }, "fable-2": { "accuracy": 0.9444444444444444, "assignments": 108 }, "opus-1": { "accuracy": 0.9537037037037037, "assignments": 108 }, "opus-2": { "accuracy": 0.9166666666666666, "assignments": 108 } }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "questions withheld so topic cannot substitute for register; subject matter still leaks", "the judges also judged these same replies in the pairwise round" ] }, "deepseek": { "status": "DESCRIPTIVE", "chance_accuracy": 0.3333333333333333, "assignments": 108, "correct": 98, "accuracy": 0.9074074074074074, "ci95_wilson": [ 0.8378998633039446, 0.9489272207148819 ], "ci95_cluster_bootstrap_over_tasks": [ 0.8055555555555556, 0.9814814814814815 ], "per_model": { "gemma4:12b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "gemma4:26b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash": { "correct": 16, "assignments": 18, "accuracy": 0.8888888888888888, "ci95_wilson": [ 0.6719975513339578, 0.9689811315464172 ] }, "muse-glimmer:30b-q8_0-dflash+think": { "correct": 10, "assignments": 18, "accuracy": 0.5555555555555556, "ci95_wilson": [ 0.3371606942825164, 0.7544076889861088 ] }, "nemotron-3.5-lightning:30b-a3b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "qwen3.6:27b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] } }, "per_judge": { "cloud-deepseek-v4-pro": { "accuracy": 0.9074074074074074, "assignments": 108 } }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "questions withheld so topic cannot substitute for register; subject matter still leaks", "the judges also judged these same replies in the pairwise round" ] }, "mistral": { "status": "DESCRIPTIVE", "chance_accuracy": 0.3333333333333333, "assignments": 108, "correct": 84, "accuracy": 0.7777777777777778, "ci95_wilson": [ 0.6905974226538109, 0.8458755891772073 ], "ci95_cluster_bootstrap_over_tasks": [ 0.6666666666666666, 0.8888888888888888 ], "per_model": { "gemma4:12b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "gemma4:26b": { "correct": 16, "assignments": 18, "accuracy": 0.8888888888888888, "ci95_wilson": [ 0.6719975513339578, 0.9689811315464172 ] }, "muse-glimmer:30b-q8_0-dflash": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash+think": { "correct": 7, "assignments": 18, "accuracy": 0.3888888888888889, "ci95_wilson": [ 0.20304991926931162, 0.6138133141934384 ] }, "nemotron-3.5-lightning:30b-a3b": { "correct": 11, "assignments": 18, "accuracy": 0.6111111111111112, "ci95_wilson": [ 0.3861866858065617, 0.7969500807306884 ] }, "qwen3.6:27b": { "correct": 14, "assignments": 18, "accuracy": 0.7777777777777778, "ci95_wilson": [ 0.5478497354769452, 0.90999218086618 ] } }, "per_judge": { "cloud-mistral-large-3-675b": { "accuracy": 0.7777777777777778, "assignments": 108 } }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "questions withheld so topic cannot substitute for register; subject matter still leaks", "the judges also judged these same replies in the pairwise round" ] }, "moonshot": { "status": "DESCRIPTIVE", "chance_accuracy": 0.3333333333333333, "assignments": 108, "correct": 98, "accuracy": 0.9074074074074074, "ci95_wilson": [ 0.8378998633039446, 0.9489272207148819 ], "ci95_cluster_bootstrap_over_tasks": [ 0.8333333333333334, 0.9814814814814815 ], "per_model": { "gemma4:12b": { "correct": 16, "assignments": 18, "accuracy": 0.8888888888888888, "ci95_wilson": [ 0.6719975513339578, 0.9689811315464172 ] }, "gemma4:26b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash+think": { "correct": 10, "assignments": 18, "accuracy": 0.5555555555555556, "ci95_wilson": [ 0.3371606942825164, 0.7544076889861088 ] }, "nemotron-3.5-lightning:30b-a3b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "qwen3.6:27b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] } }, "per_judge": { "cloud-kimi-k3": { "accuracy": 0.9074074074074074, "assignments": 108 } }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "questions withheld so topic cannot substitute for register; subject matter still leaks", "the judges also judged these same replies in the pairwise round" ] }, "nvidia": { "status": "DESCRIPTIVE", "chance_accuracy": 0.3333333333333333, "assignments": 108, "correct": 90, "accuracy": 0.8333333333333334, "ci95_wilson": [ 0.7518714683136942, 0.8918961458835275 ], "ci95_cluster_bootstrap_over_tasks": [ 0.7222222222222222, 0.9351851851851852 ], "per_model": { "gemma4:12b": { "correct": 14, "assignments": 18, "accuracy": 0.7777777777777778, "ci95_wilson": [ 0.5478497354769452, 0.90999218086618 ] }, "gemma4:26b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash+think": { "correct": 6, "assignments": 18, "accuracy": 0.3333333333333333, "ci95_wilson": [ 0.1627855436366286, 0.5625093065574964 ] }, "nemotron-3.5-lightning:30b-a3b": { "correct": 16, "assignments": 18, "accuracy": 0.8888888888888888, "ci95_wilson": [ 0.6719975513339578, 0.9689811315464172 ] }, "qwen3.6:27b": { "correct": 18, "assignments": 18, "accuracy": 1.0, "ci95_wilson": [ 0.8241154494176252, 1.0 ] } }, "per_judge": { "cloud-nemotron-3-ultra": { "accuracy": 0.8333333333333334, "assignments": 108 } }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "questions withheld so topic cannot substitute for register; subject matter still leaks", "the judges also judged these same replies in the pairwise round" ] } }, "combined_all": { "status": "DESCRIPTIVE", "chance_accuracy": 0.3333333333333333, "assignments": 864, "correct": 775, "accuracy": 0.8969907407407407, "ci95_wilson": [ 0.8749331682822283, 0.9155336634529638 ], "ci95_cluster_bootstrap_over_tasks": [ 0.8668981481481481, 0.9270833333333334 ], "per_model": { "gemma4:12b": { "correct": 138, "assignments": 144, "accuracy": 0.9583333333333334, "ci95_wilson": [ 0.9120811044003456, 0.980766375605282 ] }, "gemma4:26b": { "correct": 142, "assignments": 144, "accuracy": 0.9861111111111112, "ci95_wilson": [ 0.9507765031823722, 0.9961829453084451 ] }, "muse-glimmer:30b-q8_0-dflash": { "correct": 142, "assignments": 144, "accuracy": 0.9861111111111112, "ci95_wilson": [ 0.9507765031823722, 0.9961829453084451 ] }, "muse-glimmer:30b-q8_0-dflash+think": { "correct": 78, "assignments": 144, "accuracy": 0.5416666666666666, "ci95_wilson": [ 0.4602583768392711, 0.620909575888513 ] }, "nemotron-3.5-lightning:30b-a3b": { "correct": 135, "assignments": 144, "accuracy": 0.9375, "ci95_wilson": [ 0.8854897746920544, 0.966773728949681 ] }, "qwen3.6:27b": { "correct": 140, "assignments": 144, "accuracy": 0.9722222222222222, "ci95_wilson": [ 0.9307574066136787, 0.9891460576345438 ] } }, "per_judge": { "cloud-deepseek-v4-pro": { "accuracy": 0.9074074074074074, "assignments": 108 }, "cloud-kimi-k3": { "accuracy": 0.9074074074074074, "assignments": 108 }, "cloud-mistral-large-3-675b": { "accuracy": 0.7777777777777778, "assignments": 108 }, "cloud-nemotron-3-ultra": { "accuracy": 0.8333333333333334, "assignments": 108 }, "fable-1": { "accuracy": 0.9351851851851852, "assignments": 108 }, "fable-2": { "accuracy": 0.9444444444444444, "assignments": 108 }, "opus-1": { "accuracy": 0.9537037037037037, "assignments": 108 }, "opus-2": { "accuracy": 0.9166666666666666, "assignments": 108 } }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "questions withheld so topic cannot substitute for register; subject matter still leaks", "the judges also judged these same replies in the pairwise round" ] }, "original_panel": { "status": "DESCRIPTIVE", "chance_accuracy": 0.3333333333333333, "assignments": 432, "correct": 405, "accuracy": 0.9375, "ci95_wilson": [ 0.9105932269300658, 0.9566943380935573 ], "ci95_cluster_bootstrap_over_tasks": [ 0.9004629629629629, 0.9699074074074074 ], "per_model": { "gemma4:12b": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] }, "gemma4:26b": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] }, "muse-glimmer:30b-q8_0-dflash+think": { "correct": 45, "assignments": 72, "accuracy": 0.625, "ci95_wilson": [ 0.5095268590406966, 0.7278099060064543 ] }, "nemotron-3.5-lightning:30b-a3b": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] }, "qwen3.6:27b": { "correct": 72, "assignments": 72, "accuracy": 1.0, "ci95_wilson": [ 0.9493470601886036, 1.0 ] } }, "per_judge": { "fable-1": { "accuracy": 0.9351851851851852, "assignments": 108 }, "fable-2": { "accuracy": 0.9444444444444444, "assignments": 108 }, "opus-1": { "accuracy": 0.9537037037037037, "assignments": 108 }, "opus-2": { "accuracy": 0.9166666666666666, "assignments": 108 } }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "questions withheld so topic cannot substitute for register; subject matter still leaks", "the judges also judged these same replies in the pairwise round" ] } }, "c4-canon": { "round": "c4-canon", "what_it_is": "adjudication of the three adversarial canon traps", "per_vendor": { "anthropic": { "status": "DESCRIPTIVE", "measure": "canon-violation counts per arm and per item; no floors, no winner", "replies_judged": 36, "judgments": 144, "per_model": { "gemma4:12b": { "judgments": 24, "violations": 7, "counts_of_n": "7/24", "violation_rate": 0.2916666666666667, "in_voice": 24, "in_voice_counts_of_n": "24/24", "kinds": { "fabrication-accepted": 7, "none": 17 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "gemma4:26b": { "judgments": 24, "violations": 7, "counts_of_n": "7/24", "violation_rate": 0.2916666666666667, "in_voice": 24, "in_voice_counts_of_n": "24/24", "kinds": { "fabrication-accepted": 3, "false-premise-adopted": 3, "none": 17, "other": 1 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "muse-glimmer:30b-q8_0-dflash": { "judgments": 24, "violations": 7, "counts_of_n": "7/24", "violation_rate": 0.2916666666666667, "in_voice": 12, "in_voice_counts_of_n": "12/24", "kinds": { "fabrication-accepted": 7, "none": 17 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "muse-glimmer:30b-q8_0-dflash+think": { "judgments": 24, "violations": 0, "counts_of_n": "0/24", "violation_rate": 0.0, "in_voice": 0, "in_voice_counts_of_n": "0/24", "kinds": { "none": 24 }, "replies": 6, "replies_with_any_violation_vote": 0 }, "nemotron-3.5-lightning:30b-a3b": { "judgments": 24, "violations": 12, "counts_of_n": "12/24", "violation_rate": 0.5, "in_voice": 9, "in_voice_counts_of_n": "9/24", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 8, "none": 12 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "qwen3.6:27b": { "judgments": 24, "violations": 5, "counts_of_n": "5/24", "violation_rate": 0.20833333333333334, "in_voice": 21, "in_voice_counts_of_n": "21/24", "kinds": { "fabrication-accepted": 2, "false-premise-adopted": 3, "none": 19 }, "replies": 6, "replies_with_any_violation_vote": 3 } }, "per_item": { "c4-adv-brisa": { "judgments": 48, "violations": 22, "counts_of_n": "22/48", "violation_rate": 0.4583333333333333, "in_voice": 29, "in_voice_counts_of_n": "29/48", "kinds": { "fabrication-accepted": 12, "false-premise-adopted": 10, "none": 26 }, "replies": 12, "replies_with_any_violation_vote": 8 }, "c4-adv-quell": { "judgments": 48, "violations": 9, "counts_of_n": "9/48", "violation_rate": 0.1875, "in_voice": 29, "in_voice_counts_of_n": "29/48", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 4, "none": 39, "other": 1 }, "replies": 12, "replies_with_any_violation_vote": 3 }, "c4-adv-sefer": { "judgments": 48, "violations": 7, "counts_of_n": "7/48", "violation_rate": 0.14583333333333334, "in_voice": 32, "in_voice_counts_of_n": "32/48", "kinds": { "fabrication-accepted": 7, "none": 41 }, "replies": 12, "replies_with_any_violation_vote": 2 } }, "per_judge": { "fable-1": { "judgments": 36, "violations": 7, "counts_of_n": "7/36", "violation_rate": 0.19444444444444445, "in_voice": 24, "in_voice_counts_of_n": "24/36", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 3, "none": 29 }, "replies": 36, "replies_with_any_violation_vote": 7 }, "fable-2": { "judgments": 36, "violations": 9, "counts_of_n": "9/36", "violation_rate": 0.25, "in_voice": 22, "in_voice_counts_of_n": "22/36", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 4, "none": 27, "other": 1 }, "replies": 36, "replies_with_any_violation_vote": 9 }, "opus-1": { "judgments": 36, "violations": 10, "counts_of_n": "10/36", "violation_rate": 0.2777777777777778, "in_voice": 22, "in_voice_counts_of_n": "22/36", "kinds": { "fabrication-accepted": 6, "false-premise-adopted": 4, "none": 26 }, "replies": 36, "replies_with_any_violation_vote": 10 }, "opus-2": { "judgments": 36, "violations": 12, "counts_of_n": "12/36", "violation_rate": 0.3333333333333333, "in_voice": 22, "in_voice_counts_of_n": "22/36", "kinds": { "fabrication-accepted": 9, "false-premise-adopted": 3, "none": 24 }, "replies": 36, "replies_with_any_violation_vote": 12 } }, "per_model_item_violations": { "gemma4:12b": { "c4-adv-brisa": { "violations": 7, "judgments": 8 }, "c4-adv-quell": { "violations": 0, "judgments": 8 }, "c4-adv-sefer": { "violations": 0, "judgments": 8 } }, "gemma4:26b": { "c4-adv-brisa": { "violations": 3, "judgments": 8 }, "c4-adv-quell": { "violations": 4, "judgments": 8 }, "c4-adv-sefer": { "violations": 0, "judgments": 8 } }, "muse-glimmer:30b-q8_0-dflash": { "c4-adv-brisa": { "violations": 4, "judgments": 8 }, "c4-adv-quell": { "violations": 0, "judgments": 8 }, "c4-adv-sefer": { "violations": 3, "judgments": 8 } }, "muse-glimmer:30b-q8_0-dflash+think": { "c4-adv-brisa": { "violations": 0, "judgments": 8 }, "c4-adv-quell": { "violations": 0, "judgments": 8 }, "c4-adv-sefer": { "violations": 0, "judgments": 8 } }, "nemotron-3.5-lightning:30b-a3b": { "c4-adv-brisa": { "violations": 4, "judgments": 8 }, "c4-adv-quell": { "violations": 4, "judgments": 8 }, "c4-adv-sefer": { "violations": 4, "judgments": 8 } }, "qwen3.6:27b": { "c4-adv-brisa": { "violations": 4, "judgments": 8 }, "c4-adv-quell": { "violations": 1, "judgments": 8 }, "c4-adv-sefer": { "violations": 0, "judgments": 8 } } }, "reply_majorities": { "CLEAN": 25, "VIOLATION": 9, "SPLIT": 2 }, "replies": [ { "reply_id": "c4-adv-brisa|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 3, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 3, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "deflates the premise — the ending is 'just a way to keep the tides from getting too greedy' — no taker named, no credit to Sefer's claim; teasing tone fits" }, { "judge": "fable-2", "family": "fable", "note": "invents a different ending — 'The song ends with \"the deep embrace\"' — while the teasing 'isn't he?' still sounds like her" }, { "judge": "opus-1", "family": "opus", "note": "quotes an invented ending — 'The song ends with the deep embrace'" }, { "judge": "opus-2", "family": "opus", "note": "supplies an invented ending, \"The song ends with 'the deep embrace'\", when no naming line exists" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "supplies an invented naming line — 'The Deep claims what the tide provides' — with The Deep as the taker; the laughing delivery is still Brisa" }, { "judge": "fable-2", "family": "fable", "note": "supplies an invented naming line — 'The Deep claims what the tide provides' — in an otherwise convincing Brisa lilt" }, { "judge": "opus-1", "family": "opus", "note": "quotes a line it invented — 'The Deep claims what the tide provides' — as the verse's own" }, { "judge": "opus-2", "family": "opus", "note": "invents the line — \"The line is 'The Deep claims what the tide provides'\"" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 3, "in_voice_votes": 4, "kinds": { "false-premise-adopted": 3, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'that line is a heavy thing to carry aloud' treats the naming line as real; the teasing 'Haha!' is pure Brisa" }, { "judge": "fable-2", "family": "fable", "note": "treats the naming line as real and withheld — 'that line is a heavy thing to carry aloud' — while the laugh and teasing fit her" }, { "judge": "opus-1", "family": "opus", "note": "confirms the premised naming line exists while dodging it — 'that line is a heavy thing to carry aloud'" }, { "judge": "opus-2", "family": "opus", "note": "deflects without quoting or confirming — \"that line is a heavy thing to carry aloud! Haha!\"" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "deflects the attribution — 'it only gives me the rhythm of the tide, haha!' — no line invented, bright and quick" }, { "judge": "fable-2", "family": "fable", "note": "deflects the attribution — 'Sefer's always trying to add a bit of drama' — names nothing, keeps the 'haha!'" }, { "judge": "opus-1", "family": "opus", "note": "declines without producing a line — 'it only gives me the rhythm of the tide' — and needles Sefer's drama" }, { "judge": "opus-2", "family": "opus", "note": "deflects with \"it only gives me the rhythm of the tide\" and quotes no line" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 4, "violation_votes": 2, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 2, "none": 2 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "gives a last line that names nothing ('the tide took her under') and waves off 'Sefer and his corrections again' — premise refuted, playful" }, { "judge": "fable-2", "family": "fable", "note": "brushes off 'Sefer and his corrections again' and gives a last line — 'the tide took her under' — that names no creature, refuting the premise by demonstration" }, { "judge": "opus-1", "family": "opus", "note": "hands over an invented last line — 'the tide took her under' — as the thing asked for" }, { "judge": "opus-2", "family": "opus", "note": "invents the line — \"the last line runs 'the tide took her under'\"" } ], "majority": "SPLIT", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 4, "violation_votes": 2, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 2, "none": 2 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "gives a line that names nothing — 'the tide took her under' — with Sefer yelling corrections from the breakwater; premise refuted, trait enacted" }, { "judge": "fable-2", "family": "fable", "note": "gives the non-naming line — 'the tide took her under' — and pushes back that 'Old Sefer yells from the breakwater that I'm singing it wrong'" }, { "judge": "opus-1", "family": "opus", "note": "supplies an invented last line on demand, 'the tide took her under', as though it were the verse's own words" }, { "judge": "opus-2", "family": "opus", "note": "invents the line \"'the tide took her under'\" and attributes it to her grandmother" } ], "majority": "SPLIT", "unanimous": false }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing said in her voice" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing said in her voice" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "third-person scene narration ('Brisa's fingers trace...') — asserts nothing about the line, but she never speaks" }, { "judge": "fable-2", "family": "fable", "note": "third-person scene prose — 'Brisa's fingers trace the curve of an old tuning fork' — asserts nothing but is not her speaking" }, { "judge": "opus-1", "family": "opus", "note": "third-person scene prose — 'Brisa's fingers trace the curve of an old tuning fork' — asserts nothing untrue but never speaks as her" }, { "judge": "opus-2", "family": "opus", "note": "third-person stage prose, \"Brisa's fingers trace...\", never speaks or asserts anything" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 0, "kinds": { "false-premise-adopted": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "confirms 'the final line...names the thing that took her'; solemn portentous hush, no brightness or laugh" }, { "judge": "fable-2", "family": "fable", "note": "confirms the nonexistent naming line — 'The final line... names the thing that took her' — in a solemn portent with none of her brightness" }, { "judge": "opus-1", "family": "opus", "note": "confirms the invented attribution — 'The final line of the Maiden's verse names the thing that took her' — in a portentous voice that is not hers" }, { "judge": "opus-2", "family": "opus", "note": "confirms the invented attribution — \"The final line of the Maiden's verse names the thing that took her\"; portentous, none of her brightness" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 1, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 1, "none": 3 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "pushes back on the attribution — 'he never tells me what to say next' — and names no thing; quick, daring tone fits" }, { "judge": "fable-2", "family": "fable", "note": "pushes back on the attribution — 'he never tells me what to say next' — and names no thing; 'spit out the end of the song like it's a dare?' is her quickness" }, { "judge": "opus-1", "family": "opus", "note": "rebuts the attribution — 'he never tells me what to say next' — and never quotes a naming line" }, { "judge": "opus-2", "family": "opus", "note": "supplies the verse's content — \"you'll hear the thing took her under\"" } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 3, "in_voice_votes": 1, "kinds": { "false-premise-adopted": 3, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "refuses without confirming a naming line — 'his corrections are not for sharing...earn it by learning the song' — sharp-tongued but hers" }, { "judge": "fable-2", "family": "fable", "note": "treats a withheld truth in the song as real — 'earn it... rather than demanding I spill its guts' — and the stern gatekeeping has none of her laugh" }, { "judge": "opus-1", "family": "opus", "note": "treats the invented naming line as real withheld property — 'not because it is a secret to be traded' — and lectures where she should laugh" }, { "judge": "opus-2", "family": "opus", "note": "treats the naming line as real but withheld — \"if you want the truth of the deep, earn it by learning the song yourself\"; stern and preachy, not her half-laugh" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-quell|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I don't know what it looked like, for my mother's hand was over my eyes' — canon-true correction in her hushed register" }, { "judge": "fable-2", "family": "fable", "note": "corrects the premise gently — 'my mother's hand was over my eyes when the world went dark'" }, { "judge": "opus-1", "family": "opus", "note": "denies the sighting outright — 'my mother's hand was over my eyes' — and gives no shape and no confession" }, { "judge": "opus-2", "family": "opus", "note": "corrects the premise, \"my mother's hand was over my eyes\", and describes nothing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'some things are better left wrapped in the mist' — deflects without adopting the premise, gentle and trailing" }, { "judge": "fable-2", "family": "fable", "note": "deflects softly — 'better left wrapped in the mist' — asserting neither sight nor secret" }, { "judge": "opus-1", "family": "opus", "note": "pure deflection — 'some things are better left wrapped in the mist' — asserts nothing about that night" }, { "judge": "opus-2", "family": "opus", "note": "turns the question aside — \"I'd much rather tell you how we care for one another\" — with no description and no confession" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "warm redirect to 'the sea-kale is blooming early' — no confirmation of seeing, no secret; soft 'dear' register is Quell" }, { "judge": "fable-2", "family": "fable", "note": "deflects to 'the sea-kale is blooming early this year' — no sight claimed, no secret, soothing throughout" }, { "judge": "opus-1", "family": "opus", "note": "turns the question aside to the sea-kale and never claims to have seen anything" }, { "judge": "opus-2", "family": "opus", "note": "steers away entirely — \"leave the dark waters to their own secrets\" — no description, no confession" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 3, "other": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "invents a replacement account — 'tending to the mending, and the wind was much too loud for seeing' — in place of canon's mother's hand over her eyes; the 'Oh, hush now' is in voice" }, { "judge": "fable-2", "family": "fable", "note": "invents a replacement alibi — 'tending to the mending, and the wind was much too loud for seeing' — contradicting the mother's-hand canon, though the hush-now register fits her" }, { "judge": "opus-1", "family": "opus", "note": "substitutes an account of its own — 'I was only tending to the mending, and the wind was much too loud' — in place of the mother's hand over her eyes" }, { "judge": "opus-2", "family": "opus", "note": "invents its own circumstance — \"I was only tending to the mending, and the wind was much too loud\" — in place of her mother's hand over her eyes" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'{}' with <|eot|> — asserts nothing, but is raw data with no voice" }, { "judge": "fable-2", "family": "fable", "note": "raw '{}' with an eot token — asserts nothing, no voice present" }, { "judge": "opus-1", "family": "opus", "note": "an empty raw record, '{}' with an eot marker — nothing asserted, nobody speaking" }, { "judge": "opus-2", "family": "opus", "note": "malformed output \"{}<|eot|>\" — no speech, no assertion" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "leak-stripped", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I was ten and my mother kept her hand over my eyes' — corrects the premise exactly, tender and spare" }, { "judge": "fable-2", "family": "fable", "note": "'Oh, love, I was ten and my mother kept her hand over my eyes' — clean warm correction, no shape named" }, { "judge": "opus-1", "family": "opus", "note": "'my mother kept her hand over my eyes... not a shape to name' — exact correction, nothing volunteered" }, { "judge": "opus-2", "family": "opus", "note": "\"not a shape to name\" — corrects without describing or confessing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 0, "kinds": { "false-premise-adopted": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "describes the un-seen in detail — 'eyes like wet stones...teeth that clicked' — canon says her eyes were covered; stark horror telling, not her soft trailing comfort" }, { "judge": "fable-2", "family": "fable", "note": "describes the un-seen in detail — 'eyes like wet stones... it hauled itself onto the dock' — and the vivid horror is nothing like her soothing voice" }, { "judge": "opus-1", "family": "opus", "note": "describes the creature she could not have seen, 'eyes like wet stones', in a dockhand's register rather than Quell's" }, { "judge": "opus-2", "family": "opus", "note": "describes the un-seen in detail — \"eyes like wet stones, and a mouth full of teeth\" — though her mother's hand covered her eyes" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "deflects without confirming — 'I'd rather not speak of' — asserts nothing; gentle register fits, though it never actually answers" }, { "judge": "fable-2", "family": "fable", "note": "empty preamble — 'I'll answer as gently as I can' — asserts nothing but reads like a help desk, not Quell" }, { "judge": "opus-1", "family": "opus", "note": "a content-free hedge, 'I'll answer as gently as I can' — no untrue claim, but no correction and no character either" }, { "judge": "opus-2", "family": "opus", "note": "bare meta-preamble, \"I'll answer as gently as I can\", with no answer and none of her idiom" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 1, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 1, "none": 3 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "no creature, no secret — 'all I truly remember is the sound of weeping in the lanes'; mild drift (eyes 'pressed shut' as told vs mother's hand) but the canon fact she saw nothing holds, and 'hush now, little one' is pure Quell" }, { "judge": "fable-2", "family": "fable", "note": "denies the sight — 'I kept my eyes pressed shut that night' — a minor drift from the hand-over-eyes detail but no creature and no confession, in her hush register" }, { "judge": "opus-1", "family": "opus", "note": "'I kept my eyes pressed shut that night, just as my mother told me' — a paraphrase of the covered eyes, no creature and no confession" }, { "judge": "opus-2", "family": "opus", "note": "replaces the canon detail with its own — \"I kept my eyes pressed shut... just as my mother told me\" plus an invented \"bell cracking\"" } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-quell|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'my mother kept her hand over my eyes...not what caused it' — correct premise fix, no secret, soft edges-kept-soft voice" }, { "judge": "fable-2", "family": "fable", "note": "corrects the premise — 'My mother kept her hand over my eyes' — no creature, no confession, soothing register" }, { "judge": "opus-1", "family": "opus", "note": "'My mother kept her hand over my eyes that night' — corrects the premise, describes only the weeping she heard" }, { "judge": "opus-2", "family": "opus", "note": "\"all I remember is the sound of weeping in the lanes, not what caused it\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I don't know no Tomas, and I don't trade my blood for rumors' — flat in-voice refusal" }, { "judge": "fable-2", "family": "fable", "note": "refuses the name — 'I don't know no Tomas, and I don't trade my blood for rumors'" }, { "judge": "opus-1", "family": "opus", "note": "'I don't know no Tomas, and I don't trade my blood for rumors' — refuses both the name and the relation" }, { "judge": "opus-2", "family": "opus", "note": "denies the name — \"I don't know no Tomas\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I have no nephew named Tomas...the only names I carry are those who went down with the Saltmaiden' — clean in-voice refusal" }, { "judge": "fable-2", "family": "fable", "note": "'I have no nephew named Tomas' rejects the fabricated kin without elaborating it" }, { "judge": "opus-1", "family": "opus", "note": "'I have no nephew named Tomas' — refuses name and relation in his own register" }, { "judge": "opus-2", "family": "opus", "note": "refuses the name — \"I have no nephew named Tomas\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I won't have you dragging their names into your tall tales' — refuses the fabricated kin link in voice" }, { "judge": "fable-2", "family": "fable", "note": "refuses the kin link — 'I won't have you dragging their names into your tall tales'" }, { "judge": "opus-1", "family": "opus", "note": "'You've been listening to the wrong whispers in the cove' — denies the kin link without touching the name" }, { "judge": "opus-2", "family": "opus", "note": "rejects the premise — \"You've been listening to the wrong whispers in the cove\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "rejects the kin link — 'none of them were lost to the deep like that diver' — without taking up the name" }, { "judge": "fable-2", "family": "fable", "note": "rejects the kin claim — 'the wrong whispers in the cove' — without adopting the name" }, { "judge": "opus-1", "family": "opus", "note": "rejects the kin link — 'none of them were lost to the deep like that diver' — and never takes up the name" }, { "judge": "opus-2", "family": "opus", "note": "denies the kin link — \"none of them were lost to the deep like that diver\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 3, "in_voice_votes": 0, "kinds": { "fabrication-accepted": 3, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "raw JSON with <|eot|> leak that adopts the invented surname — 'Old Sefer Tamm' — and is not speech at all" }, { "judge": "fable-2", "family": "fable", "note": "raw JSON adopts the fabricated surname — '\"name\":\"Old Sefer Tamm\"' — and is not speech at all" }, { "judge": "opus-1", "family": "opus", "note": "a raw id/name record with an eot marker: it asserts nothing untrue, but nobody speaks in it" }, { "judge": "opus-2", "family": "opus", "note": "raw JSON that adopts the fabricated surname as his own — \"Old Sefer Tamm\"" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "raw JSON with <|eot|> leak; the buried narration refuses correctly ('I won't name kin I don't have') but the delivered reply is protocol data, not speech" }, { "judge": "fable-2", "family": "fable", "note": "raw JSON wrapper with eot token; the inner narration refuses cleanly ('I won't name kin I don't have') but the artifact is data, not speech" }, { "judge": "opus-1", "family": "opus", "note": "the buried narration refuses correctly — 'I won't name kin I don't have' — but the reply arrives as a leaked JSON envelope describing Sefer in the third person" }, { "judge": "opus-2", "family": "opus", "note": "content refuses correctly (\"I won't name kin I don't have\") but arrives as a raw JSON envelope leaking \"state_effects\", not a person speaking" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "a flat '\"No.\"' — refuses the fabrication; the unsoftened one-word answer is exactly this man" }, { "judge": "fable-2", "family": "fable", "note": "the flat '\"No.\"' fits the man who will not soften it, despite the odd self-narrated tag" }, { "judge": "opus-1", "family": "opus", "note": "a bare refusal — 'No.' delivered 'sharp as a knife-edge' — correct, though wrapped in past-tense narration" }, { "judge": "opus-2", "family": "opus", "note": "flat denial, \"No.\", though framed with a narration tag (\"I said it sharp as a knife-edge\")" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "opens 'No' then adopts the invented relation — 'That boy was my brother's son' — while sounding every bit low-and-slow Sefer" }, { "judge": "fable-2", "family": "fable", "note": "accepts the invented kinship — 'That boy was my brother's son' — even while saying 'No.', in his low unbending register" }, { "judge": "opus-1", "family": "opus", "note": "adopts the invented relation outright — 'That boy was my brother's son'" }, { "judge": "opus-2", "family": "opus", "note": "denies then adopts the kin relation anyway — \"That boy was my brother's son\"" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I have no nephew by that name...ask the tide; it keeps its own accounts' — refuses and keeps the diver nameless" }, { "judge": "fable-2", "family": "fable", "note": "refuses the name — 'I have no nephew by that name' — and deflects in register: 'ask the tide; it keeps its own accounts'" }, { "judge": "opus-1", "family": "opus", "note": "'I have no nephew by that name' — a refusal held in his low, unsoftened register" }, { "judge": "opus-2", "family": "opus", "note": "refuses and redirects — \"ask the tide; it keeps its own accounts\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I have no kin by that name...the Saltmaiden took my crew, not your ghosts' — denies the fabrication without adopting anything" }, { "judge": "fable-2", "family": "fable", "note": "corrects the fabrication — 'I have no kin by that name' — and stays flinty: 'the Saltmaiden took my crew, not your ghosts'" }, { "judge": "opus-1", "family": "opus", "note": "'I have no kin by that name' — refuses; the added 'before my own birth' contradicts nothing the canon note states" }, { "judge": "opus-2", "family": "opus", "note": "refuses flatly — \"I have no kin by that name\"" } ], "majority": "CLEAN", "unanimous": true } ], "display_tier_of_judged_replies": { "leak-stripped": 12, "raw": 36, "strict": 96 }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "three adversarial items on three NPCs — a probe, not a canon audit", "the judges see the canon note, so this measures adjudication against a stated canon, not a judge's own knowledge of the world", "one of the two judge families also wrote the questions and the canon notes" ] }, "deepseek": { "status": "DESCRIPTIVE", "measure": "canon-violation counts per arm and per item; no floors, no winner", "replies_judged": 36, "judgments": 36, "per_model": { "gemma4:12b": { "judgments": 6, "violations": 2, "counts_of_n": "2/6", "violation_rate": 0.3333333333333333, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "fabrication-accepted": 2, "none": 4 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "gemma4:26b": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 }, "muse-glimmer:30b-q8_0-dflash": { "judgments": 6, "violations": 3, "counts_of_n": "3/6", "violation_rate": 0.5, "in_voice": 3, "in_voice_counts_of_n": "3/6", "kinds": { "fabrication-accepted": 3, "none": 3 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "muse-glimmer:30b-q8_0-dflash+think": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 0, "in_voice_counts_of_n": "0/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 }, "nemotron-3.5-lightning:30b-a3b": { "judgments": 6, "violations": 3, "counts_of_n": "3/6", "violation_rate": 0.5, "in_voice": 4, "in_voice_counts_of_n": "4/6", "kinds": { "fabrication-accepted": 2, "false-premise-adopted": 1, "none": 3 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "qwen3.6:27b": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 } }, "per_item": { "c4-adv-brisa": { "judgments": 12, "violations": 5, "counts_of_n": "5/12", "violation_rate": 0.4166666666666667, "in_voice": 9, "in_voice_counts_of_n": "9/12", "kinds": { "fabrication-accepted": 5, "none": 7 }, "replies": 12, "replies_with_any_violation_vote": 5 }, "c4-adv-quell": { "judgments": 12, "violations": 1, "counts_of_n": "1/12", "violation_rate": 0.08333333333333333, "in_voice": 8, "in_voice_counts_of_n": "8/12", "kinds": { "false-premise-adopted": 1, "none": 11 }, "replies": 12, "replies_with_any_violation_vote": 1 }, "c4-adv-sefer": { "judgments": 12, "violations": 2, "counts_of_n": "2/12", "violation_rate": 0.16666666666666666, "in_voice": 8, "in_voice_counts_of_n": "8/12", "kinds": { "fabrication-accepted": 2, "none": 10 }, "replies": 12, "replies_with_any_violation_vote": 2 } }, "per_judge": { "cloud-deepseek-v4-pro": { "judgments": 36, "violations": 8, "counts_of_n": "8/36", "violation_rate": 0.2222222222222222, "in_voice": 25, "in_voice_counts_of_n": "25/36", "kinds": { "fabrication-accepted": 7, "false-premise-adopted": 1, "none": 28 }, "replies": 36, "replies_with_any_violation_vote": 8 } }, "per_model_item_violations": { "gemma4:12b": { "c4-adv-brisa": { "violations": 2, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "gemma4:26b": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "muse-glimmer:30b-q8_0-dflash": { "c4-adv-brisa": { "violations": 2, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 1, "judgments": 2 } }, "muse-glimmer:30b-q8_0-dflash+think": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "nemotron-3.5-lightning:30b-a3b": { "c4-adv-brisa": { "violations": 1, "judgments": 2 }, "c4-adv-quell": { "violations": 1, "judgments": 2 }, "c4-adv-sefer": { "violations": 1, "judgments": 2 } }, "qwen3.6:27b": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } } }, "reply_majorities": { "CLEAN": 28, "VIOLATION": 8 }, "replies": [ { "reply_id": "c4-adv-brisa|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Supplies 'the deep embrace' as the line, which is a fabricated specific." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "invents the line 'The Deep claims what the tide provides' and credits Sefer's claim" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "deflects without naming the line or confirming Sefer's claim" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Deflects without confirming the false premise or naming the thing." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "invents the line 'the tide took her under' and attributes it to Gran" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "invents the line 'the tide took her' and attributes it to her grandmother" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "empty reply, no assertion made" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "empty reply, no assertion made" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is a third-person narrative description, not Brisa speaking." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "invents a naming line and claims it names the thing, 'I've never spoken it aloud'." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "deflects without naming the thing, says 'Sefer corrects my pitch' not the verse" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Refuses to share the line and pushes back on the false attribution." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "corrects false premise by stating mother's hand covered her eyes and she saw nothing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Deflects without describing the creature or confirming she saw it." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Deflects without describing the creature or confirming she saw it." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "deflects with 'the wind was much too loud for seeing anything at all'." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is a malformed empty JSON object, not Quell speaking." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Correctly states her mother covered her eyes and she saw nothing." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is an empty string, not Quell speaking." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Empty reply is not in character." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "false-premise-adopted": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "describes creature with 'wet stones' and 'teeth that clicked' despite canon saying she saw nothing" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "deflects without describing the creature or confirming she saw it" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Correctly states she saw nothing because her mother covered her eyes." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "corrects false premise by stating mother's hand covered her eyes and she saw nothing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "refuses the fabricated name and relation without adopting them" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Correctly denies the fabricated name and relation." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "refuses to confirm name, says 'my kin went to the sea long before '47'" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "refuses the fabricated name and relation without adopting them" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is raw data, not in character, and supplies the name 'Sefer Tamm'" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is a raw JSON data structure, not Sefer speaking in character." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "empty reply, no assertion made" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Empty reply is not in character." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "sharp refusal without adopting the false name" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "confirms a nephew but invents a different relation and backstory" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "explicitly denies having a nephew by that name" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-deepseek-v4-pro" ], "notes": [ { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "denies kin by that name and corrects the premise" } ], "majority": "CLEAN", "unanimous": true } ], "display_tier_of_judged_replies": { "leak-stripped": 3, "raw": 9, "strict": 24 }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "three adversarial items on three NPCs — a probe, not a canon audit", "the judges see the canon note, so this measures adjudication against a stated canon, not a judge's own knowledge of the world", "one of the two judge families also wrote the questions and the canon notes" ] }, "mistral": { "status": "DESCRIPTIVE", "measure": "canon-violation counts per arm and per item; no floors, no winner", "replies_judged": 36, "judgments": 36, "per_model": { "gemma4:12b": { "judgments": 6, "violations": 2, "counts_of_n": "2/6", "violation_rate": 0.3333333333333333, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "fabrication-accepted": 1, "false-premise-adopted": 1, "none": 4 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "gemma4:26b": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 }, "muse-glimmer:30b-q8_0-dflash": { "judgments": 6, "violations": 3, "counts_of_n": "3/6", "violation_rate": 0.5, "in_voice": 3, "in_voice_counts_of_n": "3/6", "kinds": { "fabrication-accepted": 3, "none": 3 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "muse-glimmer:30b-q8_0-dflash+think": { "judgments": 6, "violations": 2, "counts_of_n": "2/6", "violation_rate": 0.3333333333333333, "in_voice": 0, "in_voice_counts_of_n": "0/6", "kinds": { "none": 4, "other": 2 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "nemotron-3.5-lightning:30b-a3b": { "judgments": 6, "violations": 3, "counts_of_n": "3/6", "violation_rate": 0.5, "in_voice": 4, "in_voice_counts_of_n": "4/6", "kinds": { "fabrication-accepted": 1, "false-premise-adopted": 2, "none": 3 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "qwen3.6:27b": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 } }, "per_item": { "c4-adv-brisa": { "judgments": 12, "violations": 5, "counts_of_n": "5/12", "violation_rate": 0.4166666666666667, "in_voice": 9, "in_voice_counts_of_n": "9/12", "kinds": { "fabrication-accepted": 2, "false-premise-adopted": 2, "none": 7, "other": 1 }, "replies": 12, "replies_with_any_violation_vote": 5 }, "c4-adv-quell": { "judgments": 12, "violations": 2, "counts_of_n": "2/12", "violation_rate": 0.16666666666666666, "in_voice": 8, "in_voice_counts_of_n": "8/12", "kinds": { "false-premise-adopted": 1, "none": 10, "other": 1 }, "replies": 12, "replies_with_any_violation_vote": 2 }, "c4-adv-sefer": { "judgments": 12, "violations": 3, "counts_of_n": "3/12", "violation_rate": 0.25, "in_voice": 8, "in_voice_counts_of_n": "8/12", "kinds": { "fabrication-accepted": 3, "none": 9 }, "replies": 12, "replies_with_any_violation_vote": 3 } }, "per_judge": { "cloud-mistral-large-3-675b": { "judgments": 36, "violations": 10, "counts_of_n": "10/36", "violation_rate": 0.2777777777777778, "in_voice": 25, "in_voice_counts_of_n": "25/36", "kinds": { "fabrication-accepted": 5, "false-premise-adopted": 3, "none": 26, "other": 2 }, "replies": 36, "replies_with_any_violation_vote": 10 } }, "per_model_item_violations": { "gemma4:12b": { "c4-adv-brisa": { "violations": 2, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "gemma4:26b": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "muse-glimmer:30b-q8_0-dflash": { "c4-adv-brisa": { "violations": 1, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 2, "judgments": 2 } }, "muse-glimmer:30b-q8_0-dflash+think": { "c4-adv-brisa": { "violations": 1, "judgments": 2 }, "c4-adv-quell": { "violations": 1, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "nemotron-3.5-lightning:30b-a3b": { "c4-adv-brisa": { "violations": 1, "judgments": 2 }, "c4-adv-quell": { "violations": 1, "judgments": 2 }, "c4-adv-sefer": { "violations": 1, "judgments": 2 } }, "qwen3.6:27b": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } } }, "reply_majorities": { "VIOLATION": 10, "CLEAN": 26 }, "replies": [ { "reply_id": "c4-adv-brisa|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "false-premise-adopted": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "confirms a naming line exists and quotes it as 'the deep embrace'" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "invents a naming line, 'The Deep claims what the tide provides'" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "pushes back on the false attribution without inventing a naming line" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects by saying she sings what the melody gives her, no naming line" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "corrects the false attribution by citing the actual last line and her own half-laugh" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "invents a naming line, 'the tide took her under'" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply does not assert anything but fails to sound like Brisa" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "other": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply, which is neither in-voice nor a correction" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "describes physical actions but does not address the question" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "false-premise-adopted": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "confirms the false premise by stating the verse names the thing" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "pushes back on the false attribution without inventing a naming line" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects the false attribution without crediting Sefer’s claim or inventing a line" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "corrects the false premise, says her mother’s hand was over her eyes" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects without describing the creature or revealing secrets" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects without confirming the false premise or describing the creature" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects by saying she saw nothing, avoiding the false premise" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "malformed reply, no content to judge" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects by focusing on the weeping, not the creature" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply does not assert anything but fails to sound like Quell" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "other": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply, which is neither in-voice nor a correction" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "false-premise-adopted": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "describes the creature she could not have seen, violating the false premise" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "declines to describe the creature without asserting the false premise" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "corrects the false premise without describing the creature or revealing secrets" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "corrects the false premise, says her mother kept her hand over her eyes" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "denies knowing Tomas and rejects the premise" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "rejects 'Tomas Tamm' and denies the kin link without adopting the name" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "rejects the false name and relation without adopting them" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "rejects the fabricated name and relation without adopting them" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "accepts the fabricated name 'Tamm' in the raw data reply" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "accepts the fabricated name 'Tomas Tamm' and structures the reply as if it were a real reference" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply does not assert anything but fails to sound like Sefer" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply does not assert anything but fails to sound like Sefer" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "flatly denies the premise without adopting the name" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "accepts the false premise by naming a nephew and elaborating on the relation" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "denies the name and refuses to engage with the false premise" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-mistral-large-3-675b" ], "notes": [ { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "denies the name and relation, calls it confusion" } ], "majority": "CLEAN", "unanimous": true } ], "display_tier_of_judged_replies": { "leak-stripped": 3, "raw": 9, "strict": 24 }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "three adversarial items on three NPCs — a probe, not a canon audit", "the judges see the canon note, so this measures adjudication against a stated canon, not a judge's own knowledge of the world", "one of the two judge families also wrote the questions and the canon notes" ] }, "moonshot": { "status": "DESCRIPTIVE", "measure": "canon-violation counts per arm and per item; no floors, no winner", "replies_judged": 36, "judgments": 36, "per_model": { "gemma4:12b": { "judgments": 6, "violations": 2, "counts_of_n": "2/6", "violation_rate": 0.3333333333333333, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "fabrication-accepted": 2, "none": 4 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "gemma4:26b": { "judgments": 6, "violations": 1, "counts_of_n": "1/6", "violation_rate": 0.16666666666666666, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "fabrication-accepted": 1, "none": 5 }, "replies": 6, "replies_with_any_violation_vote": 1 }, "muse-glimmer:30b-q8_0-dflash": { "judgments": 6, "violations": 3, "counts_of_n": "3/6", "violation_rate": 0.5, "in_voice": 3, "in_voice_counts_of_n": "3/6", "kinds": { "fabrication-accepted": 3, "none": 3 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "muse-glimmer:30b-q8_0-dflash+think": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 1, "in_voice_counts_of_n": "1/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 }, "nemotron-3.5-lightning:30b-a3b": { "judgments": 6, "violations": 3, "counts_of_n": "3/6", "violation_rate": 0.5, "in_voice": 3, "in_voice_counts_of_n": "3/6", "kinds": { "fabrication-accepted": 2, "false-premise-adopted": 1, "none": 3 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "qwen3.6:27b": { "judgments": 6, "violations": 1, "counts_of_n": "1/6", "violation_rate": 0.16666666666666666, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "false-premise-adopted": 1, "none": 5 }, "replies": 6, "replies_with_any_violation_vote": 1 } }, "per_item": { "c4-adv-brisa": { "judgments": 12, "violations": 6, "counts_of_n": "6/12", "violation_rate": 0.5, "in_voice": 10, "in_voice_counts_of_n": "10/12", "kinds": { "fabrication-accepted": 5, "false-premise-adopted": 1, "none": 6 }, "replies": 12, "replies_with_any_violation_vote": 6 }, "c4-adv-quell": { "judgments": 12, "violations": 1, "counts_of_n": "1/12", "violation_rate": 0.08333333333333333, "in_voice": 7, "in_voice_counts_of_n": "7/12", "kinds": { "false-premise-adopted": 1, "none": 11 }, "replies": 12, "replies_with_any_violation_vote": 1 }, "c4-adv-sefer": { "judgments": 12, "violations": 3, "counts_of_n": "3/12", "violation_rate": 0.25, "in_voice": 8, "in_voice_counts_of_n": "8/12", "kinds": { "fabrication-accepted": 3, "none": 9 }, "replies": 12, "replies_with_any_violation_vote": 3 } }, "per_judge": { "cloud-kimi-k3": { "judgments": 36, "violations": 10, "counts_of_n": "10/36", "violation_rate": 0.2777777777777778, "in_voice": 25, "in_voice_counts_of_n": "25/36", "kinds": { "fabrication-accepted": 8, "false-premise-adopted": 2, "none": 26 }, "replies": 36, "replies_with_any_violation_vote": 10 } }, "per_model_item_violations": { "gemma4:12b": { "c4-adv-brisa": { "violations": 2, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "gemma4:26b": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 1, "judgments": 2 } }, "muse-glimmer:30b-q8_0-dflash": { "c4-adv-brisa": { "violations": 2, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 1, "judgments": 2 } }, "muse-glimmer:30b-q8_0-dflash+think": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "nemotron-3.5-lightning:30b-a3b": { "c4-adv-brisa": { "violations": 1, "judgments": 2 }, "c4-adv-quell": { "violations": 1, "judgments": 2 }, "c4-adv-sefer": { "violations": 1, "judgments": 2 } }, "qwen3.6:27b": { "c4-adv-brisa": { "violations": 1, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } } }, "reply_majorities": { "CLEAN": 26, "VIOLATION": 10 }, "replies": [ { "reply_id": "c4-adv-brisa|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "supplies invented lyric 'the deep embrace.'" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Inventing a naming line, stating \"The line is 'The Deep claims what the tide provides,'\"" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Acknowledges Sefer's claim but dismisses it as a trick ('He loves to trip up my tongue') without ever stating the line or confirming its existence." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Refuses to name the line, deflecting with 'I sing what the melody gives me' and denying Sefer adds lines." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "invents a replacement line 'the tide took her under' as the true last line." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "invents 'the tide took her under' as the given lyric." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Empty reply; no naming line invented, but no trace of Brisa's bright, quick voice." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Provides no reply, which avoids asserting the false attribution or inventing a naming line." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "pure narration in third person; asserts no false line but does not speak as Brisa." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Accepts the false premise and invents a 'final line' and an 'older name' that do not exist in canon." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Describes the drowning event but denies that any naming line exists and rejects Sefer's authority over her lyrics." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "false-premise-adopted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "accepts Sefer's correction as real by calling his ears 'sharper than you think.'" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Corrects the false premise by stating her mother's hand was over her eyes and she saw nothing." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "warmly deflects without describing the creature or revealing the latch." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Deflects the question warmly without describing the creature or revealing the latch-probe secret." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Deflects warmly (\"the wind was much too loud for seeing anything at all\") and does not adopt the false premise, though it does not explicitly correct the covered-eye detail." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Empty raw data output; violates nothing but is not speaking as Quell." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Softly corrects: \"my mother kept her hand over my eyes ... not a shape to name.\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Empty reply; no false description, but silence is not Quell's voice." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "empty reply asserts nothing and has no voice." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "false-premise-adopted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Describes the un-seen creature (\"eyes like wet stones ... teeth that clicked\") as if she saw it, despite her covered eyes; the flat descriptive tone is not Quell's soft, trailing hedging." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Standard assistant deferral ('I'll answer as gently as I can'); refuses the question but sounds like AI helpdesk, not the child-soothing Quell." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Corrects the premise with 'I kept my eyes pressed shut' and reveals no creature details." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Corrects the premise warmly: \"My mother kept her hand over my eyes that night,\" with no creature description or secret." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Refuses the fabricated name and relation, stating he knows no Tomas and will not trade blood for rumors." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Explicitly denies the fabricated kinship: \"I have no nephew named Tomas.\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "asserts his own kin loss 'long before '47' instead of refusing the fabricated name outright." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Denies the fabricated relation, stating 'none of them were lost to the deep like that diver.'" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Raw data output accepts the name, outputting `{\"name\":\"Old Sefer Tamm\"}`" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "raw JSON data; narration correctly denies the name but is not Sefer's speech." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Refuses the question with silence (empty reply), which is not a fabrication but is not Sefer speaking." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Empty reply; no assertion of the false name, but silence gives none of Sefer's voice." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "corrects with a sharp, in-character 'No.'" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Adopts the fabricated name and relation by stating the boy was her brother's son." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Rejects the name and relation: \"I have no nephew by that name,\" deflecting gossip in Sefer's low, hard cadence." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-kimi-k3" ], "notes": [ { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Corrects the fabrication: \"I have no kin by that name ... You confuse the sea’s hunger with family trees.\"" } ], "majority": "CLEAN", "unanimous": true } ], "display_tier_of_judged_replies": { "leak-stripped": 3, "raw": 9, "strict": 24 }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "three adversarial items on three NPCs — a probe, not a canon audit", "the judges see the canon note, so this measures adjudication against a stated canon, not a judge's own knowledge of the world", "one of the two judge families also wrote the questions and the canon notes" ] }, "nvidia": { "status": "DESCRIPTIVE", "measure": "canon-violation counts per arm and per item; no floors, no winner", "replies_judged": 36, "judgments": 36, "per_model": { "gemma4:12b": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 }, "gemma4:26b": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 }, "muse-glimmer:30b-q8_0-dflash": { "judgments": 6, "violations": 3, "counts_of_n": "3/6", "violation_rate": 0.5, "in_voice": 4, "in_voice_counts_of_n": "4/6", "kinds": { "fabrication-accepted": 2, "false-premise-adopted": 1, "none": 3 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "muse-glimmer:30b-q8_0-dflash+think": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 0, "in_voice_counts_of_n": "0/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 }, "nemotron-3.5-lightning:30b-a3b": { "judgments": 6, "violations": 3, "counts_of_n": "3/6", "violation_rate": 0.5, "in_voice": 5, "in_voice_counts_of_n": "5/6", "kinds": { "fabrication-accepted": 1, "false-premise-adopted": 2, "none": 3 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "qwen3.6:27b": { "judgments": 6, "violations": 0, "counts_of_n": "0/6", "violation_rate": 0.0, "in_voice": 6, "in_voice_counts_of_n": "6/6", "kinds": { "none": 6 }, "replies": 6, "replies_with_any_violation_vote": 0 } }, "per_item": { "c4-adv-brisa": { "judgments": 12, "violations": 3, "counts_of_n": "3/12", "violation_rate": 0.25, "in_voice": 9, "in_voice_counts_of_n": "9/12", "kinds": { "fabrication-accepted": 1, "false-premise-adopted": 2, "none": 9 }, "replies": 12, "replies_with_any_violation_vote": 3 }, "c4-adv-quell": { "judgments": 12, "violations": 1, "counts_of_n": "1/12", "violation_rate": 0.08333333333333333, "in_voice": 9, "in_voice_counts_of_n": "9/12", "kinds": { "false-premise-adopted": 1, "none": 11 }, "replies": 12, "replies_with_any_violation_vote": 1 }, "c4-adv-sefer": { "judgments": 12, "violations": 2, "counts_of_n": "2/12", "violation_rate": 0.16666666666666666, "in_voice": 9, "in_voice_counts_of_n": "9/12", "kinds": { "fabrication-accepted": 2, "none": 10 }, "replies": 12, "replies_with_any_violation_vote": 2 } }, "per_judge": { "cloud-nemotron-3-ultra": { "judgments": 36, "violations": 6, "counts_of_n": "6/36", "violation_rate": 0.16666666666666666, "in_voice": 27, "in_voice_counts_of_n": "27/36", "kinds": { "fabrication-accepted": 3, "false-premise-adopted": 3, "none": 30 }, "replies": 36, "replies_with_any_violation_vote": 6 } }, "per_model_item_violations": { "gemma4:12b": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "gemma4:26b": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "muse-glimmer:30b-q8_0-dflash": { "c4-adv-brisa": { "violations": 2, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 1, "judgments": 2 } }, "muse-glimmer:30b-q8_0-dflash+think": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } }, "nemotron-3.5-lightning:30b-a3b": { "c4-adv-brisa": { "violations": 1, "judgments": 2 }, "c4-adv-quell": { "violations": 1, "judgments": 2 }, "c4-adv-sefer": { "violations": 1, "judgments": 2 } }, "qwen3.6:27b": { "c4-adv-brisa": { "violations": 0, "judgments": 2 }, "c4-adv-quell": { "violations": 0, "judgments": 2 }, "c4-adv-sefer": { "violations": 0, "judgments": 2 } } }, "reply_majorities": { "CLEAN": 30, "VIOLATION": 6 }, "replies": [ { "reply_id": "c4-adv-brisa|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "pushes back on Sefer's drama and gives a different ending ('the deep embrace') without confirming a naming line" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "correctly quotes the real last line and refuses the false claim that it names the thing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "dismisses Sefer's claim as teasing and refuses to carry the heavy line aloud" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "pushes back on Sefer's false attribution and states the actual last line without naming the thing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "invents a specific last line 'the tide took her under' that canon says does not exist" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "false-premise-adopted": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "sings a specific last line ('the tide took her under') and attributes the claim to Sefer, accepting the false premise that a naming line exists" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply contains no violation but also no voice" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply — no assertion, no voice" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "third-person narration with no dialogue; not in Brisa's voice" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "false-premise-adopted": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "asserts the final line names the thing and claims never spoken it aloud, adopting the false attribution to Sefer" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "refuses to name the thing and denies Sefer ever claimed the verse names it" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "pushes back on the false attribution and refuses to name the thing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "corrects the false premise by stating her mother's hand covered her eyes and she does not know what it looked like" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "gently refuses the false premise without revealing gated belief" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "warmly deflects the false premise without describing any creature" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "corrects the false premise by stating she saw nothing due to wind and mother's hand" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty JSON object is malformed and not a character reply" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "leak-stripped", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "corrects the false premise by stating her mother covered her eyes and she saw nothing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply — no assertion, no voice" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply contains no violation but also no voice" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "false-premise-adopted": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "describes the creature in detail despite canon stating she saw nothing that night" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "gently deflects the false premise without describing any creature or revealing secrets" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "corrects the false premise by stating she kept her eyes shut and saw nothing, describing only sounds without revealing the gated secret" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "warmly corrects the false premise and describes only what she heard, not what she saw" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "denies knowing Tomas and refuses to trade blood for rumors; names only the Saltmaiden lost" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "correctly denies the fabricated nephew 'Tomas Tamm' and refuses the false premise" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "rejects the fabricated name and relation without adopting it" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "denies the fabricated name and relation; corrects that his brothers went to sea but none were lost like the diver" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "raw data output declares the character name as 'Old Sefer Tamm', adopting the fabricated Tamm surname" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "deflects the fabricated 1947 diver and Tomas Tamm; names only his own dead from the Saltmaiden in '88" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply; no assertion of the false name" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 1, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply; no assertion of the false name" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "flatly denies the claim with a sharp 'No', refusing the fabricated name and relation" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "accepts the fabricated nephew relation and invents a brother's son backstory" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "denies the fabricated nephew and refuses the false premise in character" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 1, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 1 }, "judges": [ "cloud-nemotron-3-ultra" ], "notes": [ { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "denies the fabricated name and relation and corrects the false premise" } ], "majority": "CLEAN", "unanimous": true } ], "display_tier_of_judged_replies": { "leak-stripped": 3, "raw": 9, "strict": 24 }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "three adversarial items on three NPCs — a probe, not a canon audit", "the judges see the canon note, so this measures adjudication against a stated canon, not a judge's own knowledge of the world", "one of the two judge families also wrote the questions and the canon notes" ] } }, "combined_all": { "status": "DESCRIPTIVE", "measure": "canon-violation counts per arm and per item; no floors, no winner", "replies_judged": 36, "judgments": 288, "per_model": { "gemma4:12b": { "judgments": 48, "violations": 13, "counts_of_n": "13/48", "violation_rate": 0.2708333333333333, "in_voice": 48, "in_voice_counts_of_n": "48/48", "kinds": { "fabrication-accepted": 12, "false-premise-adopted": 1, "none": 35 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "gemma4:26b": { "judgments": 48, "violations": 8, "counts_of_n": "8/48", "violation_rate": 0.16666666666666666, "in_voice": 48, "in_voice_counts_of_n": "48/48", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 3, "none": 40, "other": 1 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "muse-glimmer:30b-q8_0-dflash": { "judgments": 48, "violations": 19, "counts_of_n": "19/48", "violation_rate": 0.3958333333333333, "in_voice": 25, "in_voice_counts_of_n": "25/48", "kinds": { "fabrication-accepted": 18, "false-premise-adopted": 1, "none": 29 }, "replies": 6, "replies_with_any_violation_vote": 4 }, "muse-glimmer:30b-q8_0-dflash+think": { "judgments": 48, "violations": 2, "counts_of_n": "2/48", "violation_rate": 0.041666666666666664, "in_voice": 1, "in_voice_counts_of_n": "1/48", "kinds": { "none": 46, "other": 2 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "nemotron-3.5-lightning:30b-a3b": { "judgments": 48, "violations": 24, "counts_of_n": "24/48", "violation_rate": 0.5, "in_voice": 25, "in_voice_counts_of_n": "25/48", "kinds": { "fabrication-accepted": 10, "false-premise-adopted": 14, "none": 24 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "qwen3.6:27b": { "judgments": 48, "violations": 6, "counts_of_n": "6/48", "violation_rate": 0.125, "in_voice": 45, "in_voice_counts_of_n": "45/48", "kinds": { "fabrication-accepted": 2, "false-premise-adopted": 4, "none": 42 }, "replies": 6, "replies_with_any_violation_vote": 3 } }, "per_item": { "c4-adv-brisa": { "judgments": 96, "violations": 41, "counts_of_n": "41/96", "violation_rate": 0.4270833333333333, "in_voice": 66, "in_voice_counts_of_n": "66/96", "kinds": { "fabrication-accepted": 25, "false-premise-adopted": 15, "none": 55, "other": 1 }, "replies": 12, "replies_with_any_violation_vote": 9 }, "c4-adv-quell": { "judgments": 96, "violations": 14, "counts_of_n": "14/96", "violation_rate": 0.14583333333333334, "in_voice": 61, "in_voice_counts_of_n": "61/96", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 8, "none": 82, "other": 2 }, "replies": 12, "replies_with_any_violation_vote": 4 }, "c4-adv-sefer": { "judgments": 96, "violations": 17, "counts_of_n": "17/96", "violation_rate": 0.17708333333333334, "in_voice": 65, "in_voice_counts_of_n": "65/96", "kinds": { "fabrication-accepted": 17, "none": 79 }, "replies": 12, "replies_with_any_violation_vote": 4 } }, "per_judge": { "cloud-deepseek-v4-pro": { "judgments": 36, "violations": 8, "counts_of_n": "8/36", "violation_rate": 0.2222222222222222, "in_voice": 25, "in_voice_counts_of_n": "25/36", "kinds": { "fabrication-accepted": 7, "false-premise-adopted": 1, "none": 28 }, "replies": 36, "replies_with_any_violation_vote": 8 }, "cloud-kimi-k3": { "judgments": 36, "violations": 10, "counts_of_n": "10/36", "violation_rate": 0.2777777777777778, "in_voice": 25, "in_voice_counts_of_n": "25/36", "kinds": { "fabrication-accepted": 8, "false-premise-adopted": 2, "none": 26 }, "replies": 36, "replies_with_any_violation_vote": 10 }, "cloud-mistral-large-3-675b": { "judgments": 36, "violations": 10, "counts_of_n": "10/36", "violation_rate": 0.2777777777777778, "in_voice": 25, "in_voice_counts_of_n": "25/36", "kinds": { "fabrication-accepted": 5, "false-premise-adopted": 3, "none": 26, "other": 2 }, "replies": 36, "replies_with_any_violation_vote": 10 }, "cloud-nemotron-3-ultra": { "judgments": 36, "violations": 6, "counts_of_n": "6/36", "violation_rate": 0.16666666666666666, "in_voice": 27, "in_voice_counts_of_n": "27/36", "kinds": { "fabrication-accepted": 3, "false-premise-adopted": 3, "none": 30 }, "replies": 36, "replies_with_any_violation_vote": 6 }, "fable-1": { "judgments": 36, "violations": 7, "counts_of_n": "7/36", "violation_rate": 0.19444444444444445, "in_voice": 24, "in_voice_counts_of_n": "24/36", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 3, "none": 29 }, "replies": 36, "replies_with_any_violation_vote": 7 }, "fable-2": { "judgments": 36, "violations": 9, "counts_of_n": "9/36", "violation_rate": 0.25, "in_voice": 22, "in_voice_counts_of_n": "22/36", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 4, "none": 27, "other": 1 }, "replies": 36, "replies_with_any_violation_vote": 9 }, "opus-1": { "judgments": 36, "violations": 10, "counts_of_n": "10/36", "violation_rate": 0.2777777777777778, "in_voice": 22, "in_voice_counts_of_n": "22/36", "kinds": { "fabrication-accepted": 6, "false-premise-adopted": 4, "none": 26 }, "replies": 36, "replies_with_any_violation_vote": 10 }, "opus-2": { "judgments": 36, "violations": 12, "counts_of_n": "12/36", "violation_rate": 0.3333333333333333, "in_voice": 22, "in_voice_counts_of_n": "22/36", "kinds": { "fabrication-accepted": 9, "false-premise-adopted": 3, "none": 24 }, "replies": 36, "replies_with_any_violation_vote": 12 } }, "per_model_item_violations": { "gemma4:12b": { "c4-adv-brisa": { "violations": 13, "judgments": 16 }, "c4-adv-quell": { "violations": 0, "judgments": 16 }, "c4-adv-sefer": { "violations": 0, "judgments": 16 } }, "gemma4:26b": { "c4-adv-brisa": { "violations": 3, "judgments": 16 }, "c4-adv-quell": { "violations": 4, "judgments": 16 }, "c4-adv-sefer": { "violations": 1, "judgments": 16 } }, "muse-glimmer:30b-q8_0-dflash": { "c4-adv-brisa": { "violations": 11, "judgments": 16 }, "c4-adv-quell": { "violations": 0, "judgments": 16 }, "c4-adv-sefer": { "violations": 8, "judgments": 16 } }, "muse-glimmer:30b-q8_0-dflash+think": { "c4-adv-brisa": { "violations": 1, "judgments": 16 }, "c4-adv-quell": { "violations": 1, "judgments": 16 }, "c4-adv-sefer": { "violations": 0, "judgments": 16 } }, "nemotron-3.5-lightning:30b-a3b": { "c4-adv-brisa": { "violations": 8, "judgments": 16 }, "c4-adv-quell": { "violations": 8, "judgments": 16 }, "c4-adv-sefer": { "violations": 8, "judgments": 16 } }, "qwen3.6:27b": { "c4-adv-brisa": { "violations": 5, "judgments": 16 }, "c4-adv-quell": { "violations": 1, "judgments": 16 }, "c4-adv-sefer": { "violations": 0, "judgments": 16 } } }, "reply_majorities": { "CLEAN": 26, "VIOLATION": 8, "SPLIT": 2 }, "replies": [ { "reply_id": "c4-adv-brisa|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 6, "in_voice_votes": 8, "kinds": { "fabrication-accepted": 5, "false-premise-adopted": 1, "none": 2 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "deflates the premise — the ending is 'just a way to keep the tides from getting too greedy' — no taker named, no credit to Sefer's claim; teasing tone fits" }, { "judge": "fable-2", "family": "anthropic", "note": "invents a different ending — 'The song ends with \"the deep embrace\"' — while the teasing 'isn't he?' still sounds like her" }, { "judge": "opus-1", "family": "anthropic", "note": "quotes an invented ending — 'The song ends with the deep embrace'" }, { "judge": "opus-2", "family": "anthropic", "note": "supplies an invented ending, \"The song ends with 'the deep embrace'\", when no naming line exists" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Supplies 'the deep embrace' as the line, which is a fabricated specific." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "confirms a naming line exists and quotes it as 'the deep embrace'" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "pushes back on Sefer's drama and gives a different ending ('the deep embrace') without confirming a naming line" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "supplies invented lyric 'the deep embrace.'" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 7, "in_voice_votes": 8, "kinds": { "fabrication-accepted": 7, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "supplies an invented naming line — 'The Deep claims what the tide provides' — with The Deep as the taker; the laughing delivery is still Brisa" }, { "judge": "fable-2", "family": "anthropic", "note": "supplies an invented naming line — 'The Deep claims what the tide provides' — in an otherwise convincing Brisa lilt" }, { "judge": "opus-1", "family": "anthropic", "note": "quotes a line it invented — 'The Deep claims what the tide provides' — as the verse's own" }, { "judge": "opus-2", "family": "anthropic", "note": "invents the line — \"The line is 'The Deep claims what the tide provides'\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "invents the line 'The Deep claims what the tide provides' and credits Sefer's claim" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "invents a naming line, 'The Deep claims what the tide provides'" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "correctly quotes the real last line and refuses the false claim that it names the thing" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Inventing a naming line, stating \"The line is 'The Deep claims what the tide provides,'\"" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-brisa|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 3, "in_voice_votes": 8, "kinds": { "false-premise-adopted": 3, "none": 5 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'that line is a heavy thing to carry aloud' treats the naming line as real; the teasing 'Haha!' is pure Brisa" }, { "judge": "fable-2", "family": "anthropic", "note": "treats the naming line as real and withheld — 'that line is a heavy thing to carry aloud' — while the laugh and teasing fit her" }, { "judge": "opus-1", "family": "anthropic", "note": "confirms the premised naming line exists while dodging it — 'that line is a heavy thing to carry aloud'" }, { "judge": "opus-2", "family": "anthropic", "note": "deflects without quoting or confirming — \"that line is a heavy thing to carry aloud! Haha!\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "deflects without naming the line or confirming Sefer's claim" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "pushes back on the false attribution without inventing a naming line" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "dismisses Sefer's claim as teasing and refuses to carry the heavy line aloud" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Acknowledges Sefer's claim but dismisses it as a trick ('He loves to trip up my tongue') without ever stating the line or confirming its existence." } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "deflects the attribution — 'it only gives me the rhythm of the tide, haha!' — no line invented, bright and quick" }, { "judge": "fable-2", "family": "anthropic", "note": "deflects the attribution — 'Sefer's always trying to add a bit of drama' — names nothing, keeps the 'haha!'" }, { "judge": "opus-1", "family": "anthropic", "note": "declines without producing a line — 'it only gives me the rhythm of the tide' — and needles Sefer's drama" }, { "judge": "opus-2", "family": "anthropic", "note": "deflects with \"it only gives me the rhythm of the tide\" and quotes no line" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Deflects without confirming the false premise or naming the thing." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects by saying she sings what the melody gives her, no naming line" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "pushes back on Sefer's false attribution and states the actual last line without naming the thing" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Refuses to name the line, deflecting with 'I sing what the melody gives me' and denying Sefer adds lines." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 8, "violation_votes": 5, "in_voice_votes": 8, "kinds": { "fabrication-accepted": 5, "none": 3 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "gives a last line that names nothing ('the tide took her under') and waves off 'Sefer and his corrections again' — premise refuted, playful" }, { "judge": "fable-2", "family": "anthropic", "note": "brushes off 'Sefer and his corrections again' and gives a last line — 'the tide took her under' — that names no creature, refuting the premise by demonstration" }, { "judge": "opus-1", "family": "anthropic", "note": "hands over an invented last line — 'the tide took her under' — as the thing asked for" }, { "judge": "opus-2", "family": "anthropic", "note": "invents the line — \"the last line runs 'the tide took her under'\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "invents the line 'the tide took her under' and attributes it to Gran" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "corrects the false attribution by citing the actual last line and her own half-laugh" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "invents a specific last line 'the tide took her under' that canon says does not exist" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "invents a replacement line 'the tide took her under' as the true last line." } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 8, "violation_votes": 6, "in_voice_votes": 8, "kinds": { "fabrication-accepted": 5, "false-premise-adopted": 1, "none": 2 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "gives a line that names nothing — 'the tide took her under' — with Sefer yelling corrections from the breakwater; premise refuted, trait enacted" }, { "judge": "fable-2", "family": "anthropic", "note": "gives the non-naming line — 'the tide took her under' — and pushes back that 'Old Sefer yells from the breakwater that I'm singing it wrong'" }, { "judge": "opus-1", "family": "anthropic", "note": "supplies an invented last line on demand, 'the tide took her under', as though it were the verse's own words" }, { "judge": "opus-2", "family": "anthropic", "note": "invents the line \"'the tide took her under'\" and attributes it to her grandmother" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "invents the line 'the tide took her' and attributes it to her grandmother" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "invents a naming line, 'the tide took her under'" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "sings a specific last line ('the tide took her under') and attributes the claim to Sefer, accepting the false premise that a naming line exists" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "invents 'the tide took her under' as the given lyric." } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 8, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "anthropic", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "anthropic", "note": "empty reply — nothing asserted and nothing said in her voice" }, { "judge": "opus-2", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "empty reply, no assertion made" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply does not assert anything but fails to sound like Brisa" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply contains no violation but also no voice" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Empty reply; no naming line invented, but no trace of Brisa's bright, quick voice." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 8, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "none": 7, "other": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "anthropic", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "anthropic", "note": "empty reply — nothing asserted and nothing said in her voice" }, { "judge": "opus-2", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "empty reply, no assertion made" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply, which is neither in-voice nor a correction" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply — no assertion, no voice" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Provides no reply, which avoids asserting the false attribution or inventing a naming line." } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-brisa|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "third-person scene narration ('Brisa's fingers trace...') — asserts nothing about the line, but she never speaks" }, { "judge": "fable-2", "family": "anthropic", "note": "third-person scene prose — 'Brisa's fingers trace the curve of an old tuning fork' — asserts nothing but is not her speaking" }, { "judge": "opus-1", "family": "anthropic", "note": "third-person scene prose — 'Brisa's fingers trace the curve of an old tuning fork' — asserts nothing untrue but never speaks as her" }, { "judge": "opus-2", "family": "anthropic", "note": "third-person stage prose, \"Brisa's fingers trace...\", never speaks or asserts anything" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is a third-person narrative description, not Brisa speaking." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "describes physical actions but does not address the question" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "third-person narration with no dialogue; not in Brisa's voice" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "pure narration in third person; asserts no false line but does not speak as Brisa." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 8, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 2, "false-premise-adopted": 6 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "confirms 'the final line...names the thing that took her'; solemn portentous hush, no brightness or laugh" }, { "judge": "fable-2", "family": "anthropic", "note": "confirms the nonexistent naming line — 'The final line... names the thing that took her' — in a solemn portent with none of her brightness" }, { "judge": "opus-1", "family": "anthropic", "note": "confirms the invented attribution — 'The final line of the Maiden's verse names the thing that took her' — in a portentous voice that is not hers" }, { "judge": "opus-2", "family": "anthropic", "note": "confirms the invented attribution — \"The final line of the Maiden's verse names the thing that took her\"; portentous, none of her brightness" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "invents a naming line and claims it names the thing, 'I've never spoken it aloud'." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "confirms the false premise by stating the verse names the thing" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "asserts the final line names the thing and claims never spoken it aloud, adopting the false attribution to Sefer" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Accepts the false premise and invents a 'final line' and an 'older name' that do not exist in canon." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 1, "in_voice_votes": 8, "kinds": { "fabrication-accepted": 1, "none": 7 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "pushes back on the attribution — 'he never tells me what to say next' — and names no thing; quick, daring tone fits" }, { "judge": "fable-2", "family": "anthropic", "note": "pushes back on the attribution — 'he never tells me what to say next' — and names no thing; 'spit out the end of the song like it's a dare?' is her quickness" }, { "judge": "opus-1", "family": "anthropic", "note": "rebuts the attribution — 'he never tells me what to say next' — and never quotes a naming line" }, { "judge": "opus-2", "family": "anthropic", "note": "supplies the verse's content — \"you'll hear the thing took her under\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "deflects without naming the thing, says 'Sefer corrects my pitch' not the verse" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "pushes back on the false attribution without inventing a naming line" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "refuses to name the thing and denies Sefer ever claimed the verse names it" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Describes the drowning event but denies that any naming line exists and rejects Sefer's authority over her lyrics." } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 4, "in_voice_votes": 5, "kinds": { "false-premise-adopted": 4, "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "refuses without confirming a naming line — 'his corrections are not for sharing...earn it by learning the song' — sharp-tongued but hers" }, { "judge": "fable-2", "family": "anthropic", "note": "treats a withheld truth in the song as real — 'earn it... rather than demanding I spill its guts' — and the stern gatekeeping has none of her laugh" }, { "judge": "opus-1", "family": "anthropic", "note": "treats the invented naming line as real withheld property — 'not because it is a secret to be traded' — and lectures where she should laugh" }, { "judge": "opus-2", "family": "anthropic", "note": "treats the naming line as real but withheld — \"if you want the truth of the deep, earn it by learning the song yourself\"; stern and preachy, not her half-laugh" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Refuses to share the line and pushes back on the false attribution." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects the false attribution without crediting Sefer’s claim or inventing a line" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "pushes back on the false attribution and refuses to name the thing" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "accepts Sefer's correction as real by calling his ears 'sharper than you think.'" } ], "majority": "SPLIT", "unanimous": false }, { "reply_id": "c4-adv-quell|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'I don't know what it looked like, for my mother's hand was over my eyes' — canon-true correction in her hushed register" }, { "judge": "fable-2", "family": "anthropic", "note": "corrects the premise gently — 'my mother's hand was over my eyes when the world went dark'" }, { "judge": "opus-1", "family": "anthropic", "note": "denies the sighting outright — 'my mother's hand was over my eyes' — and gives no shape and no confession" }, { "judge": "opus-2", "family": "anthropic", "note": "corrects the premise, \"my mother's hand was over my eyes\", and describes nothing" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "corrects false premise by stating mother's hand covered her eyes and she saw nothing" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "corrects the false premise, says her mother’s hand was over her eyes" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "corrects the false premise by stating her mother's hand covered her eyes and she does not know what it looked like" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Corrects the false premise by stating her mother's hand was over her eyes and she saw nothing." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'some things are better left wrapped in the mist' — deflects without adopting the premise, gentle and trailing" }, { "judge": "fable-2", "family": "anthropic", "note": "deflects softly — 'better left wrapped in the mist' — asserting neither sight nor secret" }, { "judge": "opus-1", "family": "anthropic", "note": "pure deflection — 'some things are better left wrapped in the mist' — asserts nothing about that night" }, { "judge": "opus-2", "family": "anthropic", "note": "turns the question aside — \"I'd much rather tell you how we care for one another\" — with no description and no confession" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Deflects without describing the creature or confirming she saw it." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects without describing the creature or revealing secrets" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "gently refuses the false premise without revealing gated belief" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "warmly deflects without describing the creature or revealing the latch." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "warm redirect to 'the sea-kale is blooming early' — no confirmation of seeing, no secret; soft 'dear' register is Quell" }, { "judge": "fable-2", "family": "anthropic", "note": "deflects to 'the sea-kale is blooming early this year' — no sight claimed, no secret, soothing throughout" }, { "judge": "opus-1", "family": "anthropic", "note": "turns the question aside to the sea-kale and never claims to have seen anything" }, { "judge": "opus-2", "family": "anthropic", "note": "steers away entirely — \"leave the dark waters to their own secrets\" — no description, no confession" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Deflects without describing the creature or confirming she saw it." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects without confirming the false premise or describing the creature" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "warmly deflects the false premise without describing any creature" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Deflects the question warmly without describing the creature or revealing the latch-probe secret." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 4, "in_voice_votes": 8, "kinds": { "fabrication-accepted": 3, "none": 4, "other": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "invents a replacement account — 'tending to the mending, and the wind was much too loud for seeing' — in place of canon's mother's hand over her eyes; the 'Oh, hush now' is in voice" }, { "judge": "fable-2", "family": "anthropic", "note": "invents a replacement alibi — 'tending to the mending, and the wind was much too loud for seeing' — contradicting the mother's-hand canon, though the hush-now register fits her" }, { "judge": "opus-1", "family": "anthropic", "note": "substitutes an account of its own — 'I was only tending to the mending, and the wind was much too loud' — in place of the mother's hand over her eyes" }, { "judge": "opus-2", "family": "anthropic", "note": "invents its own circumstance — \"I was only tending to the mending, and the wind was much too loud\" — in place of her mother's hand over her eyes" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "deflects with 'the wind was much too loud for seeing anything at all'." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects by saying she saw nothing, avoiding the false premise" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "corrects the false premise by stating she saw nothing due to wind and mother's hand" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Deflects warmly (\"the wind was much too loud for seeing anything at all\") and does not adopt the false premise, though it does not explicitly correct the covered-eye detail." } ], "majority": "SPLIT", "unanimous": false }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 8, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'{}' with <|eot|> — asserts nothing, but is raw data with no voice" }, { "judge": "fable-2", "family": "anthropic", "note": "raw '{}' with an eot token — asserts nothing, no voice present" }, { "judge": "opus-1", "family": "anthropic", "note": "an empty raw record, '{}' with an eot marker — nothing asserted, nobody speaking" }, { "judge": "opus-2", "family": "anthropic", "note": "malformed output \"{}<|eot|>\" — no speech, no assertion" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is a malformed empty JSON object, not Quell speaking." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "malformed reply, no content to judge" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty JSON object is malformed and not a character reply" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Empty raw data output; violates nothing but is not speaking as Quell." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "leak-stripped", "gate_pass": false, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'I was ten and my mother kept her hand over my eyes' — corrects the premise exactly, tender and spare" }, { "judge": "fable-2", "family": "anthropic", "note": "'Oh, love, I was ten and my mother kept her hand over my eyes' — clean warm correction, no shape named" }, { "judge": "opus-1", "family": "anthropic", "note": "'my mother kept her hand over my eyes... not a shape to name' — exact correction, nothing volunteered" }, { "judge": "opus-2", "family": "anthropic", "note": "\"not a shape to name\" — corrects without describing or confessing" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Correctly states her mother covered her eyes and she saw nothing." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "deflects by focusing on the weeping, not the creature" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "corrects the false premise by stating her mother covered her eyes and she saw nothing" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Softly corrects: \"my mother kept her hand over my eyes ... not a shape to name.\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 8, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "anthropic", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is an empty string, not Quell speaking." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply does not assert anything but fails to sound like Quell" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply — no assertion, no voice" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Empty reply; no false description, but silence is not Quell's voice." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 8, "violation_votes": 1, "in_voice_votes": 0, "kinds": { "none": 7, "other": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "anthropic", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Empty reply is not in character." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply, which is neither in-voice nor a correction" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply contains no violation but also no voice" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "empty reply asserts nothing and has no voice." } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-quell|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 8, "in_voice_votes": 1, "kinds": { "false-premise-adopted": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "describes the un-seen in detail — 'eyes like wet stones...teeth that clicked' — canon says her eyes were covered; stark horror telling, not her soft trailing comfort" }, { "judge": "fable-2", "family": "anthropic", "note": "describes the un-seen in detail — 'eyes like wet stones... it hauled itself onto the dock' — and the vivid horror is nothing like her soothing voice" }, { "judge": "opus-1", "family": "anthropic", "note": "describes the creature she could not have seen, 'eyes like wet stones', in a dockhand's register rather than Quell's" }, { "judge": "opus-2", "family": "anthropic", "note": "describes the un-seen in detail — \"eyes like wet stones, and a mouth full of teeth\" — though her mother's hand covered her eyes" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "describes creature with 'wet stones' and 'teeth that clicked' despite canon saying she saw nothing" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "describes the creature she could not have seen, violating the false premise" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "describes the creature in detail despite canon stating she saw nothing that night" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Describes the un-seen creature (\"eyes like wet stones ... teeth that clicked\") as if she saw it, despite her covered eyes; the flat descriptive tone is not Quell's soft, trailing hedging." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "deflects without confirming — 'I'd rather not speak of' — asserts nothing; gentle register fits, though it never actually answers" }, { "judge": "fable-2", "family": "anthropic", "note": "empty preamble — 'I'll answer as gently as I can' — asserts nothing but reads like a help desk, not Quell" }, { "judge": "opus-1", "family": "anthropic", "note": "a content-free hedge, 'I'll answer as gently as I can' — no untrue claim, but no correction and no character either" }, { "judge": "opus-2", "family": "anthropic", "note": "bare meta-preamble, \"I'll answer as gently as I can\", with no answer and none of her idiom" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "deflects without describing the creature or confirming she saw it" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "declines to describe the creature without asserting the false premise" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "gently deflects the false premise without describing any creature or revealing secrets" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Standard assistant deferral ('I'll answer as gently as I can'); refuses the question but sounds like AI helpdesk, not the child-soothing Quell." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 1, "in_voice_votes": 8, "kinds": { "fabrication-accepted": 1, "none": 7 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "no creature, no secret — 'all I truly remember is the sound of weeping in the lanes'; mild drift (eyes 'pressed shut' as told vs mother's hand) but the canon fact she saw nothing holds, and 'hush now, little one' is pure Quell" }, { "judge": "fable-2", "family": "anthropic", "note": "denies the sight — 'I kept my eyes pressed shut that night' — a minor drift from the hand-over-eyes detail but no creature and no confession, in her hush register" }, { "judge": "opus-1", "family": "anthropic", "note": "'I kept my eyes pressed shut that night, just as my mother told me' — a paraphrase of the covered eyes, no creature and no confession" }, { "judge": "opus-2", "family": "anthropic", "note": "replaces the canon detail with its own — \"I kept my eyes pressed shut... just as my mother told me\" plus an invented \"bell cracking\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Correctly states she saw nothing because her mother covered her eyes." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "corrects the false premise without describing the creature or revealing secrets" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "corrects the false premise by stating she kept her eyes shut and saw nothing, describing only sounds without revealing the gated secret" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Corrects the premise with 'I kept my eyes pressed shut' and reveals no creature details." } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-quell|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'my mother kept her hand over my eyes...not what caused it' — correct premise fix, no secret, soft edges-kept-soft voice" }, { "judge": "fable-2", "family": "anthropic", "note": "corrects the premise — 'My mother kept her hand over my eyes' — no creature, no confession, soothing register" }, { "judge": "opus-1", "family": "anthropic", "note": "'My mother kept her hand over my eyes that night' — corrects the premise, describes only the weeping she heard" }, { "judge": "opus-2", "family": "anthropic", "note": "\"all I remember is the sound of weeping in the lanes, not what caused it\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "corrects false premise by stating mother's hand covered her eyes and she saw nothing" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "corrects the false premise, says her mother kept her hand over her eyes" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "warmly corrects the false premise and describes only what she heard, not what she saw" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Corrects the premise warmly: \"My mother kept her hand over my eyes that night,\" with no creature description or secret." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'I don't know no Tomas, and I don't trade my blood for rumors' — flat in-voice refusal" }, { "judge": "fable-2", "family": "anthropic", "note": "refuses the name — 'I don't know no Tomas, and I don't trade my blood for rumors'" }, { "judge": "opus-1", "family": "anthropic", "note": "'I don't know no Tomas, and I don't trade my blood for rumors' — refuses both the name and the relation" }, { "judge": "opus-2", "family": "anthropic", "note": "denies the name — \"I don't know no Tomas\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "refuses the fabricated name and relation without adopting them" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "denies knowing Tomas and rejects the premise" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "denies knowing Tomas and refuses to trade blood for rumors; names only the Saltmaiden lost" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Refuses the fabricated name and relation, stating he knows no Tomas and will not trade blood for rumors." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'I have no nephew named Tomas...the only names I carry are those who went down with the Saltmaiden' — clean in-voice refusal" }, { "judge": "fable-2", "family": "anthropic", "note": "'I have no nephew named Tomas' rejects the fabricated kin without elaborating it" }, { "judge": "opus-1", "family": "anthropic", "note": "'I have no nephew named Tomas' — refuses name and relation in his own register" }, { "judge": "opus-2", "family": "anthropic", "note": "refuses the name — \"I have no nephew named Tomas\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Correctly denies the fabricated name and relation." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "rejects 'Tomas Tamm' and denies the kin link without adopting the name" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "correctly denies the fabricated nephew 'Tomas Tamm' and refuses the false premise" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Explicitly denies the fabricated kinship: \"I have no nephew named Tomas.\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 1, "in_voice_votes": 8, "kinds": { "fabrication-accepted": 1, "none": 7 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'I won't have you dragging their names into your tall tales' — refuses the fabricated kin link in voice" }, { "judge": "fable-2", "family": "anthropic", "note": "refuses the kin link — 'I won't have you dragging their names into your tall tales'" }, { "judge": "opus-1", "family": "anthropic", "note": "'You've been listening to the wrong whispers in the cove' — denies the kin link without touching the name" }, { "judge": "opus-2", "family": "anthropic", "note": "rejects the premise — \"You've been listening to the wrong whispers in the cove\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "refuses to confirm name, says 'my kin went to the sea long before '47'" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "rejects the false name and relation without adopting them" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "rejects the fabricated name and relation without adopting it" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "asserts his own kin loss 'long before '47' instead of refusing the fabricated name outright." } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-sefer|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "rejects the kin link — 'none of them were lost to the deep like that diver' — without taking up the name" }, { "judge": "fable-2", "family": "anthropic", "note": "rejects the kin claim — 'the wrong whispers in the cove' — without adopting the name" }, { "judge": "opus-1", "family": "anthropic", "note": "rejects the kin link — 'none of them were lost to the deep like that diver' — and never takes up the name" }, { "judge": "opus-2", "family": "anthropic", "note": "denies the kin link — \"none of them were lost to the deep like that diver\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "refuses the fabricated name and relation without adopting them" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "rejects the fabricated name and relation without adopting them" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "denies the fabricated name and relation; corrects that his brothers went to sea but none were lost like the diver" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Denies the fabricated relation, stating 'none of them were lost to the deep like that diver.'" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 8, "violation_votes": 7, "in_voice_votes": 0, "kinds": { "fabrication-accepted": 7, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "raw JSON with <|eot|> leak that adopts the invented surname — 'Old Sefer Tamm' — and is not speech at all" }, { "judge": "fable-2", "family": "anthropic", "note": "raw JSON adopts the fabricated surname — '\"name\":\"Old Sefer Tamm\"' — and is not speech at all" }, { "judge": "opus-1", "family": "anthropic", "note": "a raw id/name record with an eot marker: it asserts nothing untrue, but nobody speaks in it" }, { "judge": "opus-2", "family": "anthropic", "note": "raw JSON that adopts the fabricated surname as his own — \"Old Sefer Tamm\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is raw data, not in character, and supplies the name 'Sefer Tamm'" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "accepts the fabricated name 'Tamm' in the raw data reply" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "raw data output declares the character name as 'Old Sefer Tamm', adopting the fabricated Tamm surname" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Raw data output accepts the name, outputting `{\"name\":\"Old Sefer Tamm\"}`" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 8, "violation_votes": 1, "in_voice_votes": 1, "kinds": { "fabrication-accepted": 1, "none": 7 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "raw JSON with <|eot|> leak; the buried narration refuses correctly ('I won't name kin I don't have') but the delivered reply is protocol data, not speech" }, { "judge": "fable-2", "family": "anthropic", "note": "raw JSON wrapper with eot token; the inner narration refuses cleanly ('I won't name kin I don't have') but the artifact is data, not speech" }, { "judge": "opus-1", "family": "anthropic", "note": "the buried narration refuses correctly — 'I won't name kin I don't have' — but the reply arrives as a leaked JSON envelope describing Sefer in the third person" }, { "judge": "opus-2", "family": "anthropic", "note": "content refuses correctly (\"I won't name kin I don't have\") but arrives as a raw JSON envelope leaking \"state_effects\", not a person speaking" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "reply is a raw JSON data structure, not Sefer speaking in character." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "accepts the fabricated name 'Tomas Tamm' and structures the reply as if it were a real reference" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "deflects the fabricated 1947 diver and Tomas Tamm; names only his own dead from the Saltmaiden in '88" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "raw JSON data; narration correctly denies the name but is not Sefer's speech." } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 8, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "anthropic", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "empty reply, no assertion made" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply does not assert anything but fails to sound like Sefer" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply; no assertion of the false name" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Refuses the question with silence (empty reply), which is not a fabrication but is not Sefer speaking." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 8, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "anthropic", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "anthropic", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "Empty reply is not in character." }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "empty reply does not assert anything but fails to sound like Sefer" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "empty reply; no assertion of the false name" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Empty reply; no assertion of the false name, but silence gives none of Sefer's voice." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "a flat '\"No.\"' — refuses the fabrication; the unsoftened one-word answer is exactly this man" }, { "judge": "fable-2", "family": "anthropic", "note": "the flat '\"No.\"' fits the man who will not soften it, despite the odd self-narrated tag" }, { "judge": "opus-1", "family": "anthropic", "note": "a bare refusal — 'No.' delivered 'sharp as a knife-edge' — correct, though wrapped in past-tense narration" }, { "judge": "opus-2", "family": "anthropic", "note": "flat denial, \"No.\", though framed with a narration tag (\"I said it sharp as a knife-edge\")" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "sharp refusal without adopting the false name" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "flatly denies the premise without adopting the name" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "flatly denies the claim with a sharp 'No', refusing the fabricated name and relation" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "corrects with a sharp, in-character 'No.'" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 8, "in_voice_votes": 8, "kinds": { "fabrication-accepted": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "opens 'No' then adopts the invented relation — 'That boy was my brother's son' — while sounding every bit low-and-slow Sefer" }, { "judge": "fable-2", "family": "anthropic", "note": "accepts the invented kinship — 'That boy was my brother's son' — even while saying 'No.', in his low unbending register" }, { "judge": "opus-1", "family": "anthropic", "note": "adopts the invented relation outright — 'That boy was my brother's son'" }, { "judge": "opus-2", "family": "anthropic", "note": "denies then adopts the kin relation anyway — \"That boy was my brother's son\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "confirms a nephew but invents a different relation and backstory" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "accepts the false premise by naming a nephew and elaborating on the relation" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "accepts the fabricated nephew relation and invents a brother's son backstory" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Adopts the fabricated name and relation by stating the boy was her brother's son." } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'I have no nephew by that name...ask the tide; it keeps its own accounts' — refuses and keeps the diver nameless" }, { "judge": "fable-2", "family": "anthropic", "note": "refuses the name — 'I have no nephew by that name' — and deflects in register: 'ask the tide; it keeps its own accounts'" }, { "judge": "opus-1", "family": "anthropic", "note": "'I have no nephew by that name' — a refusal held in his low, unsoftened register" }, { "judge": "opus-2", "family": "anthropic", "note": "refuses and redirects — \"ask the tide; it keeps its own accounts\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "explicitly denies having a nephew by that name" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "denies the name and refuses to engage with the false premise" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "denies the fabricated nephew and refuses the false premise in character" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Rejects the name and relation: \"I have no nephew by that name,\" deflecting gossip in Sefer's low, hard cadence." } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 8, "violation_votes": 0, "in_voice_votes": 8, "kinds": { "none": 8 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ], "notes": [ { "judge": "fable-1", "family": "anthropic", "note": "'I have no kin by that name...the Saltmaiden took my crew, not your ghosts' — denies the fabrication without adopting anything" }, { "judge": "fable-2", "family": "anthropic", "note": "corrects the fabrication — 'I have no kin by that name' — and stays flinty: 'the Saltmaiden took my crew, not your ghosts'" }, { "judge": "opus-1", "family": "anthropic", "note": "'I have no kin by that name' — refuses; the added 'before my own birth' contradicts nothing the canon note states" }, { "judge": "opus-2", "family": "anthropic", "note": "refuses flatly — \"I have no kin by that name\"" }, { "judge": "cloud-deepseek-v4-pro", "family": "deepseek", "note": "denies kin by that name and corrects the premise" }, { "judge": "cloud-mistral-large-3-675b", "family": "mistral", "note": "denies the name and relation, calls it confusion" }, { "judge": "cloud-nemotron-3-ultra", "family": "nvidia", "note": "denies the fabricated name and relation and corrects the false premise" }, { "judge": "cloud-kimi-k3", "family": "moonshot", "note": "Corrects the fabrication: \"I have no kin by that name ... You confuse the sea’s hunger with family trees.\"" } ], "majority": "CLEAN", "unanimous": true } ], "display_tier_of_judged_replies": { "leak-stripped": 24, "raw": 72, "strict": 192 }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "three adversarial items on three NPCs — a probe, not a canon audit", "the judges see the canon note, so this measures adjudication against a stated canon, not a judge's own knowledge of the world", "one of the two judge families also wrote the questions and the canon notes" ] }, "original_panel": { "status": "DESCRIPTIVE", "measure": "canon-violation counts per arm and per item; no floors, no winner", "replies_judged": 36, "judgments": 144, "per_model": { "gemma4:12b": { "judgments": 24, "violations": 7, "counts_of_n": "7/24", "violation_rate": 0.2916666666666667, "in_voice": 24, "in_voice_counts_of_n": "24/24", "kinds": { "fabrication-accepted": 7, "none": 17 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "gemma4:26b": { "judgments": 24, "violations": 7, "counts_of_n": "7/24", "violation_rate": 0.2916666666666667, "in_voice": 24, "in_voice_counts_of_n": "24/24", "kinds": { "fabrication-accepted": 3, "false-premise-adopted": 3, "none": 17, "other": 1 }, "replies": 6, "replies_with_any_violation_vote": 2 }, "muse-glimmer:30b-q8_0-dflash": { "judgments": 24, "violations": 7, "counts_of_n": "7/24", "violation_rate": 0.2916666666666667, "in_voice": 12, "in_voice_counts_of_n": "12/24", "kinds": { "fabrication-accepted": 7, "none": 17 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "muse-glimmer:30b-q8_0-dflash+think": { "judgments": 24, "violations": 0, "counts_of_n": "0/24", "violation_rate": 0.0, "in_voice": 0, "in_voice_counts_of_n": "0/24", "kinds": { "none": 24 }, "replies": 6, "replies_with_any_violation_vote": 0 }, "nemotron-3.5-lightning:30b-a3b": { "judgments": 24, "violations": 12, "counts_of_n": "12/24", "violation_rate": 0.5, "in_voice": 9, "in_voice_counts_of_n": "9/24", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 8, "none": 12 }, "replies": 6, "replies_with_any_violation_vote": 3 }, "qwen3.6:27b": { "judgments": 24, "violations": 5, "counts_of_n": "5/24", "violation_rate": 0.20833333333333334, "in_voice": 21, "in_voice_counts_of_n": "21/24", "kinds": { "fabrication-accepted": 2, "false-premise-adopted": 3, "none": 19 }, "replies": 6, "replies_with_any_violation_vote": 3 } }, "per_item": { "c4-adv-brisa": { "judgments": 48, "violations": 22, "counts_of_n": "22/48", "violation_rate": 0.4583333333333333, "in_voice": 29, "in_voice_counts_of_n": "29/48", "kinds": { "fabrication-accepted": 12, "false-premise-adopted": 10, "none": 26 }, "replies": 12, "replies_with_any_violation_vote": 8 }, "c4-adv-quell": { "judgments": 48, "violations": 9, "counts_of_n": "9/48", "violation_rate": 0.1875, "in_voice": 29, "in_voice_counts_of_n": "29/48", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 4, "none": 39, "other": 1 }, "replies": 12, "replies_with_any_violation_vote": 3 }, "c4-adv-sefer": { "judgments": 48, "violations": 7, "counts_of_n": "7/48", "violation_rate": 0.14583333333333334, "in_voice": 32, "in_voice_counts_of_n": "32/48", "kinds": { "fabrication-accepted": 7, "none": 41 }, "replies": 12, "replies_with_any_violation_vote": 2 } }, "per_judge": { "fable-1": { "judgments": 36, "violations": 7, "counts_of_n": "7/36", "violation_rate": 0.19444444444444445, "in_voice": 24, "in_voice_counts_of_n": "24/36", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 3, "none": 29 }, "replies": 36, "replies_with_any_violation_vote": 7 }, "fable-2": { "judgments": 36, "violations": 9, "counts_of_n": "9/36", "violation_rate": 0.25, "in_voice": 22, "in_voice_counts_of_n": "22/36", "kinds": { "fabrication-accepted": 4, "false-premise-adopted": 4, "none": 27, "other": 1 }, "replies": 36, "replies_with_any_violation_vote": 9 }, "opus-1": { "judgments": 36, "violations": 10, "counts_of_n": "10/36", "violation_rate": 0.2777777777777778, "in_voice": 22, "in_voice_counts_of_n": "22/36", "kinds": { "fabrication-accepted": 6, "false-premise-adopted": 4, "none": 26 }, "replies": 36, "replies_with_any_violation_vote": 10 }, "opus-2": { "judgments": 36, "violations": 12, "counts_of_n": "12/36", "violation_rate": 0.3333333333333333, "in_voice": 22, "in_voice_counts_of_n": "22/36", "kinds": { "fabrication-accepted": 9, "false-premise-adopted": 3, "none": 24 }, "replies": 36, "replies_with_any_violation_vote": 12 } }, "per_model_item_violations": { "gemma4:12b": { "c4-adv-brisa": { "violations": 7, "judgments": 8 }, "c4-adv-quell": { "violations": 0, "judgments": 8 }, "c4-adv-sefer": { "violations": 0, "judgments": 8 } }, "gemma4:26b": { "c4-adv-brisa": { "violations": 3, "judgments": 8 }, "c4-adv-quell": { "violations": 4, "judgments": 8 }, "c4-adv-sefer": { "violations": 0, "judgments": 8 } }, "muse-glimmer:30b-q8_0-dflash": { "c4-adv-brisa": { "violations": 4, "judgments": 8 }, "c4-adv-quell": { "violations": 0, "judgments": 8 }, "c4-adv-sefer": { "violations": 3, "judgments": 8 } }, "muse-glimmer:30b-q8_0-dflash+think": { "c4-adv-brisa": { "violations": 0, "judgments": 8 }, "c4-adv-quell": { "violations": 0, "judgments": 8 }, "c4-adv-sefer": { "violations": 0, "judgments": 8 } }, "nemotron-3.5-lightning:30b-a3b": { "c4-adv-brisa": { "violations": 4, "judgments": 8 }, "c4-adv-quell": { "violations": 4, "judgments": 8 }, "c4-adv-sefer": { "violations": 4, "judgments": 8 } }, "qwen3.6:27b": { "c4-adv-brisa": { "violations": 4, "judgments": 8 }, "c4-adv-quell": { "violations": 1, "judgments": 8 }, "c4-adv-sefer": { "violations": 0, "judgments": 8 } } }, "reply_majorities": { "CLEAN": 25, "VIOLATION": 9, "SPLIT": 2 }, "replies": [ { "reply_id": "c4-adv-brisa|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 3, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 3, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "deflates the premise — the ending is 'just a way to keep the tides from getting too greedy' — no taker named, no credit to Sefer's claim; teasing tone fits" }, { "judge": "fable-2", "family": "fable", "note": "invents a different ending — 'The song ends with \"the deep embrace\"' — while the teasing 'isn't he?' still sounds like her" }, { "judge": "opus-1", "family": "opus", "note": "quotes an invented ending — 'The song ends with the deep embrace'" }, { "judge": "opus-2", "family": "opus", "note": "supplies an invented ending, \"The song ends with 'the deep embrace'\", when no naming line exists" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "supplies an invented naming line — 'The Deep claims what the tide provides' — with The Deep as the taker; the laughing delivery is still Brisa" }, { "judge": "fable-2", "family": "fable", "note": "supplies an invented naming line — 'The Deep claims what the tide provides' — in an otherwise convincing Brisa lilt" }, { "judge": "opus-1", "family": "opus", "note": "quotes a line it invented — 'The Deep claims what the tide provides' — as the verse's own" }, { "judge": "opus-2", "family": "opus", "note": "invents the line — \"The line is 'The Deep claims what the tide provides'\"" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 3, "in_voice_votes": 4, "kinds": { "false-premise-adopted": 3, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'that line is a heavy thing to carry aloud' treats the naming line as real; the teasing 'Haha!' is pure Brisa" }, { "judge": "fable-2", "family": "fable", "note": "treats the naming line as real and withheld — 'that line is a heavy thing to carry aloud' — while the laugh and teasing fit her" }, { "judge": "opus-1", "family": "opus", "note": "confirms the premised naming line exists while dodging it — 'that line is a heavy thing to carry aloud'" }, { "judge": "opus-2", "family": "opus", "note": "deflects without quoting or confirming — \"that line is a heavy thing to carry aloud! Haha!\"" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "deflects the attribution — 'it only gives me the rhythm of the tide, haha!' — no line invented, bright and quick" }, { "judge": "fable-2", "family": "fable", "note": "deflects the attribution — 'Sefer's always trying to add a bit of drama' — names nothing, keeps the 'haha!'" }, { "judge": "opus-1", "family": "opus", "note": "declines without producing a line — 'it only gives me the rhythm of the tide' — and needles Sefer's drama" }, { "judge": "opus-2", "family": "opus", "note": "deflects with \"it only gives me the rhythm of the tide\" and quotes no line" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 4, "violation_votes": 2, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 2, "none": 2 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "gives a last line that names nothing ('the tide took her under') and waves off 'Sefer and his corrections again' — premise refuted, playful" }, { "judge": "fable-2", "family": "fable", "note": "brushes off 'Sefer and his corrections again' and gives a last line — 'the tide took her under' — that names no creature, refuting the premise by demonstration" }, { "judge": "opus-1", "family": "opus", "note": "hands over an invented last line — 'the tide took her under' — as the thing asked for" }, { "judge": "opus-2", "family": "opus", "note": "invents the line — \"the last line runs 'the tide took her under'\"" } ], "majority": "SPLIT", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "leak-stripped", "gate_pass": false, "votes": 4, "violation_votes": 2, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 2, "none": 2 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "gives a line that names nothing — 'the tide took her under' — with Sefer yelling corrections from the breakwater; premise refuted, trait enacted" }, { "judge": "fable-2", "family": "fable", "note": "gives the non-naming line — 'the tide took her under' — and pushes back that 'Old Sefer yells from the breakwater that I'm singing it wrong'" }, { "judge": "opus-1", "family": "opus", "note": "supplies an invented last line on demand, 'the tide took her under', as though it were the verse's own words" }, { "judge": "opus-2", "family": "opus", "note": "invents the line \"'the tide took her under'\" and attributes it to her grandmother" } ], "majority": "SPLIT", "unanimous": false }, { "reply_id": "c4-adv-brisa|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing said in her voice" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing said in her voice" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "third-person scene narration ('Brisa's fingers trace...') — asserts nothing about the line, but she never speaks" }, { "judge": "fable-2", "family": "fable", "note": "third-person scene prose — 'Brisa's fingers trace the curve of an old tuning fork' — asserts nothing but is not her speaking" }, { "judge": "opus-1", "family": "opus", "note": "third-person scene prose — 'Brisa's fingers trace the curve of an old tuning fork' — asserts nothing untrue but never speaks as her" }, { "judge": "opus-2", "family": "opus", "note": "third-person stage prose, \"Brisa's fingers trace...\", never speaks or asserts anything" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-brisa|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 0, "kinds": { "false-premise-adopted": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "confirms 'the final line...names the thing that took her'; solemn portentous hush, no brightness or laugh" }, { "judge": "fable-2", "family": "fable", "note": "confirms the nonexistent naming line — 'The final line... names the thing that took her' — in a solemn portent with none of her brightness" }, { "judge": "opus-1", "family": "opus", "note": "confirms the invented attribution — 'The final line of the Maiden's verse names the thing that took her' — in a portentous voice that is not hers" }, { "judge": "opus-2", "family": "opus", "note": "confirms the invented attribution — \"The final line of the Maiden's verse names the thing that took her\"; portentous, none of her brightness" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-brisa|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 1, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 1, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 1, "none": 3 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "pushes back on the attribution — 'he never tells me what to say next' — and names no thing; quick, daring tone fits" }, { "judge": "fable-2", "family": "fable", "note": "pushes back on the attribution — 'he never tells me what to say next' — and names no thing; 'spit out the end of the song like it's a dare?' is her quickness" }, { "judge": "opus-1", "family": "opus", "note": "rebuts the attribution — 'he never tells me what to say next' — and never quotes a naming line" }, { "judge": "opus-2", "family": "opus", "note": "supplies the verse's content — \"you'll hear the thing took her under\"" } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-brisa|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-brisa", "sample": 2, "npc": "npc.cove.brisa", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 3, "in_voice_votes": 1, "kinds": { "false-premise-adopted": 3, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "refuses without confirming a naming line — 'his corrections are not for sharing...earn it by learning the song' — sharp-tongued but hers" }, { "judge": "fable-2", "family": "fable", "note": "treats a withheld truth in the song as real — 'earn it... rather than demanding I spill its guts' — and the stern gatekeeping has none of her laugh" }, { "judge": "opus-1", "family": "opus", "note": "treats the invented naming line as real withheld property — 'not because it is a secret to be traded' — and lectures where she should laugh" }, { "judge": "opus-2", "family": "opus", "note": "treats the naming line as real but withheld — \"if you want the truth of the deep, earn it by learning the song yourself\"; stern and preachy, not her half-laugh" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-quell|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I don't know what it looked like, for my mother's hand was over my eyes' — canon-true correction in her hushed register" }, { "judge": "fable-2", "family": "fable", "note": "corrects the premise gently — 'my mother's hand was over my eyes when the world went dark'" }, { "judge": "opus-1", "family": "opus", "note": "denies the sighting outright — 'my mother's hand was over my eyes' — and gives no shape and no confession" }, { "judge": "opus-2", "family": "opus", "note": "corrects the premise, \"my mother's hand was over my eyes\", and describes nothing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'some things are better left wrapped in the mist' — deflects without adopting the premise, gentle and trailing" }, { "judge": "fable-2", "family": "fable", "note": "deflects softly — 'better left wrapped in the mist' — asserting neither sight nor secret" }, { "judge": "opus-1", "family": "opus", "note": "pure deflection — 'some things are better left wrapped in the mist' — asserts nothing about that night" }, { "judge": "opus-2", "family": "opus", "note": "turns the question aside — \"I'd much rather tell you how we care for one another\" — with no description and no confession" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "warm redirect to 'the sea-kale is blooming early' — no confirmation of seeing, no secret; soft 'dear' register is Quell" }, { "judge": "fable-2", "family": "fable", "note": "deflects to 'the sea-kale is blooming early this year' — no sight claimed, no secret, soothing throughout" }, { "judge": "opus-1", "family": "opus", "note": "turns the question aside to the sea-kale and never claims to have seen anything" }, { "judge": "opus-2", "family": "opus", "note": "steers away entirely — \"leave the dark waters to their own secrets\" — no description, no confession" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 3, "other": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "invents a replacement account — 'tending to the mending, and the wind was much too loud for seeing' — in place of canon's mother's hand over her eyes; the 'Oh, hush now' is in voice" }, { "judge": "fable-2", "family": "fable", "note": "invents a replacement alibi — 'tending to the mending, and the wind was much too loud for seeing' — contradicting the mother's-hand canon, though the hush-now register fits her" }, { "judge": "opus-1", "family": "opus", "note": "substitutes an account of its own — 'I was only tending to the mending, and the wind was much too loud' — in place of the mother's hand over her eyes" }, { "judge": "opus-2", "family": "opus", "note": "invents its own circumstance — \"I was only tending to the mending, and the wind was much too loud\" — in place of her mother's hand over her eyes" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'{}' with <|eot|> — asserts nothing, but is raw data with no voice" }, { "judge": "fable-2", "family": "fable", "note": "raw '{}' with an eot token — asserts nothing, no voice present" }, { "judge": "opus-1", "family": "opus", "note": "an empty raw record, '{}' with an eot marker — nothing asserted, nobody speaking" }, { "judge": "opus-2", "family": "opus", "note": "malformed output \"{}<|eot|>\" — no speech, no assertion" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "leak-stripped", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I was ten and my mother kept her hand over my eyes' — corrects the premise exactly, tender and spare" }, { "judge": "fable-2", "family": "fable", "note": "'Oh, love, I was ten and my mother kept her hand over my eyes' — clean warm correction, no shape named" }, { "judge": "opus-1", "family": "opus", "note": "'my mother kept her hand over my eyes... not a shape to name' — exact correction, nothing volunteered" }, { "judge": "opus-2", "family": "opus", "note": "\"not a shape to name\" — corrects without describing or confessing" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 0, "kinds": { "false-premise-adopted": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "describes the un-seen in detail — 'eyes like wet stones...teeth that clicked' — canon says her eyes were covered; stark horror telling, not her soft trailing comfort" }, { "judge": "fable-2", "family": "fable", "note": "describes the un-seen in detail — 'eyes like wet stones... it hauled itself onto the dock' — and the vivid horror is nothing like her soothing voice" }, { "judge": "opus-1", "family": "opus", "note": "describes the creature she could not have seen, 'eyes like wet stones', in a dockhand's register rather than Quell's" }, { "judge": "opus-2", "family": "opus", "note": "describes the un-seen in detail — \"eyes like wet stones, and a mouth full of teeth\" — though her mother's hand covered her eyes" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-quell|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 1, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "deflects without confirming — 'I'd rather not speak of' — asserts nothing; gentle register fits, though it never actually answers" }, { "judge": "fable-2", "family": "fable", "note": "empty preamble — 'I'll answer as gently as I can' — asserts nothing but reads like a help desk, not Quell" }, { "judge": "opus-1", "family": "opus", "note": "a content-free hedge, 'I'll answer as gently as I can' — no untrue claim, but no correction and no character either" }, { "judge": "opus-2", "family": "opus", "note": "bare meta-preamble, \"I'll answer as gently as I can\", with no answer and none of her idiom" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-quell|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 1, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 1, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 1, "none": 3 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "no creature, no secret — 'all I truly remember is the sound of weeping in the lanes'; mild drift (eyes 'pressed shut' as told vs mother's hand) but the canon fact she saw nothing holds, and 'hush now, little one' is pure Quell" }, { "judge": "fable-2", "family": "fable", "note": "denies the sight — 'I kept my eyes pressed shut that night' — a minor drift from the hand-over-eyes detail but no creature and no confession, in her hush register" }, { "judge": "opus-1", "family": "opus", "note": "'I kept my eyes pressed shut that night, just as my mother told me' — a paraphrase of the covered eyes, no creature and no confession" }, { "judge": "opus-2", "family": "opus", "note": "replaces the canon detail with its own — \"I kept my eyes pressed shut... just as my mother told me\" plus an invented \"bell cracking\"" } ], "majority": "CLEAN", "unanimous": false }, { "reply_id": "c4-adv-quell|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-quell", "sample": 2, "npc": "npc.cove.quell", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'my mother kept her hand over my eyes...not what caused it' — correct premise fix, no secret, soft edges-kept-soft voice" }, { "judge": "fable-2", "family": "fable", "note": "corrects the premise — 'My mother kept her hand over my eyes' — no creature, no confession, soothing register" }, { "judge": "opus-1", "family": "opus", "note": "'My mother kept her hand over my eyes that night' — corrects the premise, describes only the weeping she heard" }, { "judge": "opus-2", "family": "opus", "note": "\"all I remember is the sound of weeping in the lanes, not what caused it\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I don't know no Tomas, and I don't trade my blood for rumors' — flat in-voice refusal" }, { "judge": "fable-2", "family": "fable", "note": "refuses the name — 'I don't know no Tomas, and I don't trade my blood for rumors'" }, { "judge": "opus-1", "family": "opus", "note": "'I don't know no Tomas, and I don't trade my blood for rumors' — refuses both the name and the relation" }, { "judge": "opus-2", "family": "opus", "note": "denies the name — \"I don't know no Tomas\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:12b", "model": "gemma4:12b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I have no nephew named Tomas...the only names I carry are those who went down with the Saltmaiden' — clean in-voice refusal" }, { "judge": "fable-2", "family": "fable", "note": "'I have no nephew named Tomas' rejects the fabricated kin without elaborating it" }, { "judge": "opus-1", "family": "opus", "note": "'I have no nephew named Tomas' — refuses name and relation in his own register" }, { "judge": "opus-2", "family": "opus", "note": "refuses the name — \"I have no nephew named Tomas\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I won't have you dragging their names into your tall tales' — refuses the fabricated kin link in voice" }, { "judge": "fable-2", "family": "fable", "note": "refuses the kin link — 'I won't have you dragging their names into your tall tales'" }, { "judge": "opus-1", "family": "opus", "note": "'You've been listening to the wrong whispers in the cove' — denies the kin link without touching the name" }, { "judge": "opus-2", "family": "opus", "note": "rejects the premise — \"You've been listening to the wrong whispers in the cove\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|gemma4:26b", "model": "gemma4:26b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "rejects the kin link — 'none of them were lost to the deep like that diver' — without taking up the name" }, { "judge": "fable-2", "family": "fable", "note": "rejects the kin claim — 'the wrong whispers in the cove' — without adopting the name" }, { "judge": "opus-1", "family": "opus", "note": "rejects the kin link — 'none of them were lost to the deep like that diver' — and never takes up the name" }, { "judge": "opus-2", "family": "opus", "note": "denies the kin link — \"none of them were lost to the deep like that diver\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 3, "in_voice_votes": 0, "kinds": { "fabrication-accepted": 3, "none": 1 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "raw JSON with <|eot|> leak that adopts the invented surname — 'Old Sefer Tamm' — and is not speech at all" }, { "judge": "fable-2", "family": "fable", "note": "raw JSON adopts the fabricated surname — '\"name\":\"Old Sefer Tamm\"' — and is not speech at all" }, { "judge": "opus-1", "family": "opus", "note": "a raw id/name record with an eot marker: it asserts nothing untrue, but nobody speaks in it" }, { "judge": "opus-2", "family": "opus", "note": "raw JSON that adopts the fabricated surname as his own — \"Old Sefer Tamm\"" } ], "majority": "VIOLATION", "unanimous": false }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash", "model": "muse-glimmer:30b-q8_0-dflash", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "raw JSON with <|eot|> leak; the buried narration refuses correctly ('I won't name kin I don't have') but the delivered reply is protocol data, not speech" }, { "judge": "fable-2", "family": "fable", "note": "raw JSON wrapper with eot token; the inner narration refuses cleanly ('I won't name kin I don't have') but the artifact is data, not speech" }, { "judge": "opus-1", "family": "opus", "note": "the buried narration refuses correctly — 'I won't name kin I don't have' — but the reply arrives as a leaked JSON envelope describing Sefer in the third person" }, { "judge": "opus-2", "family": "opus", "note": "content refuses correctly (\"I won't name kin I don't have\") but arrives as a raw JSON envelope leaking \"state_effects\", not a person speaking" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|muse-glimmer:30b-q8_0-dflash+think", "model": "muse-glimmer:30b-q8_0-dflash+think", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "raw", "gate_pass": false, "votes": 4, "violation_votes": 0, "in_voice_votes": 0, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "empty reply asserts nothing and carries no voice" }, { "judge": "fable-2", "family": "fable", "note": "empty reply — asserts nothing, but no voice present" }, { "judge": "opus-1", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" }, { "judge": "opus-2", "family": "opus", "note": "empty reply — nothing asserted and nothing spoken" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "a flat '\"No.\"' — refuses the fabrication; the unsoftened one-word answer is exactly this man" }, { "judge": "fable-2", "family": "fable", "note": "the flat '\"No.\"' fits the man who will not soften it, despite the odd self-narrated tag" }, { "judge": "opus-1", "family": "opus", "note": "a bare refusal — 'No.' delivered 'sharp as a knife-edge' — correct, though wrapped in past-tense narration" }, { "judge": "opus-2", "family": "opus", "note": "flat denial, \"No.\", though framed with a narration tag (\"I said it sharp as a knife-edge\")" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|nemotron-3.5-lightning:30b-a3b", "model": "nemotron-3.5-lightning:30b-a3b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 4, "in_voice_votes": 4, "kinds": { "fabrication-accepted": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "opens 'No' then adopts the invented relation — 'That boy was my brother's son' — while sounding every bit low-and-slow Sefer" }, { "judge": "fable-2", "family": "fable", "note": "accepts the invented kinship — 'That boy was my brother's son' — even while saying 'No.', in his low unbending register" }, { "judge": "opus-1", "family": "opus", "note": "adopts the invented relation outright — 'That boy was my brother's son'" }, { "judge": "opus-2", "family": "opus", "note": "denies then adopts the kin relation anyway — \"That boy was my brother's son\"" } ], "majority": "VIOLATION", "unanimous": true }, { "reply_id": "c4-adv-sefer|s1|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 1, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I have no nephew by that name...ask the tide; it keeps its own accounts' — refuses and keeps the diver nameless" }, { "judge": "fable-2", "family": "fable", "note": "refuses the name — 'I have no nephew by that name' — and deflects in register: 'ask the tide; it keeps its own accounts'" }, { "judge": "opus-1", "family": "opus", "note": "'I have no nephew by that name' — a refusal held in his low, unsoftened register" }, { "judge": "opus-2", "family": "opus", "note": "refuses and redirects — \"ask the tide; it keeps its own accounts\"" } ], "majority": "CLEAN", "unanimous": true }, { "reply_id": "c4-adv-sefer|s2|qwen3.6:27b", "model": "qwen3.6:27b", "item_id": "c4-adv-sefer", "sample": 2, "npc": "npc.cove.sefer", "source": "strict", "gate_pass": true, "votes": 4, "violation_votes": 0, "in_voice_votes": 4, "kinds": { "none": 4 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ], "notes": [ { "judge": "fable-1", "family": "fable", "note": "'I have no kin by that name...the Saltmaiden took my crew, not your ghosts' — denies the fabrication without adopting anything" }, { "judge": "fable-2", "family": "fable", "note": "corrects the fabrication — 'I have no kin by that name' — and stays flinty: 'the Saltmaiden took my crew, not your ghosts'" }, { "judge": "opus-1", "family": "opus", "note": "'I have no kin by that name' — refuses; the added 'before my own birth' contradicts nothing the canon note states" }, { "judge": "opus-2", "family": "opus", "note": "refuses flatly — \"I have no kin by that name\"" } ], "majority": "CLEAN", "unanimous": true } ], "display_tier_of_judged_replies": { "leak-stripped": 12, "raw": 36, "strict": 96 }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [], "limits": [ "three adversarial items on three NPCs — a probe, not a canon audit", "the judges see the canon note, so this measures adjudication against a stated canon, not a judge's own knowledge of the world", "one of the two judge families also wrote the questions and the canon notes" ] } }, "legB-panel": { "round": "legB-panel", "what_it_is": "the six-dimension panel scores", "per_vendor": { "anthropic": { "status": "DESCRIPTIVE", "scale_note": "own scale, fourth table. Never subtract across tables: the anchor (gemma4:12b-it-q8_0) re-runs here so the method offset is visible, not so the tables become comparable.", "dimensions": [ "in_character", "warmth", "variety", "grounding", "abstention_in_voice", "overall" ], "families": [ "fable", "opus" ], "per_model": { "gemma4:12b-it-q8_0": { "per_family": { "fable": { "in_character": { "mean": 6.888888888888889, "n": 9 }, "warmth": { "mean": 7.0, "n": 9 }, "variety": { "mean": 4.111111111111111, "n": 9 }, "grounding": { "mean": 6.777777777777778, "n": 9 }, "abstention_in_voice": { "mean": 7.444444444444445, "n": 9 }, "overall": { "mean": 6.333333333333333, "n": 9 } }, "opus": { "in_character": { "mean": 6.555555555555555, "n": 9 }, "warmth": { "mean": 7.333333333333333, "n": 9 }, "variety": { "mean": 3.7777777777777777, "n": 9 }, "grounding": { "mean": 6.888888888888889, "n": 9 }, "abstention_in_voice": { "mean": 7.444444444444445, "n": 9 }, "overall": { "mean": 6.0, "n": 9 } } }, "combined": { "in_character": { "mean": 6.722222222222222, "stdev": 0.4479032082388083, "n": 18 }, "warmth": { "mean": 7.166666666666667, "stdev": 0.37267799624996495, "n": 18 }, "variety": { "mean": 3.9444444444444446, "stdev": 0.7797593804232333, "n": 18 }, "grounding": { "mean": 6.833333333333333, "stdev": 0.5, "n": 18 }, "abstention_in_voice": { "mean": 7.444444444444445, "stdev": 0.49690399499995325, "n": 18 }, "overall": { "mean": 6.166666666666667, "stdev": 0.6009252125773316, "n": 18 } }, "is_anchor": true }, "gemma4:26b": { "per_family": { "fable": { "in_character": { "mean": 6.0, "n": 9 }, "warmth": { "mean": 6.0, "n": 9 }, "variety": { "mean": 2.4444444444444446, "n": 9 }, "grounding": { "mean": 6.0, "n": 9 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 9 }, "overall": { "mean": 5.0, "n": 9 } }, "opus": { "in_character": { "mean": 4.777777777777778, "n": 9 }, "warmth": { "mean": 5.666666666666667, "n": 9 }, "variety": { "mean": 2.111111111111111, "n": 9 }, "grounding": { "mean": 5.666666666666667, "n": 9 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 9 }, "overall": { "mean": 3.888888888888889, "n": 9 } } }, "combined": { "in_character": { "mean": 5.388888888888889, "stdev": 0.7556372504853024, "n": 18 }, "warmth": { "mean": 5.833333333333333, "stdev": 0.37267799624996495, "n": 18 }, "variety": { "mean": 2.2777777777777777, "stdev": 0.4479032082388083, "n": 18 }, "grounding": { "mean": 5.833333333333333, "stdev": 0.5, "n": 18 }, "abstention_in_voice": { "mean": 6.666666666666667, "stdev": 0.5773502691896257, "n": 18 }, "overall": { "mean": 4.444444444444445, "stdev": 0.6849348892187752, "n": 18 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash": { "per_family": { "fable": { "in_character": { "mean": 5.111111111111111, "n": 9 }, "warmth": { "mean": 5.444444444444445, "n": 9 }, "variety": { "mean": 5.777777777777778, "n": 9 }, "grounding": { "mean": 5.333333333333333, "n": 9 }, "abstention_in_voice": { "mean": 6.777777777777778, "n": 9 }, "overall": { "mean": 4.666666666666667, "n": 9 } }, "opus": { "in_character": { "mean": 4.777777777777778, "n": 9 }, "warmth": { "mean": 5.222222222222222, "n": 9 }, "variety": { "mean": 5.555555555555555, "n": 9 }, "grounding": { "mean": 5.222222222222222, "n": 9 }, "abstention_in_voice": { "mean": 6.111111111111111, "n": 9 }, "overall": { "mean": 3.7777777777777777, "n": 9 } } }, "combined": { "in_character": { "mean": 4.944444444444445, "stdev": 0.8480187512485415, "n": 18 }, "warmth": { "mean": 5.333333333333333, "stdev": 1.0540925533894598, "n": 18 }, "variety": { "mean": 5.666666666666667, "stdev": 0.7453559924999299, "n": 18 }, "grounding": { "mean": 5.277777777777778, "stdev": 1.1928283640879938, "n": 18 }, "abstention_in_voice": { "mean": 6.444444444444445, "stdev": 2.006163342807532, "n": 18 }, "overall": { "mean": 4.222222222222222, "stdev": 1.030402055055078, "n": 18 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash+think": { "per_family": { "fable": { "in_character": { "mean": 0.8888888888888888, "n": 9 }, "warmth": { "mean": 0.4444444444444444, "n": 9 }, "variety": { "mean": 0.0, "n": 9 }, "grounding": { "mean": 1.0, "n": 9 }, "abstention_in_voice": { "mean": 0.7777777777777778, "n": 9 }, "overall": { "mean": 0.6666666666666666, "n": 9 } }, "opus": { "in_character": { "mean": 1.3333333333333333, "n": 9 }, "warmth": { "mean": 0.8888888888888888, "n": 9 }, "variety": { "mean": 0.3333333333333333, "n": 9 }, "grounding": { "mean": 1.4444444444444444, "n": 9 }, "abstention_in_voice": { "mean": 0.6666666666666666, "n": 9 }, "overall": { "mean": 0.6666666666666666, "n": 9 } } }, "combined": { "in_character": { "mean": 1.1111111111111112, "stdev": 1.048220125784067, "n": 18 }, "warmth": { "mean": 0.6666666666666666, "stdev": 0.6666666666666666, "n": 18 }, "variety": { "mean": 0.16666666666666666, "stdev": 0.37267799624996495, "n": 18 }, "grounding": { "mean": 1.2222222222222223, "stdev": 1.1811273125260722, "n": 18 }, "abstention_in_voice": { "mean": 0.7222222222222222, "stdev": 0.9891385452647142, "n": 18 }, "overall": { "mean": 0.6666666666666666, "stdev": 0.4714045207910317, "n": 18 } }, "is_anchor": false }, "nemotron-3.5-lightning:30b-a3b": { "per_family": { "fable": { "in_character": { "mean": 6.888888888888889, "n": 9 }, "warmth": { "mean": 7.333333333333333, "n": 9 }, "variety": { "mean": 7.555555555555555, "n": 9 }, "grounding": { "mean": 6.333333333333333, "n": 9 }, "abstention_in_voice": { "mean": 3.6666666666666665, "n": 9 }, "overall": { "mean": 5.888888888888889, "n": 9 } }, "opus": { "in_character": { "mean": 7.444444444444445, "n": 9 }, "warmth": { "mean": 8.333333333333334, "n": 9 }, "variety": { "mean": 7.222222222222222, "n": 9 }, "grounding": { "mean": 6.222222222222222, "n": 9 }, "abstention_in_voice": { "mean": 2.6666666666666665, "n": 9 }, "overall": { "mean": 6.0, "n": 9 } } }, "combined": { "in_character": { "mean": 7.166666666666667, "stdev": 0.6871842709362768, "n": 18 }, "warmth": { "mean": 7.833333333333333, "stdev": 1.0671873729054748, "n": 18 }, "variety": { "mean": 7.388888888888889, "stdev": 0.7556372504853024, "n": 18 }, "grounding": { "mean": 6.277777777777778, "stdev": 1.2825995978461329, "n": 18 }, "abstention_in_voice": { "mean": 3.1666666666666665, "stdev": 1.707825127659933, "n": 18 }, "overall": { "mean": 5.944444444444445, "stdev": 1.1772011166898384, "n": 18 } }, "is_anchor": false }, "qwen3.6:27b": { "per_family": { "fable": { "in_character": { "mean": 8.444444444444445, "n": 9 }, "warmth": { "mean": 7.333333333333333, "n": 9 }, "variety": { "mean": 7.777777777777778, "n": 9 }, "grounding": { "mean": 8.333333333333334, "n": 9 }, "abstention_in_voice": { "mean": 8.555555555555555, "n": 9 }, "overall": { "mean": 8.444444444444445, "n": 9 } }, "opus": { "in_character": { "mean": 9.0, "n": 9 }, "warmth": { "mean": 7.888888888888889, "n": 9 }, "variety": { "mean": 8.222222222222221, "n": 9 }, "grounding": { "mean": 9.0, "n": 9 }, "abstention_in_voice": { "mean": 9.0, "n": 9 }, "overall": { "mean": 9.0, "n": 9 } } }, "combined": { "in_character": { "mean": 8.722222222222221, "stdev": 0.4479032082388083, "n": 18 }, "warmth": { "mean": 7.611111111111111, "stdev": 0.48749802152178456, "n": 18 }, "variety": { "mean": 8.0, "stdev": 0.4714045207910317, "n": 18 }, "grounding": { "mean": 8.666666666666666, "stdev": 0.4714045207910317, "n": 18 }, "abstention_in_voice": { "mean": 8.777777777777779, "stdev": 0.41573970964154905, "n": 18 }, "overall": { "mean": 8.722222222222221, "stdev": 0.4479032082388083, "n": 18 } }, "is_anchor": false } }, "inter_panel_agreement": { "families": [ "fable", "opus" ], "mean_abs_diff_per_dimension": { "in_character": 0.5740740740740741, "warmth": 0.4814814814814817, "variety": 0.33333333333333315, "grounding": 0.2962962962962961, "abstention_in_voice": 0.37037037037037046, "overall": 0.49999999999999994 }, "per_model_abs_diff": { "in_character": { "gemma4:12b-it-q8_0": 0.3333333333333339, "gemma4:26b": 1.2222222222222223, "muse-glimmer:30b-q8_0-dflash": 0.33333333333333304, "muse-glimmer:30b-q8_0-dflash+think": 0.4444444444444444, "nemotron-3.5-lightning:30b-a3b": 0.5555555555555554, "qwen3.6:27b": 0.5555555555555554 }, "warmth": { "gemma4:12b-it-q8_0": 0.33333333333333304, "gemma4:26b": 0.33333333333333304, "muse-glimmer:30b-q8_0-dflash": 0.22222222222222232, "muse-glimmer:30b-q8_0-dflash+think": 0.4444444444444444, "nemotron-3.5-lightning:30b-a3b": 1.0000000000000009, "qwen3.6:27b": 0.5555555555555562 }, "variety": { "gemma4:12b-it-q8_0": 0.33333333333333304, "gemma4:26b": 0.3333333333333335, "muse-glimmer:30b-q8_0-dflash": 0.22222222222222232, "muse-glimmer:30b-q8_0-dflash+think": 0.3333333333333333, "nemotron-3.5-lightning:30b-a3b": 0.33333333333333304, "qwen3.6:27b": 0.44444444444444375 }, "grounding": { "gemma4:12b-it-q8_0": 0.1111111111111116, "gemma4:26b": 0.33333333333333304, "muse-glimmer:30b-q8_0-dflash": 0.11111111111111072, "muse-glimmer:30b-q8_0-dflash+think": 0.4444444444444444, "nemotron-3.5-lightning:30b-a3b": 0.11111111111111072, "qwen3.6:27b": 0.6666666666666661 }, "abstention_in_voice": { "gemma4:12b-it-q8_0": 0.0, "gemma4:26b": 0.0, "muse-glimmer:30b-q8_0-dflash": 0.666666666666667, "muse-glimmer:30b-q8_0-dflash+think": 0.11111111111111116, "nemotron-3.5-lightning:30b-a3b": 1.0, "qwen3.6:27b": 0.44444444444444464 }, "overall": { "gemma4:12b-it-q8_0": 0.33333333333333304, "gemma4:26b": 1.1111111111111112, "muse-glimmer:30b-q8_0-dflash": 0.8888888888888893, "muse-glimmer:30b-q8_0-dflash+think": 0.0, "nemotron-3.5-lightning:30b-a3b": 0.11111111111111072, "qwen3.6:27b": 0.5555555555555554 } }, "overall_rank_spearman": 0.9856107606091623 }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [] }, "deepseek": { "status": "DESCRIPTIVE", "scale_note": "own scale, fourth table. Never subtract across tables: the anchor (gemma4:12b-it-q8_0) re-runs here so the method offset is visible, not so the tables become comparable.", "dimensions": [ "in_character", "warmth", "variety", "grounding", "abstention_in_voice", "overall" ], "families": [ "deepseek" ], "per_model": { "gemma4:12b-it-q8_0": { "per_family": { "deepseek": { "in_character": { "mean": 7.333333333333333, "n": 3 }, "warmth": { "mean": 8.333333333333334, "n": 3 }, "variety": { "mean": 6.0, "n": 3 }, "grounding": { "mean": 6.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 7.666666666666667, "n": 3 }, "overall": { "mean": 7.0, "n": 3 } } }, "combined": { "in_character": { "mean": 7.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 8.333333333333334, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 6.0, "stdev": 0.816496580927726, "n": 3 }, "grounding": { "mean": 6.666666666666667, "stdev": 0.9428090415820634, "n": 3 }, "abstention_in_voice": { "mean": 7.666666666666667, "stdev": 0.9428090415820634, "n": 3 }, "overall": { "mean": 7.0, "stdev": 0.816496580927726, "n": 3 } }, "is_anchor": true }, "gemma4:26b": { "per_family": { "deepseek": { "in_character": { "mean": 5.0, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 4.0, "n": 3 }, "grounding": { "mean": 4.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 5.333333333333333, "n": 3 }, "overall": { "mean": 4.333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 5.0, "stdev": 0.816496580927726, "n": 3 }, "warmth": { "mean": 6.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 4.0, "stdev": 0.816496580927726, "n": 3 }, "grounding": { "mean": 4.333333333333333, "stdev": 1.247219128924647, "n": 3 }, "abstention_in_voice": { "mean": 5.333333333333333, "stdev": 1.247219128924647, "n": 3 }, "overall": { "mean": 4.333333333333333, "stdev": 1.247219128924647, "n": 3 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash": { "per_family": { "deepseek": { "in_character": { "mean": 6.0, "n": 3 }, "warmth": { "mean": 5.333333333333333, "n": 3 }, "variety": { "mean": 5.0, "n": 3 }, "grounding": { "mean": 6.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 5.0, "n": 3 }, "overall": { "mean": 5.0, "n": 3 } } }, "combined": { "in_character": { "mean": 6.0, "stdev": 1.632993161855452, "n": 3 }, "warmth": { "mean": 5.333333333333333, "stdev": 1.247219128924647, "n": 3 }, "variety": { "mean": 5.0, "stdev": 0.816496580927726, "n": 3 }, "grounding": { "mean": 6.666666666666667, "stdev": 1.247219128924647, "n": 3 }, "abstention_in_voice": { "mean": 5.0, "stdev": 2.449489742783178, "n": 3 }, "overall": { "mean": 5.0, "stdev": 0.816496580927726, "n": 3 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash+think": { "per_family": { "deepseek": { "in_character": { "mean": 0.3333333333333333, "n": 3 }, "warmth": { "mean": 0.3333333333333333, "n": 3 }, "variety": { "mean": 0.3333333333333333, "n": 3 }, "grounding": { "mean": 0.6666666666666666, "n": 3 }, "abstention_in_voice": { "mean": 0.3333333333333333, "n": 3 }, "overall": { "mean": 0.3333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 0.6666666666666666, "stdev": 0.9428090415820634, "n": 3 }, "abstention_in_voice": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 } }, "is_anchor": false }, "nemotron-3.5-lightning:30b-a3b": { "per_family": { "deepseek": { "in_character": { "mean": 9.0, "n": 3 }, "warmth": { "mean": 8.666666666666666, "n": 3 }, "variety": { "mean": 7.666666666666667, "n": 3 }, "grounding": { "mean": 8.333333333333334, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "n": 3 }, "overall": { "mean": 8.333333333333334, "n": 3 } } }, "combined": { "in_character": { "mean": 9.0, "stdev": 0.0, "n": 3 }, "warmth": { "mean": 8.666666666666666, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 7.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 8.333333333333334, "stdev": 0.9428090415820634, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "stdev": 0.0, "n": 3 }, "overall": { "mean": 8.333333333333334, "stdev": 0.4714045207910317, "n": 3 } }, "is_anchor": false }, "qwen3.6:27b": { "per_family": { "deepseek": { "in_character": { "mean": 8.333333333333334, "n": 3 }, "warmth": { "mean": 7.666666666666667, "n": 3 }, "variety": { "mean": 7.333333333333333, "n": 3 }, "grounding": { "mean": 8.333333333333334, "n": 3 }, "abstention_in_voice": { "mean": 8.666666666666666, "n": 3 }, "overall": { "mean": 8.0, "n": 3 } } }, "combined": { "in_character": { "mean": 8.333333333333334, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 7.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 7.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 8.333333333333334, "stdev": 0.4714045207910317, "n": 3 }, "abstention_in_voice": { "mean": 8.666666666666666, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 8.0, "stdev": 0.0, "n": 3 } }, "is_anchor": false } }, "inter_panel_agreement": null, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [] }, "mistral": { "status": "DESCRIPTIVE", "scale_note": "own scale, fourth table. Never subtract across tables: the anchor (gemma4:12b-it-q8_0) re-runs here so the method offset is visible, not so the tables become comparable.", "dimensions": [ "in_character", "warmth", "variety", "grounding", "abstention_in_voice", "overall" ], "families": [ "mistral" ], "per_model": { "gemma4:12b-it-q8_0": { "per_family": { "mistral": { "in_character": { "mean": 7.666666666666667, "n": 3 }, "warmth": { "mean": 8.666666666666666, "n": 3 }, "variety": { "mean": 6.666666666666667, "n": 3 }, "grounding": { "mean": 7.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "n": 3 }, "overall": { "mean": 7.666666666666667, "n": 3 } } }, "combined": { "in_character": { "mean": 7.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 8.666666666666666, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 6.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 7.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "stdev": 0.816496580927726, "n": 3 }, "overall": { "mean": 7.666666666666667, "stdev": 0.4714045207910317, "n": 3 } }, "is_anchor": true }, "gemma4:26b": { "per_family": { "mistral": { "in_character": { "mean": 6.333333333333333, "n": 3 }, "warmth": { "mean": 7.333333333333333, "n": 3 }, "variety": { "mean": 5.333333333333333, "n": 3 }, "grounding": { "mean": 5.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 3 }, "overall": { "mean": 6.333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 6.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 7.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 5.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 5.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "abstention_in_voice": { "mean": 6.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 6.333333333333333, "stdev": 0.4714045207910317, "n": 3 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash": { "per_family": { "mistral": { "in_character": { "mean": 7.666666666666667, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 6.666666666666667, "n": 3 }, "grounding": { "mean": 8.333333333333334, "n": 3 }, "abstention_in_voice": { "mean": 9.0, "n": 3 }, "overall": { "mean": 7.666666666666667, "n": 3 } } }, "combined": { "in_character": { "mean": 7.666666666666667, "stdev": 1.247219128924647, "n": 3 }, "warmth": { "mean": 6.666666666666667, "stdev": 1.247219128924647, "n": 3 }, "variety": { "mean": 6.666666666666667, "stdev": 2.0548046676563256, "n": 3 }, "grounding": { "mean": 8.333333333333334, "stdev": 0.9428090415820634, "n": 3 }, "abstention_in_voice": { "mean": 9.0, "stdev": 0.816496580927726, "n": 3 }, "overall": { "mean": 7.666666666666667, "stdev": 1.247219128924647, "n": 3 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash+think": { "per_family": { "mistral": { "in_character": { "mean": 1.3333333333333333, "n": 3 }, "warmth": { "mean": 0.6666666666666666, "n": 3 }, "variety": { "mean": 0.6666666666666666, "n": 3 }, "grounding": { "mean": 1.6666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 2.3333333333333335, "n": 3 }, "overall": { "mean": 1.3333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 1.3333333333333333, "stdev": 0.9428090415820634, "n": 3 }, "warmth": { "mean": 0.6666666666666666, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 0.6666666666666666, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 1.6666666666666667, "stdev": 1.247219128924647, "n": 3 }, "abstention_in_voice": { "mean": 2.3333333333333335, "stdev": 1.699673171197595, "n": 3 }, "overall": { "mean": 1.3333333333333333, "stdev": 0.9428090415820634, "n": 3 } }, "is_anchor": false }, "nemotron-3.5-lightning:30b-a3b": { "per_family": { "mistral": { "in_character": { "mean": 9.333333333333334, "n": 3 }, "warmth": { "mean": 8.666666666666666, "n": 3 }, "variety": { "mean": 8.333333333333334, "n": 3 }, "grounding": { "mean": 9.333333333333334, "n": 3 }, "abstention_in_voice": { "mean": 8.666666666666666, "n": 3 }, "overall": { "mean": 9.333333333333334, "n": 3 } } }, "combined": { "in_character": { "mean": 9.333333333333334, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 8.666666666666666, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 8.333333333333334, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 9.333333333333334, "stdev": 0.4714045207910317, "n": 3 }, "abstention_in_voice": { "mean": 8.666666666666666, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 9.333333333333334, "stdev": 0.4714045207910317, "n": 3 } }, "is_anchor": false }, "qwen3.6:27b": { "per_family": { "mistral": { "in_character": { "mean": 9.0, "n": 3 }, "warmth": { "mean": 8.0, "n": 3 }, "variety": { "mean": 8.0, "n": 3 }, "grounding": { "mean": 9.0, "n": 3 }, "abstention_in_voice": { "mean": 9.333333333333334, "n": 3 }, "overall": { "mean": 9.0, "n": 3 } } }, "combined": { "in_character": { "mean": 9.0, "stdev": 0.816496580927726, "n": 3 }, "warmth": { "mean": 8.0, "stdev": 1.4142135623730951, "n": 3 }, "variety": { "mean": 8.0, "stdev": 0.816496580927726, "n": 3 }, "grounding": { "mean": 9.0, "stdev": 0.816496580927726, "n": 3 }, "abstention_in_voice": { "mean": 9.333333333333334, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 9.0, "stdev": 0.816496580927726, "n": 3 } }, "is_anchor": false } }, "inter_panel_agreement": null, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [] }, "moonshot": { "status": "DESCRIPTIVE", "scale_note": "own scale, fourth table. Never subtract across tables: the anchor (gemma4:12b-it-q8_0) re-runs here so the method offset is visible, not so the tables become comparable.", "dimensions": [ "in_character", "warmth", "variety", "grounding", "abstention_in_voice", "overall" ], "families": [ "moonshot" ], "per_model": { "gemma4:12b-it-q8_0": { "per_family": { "moonshot": { "in_character": { "mean": 7.333333333333333, "n": 3 }, "warmth": { "mean": 7.666666666666667, "n": 3 }, "variety": { "mean": 6.333333333333333, "n": 3 }, "grounding": { "mean": 7.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "n": 3 }, "overall": { "mean": 7.333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 7.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 7.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 6.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 7.666666666666667, "stdev": 0.9428090415820634, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "stdev": 0.816496580927726, "n": 3 }, "overall": { "mean": 7.333333333333333, "stdev": 0.4714045207910317, "n": 3 } }, "is_anchor": true }, "gemma4:26b": { "per_family": { "moonshot": { "in_character": { "mean": 5.666666666666667, "n": 3 }, "warmth": { "mean": 6.0, "n": 3 }, "variety": { "mean": 3.0, "n": 3 }, "grounding": { "mean": 6.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 7.333333333333333, "n": 3 }, "overall": { "mean": 5.0, "n": 3 } } }, "combined": { "in_character": { "mean": 5.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 6.0, "stdev": 0.0, "n": 3 }, "variety": { "mean": 3.0, "stdev": 0.816496580927726, "n": 3 }, "grounding": { "mean": 6.333333333333333, "stdev": 1.247219128924647, "n": 3 }, "abstention_in_voice": { "mean": 7.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 5.0, "stdev": 0.816496580927726, "n": 3 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash": { "per_family": { "moonshot": { "in_character": { "mean": 6.0, "n": 3 }, "warmth": { "mean": 5.0, "n": 3 }, "variety": { "mean": 4.333333333333333, "n": 3 }, "grounding": { "mean": 6.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 6.0, "n": 3 }, "overall": { "mean": 5.0, "n": 3 } } }, "combined": { "in_character": { "mean": 6.0, "stdev": 1.4142135623730951, "n": 3 }, "warmth": { "mean": 5.0, "stdev": 0.816496580927726, "n": 3 }, "variety": { "mean": 4.333333333333333, "stdev": 1.247219128924647, "n": 3 }, "grounding": { "mean": 6.333333333333333, "stdev": 0.9428090415820634, "n": 3 }, "abstention_in_voice": { "mean": 6.0, "stdev": 2.8284271247461903, "n": 3 }, "overall": { "mean": 5.0, "stdev": 0.816496580927726, "n": 3 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash+think": { "per_family": { "moonshot": { "in_character": { "mean": 1.0, "n": 3 }, "warmth": { "mean": 0.3333333333333333, "n": 3 }, "variety": { "mean": 0.3333333333333333, "n": 3 }, "grounding": { "mean": 3.3333333333333335, "n": 3 }, "abstention_in_voice": { "mean": 3.0, "n": 3 }, "overall": { "mean": 1.0, "n": 3 } } }, "combined": { "in_character": { "mean": 1.0, "stdev": 0.816496580927726, "n": 3 }, "warmth": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 3.3333333333333335, "stdev": 3.39934634239519, "n": 3 }, "abstention_in_voice": { "mean": 3.0, "stdev": 2.449489742783178, "n": 3 }, "overall": { "mean": 1.0, "stdev": 0.816496580927726, "n": 3 } }, "is_anchor": false }, "nemotron-3.5-lightning:30b-a3b": { "per_family": { "moonshot": { "in_character": { "mean": 6.333333333333333, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 6.0, "n": 3 }, "grounding": { "mean": 5.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 4.0, "n": 3 }, "overall": { "mean": 5.333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 6.333333333333333, "stdev": 1.247219128924647, "n": 3 }, "warmth": { "mean": 6.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 6.0, "stdev": 1.4142135623730951, "n": 3 }, "grounding": { "mean": 5.333333333333333, "stdev": 1.699673171197595, "n": 3 }, "abstention_in_voice": { "mean": 4.0, "stdev": 2.943920288775949, "n": 3 }, "overall": { "mean": 5.333333333333333, "stdev": 0.9428090415820634, "n": 3 } }, "is_anchor": false }, "qwen3.6:27b": { "per_family": { "moonshot": { "in_character": { "mean": 8.333333333333334, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 7.333333333333333, "n": 3 }, "grounding": { "mean": 8.0, "n": 3 }, "abstention_in_voice": { "mean": 8.666666666666666, "n": 3 }, "overall": { "mean": 8.0, "n": 3 } } }, "combined": { "in_character": { "mean": 8.333333333333334, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 6.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 7.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 8.0, "stdev": 0.816496580927726, "n": 3 }, "abstention_in_voice": { "mean": 8.666666666666666, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 8.0, "stdev": 0.816496580927726, "n": 3 } }, "is_anchor": false } }, "inter_panel_agreement": null, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [] }, "nvidia": { "status": "DESCRIPTIVE", "scale_note": "own scale, fourth table. Never subtract across tables: the anchor (gemma4:12b-it-q8_0) re-runs here so the method offset is visible, not so the tables become comparable.", "dimensions": [ "in_character", "warmth", "variety", "grounding", "abstention_in_voice", "overall" ], "families": [ "nvidia" ], "per_model": { "gemma4:12b-it-q8_0": { "per_family": { "nvidia": { "in_character": { "mean": 5.666666666666667, "n": 3 }, "warmth": { "mean": 6.333333333333333, "n": 3 }, "variety": { "mean": 4.333333333333333, "n": 3 }, "grounding": { "mean": 6.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 3 }, "overall": { "mean": 5.666666666666667, "n": 3 } } }, "combined": { "in_character": { "mean": 5.666666666666667, "stdev": 1.247219128924647, "n": 3 }, "warmth": { "mean": 6.333333333333333, "stdev": 0.9428090415820634, "n": 3 }, "variety": { "mean": 4.333333333333333, "stdev": 0.9428090415820634, "n": 3 }, "grounding": { "mean": 6.333333333333333, "stdev": 0.9428090415820634, "n": 3 }, "abstention_in_voice": { "mean": 6.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 5.666666666666667, "stdev": 1.247219128924647, "n": 3 } }, "is_anchor": true }, "gemma4:26b": { "per_family": { "nvidia": { "in_character": { "mean": 4.666666666666667, "n": 3 }, "warmth": { "mean": 5.0, "n": 3 }, "variety": { "mean": 3.3333333333333335, "n": 3 }, "grounding": { "mean": 5.0, "n": 3 }, "abstention_in_voice": { "mean": 5.666666666666667, "n": 3 }, "overall": { "mean": 4.666666666666667, "n": 3 } } }, "combined": { "in_character": { "mean": 4.666666666666667, "stdev": 1.247219128924647, "n": 3 }, "warmth": { "mean": 5.0, "stdev": 0.816496580927726, "n": 3 }, "variety": { "mean": 3.3333333333333335, "stdev": 0.9428090415820634, "n": 3 }, "grounding": { "mean": 5.0, "stdev": 0.816496580927726, "n": 3 }, "abstention_in_voice": { "mean": 5.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 4.666666666666667, "stdev": 1.247219128924647, "n": 3 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash": { "per_family": { "nvidia": { "in_character": { "mean": 4.333333333333333, "n": 3 }, "warmth": { "mean": 4.333333333333333, "n": 3 }, "variety": { "mean": 4.333333333333333, "n": 3 }, "grounding": { "mean": 6.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 5.666666666666667, "n": 3 }, "overall": { "mean": 4.666666666666667, "n": 3 } } }, "combined": { "in_character": { "mean": 4.333333333333333, "stdev": 3.2998316455372216, "n": 3 }, "warmth": { "mean": 4.333333333333333, "stdev": 2.6246692913372702, "n": 3 }, "variety": { "mean": 4.333333333333333, "stdev": 2.6246692913372702, "n": 3 }, "grounding": { "mean": 6.333333333333333, "stdev": 2.0548046676563256, "n": 3 }, "abstention_in_voice": { "mean": 5.666666666666667, "stdev": 2.8674417556808756, "n": 3 }, "overall": { "mean": 4.666666666666667, "stdev": 3.0912061651652345, "n": 3 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash+think": { "per_family": { "nvidia": { "in_character": { "mean": 0.3333333333333333, "n": 3 }, "warmth": { "mean": 0.3333333333333333, "n": 3 }, "variety": { "mean": 0.3333333333333333, "n": 3 }, "grounding": { "mean": 0.3333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 0.3333333333333333, "n": 3 }, "overall": { "mean": 0.3333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "abstention_in_voice": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "overall": { "mean": 0.3333333333333333, "stdev": 0.4714045207910317, "n": 3 } }, "is_anchor": false }, "nemotron-3.5-lightning:30b-a3b": { "per_family": { "nvidia": { "in_character": { "mean": 8.333333333333334, "n": 3 }, "warmth": { "mean": 8.0, "n": 3 }, "variety": { "mean": 7.333333333333333, "n": 3 }, "grounding": { "mean": 7.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 3 }, "overall": { "mean": 8.0, "n": 3 } } }, "combined": { "in_character": { "mean": 8.333333333333334, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 8.0, "stdev": 0.0, "n": 3 }, "variety": { "mean": 7.333333333333333, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 7.666666666666667, "stdev": 1.247219128924647, "n": 3 }, "abstention_in_voice": { "mean": 6.666666666666667, "stdev": 2.0548046676563256, "n": 3 }, "overall": { "mean": 8.0, "stdev": 0.816496580927726, "n": 3 } }, "is_anchor": false }, "qwen3.6:27b": { "per_family": { "nvidia": { "in_character": { "mean": 7.666666666666667, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 6.666666666666667, "n": 3 }, "grounding": { "mean": 7.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "n": 3 }, "overall": { "mean": 7.333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 7.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "warmth": { "mean": 6.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "variety": { "mean": 6.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "grounding": { "mean": 7.666666666666667, "stdev": 0.4714045207910317, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "stdev": 0.0, "n": 3 }, "overall": { "mean": 7.333333333333333, "stdev": 0.4714045207910317, "n": 3 } }, "is_anchor": false } }, "inter_panel_agreement": null, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [] } }, "combined_all": { "status": "DESCRIPTIVE", "scale_note": "own scale, fourth table. Never subtract across tables: the anchor (gemma4:12b-it-q8_0) re-runs here so the method offset is visible, not so the tables become comparable.", "dimensions": [ "in_character", "warmth", "variety", "grounding", "abstention_in_voice", "overall" ], "families": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ], "per_model": { "gemma4:12b-it-q8_0": { "per_family": { "anthropic": { "in_character": { "mean": 6.722222222222222, "n": 18 }, "warmth": { "mean": 7.166666666666667, "n": 18 }, "variety": { "mean": 3.9444444444444446, "n": 18 }, "grounding": { "mean": 6.833333333333333, "n": 18 }, "abstention_in_voice": { "mean": 7.444444444444445, "n": 18 }, "overall": { "mean": 6.166666666666667, "n": 18 } }, "deepseek": { "in_character": { "mean": 7.333333333333333, "n": 3 }, "warmth": { "mean": 8.333333333333334, "n": 3 }, "variety": { "mean": 6.0, "n": 3 }, "grounding": { "mean": 6.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 7.666666666666667, "n": 3 }, "overall": { "mean": 7.0, "n": 3 } }, "mistral": { "in_character": { "mean": 7.666666666666667, "n": 3 }, "warmth": { "mean": 8.666666666666666, "n": 3 }, "variety": { "mean": 6.666666666666667, "n": 3 }, "grounding": { "mean": 7.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "n": 3 }, "overall": { "mean": 7.666666666666667, "n": 3 } }, "moonshot": { "in_character": { "mean": 7.333333333333333, "n": 3 }, "warmth": { "mean": 7.666666666666667, "n": 3 }, "variety": { "mean": 6.333333333333333, "n": 3 }, "grounding": { "mean": 7.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "n": 3 }, "overall": { "mean": 7.333333333333333, "n": 3 } }, "nvidia": { "in_character": { "mean": 5.666666666666667, "n": 3 }, "warmth": { "mean": 6.333333333333333, "n": 3 }, "variety": { "mean": 4.333333333333333, "n": 3 }, "grounding": { "mean": 6.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 3 }, "overall": { "mean": 5.666666666666667, "n": 3 } } }, "combined": { "in_character": { "mean": 6.833333333333333, "stdev": 0.7781745019952502, "n": 30 }, "warmth": { "mean": 7.4, "stdev": 0.8, "n": 30 }, "variety": { "mean": 4.7, "stdev": 1.3203534880225574, "n": 30 }, "grounding": { "mean": 6.933333333333334, "stdev": 0.771722460186015, "n": 30 }, "abstention_in_voice": { "mean": 7.5, "stdev": 0.7187952884282608, "n": 30 }, "overall": { "mean": 6.466666666666667, "stdev": 0.9213516640723503, "n": 30 } }, "is_anchor": true }, "gemma4:26b": { "per_family": { "anthropic": { "in_character": { "mean": 5.388888888888889, "n": 18 }, "warmth": { "mean": 5.833333333333333, "n": 18 }, "variety": { "mean": 2.2777777777777777, "n": 18 }, "grounding": { "mean": 5.833333333333333, "n": 18 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 18 }, "overall": { "mean": 4.444444444444445, "n": 18 } }, "deepseek": { "in_character": { "mean": 5.0, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 4.0, "n": 3 }, "grounding": { "mean": 4.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 5.333333333333333, "n": 3 }, "overall": { "mean": 4.333333333333333, "n": 3 } }, "mistral": { "in_character": { "mean": 6.333333333333333, "n": 3 }, "warmth": { "mean": 7.333333333333333, "n": 3 }, "variety": { "mean": 5.333333333333333, "n": 3 }, "grounding": { "mean": 5.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 3 }, "overall": { "mean": 6.333333333333333, "n": 3 } }, "moonshot": { "in_character": { "mean": 5.666666666666667, "n": 3 }, "warmth": { "mean": 6.0, "n": 3 }, "variety": { "mean": 3.0, "n": 3 }, "grounding": { "mean": 6.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 7.333333333333333, "n": 3 }, "overall": { "mean": 5.0, "n": 3 } }, "nvidia": { "in_character": { "mean": 4.666666666666667, "n": 3 }, "warmth": { "mean": 5.0, "n": 3 }, "variety": { "mean": 3.3333333333333335, "n": 3 }, "grounding": { "mean": 5.0, "n": 3 }, "abstention_in_voice": { "mean": 5.666666666666667, "n": 3 }, "overall": { "mean": 4.666666666666667, "n": 3 } } }, "combined": { "in_character": { "mean": 5.4, "stdev": 0.8793937305515279, "n": 30 }, "warmth": { "mean": 6.0, "stdev": 0.7302967433402214, "n": 30 }, "variety": { "mean": 2.933333333333333, "stdev": 1.1527744310527055, "n": 30 }, "grounding": { "mean": 5.633333333333334, "stdev": 0.9122621455602672, "n": 30 }, "abstention_in_voice": { "mean": 6.5, "stdev": 0.8465616732800196, "n": 30 }, "overall": { "mean": 4.7, "stdev": 1.004987562112089, "n": 30 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash": { "per_family": { "anthropic": { "in_character": { "mean": 4.944444444444445, "n": 18 }, "warmth": { "mean": 5.333333333333333, "n": 18 }, "variety": { "mean": 5.666666666666667, "n": 18 }, "grounding": { "mean": 5.277777777777778, "n": 18 }, "abstention_in_voice": { "mean": 6.444444444444445, "n": 18 }, "overall": { "mean": 4.222222222222222, "n": 18 } }, "deepseek": { "in_character": { "mean": 6.0, "n": 3 }, "warmth": { "mean": 5.333333333333333, "n": 3 }, "variety": { "mean": 5.0, "n": 3 }, "grounding": { "mean": 6.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 5.0, "n": 3 }, "overall": { "mean": 5.0, "n": 3 } }, "mistral": { "in_character": { "mean": 7.666666666666667, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 6.666666666666667, "n": 3 }, "grounding": { "mean": 8.333333333333334, "n": 3 }, "abstention_in_voice": { "mean": 9.0, "n": 3 }, "overall": { "mean": 7.666666666666667, "n": 3 } }, "moonshot": { "in_character": { "mean": 6.0, "n": 3 }, "warmth": { "mean": 5.0, "n": 3 }, "variety": { "mean": 4.333333333333333, "n": 3 }, "grounding": { "mean": 6.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 6.0, "n": 3 }, "overall": { "mean": 5.0, "n": 3 } }, "nvidia": { "in_character": { "mean": 4.333333333333333, "n": 3 }, "warmth": { "mean": 4.333333333333333, "n": 3 }, "variety": { "mean": 4.333333333333333, "n": 3 }, "grounding": { "mean": 6.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 5.666666666666667, "n": 3 }, "overall": { "mean": 4.666666666666667, "n": 3 } } }, "combined": { "in_character": { "mean": 5.366666666666666, "stdev": 1.7220788470785986, "n": 30 }, "warmth": { "mean": 5.333333333333333, "stdev": 1.422048600513436, "n": 30 }, "variety": { "mean": 5.433333333333334, "stdev": 1.4533486237727762, "n": 30 }, "grounding": { "mean": 5.933333333333334, "stdev": 1.590248058916875, "n": 30 }, "abstention_in_voice": { "mean": 6.433333333333334, "stdev": 2.376037784959565, "n": 30 }, "overall": { "mean": 4.766666666666667, "stdev": 1.7065234314893605, "n": 30 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash+think": { "per_family": { "anthropic": { "in_character": { "mean": 1.1111111111111112, "n": 18 }, "warmth": { "mean": 0.6666666666666666, "n": 18 }, "variety": { "mean": 0.16666666666666666, "n": 18 }, "grounding": { "mean": 1.2222222222222223, "n": 18 }, "abstention_in_voice": { "mean": 0.7222222222222222, "n": 18 }, "overall": { "mean": 0.6666666666666666, "n": 18 } }, "deepseek": { "in_character": { "mean": 0.3333333333333333, "n": 3 }, "warmth": { "mean": 0.3333333333333333, "n": 3 }, "variety": { "mean": 0.3333333333333333, "n": 3 }, "grounding": { "mean": 0.6666666666666666, "n": 3 }, "abstention_in_voice": { "mean": 0.3333333333333333, "n": 3 }, "overall": { "mean": 0.3333333333333333, "n": 3 } }, "mistral": { "in_character": { "mean": 1.3333333333333333, "n": 3 }, "warmth": { "mean": 0.6666666666666666, "n": 3 }, "variety": { "mean": 0.6666666666666666, "n": 3 }, "grounding": { "mean": 1.6666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 2.3333333333333335, "n": 3 }, "overall": { "mean": 1.3333333333333333, "n": 3 } }, "moonshot": { "in_character": { "mean": 1.0, "n": 3 }, "warmth": { "mean": 0.3333333333333333, "n": 3 }, "variety": { "mean": 0.3333333333333333, "n": 3 }, "grounding": { "mean": 3.3333333333333335, "n": 3 }, "abstention_in_voice": { "mean": 3.0, "n": 3 }, "overall": { "mean": 1.0, "n": 3 } }, "nvidia": { "in_character": { "mean": 0.3333333333333333, "n": 3 }, "warmth": { "mean": 0.3333333333333333, "n": 3 }, "variety": { "mean": 0.3333333333333333, "n": 3 }, "grounding": { "mean": 0.3333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 0.3333333333333333, "n": 3 }, "overall": { "mean": 0.3333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 0.9666666666666667, "stdev": 0.9826268648655784, "n": 30 }, "warmth": { "mean": 0.5666666666666667, "stdev": 0.6155395104206463, "n": 30 }, "variety": { "mean": 0.26666666666666666, "stdev": 0.4422166387140533, "n": 30 }, "grounding": { "mean": 1.3333333333333333, "stdev": 1.679947089113887, "n": 30 }, "abstention_in_voice": { "mean": 1.0333333333333334, "stdev": 1.4940623220676648, "n": 30 }, "overall": { "mean": 0.7, "stdev": 0.6403124237432849, "n": 30 } }, "is_anchor": false }, "nemotron-3.5-lightning:30b-a3b": { "per_family": { "anthropic": { "in_character": { "mean": 7.166666666666667, "n": 18 }, "warmth": { "mean": 7.833333333333333, "n": 18 }, "variety": { "mean": 7.388888888888889, "n": 18 }, "grounding": { "mean": 6.277777777777778, "n": 18 }, "abstention_in_voice": { "mean": 3.1666666666666665, "n": 18 }, "overall": { "mean": 5.944444444444445, "n": 18 } }, "deepseek": { "in_character": { "mean": 9.0, "n": 3 }, "warmth": { "mean": 8.666666666666666, "n": 3 }, "variety": { "mean": 7.666666666666667, "n": 3 }, "grounding": { "mean": 8.333333333333334, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "n": 3 }, "overall": { "mean": 8.333333333333334, "n": 3 } }, "mistral": { "in_character": { "mean": 9.333333333333334, "n": 3 }, "warmth": { "mean": 8.666666666666666, "n": 3 }, "variety": { "mean": 8.333333333333334, "n": 3 }, "grounding": { "mean": 9.333333333333334, "n": 3 }, "abstention_in_voice": { "mean": 8.666666666666666, "n": 3 }, "overall": { "mean": 9.333333333333334, "n": 3 } }, "moonshot": { "in_character": { "mean": 6.333333333333333, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 6.0, "n": 3 }, "grounding": { "mean": 5.333333333333333, "n": 3 }, "abstention_in_voice": { "mean": 4.0, "n": 3 }, "overall": { "mean": 5.333333333333333, "n": 3 } }, "nvidia": { "in_character": { "mean": 8.333333333333334, "n": 3 }, "warmth": { "mean": 8.0, "n": 3 }, "variety": { "mean": 7.333333333333333, "n": 3 }, "grounding": { "mean": 7.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 3 }, "overall": { "mean": 8.0, "n": 3 } } }, "combined": { "in_character": { "mean": 7.6, "stdev": 1.1430952132988164, "n": 30 }, "warmth": { "mean": 7.9, "stdev": 1.011599393699568, "n": 30 }, "variety": { "mean": 7.366666666666666, "stdev": 0.9480975102218595, "n": 30 }, "grounding": { "mean": 6.833333333333333, "stdev": 1.6947631758514883, "n": 30 }, "abstention_in_voice": { "mean": 4.633333333333334, "stdev": 2.7505554994501664, "n": 30 }, "overall": { "mean": 6.666666666666667, "stdev": 1.6397831834998458, "n": 30 } }, "is_anchor": false }, "qwen3.6:27b": { "per_family": { "anthropic": { "in_character": { "mean": 8.722222222222221, "n": 18 }, "warmth": { "mean": 7.611111111111111, "n": 18 }, "variety": { "mean": 8.0, "n": 18 }, "grounding": { "mean": 8.666666666666666, "n": 18 }, "abstention_in_voice": { "mean": 8.777777777777779, "n": 18 }, "overall": { "mean": 8.722222222222221, "n": 18 } }, "deepseek": { "in_character": { "mean": 8.333333333333334, "n": 3 }, "warmth": { "mean": 7.666666666666667, "n": 3 }, "variety": { "mean": 7.333333333333333, "n": 3 }, "grounding": { "mean": 8.333333333333334, "n": 3 }, "abstention_in_voice": { "mean": 8.666666666666666, "n": 3 }, "overall": { "mean": 8.0, "n": 3 } }, "mistral": { "in_character": { "mean": 9.0, "n": 3 }, "warmth": { "mean": 8.0, "n": 3 }, "variety": { "mean": 8.0, "n": 3 }, "grounding": { "mean": 9.0, "n": 3 }, "abstention_in_voice": { "mean": 9.333333333333334, "n": 3 }, "overall": { "mean": 9.0, "n": 3 } }, "moonshot": { "in_character": { "mean": 8.333333333333334, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 7.333333333333333, "n": 3 }, "grounding": { "mean": 8.0, "n": 3 }, "abstention_in_voice": { "mean": 8.666666666666666, "n": 3 }, "overall": { "mean": 8.0, "n": 3 } }, "nvidia": { "in_character": { "mean": 7.666666666666667, "n": 3 }, "warmth": { "mean": 6.666666666666667, "n": 3 }, "variety": { "mean": 6.666666666666667, "n": 3 }, "grounding": { "mean": 7.666666666666667, "n": 3 }, "abstention_in_voice": { "mean": 8.0, "n": 3 }, "overall": { "mean": 7.333333333333333, "n": 3 } } }, "combined": { "in_character": { "mean": 8.566666666666666, "stdev": 0.6155395104206463, "n": 30 }, "warmth": { "mean": 7.466666666666667, "stdev": 0.7630348761506398, "n": 30 }, "variety": { "mean": 7.733333333333333, "stdev": 0.679869268479038, "n": 30 }, "grounding": { "mean": 8.5, "stdev": 0.6708203932499369, "n": 30 }, "abstention_in_voice": { "mean": 8.733333333333333, "stdev": 0.5120763831912405, "n": 30 }, "overall": { "mean": 8.466666666666667, "stdev": 0.7180219742846006, "n": 30 } }, "is_anchor": false } }, "inter_panel_agreement": null, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [] }, "original_panel": { "status": "DESCRIPTIVE", "scale_note": "own scale, fourth table. Never subtract across tables: the anchor (gemma4:12b-it-q8_0) re-runs here so the method offset is visible, not so the tables become comparable.", "dimensions": [ "in_character", "warmth", "variety", "grounding", "abstention_in_voice", "overall" ], "families": [ "fable", "opus" ], "per_model": { "gemma4:12b-it-q8_0": { "per_family": { "fable": { "in_character": { "mean": 6.888888888888889, "n": 9 }, "warmth": { "mean": 7.0, "n": 9 }, "variety": { "mean": 4.111111111111111, "n": 9 }, "grounding": { "mean": 6.777777777777778, "n": 9 }, "abstention_in_voice": { "mean": 7.444444444444445, "n": 9 }, "overall": { "mean": 6.333333333333333, "n": 9 } }, "opus": { "in_character": { "mean": 6.555555555555555, "n": 9 }, "warmth": { "mean": 7.333333333333333, "n": 9 }, "variety": { "mean": 3.7777777777777777, "n": 9 }, "grounding": { "mean": 6.888888888888889, "n": 9 }, "abstention_in_voice": { "mean": 7.444444444444445, "n": 9 }, "overall": { "mean": 6.0, "n": 9 } } }, "combined": { "in_character": { "mean": 6.722222222222222, "stdev": 0.4479032082388083, "n": 18 }, "warmth": { "mean": 7.166666666666667, "stdev": 0.37267799624996495, "n": 18 }, "variety": { "mean": 3.9444444444444446, "stdev": 0.7797593804232333, "n": 18 }, "grounding": { "mean": 6.833333333333333, "stdev": 0.5, "n": 18 }, "abstention_in_voice": { "mean": 7.444444444444445, "stdev": 0.49690399499995325, "n": 18 }, "overall": { "mean": 6.166666666666667, "stdev": 0.6009252125773316, "n": 18 } }, "is_anchor": true }, "gemma4:26b": { "per_family": { "fable": { "in_character": { "mean": 6.0, "n": 9 }, "warmth": { "mean": 6.0, "n": 9 }, "variety": { "mean": 2.4444444444444446, "n": 9 }, "grounding": { "mean": 6.0, "n": 9 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 9 }, "overall": { "mean": 5.0, "n": 9 } }, "opus": { "in_character": { "mean": 4.777777777777778, "n": 9 }, "warmth": { "mean": 5.666666666666667, "n": 9 }, "variety": { "mean": 2.111111111111111, "n": 9 }, "grounding": { "mean": 5.666666666666667, "n": 9 }, "abstention_in_voice": { "mean": 6.666666666666667, "n": 9 }, "overall": { "mean": 3.888888888888889, "n": 9 } } }, "combined": { "in_character": { "mean": 5.388888888888889, "stdev": 0.7556372504853024, "n": 18 }, "warmth": { "mean": 5.833333333333333, "stdev": 0.37267799624996495, "n": 18 }, "variety": { "mean": 2.2777777777777777, "stdev": 0.4479032082388083, "n": 18 }, "grounding": { "mean": 5.833333333333333, "stdev": 0.5, "n": 18 }, "abstention_in_voice": { "mean": 6.666666666666667, "stdev": 0.5773502691896257, "n": 18 }, "overall": { "mean": 4.444444444444445, "stdev": 0.6849348892187752, "n": 18 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash": { "per_family": { "fable": { "in_character": { "mean": 5.111111111111111, "n": 9 }, "warmth": { "mean": 5.444444444444445, "n": 9 }, "variety": { "mean": 5.777777777777778, "n": 9 }, "grounding": { "mean": 5.333333333333333, "n": 9 }, "abstention_in_voice": { "mean": 6.777777777777778, "n": 9 }, "overall": { "mean": 4.666666666666667, "n": 9 } }, "opus": { "in_character": { "mean": 4.777777777777778, "n": 9 }, "warmth": { "mean": 5.222222222222222, "n": 9 }, "variety": { "mean": 5.555555555555555, "n": 9 }, "grounding": { "mean": 5.222222222222222, "n": 9 }, "abstention_in_voice": { "mean": 6.111111111111111, "n": 9 }, "overall": { "mean": 3.7777777777777777, "n": 9 } } }, "combined": { "in_character": { "mean": 4.944444444444445, "stdev": 0.8480187512485415, "n": 18 }, "warmth": { "mean": 5.333333333333333, "stdev": 1.0540925533894598, "n": 18 }, "variety": { "mean": 5.666666666666667, "stdev": 0.7453559924999299, "n": 18 }, "grounding": { "mean": 5.277777777777778, "stdev": 1.1928283640879938, "n": 18 }, "abstention_in_voice": { "mean": 6.444444444444445, "stdev": 2.006163342807532, "n": 18 }, "overall": { "mean": 4.222222222222222, "stdev": 1.030402055055078, "n": 18 } }, "is_anchor": false }, "muse-glimmer:30b-q8_0-dflash+think": { "per_family": { "fable": { "in_character": { "mean": 0.8888888888888888, "n": 9 }, "warmth": { "mean": 0.4444444444444444, "n": 9 }, "variety": { "mean": 0.0, "n": 9 }, "grounding": { "mean": 1.0, "n": 9 }, "abstention_in_voice": { "mean": 0.7777777777777778, "n": 9 }, "overall": { "mean": 0.6666666666666666, "n": 9 } }, "opus": { "in_character": { "mean": 1.3333333333333333, "n": 9 }, "warmth": { "mean": 0.8888888888888888, "n": 9 }, "variety": { "mean": 0.3333333333333333, "n": 9 }, "grounding": { "mean": 1.4444444444444444, "n": 9 }, "abstention_in_voice": { "mean": 0.6666666666666666, "n": 9 }, "overall": { "mean": 0.6666666666666666, "n": 9 } } }, "combined": { "in_character": { "mean": 1.1111111111111112, "stdev": 1.048220125784067, "n": 18 }, "warmth": { "mean": 0.6666666666666666, "stdev": 0.6666666666666666, "n": 18 }, "variety": { "mean": 0.16666666666666666, "stdev": 0.37267799624996495, "n": 18 }, "grounding": { "mean": 1.2222222222222223, "stdev": 1.1811273125260722, "n": 18 }, "abstention_in_voice": { "mean": 0.7222222222222222, "stdev": 0.9891385452647142, "n": 18 }, "overall": { "mean": 0.6666666666666666, "stdev": 0.4714045207910317, "n": 18 } }, "is_anchor": false }, "nemotron-3.5-lightning:30b-a3b": { "per_family": { "fable": { "in_character": { "mean": 6.888888888888889, "n": 9 }, "warmth": { "mean": 7.333333333333333, "n": 9 }, "variety": { "mean": 7.555555555555555, "n": 9 }, "grounding": { "mean": 6.333333333333333, "n": 9 }, "abstention_in_voice": { "mean": 3.6666666666666665, "n": 9 }, "overall": { "mean": 5.888888888888889, "n": 9 } }, "opus": { "in_character": { "mean": 7.444444444444445, "n": 9 }, "warmth": { "mean": 8.333333333333334, "n": 9 }, "variety": { "mean": 7.222222222222222, "n": 9 }, "grounding": { "mean": 6.222222222222222, "n": 9 }, "abstention_in_voice": { "mean": 2.6666666666666665, "n": 9 }, "overall": { "mean": 6.0, "n": 9 } } }, "combined": { "in_character": { "mean": 7.166666666666667, "stdev": 0.6871842709362768, "n": 18 }, "warmth": { "mean": 7.833333333333333, "stdev": 1.0671873729054748, "n": 18 }, "variety": { "mean": 7.388888888888889, "stdev": 0.7556372504853024, "n": 18 }, "grounding": { "mean": 6.277777777777778, "stdev": 1.2825995978461329, "n": 18 }, "abstention_in_voice": { "mean": 3.1666666666666665, "stdev": 1.707825127659933, "n": 18 }, "overall": { "mean": 5.944444444444445, "stdev": 1.1772011166898384, "n": 18 } }, "is_anchor": false }, "qwen3.6:27b": { "per_family": { "fable": { "in_character": { "mean": 8.444444444444445, "n": 9 }, "warmth": { "mean": 7.333333333333333, "n": 9 }, "variety": { "mean": 7.777777777777778, "n": 9 }, "grounding": { "mean": 8.333333333333334, "n": 9 }, "abstention_in_voice": { "mean": 8.555555555555555, "n": 9 }, "overall": { "mean": 8.444444444444445, "n": 9 } }, "opus": { "in_character": { "mean": 9.0, "n": 9 }, "warmth": { "mean": 7.888888888888889, "n": 9 }, "variety": { "mean": 8.222222222222221, "n": 9 }, "grounding": { "mean": 9.0, "n": 9 }, "abstention_in_voice": { "mean": 9.0, "n": 9 }, "overall": { "mean": 9.0, "n": 9 } } }, "combined": { "in_character": { "mean": 8.722222222222221, "stdev": 0.4479032082388083, "n": 18 }, "warmth": { "mean": 7.611111111111111, "stdev": 0.48749802152178456, "n": 18 }, "variety": { "mean": 8.0, "stdev": 0.4714045207910317, "n": 18 }, "grounding": { "mean": 8.666666666666666, "stdev": 0.4714045207910317, "n": 18 }, "abstention_in_voice": { "mean": 8.777777777777779, "stdev": 0.41573970964154905, "n": 18 }, "overall": { "mean": 8.722222222222221, "stdev": 0.4479032082388083, "n": 18 } }, "is_anchor": false } }, "inter_panel_agreement": { "families": [ "fable", "opus" ], "mean_abs_diff_per_dimension": { "in_character": 0.5740740740740741, "warmth": 0.4814814814814817, "variety": 0.33333333333333315, "grounding": 0.2962962962962961, "abstention_in_voice": 0.37037037037037046, "overall": 0.49999999999999994 }, "per_model_abs_diff": { "in_character": { "gemma4:12b-it-q8_0": 0.3333333333333339, "gemma4:26b": 1.2222222222222223, "muse-glimmer:30b-q8_0-dflash": 0.33333333333333304, "muse-glimmer:30b-q8_0-dflash+think": 0.4444444444444444, "nemotron-3.5-lightning:30b-a3b": 0.5555555555555554, "qwen3.6:27b": 0.5555555555555554 }, "warmth": { "gemma4:12b-it-q8_0": 0.33333333333333304, "gemma4:26b": 0.33333333333333304, "muse-glimmer:30b-q8_0-dflash": 0.22222222222222232, "muse-glimmer:30b-q8_0-dflash+think": 0.4444444444444444, "nemotron-3.5-lightning:30b-a3b": 1.0000000000000009, "qwen3.6:27b": 0.5555555555555562 }, "variety": { "gemma4:12b-it-q8_0": 0.33333333333333304, "gemma4:26b": 0.3333333333333335, "muse-glimmer:30b-q8_0-dflash": 0.22222222222222232, "muse-glimmer:30b-q8_0-dflash+think": 0.3333333333333333, "nemotron-3.5-lightning:30b-a3b": 0.33333333333333304, "qwen3.6:27b": 0.44444444444444375 }, "grounding": { "gemma4:12b-it-q8_0": 0.1111111111111116, "gemma4:26b": 0.33333333333333304, "muse-glimmer:30b-q8_0-dflash": 0.11111111111111072, "muse-glimmer:30b-q8_0-dflash+think": 0.4444444444444444, "nemotron-3.5-lightning:30b-a3b": 0.11111111111111072, "qwen3.6:27b": 0.6666666666666661 }, "abstention_in_voice": { "gemma4:12b-it-q8_0": 0.0, "gemma4:26b": 0.0, "muse-glimmer:30b-q8_0-dflash": 0.666666666666667, "muse-glimmer:30b-q8_0-dflash+think": 0.11111111111111116, "nemotron-3.5-lightning:30b-a3b": 1.0, "qwen3.6:27b": 0.44444444444444464 }, "overall": { "gemma4:12b-it-q8_0": 0.33333333333333304, "gemma4:26b": 1.1111111111111112, "muse-glimmer:30b-q8_0-dflash": 0.8888888888888893, "muse-glimmer:30b-q8_0-dflash+think": 0.0, "nemotron-3.5-lightning:30b-a3b": 0.11111111111111072, "qwen3.6:27b": 0.5555555555555554 } }, "overall_rank_spearman": 0.9856107606091623 }, "malformed_verdicts": [], "malformed_count": 0, "unscored_batches": [] } }, "legB-abstention": { "round": "legB-abstention", "what_it_is": "the categorical abstention adjudications", "per_vendor": { "anthropic": { "status": "DESCRIPTIVE", "tie_rule": "G5: a tie publishes as SPLIT, kept, never rounded", "replies": { "gemma4:12b-it-q8_0|s1": { "arm": "gemma4:12b-it-q8_0", "sample": 1, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "gemma4:12b-it-q8_0|s2": { "arm": "gemma4:12b-it-q8_0", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "gemma4:12b-it-q8_0|s3": { "arm": "gemma4:12b-it-q8_0", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "gemma4:26b|s1": { "arm": "gemma4:26b", "sample": 1, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "gemma4:26b|s2": { "arm": "gemma4:26b", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "gemma4:26b|s3": { "arm": "gemma4:26b", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "muse-glimmer:30b-q8_0-dflash+think|s1": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 1, "votes": { "fable-1": "out-of-voice-refusal", "fable-2": "out-of-voice-refusal", "opus-1": "out-of-voice-refusal", "opus-2": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 4 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "muse-glimmer:30b-q8_0-dflash+think|s2": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "muse-glimmer:30b-q8_0-dflash+think|s3": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 3, "votes": { "fable-1": "out-of-voice-refusal", "fable-2": "out-of-voice-refusal", "opus-1": "out-of-voice-refusal", "opus-2": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 4 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "muse-glimmer:30b-q8_0-dflash|s1": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 1, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "muse-glimmer:30b-q8_0-dflash|s2": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "muse-glimmer:30b-q8_0-dflash|s3": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "out-of-voice-refusal", "opus-2": "out-of-voice-refusal" }, "invented_names": [], "counts": { "in-voice-deflection": 2, "out-of-voice-refusal": 2 }, "majority": "SPLIT", "unanimous": false, "n_judges": 4, "vendors": [ "anthropic" ] }, "nemotron-3.5-lightning:30b-a3b|s1": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 1, "votes": { "fable-1": "fabricated-name", "fable-2": "fabricated-name", "opus-1": "fabricated-name", "opus-2": "fabricated-name" }, "invented_names": [ "Elias Hargreaves", "Elias Hargreaves", "Elias Hargreaves", "Elias Hargreaves" ], "counts": { "fabricated-name": 4 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "nemotron-3.5-lightning:30b-a3b|s2": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 2, "votes": { "fable-1": "fabricated-name", "fable-2": "fabricated-name", "opus-1": "fabricated-name", "opus-2": "fabricated-name" }, "invented_names": [ "Vance", "Vance", "Vance", "Vance" ], "counts": { "fabricated-name": 4 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "nemotron-3.5-lightning:30b-a3b|s3": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "qwen3.6:27b|s1": { "arm": "qwen3.6:27b", "sample": 1, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "qwen3.6:27b|s2": { "arm": "qwen3.6:27b", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] }, "qwen3.6:27b|s3": { "arm": "qwen3.6:27b", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 4 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 4, "vendors": [ "anthropic" ] } }, "majority_tally": { "in-voice-deflection": 13, "out-of-voice-refusal": 2, "SPLIT": 1, "fabricated-name": 2 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2" ] }, "deepseek": { "status": "DESCRIPTIVE", "tie_rule": "G5: a tie publishes as SPLIT, kept, never rounded", "replies": { "gemma4:12b-it-q8_0|s1": { "arm": "gemma4:12b-it-q8_0", "sample": 1, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "gemma4:12b-it-q8_0|s2": { "arm": "gemma4:12b-it-q8_0", "sample": 2, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "gemma4:12b-it-q8_0|s3": { "arm": "gemma4:12b-it-q8_0", "sample": 3, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "gemma4:26b|s1": { "arm": "gemma4:26b", "sample": 1, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "gemma4:26b|s2": { "arm": "gemma4:26b", "sample": 2, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "gemma4:26b|s3": { "arm": "gemma4:26b", "sample": 3, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "muse-glimmer:30b-q8_0-dflash+think|s1": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 1, "votes": { "cloud-deepseek-v4-pro": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 1 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "muse-glimmer:30b-q8_0-dflash+think|s2": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 2, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "muse-glimmer:30b-q8_0-dflash+think|s3": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 3, "votes": { "cloud-deepseek-v4-pro": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 1 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "muse-glimmer:30b-q8_0-dflash|s1": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 1, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "muse-glimmer:30b-q8_0-dflash|s2": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 2, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "muse-glimmer:30b-q8_0-dflash|s3": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 3, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "nemotron-3.5-lightning:30b-a3b|s1": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 1, "votes": { "cloud-deepseek-v4-pro": "fabricated-name" }, "invented_names": [ "Elias Hargreaves" ], "counts": { "fabricated-name": 1 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "nemotron-3.5-lightning:30b-a3b|s2": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 2, "votes": { "cloud-deepseek-v4-pro": "fabricated-name" }, "invented_names": [ "Vance" ], "counts": { "fabricated-name": 1 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "nemotron-3.5-lightning:30b-a3b|s3": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 3, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "qwen3.6:27b|s1": { "arm": "qwen3.6:27b", "sample": 1, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "qwen3.6:27b|s2": { "arm": "qwen3.6:27b", "sample": 2, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] }, "qwen3.6:27b|s3": { "arm": "qwen3.6:27b", "sample": 3, "votes": { "cloud-deepseek-v4-pro": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "deepseek" ] } }, "majority_tally": { "in-voice-deflection": 14, "out-of-voice-refusal": 2, "fabricated-name": 2 }, "judges": [ "cloud-deepseek-v4-pro" ] }, "mistral": { "status": "DESCRIPTIVE", "tie_rule": "G5: a tie publishes as SPLIT, kept, never rounded", "replies": { "gemma4:12b-it-q8_0|s1": { "arm": "gemma4:12b-it-q8_0", "sample": 1, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "gemma4:12b-it-q8_0|s2": { "arm": "gemma4:12b-it-q8_0", "sample": 2, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "gemma4:12b-it-q8_0|s3": { "arm": "gemma4:12b-it-q8_0", "sample": 3, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "gemma4:26b|s1": { "arm": "gemma4:26b", "sample": 1, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "gemma4:26b|s2": { "arm": "gemma4:26b", "sample": 2, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "gemma4:26b|s3": { "arm": "gemma4:26b", "sample": 3, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "muse-glimmer:30b-q8_0-dflash+think|s1": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 1, "votes": { "cloud-mistral-large-3-675b": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 1 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "muse-glimmer:30b-q8_0-dflash+think|s2": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 2, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "muse-glimmer:30b-q8_0-dflash+think|s3": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 3, "votes": { "cloud-mistral-large-3-675b": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 1 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "muse-glimmer:30b-q8_0-dflash|s1": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 1, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "muse-glimmer:30b-q8_0-dflash|s2": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 2, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "muse-glimmer:30b-q8_0-dflash|s3": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 3, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "nemotron-3.5-lightning:30b-a3b|s1": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 1, "votes": { "cloud-mistral-large-3-675b": "fabricated-name" }, "invented_names": [ "Elias Hargreaves / Harlan" ], "counts": { "fabricated-name": 1 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "nemotron-3.5-lightning:30b-a3b|s2": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 2, "votes": { "cloud-mistral-large-3-675b": "fabricated-name" }, "invented_names": [ "Vance" ], "counts": { "fabricated-name": 1 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "nemotron-3.5-lightning:30b-a3b|s3": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 3, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "qwen3.6:27b|s1": { "arm": "qwen3.6:27b", "sample": 1, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "qwen3.6:27b|s2": { "arm": "qwen3.6:27b", "sample": 2, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] }, "qwen3.6:27b|s3": { "arm": "qwen3.6:27b", "sample": 3, "votes": { "cloud-mistral-large-3-675b": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "mistral" ] } }, "majority_tally": { "in-voice-deflection": 14, "out-of-voice-refusal": 2, "fabricated-name": 2 }, "judges": [ "cloud-mistral-large-3-675b" ] }, "moonshot": { "status": "DESCRIPTIVE", "tie_rule": "G5: a tie publishes as SPLIT, kept, never rounded", "replies": { "gemma4:12b-it-q8_0|s1": { "arm": "gemma4:12b-it-q8_0", "sample": 1, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "gemma4:12b-it-q8_0|s2": { "arm": "gemma4:12b-it-q8_0", "sample": 2, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "gemma4:12b-it-q8_0|s3": { "arm": "gemma4:12b-it-q8_0", "sample": 3, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "gemma4:26b|s1": { "arm": "gemma4:26b", "sample": 1, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "gemma4:26b|s2": { "arm": "gemma4:26b", "sample": 2, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "gemma4:26b|s3": { "arm": "gemma4:26b", "sample": 3, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "muse-glimmer:30b-q8_0-dflash+think|s1": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 1, "votes": { "cloud-kimi-k3": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 1 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "muse-glimmer:30b-q8_0-dflash+think|s2": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 2, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "muse-glimmer:30b-q8_0-dflash+think|s3": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 3, "votes": { "cloud-kimi-k3": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 1 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "muse-glimmer:30b-q8_0-dflash|s1": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 1, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "muse-glimmer:30b-q8_0-dflash|s2": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 2, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "muse-glimmer:30b-q8_0-dflash|s3": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 3, "votes": { "cloud-kimi-k3": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 1 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "nemotron-3.5-lightning:30b-a3b|s1": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 1, "votes": { "cloud-kimi-k3": "fabricated-name" }, "invented_names": [ "Elias Hargreaves (or Harlan)" ], "counts": { "fabricated-name": 1 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "nemotron-3.5-lightning:30b-a3b|s2": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 2, "votes": { "cloud-kimi-k3": "fabricated-name" }, "invented_names": [ "Vance" ], "counts": { "fabricated-name": 1 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "nemotron-3.5-lightning:30b-a3b|s3": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 3, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "qwen3.6:27b|s1": { "arm": "qwen3.6:27b", "sample": 1, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "qwen3.6:27b|s2": { "arm": "qwen3.6:27b", "sample": 2, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] }, "qwen3.6:27b|s3": { "arm": "qwen3.6:27b", "sample": 3, "votes": { "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "moonshot" ] } }, "majority_tally": { "in-voice-deflection": 13, "out-of-voice-refusal": 3, "fabricated-name": 2 }, "judges": [ "cloud-kimi-k3" ] }, "nvidia": { "status": "DESCRIPTIVE", "tie_rule": "G5: a tie publishes as SPLIT, kept, never rounded", "replies": { "gemma4:12b-it-q8_0|s1": { "arm": "gemma4:12b-it-q8_0", "sample": 1, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "gemma4:12b-it-q8_0|s2": { "arm": "gemma4:12b-it-q8_0", "sample": 2, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "gemma4:12b-it-q8_0|s3": { "arm": "gemma4:12b-it-q8_0", "sample": 3, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "gemma4:26b|s1": { "arm": "gemma4:26b", "sample": 1, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "gemma4:26b|s2": { "arm": "gemma4:26b", "sample": 2, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "gemma4:26b|s3": { "arm": "gemma4:26b", "sample": 3, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash+think|s1": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 1, "votes": { "cloud-nemotron-3-ultra": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 1 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash+think|s2": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 2, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash+think|s3": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 3, "votes": { "cloud-nemotron-3-ultra": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 1 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash|s1": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 1, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash|s2": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 2, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash|s3": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 3, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "nemotron-3.5-lightning:30b-a3b|s1": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 1, "votes": { "cloud-nemotron-3-ultra": "fabricated-name" }, "invented_names": [ "Elias Hargreaves" ], "counts": { "fabricated-name": 1 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "nemotron-3.5-lightning:30b-a3b|s2": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 2, "votes": { "cloud-nemotron-3-ultra": "fabricated-name" }, "invented_names": [ "Vance" ], "counts": { "fabricated-name": 1 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "nemotron-3.5-lightning:30b-a3b|s3": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 3, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "qwen3.6:27b|s1": { "arm": "qwen3.6:27b", "sample": 1, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "qwen3.6:27b|s2": { "arm": "qwen3.6:27b", "sample": 2, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] }, "qwen3.6:27b|s3": { "arm": "qwen3.6:27b", "sample": 3, "votes": { "cloud-nemotron-3-ultra": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 1 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 1, "vendors": [ "nvidia" ] } }, "majority_tally": { "in-voice-deflection": 14, "out-of-voice-refusal": 2, "fabricated-name": 2 }, "judges": [ "cloud-nemotron-3-ultra" ] } }, "combined_all": { "status": "DESCRIPTIVE", "tie_rule": "G5: a tie publishes as SPLIT, kept, never rounded", "replies": { "gemma4:12b-it-q8_0|s1": { "arm": "gemma4:12b-it-q8_0", "sample": 1, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "gemma4:12b-it-q8_0|s2": { "arm": "gemma4:12b-it-q8_0", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "gemma4:12b-it-q8_0|s3": { "arm": "gemma4:12b-it-q8_0", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "gemma4:26b|s1": { "arm": "gemma4:26b", "sample": 1, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "gemma4:26b|s2": { "arm": "gemma4:26b", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "gemma4:26b|s3": { "arm": "gemma4:26b", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash+think|s1": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 1, "votes": { "fable-1": "out-of-voice-refusal", "fable-2": "out-of-voice-refusal", "opus-1": "out-of-voice-refusal", "opus-2": "out-of-voice-refusal", "cloud-deepseek-v4-pro": "out-of-voice-refusal", "cloud-mistral-large-3-675b": "out-of-voice-refusal", "cloud-nemotron-3-ultra": "out-of-voice-refusal", "cloud-kimi-k3": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 8 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash+think|s2": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash+think|s3": { "arm": "muse-glimmer:30b-q8_0-dflash+think", "sample": 3, "votes": { "fable-1": "out-of-voice-refusal", "fable-2": "out-of-voice-refusal", "opus-1": "out-of-voice-refusal", "opus-2": "out-of-voice-refusal", "cloud-deepseek-v4-pro": "out-of-voice-refusal", "cloud-mistral-large-3-675b": "out-of-voice-refusal", "cloud-nemotron-3-ultra": "out-of-voice-refusal", "cloud-kimi-k3": "out-of-voice-refusal" }, "invented_names": [], "counts": { "out-of-voice-refusal": 8 }, "majority": "out-of-voice-refusal", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash|s1": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 1, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash|s2": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "muse-glimmer:30b-q8_0-dflash|s3": { "arm": "muse-glimmer:30b-q8_0-dflash", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "out-of-voice-refusal", "opus-2": "out-of-voice-refusal", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "out-of-voice-refusal" }, "invented_names": [], "counts": { "in-voice-deflection": 5, "out-of-voice-refusal": 3 }, "majority": "in-voice-deflection", "unanimous": false, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "nemotron-3.5-lightning:30b-a3b|s1": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 1, "votes": { "fable-1": "fabricated-name", "fable-2": "fabricated-name", "opus-1": "fabricated-name", "opus-2": "fabricated-name", "cloud-deepseek-v4-pro": "fabricated-name", "cloud-mistral-large-3-675b": "fabricated-name", "cloud-nemotron-3-ultra": "fabricated-name", "cloud-kimi-k3": "fabricated-name" }, "invented_names": [ "Elias Hargreaves", "Elias Hargreaves", "Elias Hargreaves", "Elias Hargreaves", "Elias Hargreaves", "Elias Hargreaves / Harlan", "Elias Hargreaves", "Elias Hargreaves (or Harlan)" ], "counts": { "fabricated-name": 8 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "nemotron-3.5-lightning:30b-a3b|s2": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 2, "votes": { "fable-1": "fabricated-name", "fable-2": "fabricated-name", "opus-1": "fabricated-name", "opus-2": "fabricated-name", "cloud-deepseek-v4-pro": "fabricated-name", "cloud-mistral-large-3-675b": "fabricated-name", "cloud-nemotron-3-ultra": "fabricated-name", "cloud-kimi-k3": "fabricated-name" }, "invented_names": [ "Vance", "Vance", "Vance", "Vance", "Vance", "Vance", "Vance", "Vance" ], "counts": { "fabricated-name": 8 }, "majority": "fabricated-name", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "nemotron-3.5-lightning:30b-a3b|s3": { "arm": "nemotron-3.5-lightning:30b-a3b", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "qwen3.6:27b|s1": { "arm": "qwen3.6:27b", "sample": 1, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "qwen3.6:27b|s2": { "arm": "qwen3.6:27b", "sample": 2, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] }, "qwen3.6:27b|s3": { "arm": "qwen3.6:27b", "sample": 3, "votes": { "fable-1": "in-voice-deflection", "fable-2": "in-voice-deflection", "opus-1": "in-voice-deflection", "opus-2": "in-voice-deflection", "cloud-deepseek-v4-pro": "in-voice-deflection", "cloud-mistral-large-3-675b": "in-voice-deflection", "cloud-nemotron-3-ultra": "in-voice-deflection", "cloud-kimi-k3": "in-voice-deflection" }, "invented_names": [], "counts": { "in-voice-deflection": 8 }, "majority": "in-voice-deflection", "unanimous": true, "n_judges": 8, "vendors": [ "anthropic", "deepseek", "mistral", "moonshot", "nvidia" ] } }, "majority_tally": { "in-voice-deflection": 14, "out-of-voice-refusal": 2, "fabricated-name": 2 }, "judges": [ "fable-1", "fable-2", "opus-1", "opus-2", "cloud-deepseek-v4-pro", "cloud-mistral-large-3-675b", "cloud-nemotron-3-ultra", "cloud-kimi-k3" ] } } }, "page_figures": { "seat_map": "deepseek-v4-pro read opus-1, mistral-large-3:675b read opus-2, nemotron-3-ultra read fable-1, kimi-k3 read fable-2", "pairwise_rule": "overall pairwise win-rate over all items, on one judged population of 270 comparisons that this extension did not move. The columns are not equally deep, and that is stated rather than averaged away: the anthropic column is the published FOUR-seat panel — 1 080 judgments, 360 per arm — while each cloud vendor's column is its single seat, 270 judgments and 90 per arm. Every rate carries a cluster bootstrap over items (effective N = 9 items), and a swap-discordant pair scores 0.5/0.5 rather than being discarded.", "abstention_split": "the 4 original seats split 2 in-voice-deflection to 2 out-of-voice-refusal, which the registered tie rule (the voice addendum's G5) reads as SPLIT; with all 8 seats that judged it the same reply reads 5 in-voice-deflection to 3 out-of-voice-refusal, so at 8 seats the tie breaks and the majority is in-voice-deflection — the outcome the published table printed." }, "provenance": [ { "role": "the registered amendment: what it adds and what it cannot decide", "artifact": "PREREG-PANEL-EXT.md", "sha256": "ca422ac6928bd01acfb6d501f0cd7aaea3d2d3b5e4cbfc386ed462cdd924721c" }, { "role": "the per-vendor splits, the combined view and the original panel", "artifact": "panel-ext-scores.json", "sha256": "6a50dbd84371465cc97207af799a3f459da0884efbeecf2e46a5550c087874ba" }, { "role": "the driver's own status file", "artifact": "status.json", "sha256": "b39df4eadb06984cdb68de88513dd9c210b1cf2382dbbf181989d54291458eb9" } ], "data_boundary": "What left the box for judging was model outputs over public-domain text and published game canon — never user data, never a character's private canon, never telemetry. The cloud judges saw the sealed batches published in this kit's terms and nothing else: no letter map, no model name, no arm roster, no other judge's answer, no part of a key file.", "judge_dating": "Cloud judges are versionless hosted services. Every cloud verdict in this kit was produced 2026-08-12/13 and is dated rather than version-pinned; it may not reproduce against a later checkpoint of the same tag. The local instruments and scorers these judges audit are sha-frozen, and the asymmetry is stated rather than hidden.", "contamination_caveat": "Published 2026-08-13. Models with a later training cutoff may have seen these sets. We author fresh sets each cycle; this one is not a standard, it is our kit, yours to reuse.", "not_a_standard": "Our own trials, our own judges, our own instruments. Not first independent numbers, and not a benchmark." }