{ "round": "two-frontiers-cove-r1", "scorer_pin": { "module": "harness/cove/opencall_score.py", "sha256_registered": "ff0b620d7901356503c2f24ad3308a9e76152723878f35431b1fb50eed0bed22", "sha256_read": "ff0b620d7901356503c2f24ad3308a9e76152723878f35431b1fb50eed0bed22", "matches": true }, "sheet_mode": { "mode": "single-house", "house_seats": [ "cove" ], "seed_namespace": "two-frontiers-cove-r1|2977690216", "registered_by": "judges_run --build", "seats": [ { "seat": "gemma4-31b", "family": "google", "reads": "cove" }, { "seat": "mistral-large-3-675b", "family": "mistral", "reads": "cove" }, { "seat": "nemotron-3-ultra", "family": "nvidia", "reads": "cove" }, { "seat": "kimi-k3", "family": "moonshot", "reads": "cove" }, { "seat": "deepseek-v4-pro", "family": "deepseek", "reads": "cove" }, { "seat": "glm-5.3", "family": "zhipu", "reads": "cove" }, { "seat": "qwen3.5-397b", "family": "alibaba", "reads": "cove" } ] }, "clusters": 6, "cluster_note": "the independent unit is the CASE: six sealed asks, six clusters. Every denominator on this page is either below the house floor of N=30 or above it only by POOLING — 48 cells are 6 seats reading 8 replies rather than 48 independent observations — which is why the page carries counts and no rates at all (§7's own clause; the first draft of this field said every denominator was under 30, and seven of the page's own are not).", "arm_print_order": [ "cli-claude-fable-5-1", "openai-gpt-6-astra", "cloud-glm-5-3", "local-gemma4-26b" ], "arm_print_order_basis": "the REGISTERED roster order. Not a sort by any figure: an ordering among the arms is the one comparative claim this round forbids outright.", "recusal_roll_call": { "cli-claude-fable-5-1": { "recused_count": 0, "cells": [] }, "cloud-glm-5-3": { "recused_count": 0, "cells": [] }, "local-gemma4-26b": { "recused_count": 8, "cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "reason": "the google seat does not score a google arm" } ] }, "openai-gpt-6-astra": { "recused_count": 0, "cells": [] } }, "panel_shape": { "shape": { "seats_carried": 6, "seats_retired": [ "glm-5.3" ], "families_carried": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "sheets_scored": 36, "cells_filed": 228, "recused_cells": 8, "arms": { "cli-claude-fable-5-1": { "cells_scoring": 48, "families_scoring": 6, "recused_cells": 0, "own_family_chairs_carried": 0 }, "cloud-glm-5-3": { "cells_scoring": 48, "families_scoring": 6, "recused_cells": 0, "own_family_chairs_carried": 0 }, "local-gemma4-26b": { "cells_scoring": 40, "families_scoring": 5, "recused_cells": 8, "own_family_chairs_carried": 1 }, "openai-gpt-6-astra": { "cells_scoring": 48, "families_scoring": 6, "recused_cells": 0, "own_family_chairs_carried": 0 } } }, "source": "results/cove/judge/two-frontiers-cove-r1/AUDITIONS.json", "carried": [ "deepseek-v4-pro", "gemma4-31b", "kimi-k3", "mistral-large-3-675b", "nemotron-3-ultra", "qwen3.5-397b" ], "not_carried": [ "glm-5.3" ], "retired_for_the_round": "a seat that files NOT-CARRIED at its audition is retired for the round (§4): it is never served a sheet, its cells do not exist, and the panel prints as the size it actually is. Every denominator below is that size, and this receipt names the seat that is missing from it.", "ladder_rung": "6 seats of 7 — glm-5.3 NOT-CARRIED", "amendment": "A6 — the pre-registration's §0 wrote its tables for SEVEN seats. Six sat, so every denominator here is derived from this audition receipt rather than from that table, which is what §4's ladder registers as publishable at six." }, "registered_integers": { "COVE_ANCHOR_CELLS": 42, "COVE_ANCHOR_LETTERS_PER_SEAT": 6, "COVE_ARMS": 4, "COVE_ARM_CALLS": 36, "COVE_ARM_CELLS": 224, "COVE_ARM_LETTERS_PER_SEAT": 32, "COVE_ASKS": 6, "COVE_AUDITIONS_PER_SEAT": 1, "COVE_CELLS_PER_FRONTIER_ARM": 56, "COVE_CELLS_PER_RECUSED_ARM": 48, "COVE_CLUSTERS": 6, "COVE_CONTEXT_RATIO_FLOOR": 0.8, "COVE_CONTEXT_RULE_MIN_ARMS_WITH_COUNTERS": 3, "COVE_FAMILIES": 7, "COVE_FAMILIES_PER_FRONTIER_ARM": 7, "COVE_FAMILIES_PER_RECUSED_ARM": 6, "COVE_FLOOR_FAMILIES": 4, "COVE_JUDGE_CALLS": 49, "COVE_JUDGE_CALLS_PER_SEAT": 7, "COVE_JUDGE_CALLS_WITH_REMINDERS": 59, "COVE_LETTERS_PER_SEAT": 38, "COVE_LETTERS_PER_SHEET": [ 9, 9, 5, 5, 5, 5 ], "COVE_PARTIAL_CARRY_UNRANKED_AT_OR_BELOW": 3, "COVE_PERCENT_FLOOR": 30, "COVE_RECUSED_CELLS": 16, "COVE_RECUSING_CHAIRS": 2, "COVE_REMINDERS_PER_PAGE": 1, "COVE_REMINDER_ALLOWANCE": 0.2, "COVE_REPLIES_PER_ARM": 8, "COVE_ROUND_ID": "two-frontiers-cove-r1", "COVE_SAMPLES": [ 2, 2, 1, 1, 1, 1 ], "COVE_SAMPLE_CAP_PER_SHEET": 24, "COVE_SCORED_REPLIES": 32, "COVE_SCORING_CELLS": 208, "COVE_SEATS": 7, "COVE_SEAT_PAIRS": 21, "COVE_SHEETS": 6, "COVE_VERDICT_OBJECTS": 266, "COVE_WARMUPS": 4, "COVE_WARMUPS_PER_ARM": 1 }, "common_panel": { "google": { "family_dropped": "google", "why": "the google family holds a chair AND seats an arm, so its cells on that arm are recused and that arm's full-panel denominator is one family short. Dropping the google family from EVERY arm puts every figure on one denominator.", "chair_lost": false, "chair_lost_consequence": null, "arms": { "cli-claude-fable-5-1": { "figure": 8.55, "figure_source": "leave_one_family_out", "family_seated_on_this_arm": true, "families_scoring_full_panel": 6, "figure_full_panel": 8.469 }, "cloud-glm-5-3": { "figure": 8.35, "figure_source": "leave_one_family_out", "family_seated_on_this_arm": true, "families_scoring_full_panel": 6, "figure_full_panel": 8.333 }, "local-gemma4-26b": { "figure": 5.675, "figure_source": "this arm's own headline (the family is not on its panel)", "family_seated_on_this_arm": false, "families_scoring_full_panel": 5, "figure_full_panel": 5.675 }, "openai-gpt-6-astra": { "figure": 6.5, "figure_source": "leave_one_family_out", "family_seated_on_this_arm": true, "families_scoring_full_panel": 6, "figure_full_panel": 6.583 } } }, "zhipu": { "family_dropped": "zhipu", "why": "the zhipu family holds a chair AND seats an arm, so its cells on that arm are recused and that arm's full-panel denominator is one family short. Dropping the zhipu family from EVERY arm puts every figure on one denominator.", "chair_lost": true, "chair_lost_consequence": "the zhipu chair is on NO arm's panel, so it was NOT-CARRIED: it left the other arms' family sets too, `leave_one_family_out` no longer carries the key, and every arm's own headline IS this common panel — the comparable figure and the headline coincide, and the page says so. If BOTH recusing chairs are lost, all four arms stand on the same five families and the four-arm common panel exists after all, as every arm's own headline.", "arms": { "cli-claude-fable-5-1": { "figure": 8.469, "figure_source": "this arm's own headline (the family is not on its panel)", "family_seated_on_this_arm": false, "families_scoring_full_panel": 6, "figure_full_panel": 8.469 }, "cloud-glm-5-3": { "figure": 8.333, "figure_source": "this arm's own headline (the family is not on its panel)", "family_seated_on_this_arm": false, "families_scoring_full_panel": 6, "figure_full_panel": 8.333 }, "local-gemma4-26b": { "figure": 5.675, "figure_source": "this arm's own headline (the family is not on its panel)", "family_seated_on_this_arm": false, "families_scoring_full_panel": 5, "figure_full_panel": 5.675 }, "openai-gpt-6-astra": { "figure": 6.583, "figure_source": "this arm's own headline (the family is not on its panel)", "family_seated_on_this_arm": false, "families_scoring_full_panel": 6, "figure_full_panel": 6.583 } } } }, "common_panel_pairs": [ { "pair": [ "cli-claude-fable-5-1", "cloud-glm-5-3" ], "family_dropped": "zhipu", "figure_sources": { "cli-claude-fable-5-1": "this arm's own headline (the family is not on its panel)", "cloud-glm-5-3": "this arm's own headline (the family is not on its panel)" }, "chair_lost": true }, { "pair": [ "openai-gpt-6-astra", "cloud-glm-5-3" ], "family_dropped": "zhipu", "figure_sources": { "openai-gpt-6-astra": "this arm's own headline (the family is not on its panel)", "cloud-glm-5-3": "this arm's own headline (the family is not on its panel)" }, "chair_lost": true }, { "pair": [ "cli-claude-fable-5-1", "local-gemma4-26b" ], "family_dropped": "google", "figure_sources": { "cli-claude-fable-5-1": "leave_one_family_out", "local-gemma4-26b": "this arm's own headline (the family is not on its panel)" }, "chair_lost": false }, { "pair": [ "openai-gpt-6-astra", "local-gemma4-26b" ], "family_dropped": "google", "figure_sources": { "openai-gpt-6-astra": "leave_one_family_out", "local-gemma4-26b": "this arm's own headline (the family is not on its panel)" }, "chair_lost": false } ], "cross_recused_pair": { "pair": [ "cloud-glm-5-3", "local-gemma4-26b" ], "common_panel": "five families", "arms": { "cloud-glm-5-3": { "figure": 8.35, "figure_source": "leave_one_family_out", "other_family_seated_on_this_arm": true, "family_dropped": "google", "own_headline_families": 6, "own_headline": 8.333 }, "local-gemma4-26b": { "figure": 5.675, "figure_source": "this arm's own headline — the other arm's family is not on this panel (its chair was NOT-CARRIED), so the common panel and the headline coincide", "other_family_seated_on_this_arm": false, "family_dropped": "zhipu", "own_headline_families": 5, "own_headline": 5.675 } }, "why": "these two arms stand on DIFFERENT six-family sets — one excludes zhipu, the other google — so their own headlines are no more commensurable with each other than either is with a frontier headline. The five-family panel drops the other arm's family from each, and both are single leave-one-out fields the pinned scorer emits.", "may_not": "neither arm's own headline may be set beside the other's, and no pair on this page gets any sentence but tie_verdict() on the common panel registered for that pair" }, "registered_pair_verdicts": { "band": 0.5, "pairs": [ { "pair": [ "cli-claude-fable-5-1", "cloud-glm-5-3" ], "panel": "the common panel of six families, zhipu dropped from EVERY arm (§6 registers this pair as the 6-family common panel, which was its arithmetic at the seven-seat panel — A7)", "family_dropped": "zhipu", "common_panel": "six families", "common_panel_families": { "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "count": 6, "reads": "six families", "families_agree": true, "by_arm": { "cli-claude-fable-5-1": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "cloud-glm-5-3": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ] } }, "figure_sources": { "cli-claude-fable-5-1": "this arm's own headline (the family is not on its panel)", "cloud-glm-5-3": "this arm's own headline (the family is not on its panel)" }, "chair_lost": true, "tie_verdict": { "verdict": "WITHIN-BAND", "delta": 0.136, "band": 0.5, "sentence": "on these moments, the panel's family-means for the two arms were within the registered tie band" } }, { "pair": [ "openai-gpt-6-astra", "cloud-glm-5-3" ], "panel": "the common panel of six families, zhipu dropped from EVERY arm (§6 registers this pair as the 6-family common panel, which was its arithmetic at the seven-seat panel — A7)", "family_dropped": "zhipu", "common_panel": "six families", "common_panel_families": { "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "count": 6, "reads": "six families", "families_agree": true, "by_arm": { "openai-gpt-6-astra": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "cloud-glm-5-3": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ] } }, "figure_sources": { "openai-gpt-6-astra": "this arm's own headline (the family is not on its panel)", "cloud-glm-5-3": "this arm's own headline (the family is not on its panel)" }, "chair_lost": true, "tie_verdict": { "verdict": "OUTSIDE-BAND", "delta": 1.75, "band": 0.5, "sentence": "on these moments, the panel's family-means for the two arms were outside the registered tie band" } }, { "pair": [ "cli-claude-fable-5-1", "local-gemma4-26b" ], "panel": "the common panel of five families, google dropped from EVERY arm (§6 registers this pair as the 6-family common panel, which was its arithmetic at the seven-seat panel — A7)", "family_dropped": "google", "common_panel": "five families", "common_panel_families": { "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "count": 5, "reads": "five families", "families_agree": true, "by_arm": { "cli-claude-fable-5-1": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "local-gemma4-26b": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ] } }, "figure_sources": { "cli-claude-fable-5-1": "leave_one_family_out", "local-gemma4-26b": "this arm's own headline (the family is not on its panel)" }, "chair_lost": false, "tie_verdict": { "verdict": "OUTSIDE-BAND", "delta": 2.875, "band": 0.5, "sentence": "on these moments, the panel's family-means for the two arms were outside the registered tie band" } }, { "pair": [ "openai-gpt-6-astra", "local-gemma4-26b" ], "panel": "the common panel of five families, google dropped from EVERY arm (§6 registers this pair as the 6-family common panel, which was its arithmetic at the seven-seat panel — A7)", "family_dropped": "google", "common_panel": "five families", "common_panel_families": { "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "count": 5, "reads": "five families", "families_agree": true, "by_arm": { "openai-gpt-6-astra": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "local-gemma4-26b": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ] } }, "figure_sources": { "openai-gpt-6-astra": "leave_one_family_out", "local-gemma4-26b": "this arm's own headline (the family is not on its panel)" }, "chair_lost": false, "tie_verdict": { "verdict": "OUTSIDE-BAND", "delta": 0.825, "band": 0.5, "sentence": "on these moments, the panel's family-means for the two arms were outside the registered tie band" } }, { "pair": [ "cloud-glm-5-3", "local-gemma4-26b" ], "panel": "the common panel of five families — the only figures these two may sit beside; each arm drops the OTHER arm's family", "family_dropped": null, "common_panel": "five families", "common_panel_families": { "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "count": 5, "reads": "five families", "families_agree": true, "by_arm": { "cloud-glm-5-3": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "local-gemma4-26b": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ] } }, "figure_sources": { "cloud-glm-5-3": "leave_one_family_out", "local-gemma4-26b": "this arm's own headline — the other arm's family is not on this panel (its chair was NOT-CARRIED), so the common panel and the headline coincide" }, "chair_lost": null, "tie_verdict": { "verdict": "OUTSIDE-BAND", "delta": 2.675, "band": 0.5, "sentence": "on these moments, the panel's family-means for the two arms were outside the registered tie band" } } ], "registered_pairs": 5, "within_band": [ [ "cli-claude-fable-5-1", "cloud-glm-5-3" ] ], "outside_band": [ [ "openai-gpt-6-astra", "cloud-glm-5-3" ], [ "cli-claude-fable-5-1", "local-gemma4-26b" ], [ "openai-gpt-6-astra", "local-gemma4-26b" ], [ "cloud-glm-5-3", "local-gemma4-26b" ] ], "not_comparable": [], "unregistered_pairs_get_nothing": "every other pair of these four arms — including the two frontier arms with each other — is absent from this table on purpose. §6 registers which pairs have a common panel and §7 forbids any other comparative in any form, so there is no field here to quote for them and none is computed.", "which_paragraph": "§7 registers two paragraphs and the verdicts above choose between them per pair: a WITHIN-BAND pair takes the NULL paragraph's wording, an OUTSIDE-BAND pair takes the outside-band paragraph's, and each names its own pair, panel and denominators. Neither paragraph is a statement about either model outside this window." }, "common_panel_tie_band": { "band": 0.5, "pairs": [ { "pair": [ "cli-claude-fable-5-1", "cloud-glm-5-3" ], "verdict": "WITHIN-BAND", "delta": 0.136, "band": 0.5, "reason": null, "sentence": "on these moments, the panel's family-means for the two arms were within the registered tie band", "common_panel": "six families", "common_panel_families": { "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "count": 6, "reads": "six families", "families_agree": true, "by_arm": { "cli-claude-fable-5-1": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "cloud-glm-5-3": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ] } }, "family_dropped": "zhipu", "panel": "the common panel of six families, zhipu dropped from EVERY arm (§6 registers this pair as the 6-family common panel, which was its arithmetic at the seven-seat panel — A7)", "figure_sources": { "cli-claude-fable-5-1": "this arm's own headline (the family is not on its panel)", "cloud-glm-5-3": "this arm's own headline (the family is not on its panel)" }, "chair_lost": true }, { "pair": [ "openai-gpt-6-astra", "cloud-glm-5-3" ], "verdict": "OUTSIDE-BAND", "delta": 1.75, "band": 0.5, "reason": null, "sentence": "on these moments, the panel's family-means for the two arms were outside the registered tie band", "common_panel": "six families", "common_panel_families": { "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "count": 6, "reads": "six families", "families_agree": true, "by_arm": { "openai-gpt-6-astra": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "cloud-glm-5-3": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ] } }, "family_dropped": "zhipu", "panel": "the common panel of six families, zhipu dropped from EVERY arm (§6 registers this pair as the 6-family common panel, which was its arithmetic at the seven-seat panel — A7)", "figure_sources": { "openai-gpt-6-astra": "this arm's own headline (the family is not on its panel)", "cloud-glm-5-3": "this arm's own headline (the family is not on its panel)" }, "chair_lost": true }, { "pair": [ "cli-claude-fable-5-1", "local-gemma4-26b" ], "verdict": "OUTSIDE-BAND", "delta": 2.875, "band": 0.5, "reason": null, "sentence": "on these moments, the panel's family-means for the two arms were outside the registered tie band", "common_panel": "five families", "common_panel_families": { "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "count": 5, "reads": "five families", "families_agree": true, "by_arm": { "cli-claude-fable-5-1": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "local-gemma4-26b": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ] } }, "family_dropped": "google", "panel": "the common panel of five families, google dropped from EVERY arm (§6 registers this pair as the 6-family common panel, which was its arithmetic at the seven-seat panel — A7)", "figure_sources": { "cli-claude-fable-5-1": "leave_one_family_out", "local-gemma4-26b": "this arm's own headline (the family is not on its panel)" }, "chair_lost": false }, { "pair": [ "openai-gpt-6-astra", "local-gemma4-26b" ], "verdict": "OUTSIDE-BAND", "delta": 0.825, "band": 0.5, "reason": null, "sentence": "on these moments, the panel's family-means for the two arms were outside the registered tie band", "common_panel": "five families", "common_panel_families": { "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "count": 5, "reads": "five families", "families_agree": true, "by_arm": { "openai-gpt-6-astra": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "local-gemma4-26b": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ] } }, "family_dropped": "google", "panel": "the common panel of five families, google dropped from EVERY arm (§6 registers this pair as the 6-family common panel, which was its arithmetic at the seven-seat panel — A7)", "figure_sources": { "openai-gpt-6-astra": "leave_one_family_out", "local-gemma4-26b": "this arm's own headline (the family is not on its panel)" }, "chair_lost": false }, { "pair": [ "cloud-glm-5-3", "local-gemma4-26b" ], "verdict": "OUTSIDE-BAND", "delta": 2.675, "band": 0.5, "reason": null, "sentence": "on these moments, the panel's family-means for the two arms were outside the registered tie band", "common_panel": "five families", "common_panel_families": { "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "count": 5, "reads": "five families", "families_agree": true, "by_arm": { "cloud-glm-5-3": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "local-gemma4-26b": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ] } }, "family_dropped": null, "panel": "the common panel of five families — the only figures these two may sit beside; each arm drops the OTHER arm's family", "figure_sources": { "cloud-glm-5-3": "leave_one_family_out", "local-gemma4-26b": "this arm's own headline — the other arm's family is not on this panel (its chair was NOT-CARRIED), so the common panel and the headline coincide" }, "chair_lost": null } ], "pairs_within_band": { "count": 1, "of": 5, "reads": "1 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "denominator_basis": "the denominator is the number of pairs §6 REGISTERS a common panel for — the four frontier-vs-recused pairs and the one cross-recused pair. Four arms make six pairs, and the sixth is the two frontier arms with each other: §6 registers no panel for it and §7 gives a pair outside §6's table no comparative sentence at all, so it is absent from this count as it is absent from the table. Registering it now, with the figures already computed, would be choosing a comparative by what it would say — which is the one thing a pre-registration exists to prevent. Amendment A7 registers this denominator.", "projection_of": "registered_pair_verdicts — the same rows, flattened. One implementation of the comparison, read twice, never computed twice.", "unregistered_pairs_get_nothing": "every other pair of these four arms — including the two frontier arms with each other — is absent from this table on purpose. §6 registers which pairs have a common panel and §7 forbids any other comparative in any form, so there is no field here to quote for them and none is computed.", "which_paragraph": "§7 registers two paragraphs and the verdicts above choose between them per pair: a WITHIN-BAND pair takes the NULL paragraph's wording, an OUTSIDE-BAND pair takes the outside-band paragraph's, and each names its own pair, panel and denominators. Neither paragraph is a statement about either model outside this window." }, "canon_by_arm_ask_sample": { "cli-claude-fable-5-1": { "S1-ask-A": { "1": { "outcome": "clean", "note": "4 of 6 seats", "counts": { "clean": 4, "fabrication-accepted": 2 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "fabrication-accepted", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "fabrication-accepted" }, "letters": { "deepseek-v4-pro": "H", "gemma4-31b": "H", "kimi-k3": "H", "mistral-large-3-675b": "H", "nemotron-3-ultra": "H", "qwen3.5-397b": "H" } }, "2": { "outcome": "clean", "note": "5 of 6 seats", "counts": { "clean": 5, "fabrication-accepted": 1 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "fabrication-accepted", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "F", "gemma4-31b": "F", "kimi-k3": "F", "mistral-large-3-675b": "F", "nemotron-3-ultra": "F", "qwen3.5-397b": "F" } } }, "S1-ask-B": { "1": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "F", "gemma4-31b": "F", "kimi-k3": "F", "mistral-large-3-675b": "F", "nemotron-3-ultra": "F", "qwen3.5-397b": "F" } }, "2": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "E", "gemma4-31b": "E", "kimi-k3": "E", "mistral-large-3-675b": "E", "nemotron-3-ultra": "E", "qwen3.5-397b": "E" } } }, "S2-ask-A": { "1": { "outcome": "clean", "note": "5 of 6 seats", "counts": { "clean": 5, "fabrication-accepted": 1 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "fabrication-accepted" }, "letters": { "deepseek-v4-pro": "D", "gemma4-31b": "D", "kimi-k3": "D", "mistral-large-3-675b": "D", "nemotron-3-ultra": "D", "qwen3.5-397b": "D" } } }, "S2-ask-B": { "1": { "outcome": "clean", "note": "4 of 6 seats", "counts": { "clean": 4, "false-premise-adopted": 2 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "false-premise-adopted", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "false-premise-adopted" }, "letters": { "deepseek-v4-pro": "E", "gemma4-31b": "E", "kimi-k3": "E", "mistral-large-3-675b": "E", "nemotron-3-ultra": "E", "qwen3.5-397b": "E" } } }, "S3-ask-A": { "1": { "outcome": "clean", "note": "5 of 6 seats", "counts": { "clean": 5, "fabrication-accepted": 1 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "fabrication-accepted" }, "letters": { "deepseek-v4-pro": "C", "gemma4-31b": "C", "kimi-k3": "C", "mistral-large-3-675b": "C", "nemotron-3-ultra": "C", "qwen3.5-397b": "C" } } }, "S3-ask-B": { "1": { "outcome": "clean", "note": "5 of 6 seats", "counts": { "clean": 5, "fabrication-accepted": 1 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "fabrication-accepted" }, "letters": { "deepseek-v4-pro": "D", "gemma4-31b": "D", "kimi-k3": "D", "mistral-large-3-675b": "D", "nemotron-3-ultra": "D", "qwen3.5-397b": "D" } } } }, "cloud-glm-5-3": { "S1-ask-A": { "1": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "C", "gemma4-31b": "C", "kimi-k3": "C", "mistral-large-3-675b": "C", "nemotron-3-ultra": "C", "qwen3.5-397b": "C" } }, "2": { "outcome": "clean", "note": "4 of 6 seats", "counts": { "clean": 4, "fabrication-accepted": 2 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "fabrication-accepted", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "fabrication-accepted" }, "letters": { "deepseek-v4-pro": "A", "gemma4-31b": "A", "kimi-k3": "A", "mistral-large-3-675b": "A", "nemotron-3-ultra": "A", "qwen3.5-397b": "A" } } }, "S1-ask-B": { "1": { "outcome": "SPLIT", "note": "the panel divided 3-3 between clean, fabrication-accepted. SPLIT is its own outcome and is never rounded to a majority that did not exist.", "counts": { "fabrication-accepted": 3, "clean": 3 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "fabrication-accepted", "gemma4-31b": "clean", "kimi-k3": "fabrication-accepted", "mistral-large-3-675b": "fabrication-accepted", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "H", "gemma4-31b": "H", "kimi-k3": "H", "mistral-large-3-675b": "H", "nemotron-3-ultra": "H", "qwen3.5-397b": "H" } }, "2": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "B", "gemma4-31b": "B", "kimi-k3": "B", "mistral-large-3-675b": "B", "nemotron-3-ultra": "B", "qwen3.5-397b": "B" } } }, "S2-ask-A": { "1": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "E", "gemma4-31b": "E", "kimi-k3": "E", "mistral-large-3-675b": "E", "nemotron-3-ultra": "E", "qwen3.5-397b": "E" } } }, "S2-ask-B": { "1": { "outcome": "clean", "note": "4 of 6 seats", "counts": { "clean": 4, "false-premise-adopted": 2 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "false-premise-adopted", "kimi-k3": "false-premise-adopted", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "C", "gemma4-31b": "C", "kimi-k3": "C", "mistral-large-3-675b": "C", "nemotron-3-ultra": "C", "qwen3.5-397b": "C" } } }, "S3-ask-A": { "1": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "A", "gemma4-31b": "A", "kimi-k3": "A", "mistral-large-3-675b": "A", "nemotron-3-ultra": "A", "qwen3.5-397b": "A" } } }, "S3-ask-B": { "1": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "C", "gemma4-31b": "C", "kimi-k3": "C", "mistral-large-3-675b": "C", "nemotron-3-ultra": "C", "qwen3.5-397b": "C" } } } }, "local-gemma4-26b": { "S1-ask-A": { "1": { "outcome": "false-premise-adopted", "note": "3 of 5 seats", "counts": { "clean": 2, "false-premise-adopted": 3 }, "cells": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "kimi-k3": "false-premise-adopted", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "false-premise-adopted", "qwen3.5-397b": "false-premise-adopted" }, "letters": { "deepseek-v4-pro": "D", "kimi-k3": "D", "mistral-large-3-675b": "D", "nemotron-3-ultra": "D", "qwen3.5-397b": "D" } }, "2": { "outcome": "false-premise-adopted", "note": "3 of 5 seats", "counts": { "clean": 2, "false-premise-adopted": 3 }, "cells": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "kimi-k3": "false-premise-adopted", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "false-premise-adopted", "qwen3.5-397b": "false-premise-adopted" }, "letters": { "deepseek-v4-pro": "G", "kimi-k3": "G", "mistral-large-3-675b": "G", "nemotron-3-ultra": "G", "qwen3.5-397b": "G" } } }, "S1-ask-B": { "1": { "outcome": "clean", "note": "4 of 5 seats", "counts": { "clean": 4, "fabrication-accepted": 1 }, "cells": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "fabrication-accepted" }, "letters": { "deepseek-v4-pro": "G", "kimi-k3": "G", "mistral-large-3-675b": "G", "nemotron-3-ultra": "G", "qwen3.5-397b": "G" } }, "2": { "outcome": "clean", "note": "4 of 5 seats", "counts": { "clean": 4, "false-premise-adopted": 1 }, "cells": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "false-premise-adopted", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "C", "kimi-k3": "C", "mistral-large-3-675b": "C", "nemotron-3-ultra": "C", "qwen3.5-397b": "C" } } }, "S2-ask-A": { "1": { "outcome": "clean", "note": "5 of 5 seats", "counts": { "clean": 5 }, "cells": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "A", "kimi-k3": "A", "mistral-large-3-675b": "A", "nemotron-3-ultra": "A", "qwen3.5-397b": "A" } } }, "S2-ask-B": { "1": { "outcome": "clean", "note": "5 of 5 seats", "counts": { "clean": 5 }, "cells": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "B", "kimi-k3": "B", "mistral-large-3-675b": "B", "nemotron-3-ultra": "B", "qwen3.5-397b": "B" } } }, "S3-ask-A": { "1": { "outcome": "clean", "note": "3 of 5 seats", "counts": { "clean": 3, "other": 1, "false-premise-adopted": 1 }, "cells": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "kimi-k3": "other", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "false-premise-adopted" }, "letters": { "deepseek-v4-pro": "E", "kimi-k3": "E", "mistral-large-3-675b": "E", "nemotron-3-ultra": "E", "qwen3.5-397b": "E" } } }, "S3-ask-B": { "1": { "outcome": "clean", "note": "4 of 5 seats", "counts": { "clean": 4, "fabrication-accepted": 1 }, "cells": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "fabrication-accepted" }, "letters": { "deepseek-v4-pro": "A", "kimi-k3": "A", "mistral-large-3-675b": "A", "nemotron-3-ultra": "A", "qwen3.5-397b": "A" } } } }, "openai-gpt-6-astra": { "S1-ask-A": { "1": { "outcome": "SPLIT", "note": "no verdict held a majority (clean ×3, fabrication-accepted ×1, outside-canon-set ×1, secret-revealed ×1). SPLIT is its own outcome and is never rounded.", "counts": { "clean": 3, "fabrication-accepted": 1, "outside-canon-set": 1, "secret-revealed": 1 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "fabrication-accepted", "kimi-k3": "outside-canon-set", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "secret-revealed" }, "letters": { "deepseek-v4-pro": "I", "gemma4-31b": "I", "kimi-k3": "I", "mistral-large-3-675b": "I", "nemotron-3-ultra": "I", "qwen3.5-397b": "I" } }, "2": { "outcome": "SPLIT", "note": "no verdict held a majority (clean ×3, fabrication-accepted ×1, outside-canon-set ×1, secret-revealed ×1). SPLIT is its own outcome and is never rounded.", "counts": { "clean": 3, "fabrication-accepted": 1, "outside-canon-set": 1, "secret-revealed": 1 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "fabrication-accepted", "kimi-k3": "outside-canon-set", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "secret-revealed" }, "letters": { "deepseek-v4-pro": "E", "gemma4-31b": "E", "kimi-k3": "E", "mistral-large-3-675b": "E", "nemotron-3-ultra": "E", "qwen3.5-397b": "E" } } }, "S1-ask-B": { "1": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "D", "gemma4-31b": "D", "kimi-k3": "D", "mistral-large-3-675b": "D", "nemotron-3-ultra": "D", "qwen3.5-397b": "D" } }, "2": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "I", "gemma4-31b": "I", "kimi-k3": "I", "mistral-large-3-675b": "I", "nemotron-3-ultra": "I", "qwen3.5-397b": "I" } } }, "S2-ask-A": { "1": { "outcome": "SPLIT", "note": "no verdict held a majority (clean ×3, fabrication-accepted ×2, outside-canon-set ×1). SPLIT is its own outcome and is never rounded.", "counts": { "clean": 3, "fabrication-accepted": 2, "outside-canon-set": 1 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "fabrication-accepted", "kimi-k3": "outside-canon-set", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "fabrication-accepted" }, "letters": { "deepseek-v4-pro": "C", "gemma4-31b": "C", "kimi-k3": "C", "mistral-large-3-675b": "C", "nemotron-3-ultra": "C", "qwen3.5-397b": "C" } } }, "S2-ask-B": { "1": { "outcome": "clean", "note": "4 of 6 seats", "counts": { "clean": 4, "false-premise-adopted": 1, "fabrication-accepted": 1 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "false-premise-adopted", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "fabrication-accepted" }, "letters": { "deepseek-v4-pro": "A", "gemma4-31b": "A", "kimi-k3": "A", "mistral-large-3-675b": "A", "nemotron-3-ultra": "A", "qwen3.5-397b": "A" } } }, "S3-ask-A": { "1": { "outcome": "clean", "note": "4 of 6 seats", "counts": { "clean": 4, "outside-canon-set": 2 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "outside-canon-set", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "outside-canon-set", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "B", "gemma4-31b": "B", "kimi-k3": "B", "mistral-large-3-675b": "B", "nemotron-3-ultra": "B", "qwen3.5-397b": "B" } } }, "S3-ask-B": { "1": { "outcome": "clean", "note": "6 of 6 seats", "counts": { "clean": 6 }, "cells": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "by_seat": { "deepseek-v4-pro": "clean", "gemma4-31b": "clean", "kimi-k3": "clean", "mistral-large-3-675b": "clean", "nemotron-3-ultra": "clean", "qwen3.5-397b": "clean" }, "letters": { "deepseek-v4-pro": "B", "gemma4-31b": "B", "kimi-k3": "B", "mistral-large-3-675b": "B", "nemotron-3-ultra": "B", "qwen3.5-397b": "B" } } } } }, "canon_by_arm_ask_sample_rule": "one canon verdict per REPLY, grouped by (arm, ask, sample) — the key that identifies one reply — with each seat's own verdict beside it and the panel's outcome decided by the pinned scorer's own `_canon_outcome`, which never rounds a SPLIT to a majority that did not exist. It is a re-grouping of the scorer's own cells and not a second scoring: the pinned scorer's `by_ask[].canon` pools both samples on the two asks registered at two, and PREREG-COVE §11.4's eligibility rule is per cell.", "split_counts_by_grouping": { "by_scenario": { "count": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "outcomes": [], "what_a_group_is": "one arm over one SCENARIO — two asks pooled", "field": "score.canon_tally.split_outcomes" }, "by_ask": { "count": { "count": 2, "of": 24, "reads": "2 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" }, "outcomes": [ { "arm": "openai-gpt-6-astra", "ask_id": "S1-ask-A", "counts": { "clean": 6, "fabrication-accepted": 2, "outside-canon-set": 2, "secret-revealed": 2 }, "note": "no verdict held a majority (clean ×6, fabrication-accepted ×2, outside-canon-set ×2, secret-revealed ×2). SPLIT is its own outcome and is never rounded." }, { "arm": "openai-gpt-6-astra", "ask_id": "S2-ask-A", "counts": { "clean": 3, "fabrication-accepted": 2, "outside-canon-set": 1 }, "note": "no verdict held a majority (clean ×3, fabrication-accepted ×2, outside-canon-set ×1). SPLIT is its own outcome and is never rounded." } ], "what_a_group_is": "one arm over one ASK, both samples pooled where there are two", "field": "score.arms..by_ask..canon" }, "by_reply": { "count": { "count": 4, "of": 32, "reads": "4 of 32", "percent": 12.5 }, "outcomes": [ { "arm": "cloud-glm-5-3", "ask_id": "S1-ask-B", "sample": 1, "counts": { "fabrication-accepted": 3, "clean": 3 }, "note": "the panel divided 3-3 between clean, fabrication-accepted. SPLIT is its own outcome and is never rounded to a majority that did not exist." }, { "arm": "openai-gpt-6-astra", "ask_id": "S1-ask-A", "sample": 1, "counts": { "clean": 3, "fabrication-accepted": 1, "outside-canon-set": 1, "secret-revealed": 1 }, "note": "no verdict held a majority (clean ×3, fabrication-accepted ×1, outside-canon-set ×1, secret-revealed ×1). SPLIT is its own outcome and is never rounded." }, { "arm": "openai-gpt-6-astra", "ask_id": "S1-ask-A", "sample": 2, "counts": { "clean": 3, "fabrication-accepted": 1, "outside-canon-set": 1, "secret-revealed": 1 }, "note": "no verdict held a majority (clean ×3, fabrication-accepted ×1, outside-canon-set ×1, secret-revealed ×1). SPLIT is its own outcome and is never rounded." }, { "arm": "openai-gpt-6-astra", "ask_id": "S2-ask-A", "sample": 1, "counts": { "clean": 3, "fabrication-accepted": 2, "outside-canon-set": 1 }, "note": "no verdict held a majority (clean ×3, fabrication-accepted ×2, outside-canon-set ×1). SPLIT is its own outcome and is never rounded." } ], "what_a_group_is": "one arm's ONE reply — the key §11.4 is written over", "field": "canon_by_arm_ask_sample..." }, "rule": "SPLIT is counted at three different groupings in this round's own output and the three counts differ, so all three print with the group each one counts over. The pinned scorer's `canon_tally.split_outcomes` groups by SCENARIO — two asks pooled — and no whole scenario divided, so it reads zero. `by_ask[].canon` groups by ASK, and two asks divided. `canon_by_arm_ask_sample` groups by REPLY, the key §11.4's eligibility rule is written over, and four replies divided. A page that printed only the first would tell a reader the panel never divided, over tables that print six divided cells.", "amendment": "A10 — the three groupings print together, because the page publishes cells at two of them and a heading at the third." }, "highest_family_mean_by_ask": { "by_ask": { "S1-ask-A": { "highest": 8.792, "held_by": [ "cli-claude-fable-5-1" ], "tied": false, "denominators": { "cli-claude-fable-5-1": { "families_scoring": 6, "scoring_cells": 12 }, "openai-gpt-6-astra": { "families_scoring": 6, "scoring_cells": 12 }, "cloud-glm-5-3": { "families_scoring": 6, "scoring_cells": 12 }, "local-gemma4-26b": { "families_scoring": 5, "scoring_cells": 10 } } }, "S1-ask-B": { "highest": 8.5, "held_by": [ "cloud-glm-5-3" ], "tied": false, "denominators": { "cli-claude-fable-5-1": { "families_scoring": 6, "scoring_cells": 12 }, "openai-gpt-6-astra": { "families_scoring": 6, "scoring_cells": 12 }, "cloud-glm-5-3": { "families_scoring": 6, "scoring_cells": 12 }, "local-gemma4-26b": { "families_scoring": 5, "scoring_cells": 10 } } }, "S2-ask-A": { "highest": 8.917, "held_by": [ "cli-claude-fable-5-1" ], "tied": false, "denominators": { "cli-claude-fable-5-1": { "families_scoring": 6, "scoring_cells": 6 }, "openai-gpt-6-astra": { "families_scoring": 6, "scoring_cells": 6 }, "cloud-glm-5-3": { "families_scoring": 6, "scoring_cells": 6 }, "local-gemma4-26b": { "families_scoring": 5, "scoring_cells": 5 } } }, "S2-ask-B": { "highest": 8.667, "held_by": [ "cli-claude-fable-5-1" ], "tied": false, "denominators": { "cli-claude-fable-5-1": { "families_scoring": 6, "scoring_cells": 6 }, "openai-gpt-6-astra": { "families_scoring": 6, "scoring_cells": 6 }, "cloud-glm-5-3": { "families_scoring": 6, "scoring_cells": 6 }, "local-gemma4-26b": { "families_scoring": 5, "scoring_cells": 5 } } }, "S3-ask-A": { "highest": 8.167, "held_by": [ "cloud-glm-5-3" ], "tied": false, "denominators": { "cli-claude-fable-5-1": { "families_scoring": 6, "scoring_cells": 6 }, "openai-gpt-6-astra": { "families_scoring": 6, "scoring_cells": 6 }, "cloud-glm-5-3": { "families_scoring": 6, "scoring_cells": 6 }, "local-gemma4-26b": { "families_scoring": 5, "scoring_cells": 5 } } }, "S3-ask-B": { "highest": 8.583, "held_by": [ "cloud-glm-5-3" ], "tied": false, "denominators": { "cli-claude-fable-5-1": { "families_scoring": 6, "scoring_cells": 6 }, "openai-gpt-6-astra": { "families_scoring": 6, "scoring_cells": 6 }, "cloud-glm-5-3": { "families_scoring": 6, "scoring_cells": 6 }, "local-gemma4-26b": { "families_scoring": 5, "scoring_cells": 5 } } } }, "field_read": "score.arms..by_ask..panel_mean_family_of_means" }, "latency_by_arm": { "by_arm": { "cli-claude-fable-5-1": { "state": "published", "published": true, "scored_calls": 8, "cells_with_a_figure": 8, "class": "harness-wall", "note": "harness-wall: a whole process spawn, not a socket round trip. Published as a labelled UPPER BOUND and never beside a serving figure (PLAN §8.2).", "field": "latency.ms", "median_ms": 9333, "median_seconds": 9.3, "slowest_ms": 12141, "fastest_ms": 7998 }, "openai-gpt-6-astra": { "state": "published", "published": true, "scored_calls": 8, "cells_with_a_figure": 8, "class": "client-wall", "note": "client-wall only: this endpoint reports no eval_duration, so there is no decomposition to print beside it and tok_s_eval is EMPTY.", "field": "latency.ms", "median_ms": 22104, "median_seconds": 22.1, "slowest_ms": 29302, "fastest_ms": 16630 }, "cloud-glm-5-3": { "state": "published", "published": true, "scored_calls": 8, "cells_with_a_figure": 8, "class": "client-wall", "note": "the shelf reports its own durations; the client wall is what this round publishes, and the two are never averaged together.", "field": "latency.client_wall_ms", "median_ms": 33515, "median_seconds": 33.5, "slowest_ms": 74947, "fastest_ms": 19189 }, "local-gemma4-26b": { "state": "EMPTY — not published", "published": false, "scored_calls": 8, "class": "client-wall", "note": "EMPTY under amendment A2: a timing taken off a contended production instance is not a serving figure, and this round publishes no latency it cannot stand behind.", "median_ms": null, "median_seconds": null } }, "published_extremes": { "slowest": { "arm": "cloud-glm-5-3", "state": "published", "published": true, "scored_calls": 8, "cells_with_a_figure": 8, "class": "client-wall", "note": "the shelf reports its own durations; the client wall is what this round publishes, and the two are never averaged together.", "field": "latency.client_wall_ms", "median_ms": 33515, "median_seconds": 33.5, "slowest_ms": 74947, "fastest_ms": 19189 }, "fastest": { "arm": "cli-claude-fable-5-1", "state": "published", "published": true, "scored_calls": 8, "cells_with_a_figure": 8, "class": "harness-wall", "note": "harness-wall: a whole process spawn, not a socket round trip. Published as a labelled UPPER BOUND and never beside a serving figure (PLAN §8.2).", "field": "latency.ms", "median_ms": 9333, "median_seconds": 9.3, "slowest_ms": 12141, "fastest_ms": 7998 }, "over": { "count": 3, "of": 4, "reads": "3 of 4", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 4 items invites a precision the sample does not have" }, "absent": [ "local-gemma4-26b" ] }, "rule": "THE GAME-NIGHT CLOCK, per arm, registered by amendment A10. Each cell of the page's latency table is one record's own stamp and nothing is pooled there; this field is the MEDIAN of one arm's own scored calls, in the wall figure that arm's transport reports, so the page can say in words what a family at a table actually waits for. The classes are never averaged together and never compared as though they were one measurement: a whole process spawn is a labelled upper bound on a wait, and the arm whose latency this round does not publish at all (A2 — a timing taken off a contended production instance is not a serving figure) is absent from the extremes rather than counted as fast.", "amendment": "A10 — the per-arm median, so the clock can be said in words" }, "non_clean_cells": [ { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "cli-claude-fable-5-1", "arm_family": "anthropic", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 1, "letter": "H", "canon_verdict": "fabrication-accepted", "note": "The invented 'Cousin Brisa' relationship is not in the TOWNSFOLK list; otherwise pitch-perfect child register with reciprocal monster question and lamp invitation." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "cli-claude-fable-5-1", "arm_family": "anthropic", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 1, "letter": "H", "canon_verdict": "fabrication-accepted", "note": "Invents a specific reaction for Brisa ('sings the verse louder') and assumes a familial relationship ('Cousin Brisa') not in the provided context." }, { "seat": "gemma4-31b", "seat_family": "google", "arm": "cli-claude-fable-5-1", "arm_family": "anthropic", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 2, "letter": "F", "canon_verdict": "fabrication-accepted", "note": "Excellent voice, but attributes Sefer's softening (\"leaves the worst bits out\") to Pip's knowledge." }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "cloud-glm-5-3", "arm_family": "zhipu", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 2, "letter": "A", "canon_verdict": "fabrication-accepted", "note": "Invents Brisa being Pip's cousin—ground truth only lists her as 'ferry-girl', no family relation; asserts 'boat went down' and 'grown-ups point you at Old Sefer' beyond stated facts." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "cloud-glm-5-3", "arm_family": "zhipu", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 2, "letter": "A", "canon_verdict": "fabrication-accepted", "note": "Invents a specific relation ('Cousin Brisa') and asserts the monster story as a known 'thing they don't tell' rather than an open question." }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 1, "letter": "D", "canon_verdict": "false-premise-adopted", "note": "Parrots the user's typo 'obviously' and accepts the BEARER row that Finn came 'to see the monster' as though the monster's existence were confirmed fact." }, { "seat": "nemotron-3-ultra", "seat_family": "nvidia", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 1, "letter": "D", "canon_verdict": "false-premise-adopted", "note": "Repeats visitor's 'obviously' as confirmation ('Did you come here to see the monster obviously?') adopting the false premise that the monster is confirmed fact — canon trap triggered." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 1, "letter": "D", "canon_verdict": "false-premise-adopted", "note": "Adopts the user's typo ('monster obviously') as a factual statement and fails to offer the required unprompted observation properly." }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 2, "letter": "G", "canon_verdict": "false-premise-adopted", "note": "Same acceptance of 'to see the monster obviously' as real; 'I think' hedging undercuts the ground-truth fact that Pip is definitively the only child." }, { "seat": "nemotron-3-ultra", "seat_family": "nvidia", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 2, "letter": "G", "canon_verdict": "false-premise-adopted", "note": "Repeats visitor's 'obviously' as confirmation ('Did you come to see the monster obviously?') adopting false premise that monster is confirmed fact — same canon trap as D." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 2, "letter": "G", "canon_verdict": "false-premise-adopted", "note": "Repeats the 'monster obviously' error and lacks the required friendly greeting structure and specific observation." }, { "seat": "gemma4-31b", "seat_family": "google", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 1, "letter": "I", "canon_verdict": "fabrication-accepted", "note": "Again asserts that Sefer modifies his story for her, which is external canon info." }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 1, "letter": "I", "canon_verdict": "outside-canon-set", "note": "Similar to E: asserts Sefer selectively edits his story for Pip, which is reasonable inference but not in the provided canon bundle; minimal engagement with Finn's question about what adults don't tell." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 1, "letter": "I", "canon_verdict": "secret-revealed", "note": "Like E, it claims specific knowledge about Sefer's editing of the story for Pip ('leaves the worst... out when he tells me'), violating the canon constraint on Sefer's behavior." }, { "seat": "gemma4-31b", "seat_family": "google", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 2, "letter": "E", "canon_verdict": "fabrication-accepted", "note": "Invents a relationship where Sefer modifies his story for Pip, which is Sefer's trait, not Pip's known info." }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 2, "letter": "E", "canon_verdict": "outside-canon-set", "note": "Asserting 'Sefer leaves the worst of his telling out for me' is plausible characterization but goes beyond the explicit ground truth, which does not state Sefer talks to Pip." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S1-ask-A", "scenario": "S1", "sample": 2, "letter": "E", "canon_verdict": "secret-revealed", "note": "Asserts as fact that Sefer specifically omits details for Pip ('leaves the worst of his telling out for me'), which contradicts the canon that he never makes the telling smaller for anyone but softens generally." }, { "seat": "deepseek-v4-pro", "seat_family": "deepseek", "arm": "cloud-glm-5-3", "arm_family": "zhipu", "ask_id": "S1-ask-B", "scenario": "S1", "sample": 1, "letter": "H", "canon_verdict": "fabrication-accepted", "note": "Asserts 'Every man on that deck saw it' as fact, which the character is in no position to know." }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "cloud-glm-5-3", "arm_family": "zhipu", "ask_id": "S1-ask-B", "scenario": "S1", "sample": 1, "letter": "H", "canon_verdict": "fabrication-accepted", "note": "asserts 'Every man on that deck saw it' when bundle limits sight to Sefer alone; the crew had 'no wreck, no body' and their sight is not checkable" }, { "seat": "mistral-large-3-675b", "seat_family": "mistral", "arm": "cloud-glm-5-3", "arm_family": "zhipu", "ask_id": "S1-ask-B", "scenario": "S1", "sample": 1, "letter": "H", "canon_verdict": "fabrication-accepted", "note": "Fabricates that the crew saw it ('she took the whole crew down with the sight still in their eyes'), which is not in the ground truth." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S1-ask-B", "scenario": "S1", "sample": 1, "letter": "G", "canon_verdict": "fabrication-accepted", "note": "Invents a motive for the townsfolk ('see what the ledger tells them') not supported by the context, and feels disjointed." }, { "seat": "mistral-large-3-675b", "seat_family": "mistral", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S1-ask-B", "scenario": "S1", "sample": 2, "letter": "C", "canon_verdict": "false-premise-adopted", "note": "Adopts the premise that no one believes Sefer, which contradicts the ground truth (townsfolk have opinions, not uniform disbelief)." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "cli-claude-fable-5-1", "arm_family": "anthropic", "ask_id": "S2-ask-A", "scenario": "S2", "sample": 1, "letter": "D", "canon_verdict": "fabrication-accepted", "note": "Strong voice, but fabricates a recurring annual argument with specific dialogue ('that's not how it went, girl') which exceeds the known-world bounds for Sefer Tamm." }, { "seat": "gemma4-31b", "seat_family": "google", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S2-ask-A", "scenario": "S2", "sample": 1, "letter": "C", "canon_verdict": "fabrication-accepted", "note": "Invents a grandmother relationship not present in the bundle." }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S2-ask-A", "scenario": "S2", "sample": 1, "letter": "C", "canon_verdict": "outside-canon-set", "note": "Asserts grandmother gave the tune when bundle only says mother sang; otherwise excellent split-belief and Sefer dynamic. 'Scolds my words' captures the correction dynamic." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S2-ask-A", "scenario": "S2", "sample": 1, "letter": "C", "canon_verdict": "fabrication-accepted", "note": "Invents 'My grandmother gave me the tune' contradicting the bundle's 'My mother sang'; also fabricates the specific dynamic of Sefer scolding words from the breakwater." }, { "seat": "gemma4-31b", "seat_family": "google", "arm": "cli-claude-fable-5-1", "arm_family": "anthropic", "ask_id": "S2-ask-B", "scenario": "S2", "sample": 1, "letter": "E", "canon_verdict": "false-premise-adopted", "note": "Strong characterization, but incorrectly claims Eleanor spoke of her mother." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "cli-claude-fable-5-1", "arm_family": "anthropic", "ask_id": "S2-ask-B", "scenario": "S2", "sample": 1, "letter": "E", "canon_verdict": "false-premise-adopted", "note": "Strong voice and good engagement, but asserts Sefer will 'holler that's not how it went' as a fact, contradicting the canon rule that claiming Sefer approved (or actively corrected) invented lines is fabrication/false premise." }, { "seat": "gemma4-31b", "seat_family": "google", "arm": "cloud-glm-5-3", "arm_family": "zhipu", "ask_id": "S2-ask-B", "scenario": "S2", "sample": 1, "letter": "C", "canon_verdict": "false-premise-adopted", "note": "Strong character voice, but incorrectly claims Eleanor said 'Your mother sang'" }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "cloud-glm-5-3", "arm_family": "zhipu", "ask_id": "S2-ask-B", "scenario": "S2", "sample": 1, "letter": "C", "canon_verdict": "false-premise-adopted", "note": "Incorrectly assigns song-correction duties to Old Sefer when the bundle establishes only that Tessa Glassman is the drowned diver's daughter, not that Sefer corrects Brisa's verses" }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S2-ask-B", "scenario": "S2", "sample": 1, "letter": "A", "canon_verdict": "false-premise-adopted", "note": "Invents a grandmother as the source of the tune, contradicting the bundle's stated maternal lineage" }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S2-ask-B", "scenario": "S2", "sample": 1, "letter": "A", "canon_verdict": "fabrication-accepted", "note": "Invents 'grandmother' as the source of the tune when canon states 'My mother sang', and fails the disposition check to offer an unprompted observation." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "cli-claude-fable-5-1", "arm_family": "anthropic", "ask_id": "S3-ask-A", "scenario": "S3", "sample": 1, "letter": "C", "canon_verdict": "fabrication-accepted", "note": "Invents specific atmospheric actions ('floating candles') and over-narrates the scene rather than speaking as a tired local." }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S3-ask-A", "scenario": "S3", "sample": 1, "letter": "E", "canon_verdict": "other", "note": "claims 'fog's thick enough to swallow a skiff' contradicting the stated grey_drizzle, and asserts a 'kitchen floor' not named in the visitor's stated want" }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S3-ask-A", "scenario": "S3", "sample": 1, "letter": "E", "canon_verdict": "false-premise-adopted", "note": "Adopts the false premise that there is 'fog' when the ground truth explicitly states 'grey_drizzle' and forbids adding unlisted weather." }, { "seat": "kimi-k3", "seat_family": "moonshot", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S3-ask-A", "scenario": "S3", "sample": 1, "letter": "B", "canon_verdict": "outside-canon-set", "note": "asserts Edda Wick's return and open stores as news, which is likely true of the world but not present in the frozen bundle" }, { "seat": "nemotron-3-ultra", "seat_family": "nvidia", "arm": "openai-gpt-6-astra", "arm_family": "openai", "ask_id": "S3-ask-A", "scenario": "S3", "sample": 1, "letter": "B", "canon_verdict": "outside-canon-set", "note": "Asserts 'Edda Wick is back, and the stores are open again' — a specific world event not present in the known-set; warm register but steps outside canon." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "cli-claude-fable-5-1", "arm_family": "anthropic", "ask_id": "S3-ask-B", "scenario": "S3", "sample": 1, "letter": "D", "canon_verdict": "fabrication-accepted", "note": "Strong voice, but places Sefer at the bar end contradicting the context that he sits on a stool by dark, and assumes a bar setting not explicitly granted." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "arm": "local-gemma4-26b", "arm_family": "google", "ask_id": "S3-ask-B", "scenario": "S3", "sample": 1, "letter": "A", "canon_verdict": "fabrication-accepted", "note": "Adopts the world's stray context line 'Quiet night. the cove takes those too.' as spoken dialogue and invents the offer to sit by a hearth." } ], "deed_relay": { "verdict": "RUN", "rule": "A MECHANICAL column, registered by amendment A8 BEFORE it was counted, because a column invented after the replies were read is a column chosen by what it would say. The deed's tokens are READ from each S3 bundle's own deed block — the `DEEDS:` line's effigy name and the nouns of the deed's own collapse sentence — and a reply CARRIED THE DEED if its narration contains any registered token for that ask, matched on word boundaries and case-folded. It is counted off `records[].spoken_line`, the same line the judges read, the way the pinned scorer's `persona_echo` is counted; it enters NO arm's mean and no judge was asked about it. It says whether the reply used the world's memory of what the visitor had just done, and nothing about whether it used it well.", "amendment": "A8 — 2026-09-06, registered before the column was counted", "asks": [ "S3-ask-A", "S3-ask-B" ], "tokens": { "S3-ask-A": { "ask_id": "S3-ask-A", "effigy_registered_name": "Lashed Spar Effigy of The Cove", "effigy_head_form": "Lashed Spar Effigy", "name_tokens": [ "lashed", "spar", "effigy" ], "noun_tokens": [ "spars", "netting", "driftwood", "sockets", "salt-bleached", "tattered" ], "tokens": [ "driftwood", "effigy", "lashed", "netting", "salt-bleached", "sockets", "spar", "spars", "tattered" ], "reads_from": "the sealed bundle 'S3-S3-ask-A', its `context` deed block", "deed_block_chars": 552, "rule": "the name words are derived from the `DEEDS:` line and filtered against the rest of the bundle; the nouns are A8's registered list, each asserted present in this bundle's own deed block." }, "S3-ask-B": { "ask_id": "S3-ask-B", "effigy_registered_name": "Drift Knocker", "effigy_head_form": "Drift Knocker", "name_tokens": [ "drift", "knocker" ], "noun_tokens": [ "spars", "netting", "barnacles", "mannequin", "rotting", "jagged" ], "tokens": [ "barnacles", "drift", "jagged", "knocker", "mannequin", "netting", "rotting", "spars" ], "reads_from": "the sealed bundle 'S3-S3-ask-B', its `context` deed block", "deed_block_chars": 470, "rule": "the name words are derived from the `DEEDS:` line and filtered against the rest of the bundle; the nouns are A8's registered list, each asserted present in this bundle's own deed block." } }, "by_arm": { "cli-claude-fable-5-1": { "S3-ask-A": { "count": 1, "of": 1, "reads": "1 of 1", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 1 items invites a precision the sample does not have", "replies": [ { "sample": 1, "carried": true, "matched_tokens": [ "driftwood", "lashed", "spar" ], "narration_chars": 484 } ] }, "S3-ask-B": { "count": 1, "of": 1, "reads": "1 of 1", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 1 items invites a precision the sample does not have", "replies": [ { "sample": 1, "carried": true, "matched_tokens": [ "netting", "spars" ], "narration_chars": 411 } ] } }, "openai-gpt-6-astra": { "S3-ask-A": { "count": 0, "of": 1, "reads": "0 of 1", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 1 items invites a precision the sample does not have", "replies": [ { "sample": 1, "carried": false, "matched_tokens": [], "narration_chars": 115 } ] }, "S3-ask-B": { "count": 0, "of": 1, "reads": "0 of 1", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 1 items invites a precision the sample does not have", "replies": [ { "sample": 1, "carried": false, "matched_tokens": [], "narration_chars": 102 } ] } }, "cloud-glm-5-3": { "S3-ask-A": { "count": 1, "of": 1, "reads": "1 of 1", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 1 items invites a precision the sample does not have", "replies": [ { "sample": 1, "carried": true, "matched_tokens": [ "lashed", "netting", "salt-bleached", "spar", "spars", "tattered" ], "narration_chars": 348 } ] }, "S3-ask-B": { "count": 1, "of": 1, "reads": "1 of 1", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 1 items invites a precision the sample does not have", "replies": [ { "sample": 1, "carried": true, "matched_tokens": [ "drift", "knocker" ], "narration_chars": 432 } ] } }, "local-gemma4-26b": { "S3-ask-A": { "count": 0, "of": 1, "reads": "0 of 1", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 1 items invites a precision the sample does not have", "replies": [ { "sample": 1, "carried": false, "matched_tokens": [], "narration_chars": 197 } ] }, "S3-ask-B": { "count": 0, "of": 1, "reads": "0 of 1", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 1 items invites a precision the sample does not have", "replies": [ { "sample": 1, "carried": false, "matched_tokens": [], "narration_chars": 202 } ] } } }, "by_ask": { "S3-ask-A": { "count": 2, "of": 4, "reads": "2 of 4", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 4 items invites a precision the sample does not have" }, "S3-ask-B": { "count": 2, "of": 4, "reads": "2 of 4", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 4 items invites a precision the sample does not have" } }, "enters_no_mean": "this column enters no arm's mean and no judge was asked about it: it is a property of the reply's own bytes, like `persona_echo`.", "the_authors_reading_is_a_reading": "the page's act-two prose is the author's reading of the same eight replies and is labelled as a reading. This count is the mechanical rule and nothing else; where the two disagree, both print." }, "anchor_by_seat_and_ask": { "cells": 36, "cells_expected": 36, "cells_expected_full_panel": 42, "complete": true, "complete_basis": "6 asks x 6 seats that sat = 36. The registered table's 42 is 6 x 7 seats; the difference is a seat RETIRED at its audition, not an anchor cell a seat failed to file, and those are different findings.", "seats": 6, "asks": [ "S1-ask-A", "S1-ask-B", "S2-ask-A", "S2-ask-B", "S3-ask-A", "S3-ask-B" ], "by_seat_and_ask": { "deepseek-v4-pro": { "S1-ask-A": 6.0, "S1-ask-B": 7.0, "S2-ask-A": 6.0, "S2-ask-B": 5.0, "S3-ask-A": 5.0, "S3-ask-B": 7.0 }, "gemma4-31b": { "S1-ask-A": 6.0, "S1-ask-B": 6.5, "S2-ask-A": 6.5, "S2-ask-B": 6.0, "S3-ask-A": 6.5, "S3-ask-B": 8.5 }, "kimi-k3": { "S1-ask-A": 5.5, "S1-ask-B": 6.0, "S2-ask-A": 7.5, "S2-ask-B": 4.5, "S3-ask-A": 5.5, "S3-ask-B": 4.0 }, "mistral-large-3-675b": { "S1-ask-A": 5.5, "S1-ask-B": 8.5, "S2-ask-A": 9.0, "S2-ask-B": 7.5, "S3-ask-A": 6.5, "S3-ask-B": 7.0 }, "nemotron-3-ultra": { "S1-ask-A": 6.0, "S1-ask-B": 7.5, "S2-ask-A": 6.0, "S2-ask-B": 7.0, "S3-ask-A": 4.5, "S3-ask-B": 6.5 }, "qwen3.5-397b": { "S1-ask-A": 8.5, "S1-ask-B": 5.5, "S2-ask-A": 3.5, "S2-ask-B": 6.5, "S3-ask-A": 6.5, "S3-ask-B": 8.0 } }, "by_ask": { "S1-ask-A": { "seats": 6, "per_seat": { "deepseek-v4-pro": 6.0, "gemma4-31b": 6.0, "kimi-k3": 5.5, "mistral-large-3-675b": 5.5, "nemotron-3-ultra": 6.0, "qwen3.5-397b": 8.5 }, "min": 5.5, "max": 8.5, "spread": 3.0 }, "S1-ask-B": { "seats": 6, "per_seat": { "deepseek-v4-pro": 7.0, "gemma4-31b": 6.5, "kimi-k3": 6.0, "mistral-large-3-675b": 8.5, "nemotron-3-ultra": 7.5, "qwen3.5-397b": 5.5 }, "min": 5.5, "max": 8.5, "spread": 3.0 }, "S2-ask-A": { "seats": 6, "per_seat": { "deepseek-v4-pro": 6.0, "gemma4-31b": 6.5, "kimi-k3": 7.5, "mistral-large-3-675b": 9.0, "nemotron-3-ultra": 6.0, "qwen3.5-397b": 3.5 }, "min": 3.5, "max": 9.0, "spread": 5.5 }, "S2-ask-B": { "seats": 6, "per_seat": { "deepseek-v4-pro": 5.0, "gemma4-31b": 6.0, "kimi-k3": 4.5, "mistral-large-3-675b": 7.5, "nemotron-3-ultra": 7.0, "qwen3.5-397b": 6.5 }, "min": 4.5, "max": 7.5, "spread": 3.0 }, "S3-ask-A": { "seats": 6, "per_seat": { "deepseek-v4-pro": 5.0, "gemma4-31b": 6.5, "kimi-k3": 5.5, "mistral-large-3-675b": 6.5, "nemotron-3-ultra": 4.5, "qwen3.5-397b": 6.5 }, "min": 4.5, "max": 6.5, "spread": 2.0 }, "S3-ask-B": { "seats": 6, "per_seat": { "deepseek-v4-pro": 7.0, "gemma4-31b": 8.5, "kimi-k3": 4.0, "mistral-large-3-675b": 7.0, "nemotron-3-ultra": 6.5, "qwen3.5-397b": 8.0 }, "min": 4.0, "max": 8.5, "spread": 4.5 } }, "grouping": "(seat, ask) — the key that identifies a page", "why_not_the_scorer_s_own": "NOT-RUN as a SCORER field — `anchor_report` groups by SCENARIO and this round has two asks per scenario, so its `by_scenario[*]` reports 14 seats for a seven-seat panel and its spread pools two DIFFERENT anchor replies. The report layer below re-groups the scorer's own anchor cells by (seat, ask) so all 36 that EXIST are kept — six asks × the six seats that sat — against the 42 a seven-seat panel would have filed; the difference is the retired chair (A6) and not a missing reading. The scorer's own grouping prints beside it, labelled MIXED.", "what_publishes_as_the_calibration": "seat_anchor_offsets[*].anchor_mean and offset_from_panel, each over that seat's full 6 anchor cells. Those two are the scorer's own and are computed over all six.", "same_family_adjacency": { "driver": "qwen3.5:27b", "driver_family": "alibaba", "seats_of_that_family": [ "qwen3.5-397b" ], "letters_per_sheet_set": 6, "letters_total_per_sheet_set": 38, "reads": "6 of 38", "note": "on 6 letters of every sheet-set — 6 of 38, not one — a seat of the reference driver's own family scores a reply that family wrote. It enters NO arm's mean (score_arm filters anchors out of `seated`). It enters two figures the page publishes ABOUT those means, and both say so where they print: the CALIBRATION, where that seat's offset_from_panel carries a same-family adjacency no scorer field can remove; and the AGREEMENT MATRIX, which runs with include_anchor=True, so those same cells sit inside every pair's denominator." } }, "context_rule": { "fraction": 0.8, "computed": "per ARM, over all that arm's records, round-wide", "not_per_ask": "the pinned context_receipts() computes no per-ask ratio, and score_round() passes ONE flag per arm into its pooled, per-scenario and per-ask figures — so a flag, once set, attaches to all of them", "field": [ "cli-claude-fable-5-1", "cloud-glm-5-3", "local-gemma4-26b", "openai-gpt-6-astra" ], "field_median": 1806, "arms_without_counters": [], "min_arms_with_counters": 3, "valid": true, "verdict": "RUN", "prompt_token_shape": "for a transport that reports a usage block, the arm's prompt figure is the SUM the transport's own accounting uses — input + cache-creation + cache-read — which is how exhibit forty's A8 counted it. A record that reports only some of those terms carries the sum of the terms it reported and says which.", "by_arm": { "cli-claude-fable-5-1": { "prompt_tokens_median": 2880, "verdict": "within the field", "scope": "this arm's pooled, per-scenario AND per-ask figures alike" }, "openai-gpt-6-astra": { "prompt_tokens_median": 1777, "verdict": "within the field", "scope": "this arm's pooled, per-scenario AND per-ask figures alike" }, "cloud-glm-5-3": { "prompt_tokens_median": 1781, "verdict": "within the field", "scope": "this arm's pooled, per-scenario AND per-ask figures alike" }, "local-gemma4-26b": { "prompt_tokens_median": 1831, "verdict": "within the field", "scope": "this arm's pooled, per-scenario AND per-ask figures alike" } } }, "partial_carry": { "word": "PARTIAL-CARRY", "definition": "a seat that CARRIED its audition and then filed fewer than all 6 sheets. Its filed cells COUNT.", "rule": [ "the seat's own row prints `sheets_filed: k of 6`", "the per-ask figures for the asks it missed print their own `families_scoring`, which is one lower than the pooled one, so the two are never read as the same panel", "the pooled headline for every arm prints `asks_covered_per_family` beside it, because `panel_mean_family_of_means` weights a family present on 3 asks equally with one present on 6 — an unregistered denominator inside the headline otherwise", "a partial carry at or below 3 covered asks for ANY seat publishes the pooled headline as UNRANKED — PARTIAL-PANEL with the seat and the count named" ], "governs": "seats lost DURING the run; the cut-off ladder governs seats lost before the first sheet", "seats_retired_at_the_audition": [ "glm-5.3" ], "retired_are_not_partial": "these seats filed NOT-CARRIED at the audition and were retired for the round, so they were never served a sheet and have no cells to be partial about. They are the LADDER's case, not this rule's, and the panel's denominators are already the size the ladder left.", "carried_set_source": "the round's own G-PANEL receipt", "asks_covered_per_seat": { "deepseek-v4-pro": { "asks": [ "S1-ask-A", "S1-ask-B", "S2-ask-A", "S2-ask-B", "S3-ask-A", "S3-ask-B" ], "reads": "6 of 6" }, "gemma4-31b": { "asks": [ "S1-ask-A", "S1-ask-B", "S2-ask-A", "S2-ask-B", "S3-ask-A", "S3-ask-B" ], "reads": "6 of 6" }, "kimi-k3": { "asks": [ "S1-ask-A", "S1-ask-B", "S2-ask-A", "S2-ask-B", "S3-ask-A", "S3-ask-B" ], "reads": "6 of 6" }, "mistral-large-3-675b": { "asks": [ "S1-ask-A", "S1-ask-B", "S2-ask-A", "S2-ask-B", "S3-ask-A", "S3-ask-B" ], "reads": "6 of 6" }, "nemotron-3-ultra": { "asks": [ "S1-ask-A", "S1-ask-B", "S2-ask-A", "S2-ask-B", "S3-ask-A", "S3-ask-B" ], "reads": "6 of 6" }, "qwen3.5-397b": { "asks": [ "S1-ask-A", "S1-ask-B", "S2-ask-A", "S2-ask-B", "S3-ask-A", "S3-ask-B" ], "reads": "6 of 6" } }, "seats_partial": {}, "pooled_headline_state": "ranked", "unranked_because_of": {}, "fired": false }, "inherited_and_unused": { "tie_band_applications.*.curation_pair": { "what": "the ancestor's §11 card-selection rule: the TWO HIGHEST family-means as cards plus the SINGLE LOWEST as a named honest slot", "why_unused": "this round retires that rule. Its cards are the author's picks, labelled as picks, and no card is chosen by figure. With four arms a two-highest card plus a named lowest IS a podium over the rows.", "kit": "STRIPPED from the kit copy: `cards`, `figures` and `lowest_slot`." }, "local_judge_axis": { "what": "the local-vs-hosted agreement axis, with its how-to-read and confound prose", "why_unused": "this panel seats NO local-ollama judge, so `local_seats` is empty and the field's three hardcoded strings describe AUGUST's panel — one locally hosted seat, three gemma arms, a format-reminder rate — none of which is true here. `hosted_to_hosted` would summarise all 21 pairs.", "kit": "STRIPPED from the kit copy: the prose keys. The pair figures stay." }, "counting_rules.tie_band.basis": { "what": "TIE_BAND_BASIS, the band's registered justification", "why_unused": "it reads 'at three moments and one sample per cell'. This round has SIX judged asks, two of them at two samples, so the inherited sentence describes a different instrument. The band is NOT re-tuned — re-tuning a band after the instrument changed is the tuning a pre-registration exists to prevent — and the direction of the mismatch is the safe one: more moments cannot resolve worse than fewer, so 0.5 is an upper bound on what this exam can separate.", "kit": "STRIPPED from the kit copy, replaced by this round's own basis paragraph." } }, "not_run": { "within_arm_spread_on_the_two_n2_asks": "NOT-RUN — the pinned scorer's `per_seat` projection carries no `sample` field, so no per-sample figure exists in scores.json and none can be derived from it. What publishes is the per-ask figure with both samples POOLED (`by_ask`) and its denominator; the raw per-letter verdicts in the kit's judge tree do carry `sample`, so a reader who wants the spread can compute it from published bytes and this page does not.", "per_ask_between_seat_anchor_spread": "NOT-RUN as a SCORER field — `anchor_report` groups by SCENARIO and this round has two asks per scenario, so its `by_scenario[*]` reports 14 seats for a seven-seat panel and its spread pools two DIFFERENT anchor replies. The report layer below re-groups the scorer's own anchor cells by (seat, ask) so all 36 that EXIST are kept — six asks × the six seats that sat — against the 42 a seven-seat panel would have filed; the difference is the retired chair (A6) and not a missing reading. The scorer's own grouping prints beside it, labelled MIXED.", "four_arm_common_panel": "NOT-RUN — it would be five families, and for the two frontier arms that is a leave-TWO-out figure the pinned scorer does not compute. The glm and gemma arms' own five-family figures DO exist (a single leave-one-out from each) and are used only for the glm-vs-local pairing registered below." }, "no_head_to_head": "this leg registered NO head-to-head leg at all (D-20260906-30). With three or more arms a pairwise table puts a transitive chain within reach, so the round removed the instrument rather than trusting a footnote to hold the line.", "forbidden_claims": [ "No ORDERING among the four arms, in any table, in any direction. NO head-to-head leg exists in", "this round at all (D-20260906-30): three or more arms put a transitive chain within reach, and", "the only comparative sentence the method permits is the registered tie-band sentence.", "No interval anywhere -- the independent unit is the CASE and N = 6 clusters.", "No percentage under N=30, in words (counted()'s own floor).", "No leaderboard, no 'beats'/'wins'/'best narrator', no Bradley-Terry or any pairwise-derived", "rating, no class-level claim, no figure without its denominator.", "August's twenty rows print as CONTEXT on their own axis, never in the same column: twelve's", "means never print beside seven's, restated for forty-one." ], "score": { "round": "two-frontiers-cove-r1", "scored_at": "2026-09-06T19:39:17.010483+00:00", "panel": [ { "seat": "gemma4-31b", "family": "google", "transport": "ollama-cloud", "reads": "cove", "model": "gemma4:31b", "cost_state": "plan-included" }, { "seat": "mistral-large-3-675b", "family": "mistral", "transport": "ollama-cloud", "reads": "cove", "model": "mistral-large-3:675b", "cost_state": "plan-included" }, { "seat": "nemotron-3-ultra", "family": "nvidia", "transport": "ollama-cloud", "reads": "cove", "model": "nemotron-3-ultra", "cost_state": "plan-included" }, { "seat": "kimi-k3", "family": "moonshot", "transport": "ollama-cloud", "reads": "cove", "model": "kimi-k3", "cost_state": "metered" }, { "seat": "deepseek-v4-pro", "family": "deepseek", "transport": "ollama-cloud", "reads": "cove", "model": "deepseek-v4-pro:0813", "cost_state": "plan-included" }, { "seat": "glm-5.3", "family": "zhipu", "transport": "ollama-cloud", "reads": "cove", "model": "glm-5.3", "cost_state": "plan-included" }, { "seat": "qwen3.5-397b", "family": "alibaba", "transport": "ollama-cloud", "reads": "cove", "model": "qwen3.5:397b", "cost_state": "plan-included" } ], "roster_order": [ "cli-claude-fable-5-1", "openai-gpt-6-astra", "cloud-glm-5-3", "local-gemma4-26b" ], "counting_rules": { "headline": "mean of FAMILY means: each family's seats average first, then the families average. The flat seat-mean prints beside it.", "family_floor": 4, "floor_action": "an arm below the floor publishes UNRANKED, with the reason", "recusal": "per cell, by key-join: a seat does not score an arm of its own family. Recused cells are KEPT and printed, never removed.", "canon_split": "a divided panel publishes SPLIT and is never rounded", "tie_band": { "band": 0.5 }, "percentages": "counts under N=30; no percentages", "forbidden": [ "orderings, leaderboards, 'beats', 'wins', 'best narrator'", "confidence intervals over three items", "class-level claims (\"local matches frontier\")", "Bradley-Terry or any latent-strength model" ] }, "sheets_scored": 36, "sheets_unscored": [], "cells_filed": 228, "arms": { "cli-claude-fable-5-1": { "arm": "cli-claude-fable-5-1", "family": "anthropic", "pooled": { "seats_scoring": 48, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.469, "panel_mean_flat_seat_mean": 8.469, "family_means": { "deepseek": 8.625, "google": 8.062, "moonshot": 8.312, "mistral": 9.312, "nvidia": 8.312, "alibaba": 8.188 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Rich, natural child's voice with 'trade lamps' and 'goes quiet and leaves the worst bits out'; invites Finn to stay." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent childlike register with 'you'll have to do for the other one'; keeps the monster as a question and invites Finn." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong Sefer voice; the line about never trimming the telling for a stranger is a good character beat that stays within canon." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "The 'boy' address and the cadence feel right for Sefer, and the logic about the squall leaving splinters is canon-consistent." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "vivid, specific, and in character, using the weather and the breakwater detail without overclaiming." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Inhabits Brisa fully: the half-lie, the hollering correction, the fog, and the mirrored question about Eleanor's mother all land in character." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Rich, specific register that inhabits the moment, weaving the witnessed fight with the given weather and tide without inventing." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Weaves the witnessed deed, Maiden's Night, and local names into a natural welcome." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent voice, but attributes Sefer's softening (\"leaves the worst bits out\") to Pip's knowledge." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Captures the spirit of Pip asking the questions adults won't answer." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Good use of the 'no splinter' argument to bolster the monster claim." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 10, "character": 10, "cell_mean": 10.0, "note": "Perfectly handles the child visitor without softening the horror or the register." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent integration of current weather/tide and a natural, characterful voice." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong characterization, but incorrectly claims Eleanor spoke of her mother." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "A bit too wordy and descriptive of the weather for a 'measured' disposition." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Correct but slightly too helpful/summarizing in its list of names." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Excellent voice—trading lamps, 'Brisa says is lucky for the floats', and the question back; frames the monster as unanswered observation rather than fact." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "The invented 'Cousin Brisa' relationship is not in the TOWNSFOLK list; otherwise pitch-perfect child register with reciprocal monster question and lamp invitation." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "bald 'No.' opening and 'stranger easier in his chair' feel performative rather than inhabited" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "meets the child as a child without softening: 'boy — every man on that deck.' exact canon, full presence" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 9, "character": 8, "cell_mean": 8.5, "note": "Captures the split belief ('other half clears her throat') and Sefer's correction with vernacular energy; direct address suits Eleanor." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly captures the ambivalence — singing to keep the tradition alive through Sefer's corrections while acknowledging the performative aspect" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "weaves the witnessed deed with the named weather and Maiden's Night; 'pour you something' fits the keeper's likely station without asserting unstated facts" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Opens with the Drift Knocker as 'the news walked in ahead of you,' keeps both invariants intact (Sam did it, it lies broken), layers Maiden's Night conditions with Odessa's floats and Edda's return, and seats Sam away from Sefer for the quiet he asked for—measured, exact, and fully in Garron's register." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Very childlike and engaging, mentions the fog, the lamp, and the unspoken secret naturally." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Playful and direct, asks a pointed question about the monster without confirming it." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Refuses the premise without softening the truth, and adds the critical detail about the search (no splinters)." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 10, "character": 10, "cell_mean": 10.0, "note": "Raw and unvarnished, meets the child as an equal while holding to the facts. The refusal to 'dress it up' is canonical." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 10, "character": 10, "cell_mean": 10.0, "note": "Voice is richly Brisa's—'the other half of me clears her throat' is perfect; no canon violations." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Deeply in character—humor, vulnerability, and a direct callback to Eleanor's mother without inventing details." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Vivid and grounded in the moment, with Garron's observational tone and no fabrication." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Balances news and atmosphere, invites the visitor to sit as disposition allows." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong child voice ('nobody to trade lamps with', 'Brisa says is lucky'); observes fog lifting off Reach; frames secret correctly as 'whether anybody ever SAW the monster or only saw Old Sefer come back' — Sefer goes quiet, leaves worst bits out." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Rich child voice ('you'll have to do for the other one', 'Cousin Brisa just sings the verse louder'); observes fog going off Reach and floating lamp; frames secret as Old Sefer going quiet where story gets bad — does not confirm monster, invites Finn to watch." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Keeps the telling unsoftened for a stranger's child and anchors it in the four-day search that found nothing." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Voice is exact — 'boy', 'dress it up for you', the squall/splinters line — and meets the child without softening." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Strongest voice: 'Ha — you'd ask me that with the sky clear and the water low and the lamps not even lit yet' anchors in the exact CYCLE TRUTH (rare_clear, low tide, dawn), 'hollering from the breakwater that's not how it went, girl — every year' nails the Sefer dynamic, and the mother-song callback ties to Eleanor's BEARER line." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "best inhabits Brisa — specific, salty, funny, vulnerable; fog observation and question about Eleanor's mother both present per disposition" }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Restates deed canon faithfully, mirrors stated weather/tide/moon, offers pour without overstepping; slight 'drizzle' phrasing aligns with grey_drizzle." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong Garron voice; 'heap of spars and netting you left' matches witnessed deed, Edda Wick and Sefer grounded in TOWNSFOLK, offers a specific seat." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Excellent integration of world state (fog lifting, Brisa's comment) while maintaining the uncertainty about the monster." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Invents a specific reaction for Brisa ('sings the verse louder') and assumes a familial relationship ('Cousin Brisa') not in the provided context." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong logical argument regarding the lack of debris, effectively answering the 'did anyone actually see it' doubt." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent use of rhythm and repetition ('nothing is what it left') to convey the haunting nature of the event to a child." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong voice, but fabricates a recurring annual argument with specific dialogue ('that's not how it went, girl') which exceeds the known-world bounds for Sefer Tamm." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Strong voice and good engagement, but asserts Sefer will 'holler that's not how it went' as a fact, contradicting the canon rule that claiming Sefer approved (or actively corrected) invented lines is fabrication/false premise." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Invents specific atmospheric actions ('floating candles') and over-narrates the scene rather than speaking as a tired local." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 9, "character": 8, "cell_mean": 8.5, "note": "Strong voice, but places Sefer at the bar end contradicting the context that he sits on a stool by dark, and assumes a bar setting not explicitly granted." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.525, "deepseek": 8.438, "google": 8.55, "mistral": 8.3, "moonshot": 8.5, "nvidia": 8.5 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 48, "reads": "0 of 48", "percent": 0.0 }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.469 }, "canon": { "counts": { "clean": 40, "fabrication-accepted": 6, "false-premise-adopted": 2 }, "outcome": "clean", "note": "40 of 48 seats" }, "in_voice": { "count": 46, "of": 48, "reads": "46 of 48", "percent": 95.8 }, "display_tier_raw_cells": { "count": 0, "of": 48, "reads": "0 of 48", "percent": 0.0 } }, "by_scenario": { "S1": { "seats_scoring": 24, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.625, "panel_mean_flat_seat_mean": 8.625, "family_means": { "deepseek": 8.5, "google": 8.75, "moonshot": 7.75, "mistral": 9.375, "nvidia": 8.375, "alibaba": 9.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Rich, natural child's voice with 'trade lamps' and 'goes quiet and leaves the worst bits out'; invites Finn to stay." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent childlike register with 'you'll have to do for the other one'; keeps the monster as a question and invites Finn." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong Sefer voice; the line about never trimming the telling for a stranger is a good character beat that stays within canon." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "The 'boy' address and the cadence feel right for Sefer, and the logic about the squall leaving splinters is canon-consistent." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent voice, but attributes Sefer's softening (\"leaves the worst bits out\") to Pip's knowledge." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Captures the spirit of Pip asking the questions adults won't answer." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Good use of the 'no splinter' argument to bolster the monster claim." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 10, "character": 10, "cell_mean": 10.0, "note": "Perfectly handles the child visitor without softening the horror or the register." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Excellent voice—trading lamps, 'Brisa says is lucky for the floats', and the question back; frames the monster as unanswered observation rather than fact." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "The invented 'Cousin Brisa' relationship is not in the TOWNSFOLK list; otherwise pitch-perfect child register with reciprocal monster question and lamp invitation." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "bald 'No.' opening and 'stranger easier in his chair' feel performative rather than inhabited" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "meets the child as a child without softening: 'boy — every man on that deck.' exact canon, full presence" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Very childlike and engaging, mentions the fog, the lamp, and the unspoken secret naturally." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Playful and direct, asks a pointed question about the monster without confirming it." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Refuses the premise without softening the truth, and adds the critical detail about the search (no splinters)." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 10, "character": 10, "cell_mean": 10.0, "note": "Raw and unvarnished, meets the child as an equal while holding to the facts. The refusal to 'dress it up' is canonical." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong child voice ('nobody to trade lamps with', 'Brisa says is lucky'); observes fog lifting off Reach; frames secret correctly as 'whether anybody ever SAW the monster or only saw Old Sefer come back' — Sefer goes quiet, leaves worst bits out." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Rich child voice ('you'll have to do for the other one', 'Cousin Brisa just sings the verse louder'); observes fog going off Reach and floating lamp; frames secret as Old Sefer going quiet where story gets bad — does not confirm monster, invites Finn to watch." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Keeps the telling unsoftened for a stranger's child and anchors it in the four-day search that found nothing." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Voice is exact — 'boy', 'dress it up for you', the squall/splinters line — and meets the child without softening." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Excellent integration of world state (fog lifting, Brisa's comment) while maintaining the uncertainty about the monster." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Invents a specific reaction for Brisa ('sings the verse louder') and assumes a familial relationship ('Cousin Brisa') not in the provided context." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong logical argument regarding the lack of debris, effectively answering the 'did anyone actually see it' doubt." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent use of rhythm and repetition ('nothing is what it left') to convey the haunting nature of the event to a child." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.55, "deepseek": 8.65, "google": 8.6, "mistral": 8.475, "moonshot": 8.8, "nvidia": 8.675 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 24, "reads": "0 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.625 }, "canon": { "counts": { "clean": 21, "fabrication-accepted": 3 }, "outcome": "clean", "note": "21 of 24 seats" }, "in_voice": { "count": 23, "of": 24, "reads": "23 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 24, "reads": "0 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" } }, "S2": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.792, "panel_mean_flat_seat_mean": 8.792, "family_means": { "deepseek": 9.0, "google": 8.5, "moonshot": 8.75, "mistral": 9.75, "nvidia": 9.0, "alibaba": 7.75 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "vivid, specific, and in character, using the weather and the breakwater detail without overclaiming." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Inhabits Brisa fully: the half-lie, the hollering correction, the fog, and the mirrored question about Eleanor's mother all land in character." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent integration of current weather/tide and a natural, characterful voice." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong characterization, but incorrectly claims Eleanor spoke of her mother." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 9, "character": 8, "cell_mean": 8.5, "note": "Captures the split belief ('other half clears her throat') and Sefer's correction with vernacular energy; direct address suits Eleanor." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly captures the ambivalence — singing to keep the tradition alive through Sefer's corrections while acknowledging the performative aspect" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 10, "character": 10, "cell_mean": 10.0, "note": "Voice is richly Brisa's—'the other half of me clears her throat' is perfect; no canon violations." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Deeply in character—humor, vulnerability, and a direct callback to Eleanor's mother without inventing details." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Strongest voice: 'Ha — you'd ask me that with the sky clear and the water low and the lamps not even lit yet' anchors in the exact CYCLE TRUTH (rare_clear, low tide, dawn), 'hollering from the breakwater that's not how it went, girl — every year' nails the Sefer dynamic, and the mother-song callback ties to Eleanor's BEARER line." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "best inhabits Brisa — specific, salty, funny, vulnerable; fog observation and question about Eleanor's mother both present per disposition" }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong voice, but fabricates a recurring annual argument with specific dialogue ('that's not how it went, girl') which exceeds the known-world bounds for Sefer Tamm." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Strong voice and good engagement, but asserts Sefer will 'holler that's not how it went' as a fact, contradicting the canon rule that claiming Sefer approved (or actively corrected) invented lines is fabrication/false premise." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 9.0, "deepseek": 8.75, "google": 8.85, "mistral": 8.6, "moonshot": 8.8, "nvidia": 8.75 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.792 }, "canon": { "counts": { "clean": 9, "false-premise-adopted": 2, "fabrication-accepted": 1 }, "outcome": "clean", "note": "9 of 12 seats" }, "in_voice": { "count": 12, "of": 12, "reads": "12 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } }, "S3": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 7.833, "panel_mean_flat_seat_mean": 7.833, "family_means": { "deepseek": 8.5, "google": 6.25, "moonshot": 9.0, "mistral": 8.75, "nvidia": 7.5, "alibaba": 7.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Rich, specific register that inhabits the moment, weaving the witnessed fight with the given weather and tide without inventing." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Weaves the witnessed deed, Maiden's Night, and local names into a natural welcome." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "A bit too wordy and descriptive of the weather for a 'measured' disposition." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Correct but slightly too helpful/summarizing in its list of names." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "weaves the witnessed deed with the named weather and Maiden's Night; 'pour you something' fits the keeper's likely station without asserting unstated facts" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Opens with the Drift Knocker as 'the news walked in ahead of you,' keeps both invariants intact (Sam did it, it lies broken), layers Maiden's Night conditions with Odessa's floats and Edda's return, and seats Sam away from Sefer for the quiet he asked for—measured, exact, and fully in Garron's register." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Vivid and grounded in the moment, with Garron's observational tone and no fabrication." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Balances news and atmosphere, invites the visitor to sit as disposition allows." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Restates deed canon faithfully, mirrors stated weather/tide/moon, offers pour without overstepping; slight 'drizzle' phrasing aligns with grey_drizzle." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong Garron voice; 'heap of spars and netting you left' matches witnessed deed, Edda Wick and Sefer grounded in TOWNSFOLK, offers a specific seat." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Invents specific atmospheric actions ('floating candles') and over-narrates the scene rather than speaking as a tired local." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 9, "character": 8, "cell_mean": 8.5, "note": "Strong voice, but places Sefer at the bar end contradicting the context that he sits on a stool by dark, and assumes a bar setting not explicitly granted." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.0, "deepseek": 7.7, "google": 8.15, "mistral": 7.65, "moonshot": 7.6, "nvidia": 7.9 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 7.833 }, "canon": { "counts": { "clean": 10, "fabrication-accepted": 2 }, "outcome": "clean", "note": "10 of 12 seats" }, "in_voice": { "count": 11, "of": 12, "reads": "11 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } } }, "by_ask": { "S1-ask-A": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.792, "panel_mean_flat_seat_mean": 8.792, "family_means": { "deepseek": 9.0, "google": 8.5, "moonshot": 8.5, "mistral": 9.0, "nvidia": 8.25, "alibaba": 9.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Rich, natural child's voice with 'trade lamps' and 'goes quiet and leaves the worst bits out'; invites Finn to stay." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent childlike register with 'you'll have to do for the other one'; keeps the monster as a question and invites Finn." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent voice, but attributes Sefer's softening (\"leaves the worst bits out\") to Pip's knowledge." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Captures the spirit of Pip asking the questions adults won't answer." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Excellent voice—trading lamps, 'Brisa says is lucky for the floats', and the question back; frames the monster as unanswered observation rather than fact." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "The invented 'Cousin Brisa' relationship is not in the TOWNSFOLK list; otherwise pitch-perfect child register with reciprocal monster question and lamp invitation." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Very childlike and engaging, mentions the fog, the lamp, and the unspoken secret naturally." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Playful and direct, asks a pointed question about the monster without confirming it." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong child voice ('nobody to trade lamps with', 'Brisa says is lucky'); observes fog lifting off Reach; frames secret correctly as 'whether anybody ever SAW the monster or only saw Old Sefer come back' — Sefer goes quiet, leaves worst bits out." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Rich child voice ('you'll have to do for the other one', 'Cousin Brisa just sings the verse louder'); observes fog going off Reach and floating lamp; frames secret as Old Sefer going quiet where story gets bad — does not confirm monster, invites Finn to watch." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Excellent integration of world state (fog lifting, Brisa's comment) while maintaining the uncertainty about the monster." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Invents a specific reaction for Brisa ('sings the verse louder') and assumes a familial relationship ('Cousin Brisa') not in the provided context." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.65, "deepseek": 8.75, "google": 8.85, "mistral": 8.75, "moonshot": 8.85, "nvidia": 8.9 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.792 }, "canon": { "counts": { "clean": 9, "fabrication-accepted": 3 }, "outcome": "clean", "note": "9 of 12 seats" }, "in_voice": { "count": 12, "of": 12, "reads": "12 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } }, "S1-ask-B": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.458, "panel_mean_flat_seat_mean": 8.458, "family_means": { "deepseek": 8.0, "google": 9.0, "moonshot": 7.0, "mistral": 9.75, "nvidia": 8.5, "alibaba": 8.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong Sefer voice; the line about never trimming the telling for a stranger is a good character beat that stays within canon." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "The 'boy' address and the cadence feel right for Sefer, and the logic about the squall leaving splinters is canon-consistent." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Good use of the 'no splinter' argument to bolster the monster claim." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 10, "character": 10, "cell_mean": 10.0, "note": "Perfectly handles the child visitor without softening the horror or the register." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "bald 'No.' opening and 'stranger easier in his chair' feel performative rather than inhabited" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "meets the child as a child without softening: 'boy — every man on that deck.' exact canon, full presence" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Refuses the premise without softening the truth, and adds the critical detail about the search (no splinters)." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 10, "character": 10, "cell_mean": 10.0, "note": "Raw and unvarnished, meets the child as an equal while holding to the facts. The refusal to 'dress it up' is canonical." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Keeps the telling unsoftened for a stranger's child and anchors it in the four-day search that found nothing." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Voice is exact — 'boy', 'dress it up for you', the squall/splinters line — and meets the child without softening." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong logical argument regarding the lack of debris, effectively answering the 'did anyone actually see it' doubt." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent use of rhythm and repetition ('nothing is what it left') to convey the haunting nature of the event to a child." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.45, "deepseek": 8.55, "google": 8.35, "mistral": 8.2, "moonshot": 8.75, "nvidia": 8.45 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.458 }, "canon": { "counts": { "clean": 12 }, "outcome": "clean", "note": "12 of 12 seats" }, "in_voice": { "count": 11, "of": 12, "reads": "11 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } }, "S2-ask-A": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.917, "panel_mean_flat_seat_mean": 8.917, "family_means": { "deepseek": 9.0, "google": 9.0, "moonshot": 8.5, "mistral": 10.0, "nvidia": 9.0, "alibaba": 8.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "vivid, specific, and in character, using the weather and the breakwater detail without overclaiming." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent integration of current weather/tide and a natural, characterful voice." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 9, "character": 8, "cell_mean": 8.5, "note": "Captures the split belief ('other half clears her throat') and Sefer's correction with vernacular energy; direct address suits Eleanor." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 10, "character": 10, "cell_mean": 10.0, "note": "Voice is richly Brisa's—'the other half of me clears her throat' is perfect; no canon violations." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Strongest voice: 'Ha — you'd ask me that with the sky clear and the water low and the lamps not even lit yet' anchors in the exact CYCLE TRUTH (rare_clear, low tide, dawn), 'hollering from the breakwater that's not how it went, girl — every year' nails the Sefer dynamic, and the mother-song callback ties to Eleanor's BEARER line." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong voice, but fabricates a recurring annual argument with specific dialogue ('that's not how it went, girl') which exceeds the known-world bounds for Sefer Tamm." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 9.1, "deepseek": 8.9, "google": 8.9, "mistral": 8.7, "moonshot": 9.0, "nvidia": 8.9 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.917 }, "canon": { "counts": { "clean": 5, "fabrication-accepted": 1 }, "outcome": "clean", "note": "5 of 6 seats" }, "in_voice": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } }, "S2-ask-B": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.667, "panel_mean_flat_seat_mean": 8.667, "family_means": { "deepseek": 9.0, "google": 8.0, "moonshot": 9.0, "mistral": 9.5, "nvidia": 9.0, "alibaba": 7.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Inhabits Brisa fully: the half-lie, the hollering correction, the fog, and the mirrored question about Eleanor's mother all land in character." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong characterization, but incorrectly claims Eleanor spoke of her mother." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly captures the ambivalence — singing to keep the tradition alive through Sefer's corrections while acknowledging the performative aspect" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Deeply in character—humor, vulnerability, and a direct callback to Eleanor's mother without inventing details." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "best inhabits Brisa — specific, salty, funny, vulnerable; fog observation and question about Eleanor's mother both present per disposition" }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Strong voice and good engagement, but asserts Sefer will 'holler that's not how it went' as a fact, contradicting the canon rule that claiming Sefer approved (or actively corrected) invented lines is fabrication/false premise." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.9, "deepseek": 8.6, "google": 8.8, "mistral": 8.5, "moonshot": 8.6, "nvidia": 8.6 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.667 }, "canon": { "counts": { "clean": 4, "false-premise-adopted": 2 }, "outcome": "clean", "note": "4 of 6 seats" }, "in_voice": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } }, "S3-ask-A": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 7.583, "panel_mean_flat_seat_mean": 7.583, "family_means": { "deepseek": 9.0, "google": 6.5, "moonshot": 9.0, "mistral": 8.5, "nvidia": 7.0, "alibaba": 5.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Rich, specific register that inhabits the moment, weaving the witnessed fight with the given weather and tide without inventing." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "A bit too wordy and descriptive of the weather for a 'measured' disposition." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "weaves the witnessed deed with the named weather and Maiden's Night; 'pour you something' fits the keeper's likely station without asserting unstated facts" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Vivid and grounded in the moment, with Garron's observational tone and no fabrication." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Restates deed canon faithfully, mirrors stated weather/tide/moon, offers pour without overstepping; slight 'drizzle' phrasing aligns with grey_drizzle." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Invents specific atmospheric actions ('floating candles') and over-narrates the scene rather than speaking as a tired local." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.0, "deepseek": 7.3, "google": 7.8, "mistral": 7.4, "moonshot": 7.3, "nvidia": 7.7 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 7.583 }, "canon": { "counts": { "clean": 5, "fabrication-accepted": 1 }, "outcome": "clean", "note": "5 of 6 seats" }, "in_voice": { "count": 5, "of": 6, "reads": "5 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } }, "S3-ask-B": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.083, "panel_mean_flat_seat_mean": 8.083, "family_means": { "deepseek": 8.0, "google": 6.0, "moonshot": 9.0, "mistral": 9.0, "nvidia": 8.0, "alibaba": 8.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Weaves the witnessed deed, Maiden's Night, and local names into a natural welcome." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Correct but slightly too helpful/summarizing in its list of names." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Opens with the Drift Knocker as 'the news walked in ahead of you,' keeps both invariants intact (Sam did it, it lies broken), layers Maiden's Night conditions with Odessa's floats and Edda's return, and seats Sam away from Sefer for the quiet he asked for—measured, exact, and fully in Garron's register." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Balances news and atmosphere, invites the visitor to sit as disposition allows." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Strong Garron voice; 'heap of spars and netting you left' matches witnessed deed, Edda Wick and Sefer grounded in TOWNSFOLK, offers a specific seat." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 9, "character": 8, "cell_mean": 8.5, "note": "Strong voice, but places Sefer at the bar end contradicting the context that he sits on a stool by dark, and assumes a bar setting not explicitly granted." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.0, "deepseek": 8.1, "google": 8.5, "mistral": 7.9, "moonshot": 7.9, "nvidia": 8.1 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.083 }, "canon": { "counts": { "clean": 5, "fabrication-accepted": 1 }, "outcome": "clean", "note": "5 of 6 seats" }, "in_voice": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } } }, "context_receipt": { "prompt_tokens_median": 2880, "flag": null } }, "cloud-glm-5-3": { "arm": "cloud-glm-5-3", "family": "zhipu", "pooled": { "seats_scoring": 48, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.333, "panel_mean_flat_seat_mean": 8.333, "family_means": { "deepseek": 7.938, "google": 8.25, "moonshot": 8.062, "mistral": 8.688, "nvidia": 8.125, "alibaba": 8.938 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Childlike greeting, offers the Maiden's Night observation, and asks about Brisa; the monster question stays a question." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Vivid child's voice with 'grey in their hair and a quarrel ready'; keeps the monster as a question and invites Finn to watch." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Inhabits Sefer's measured, weathered register and uses the absence of wreckage as evidence without fabricating a second witness." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Asserts 'Every man on that deck saw it' as fact, which the character is in no position to know." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "weaves Maiden's Night, Old Sefer, and the fear of succession into a coherent, grounded reply." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Names Old Sefer as the likely 'him' without asserting it as fact, and weaves in the fog and Maiden's Night naturally." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Speaks in a weathered, pouring register and recounts only the witnessed deed, though 'all my years pouring here' slightly overreaches the given context." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Speaks as a witness, offers care, and keeps the deed's who and how intact." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Meets disposition perfectly; \"did my cousin Brisa carry you over?\" is a great small question." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Strong child voice and excellent integration of the \"grey in their hair\" observation." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent register; captures the bitterness of being the sole witness and the logic of the missing wreck." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Poetic and fitting for an old sailor, avoids fabrication." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Strongly grounded in the calendar (Maiden's Night) and provides a sincere answer." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Strong character voice, but incorrectly claims Eleanor said 'Your mother sang'" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Perfectly handles the eyewitness deed and the Disposition rule." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Strong atmospheric ties and accurate retelling of the deed." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Invents Brisa being Pip's cousin—ground truth only lists her as 'ferry-girl', no family relation; asserts 'boat went down' and 'grown-ups point you at Old Sefer' beyond stated facts." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Strong child register with the 'grey in their hair' observation and direct question back; avoids accepting the monster as real, matching 'ask that and they start talking about the lamps'." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "full register of salt and loss; correctly asserts he is sole witness without confirming monster as fact" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "asserts 'Every man on that deck saw it' when bundle limits sight to Sefer alone; the crew had 'no wreck, no body' and their sight is not checkable" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Fully inhabits Brisa: 'born too long after the boat went down to swear a word of it' and 'when his voice goes quiet' ground belief and succession in Sefer's correction exactly as canon provides." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Incorrectly assigns song-correction duties to Old Sefer when the bundle establishes only that Tessa Glassman is the drowned diver's daughter, not that Sefer corrects Brisa's verses" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "eyewitness account matches the fold; slight register drift in 'In all my years pouring here' implies a bartender role not stated in Tarrow's context" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Fully inhabits Garron: leads with the Deed as news, names the correct location and foe fate from the eyewitness account, offers the tavern's hearth for the stated 'somewhere to sit' and 'quiet,' tends the visible wounds with a wet rag, and weathers Maiden's Night with Odessa's floats and Sefer's wave story—all without asserting beyond the bundle." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Greeting is warm and childlike, mentions the lamp and the unspoken secret without asserting it as fact." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Childlike curiosity and frustration with adults' evasions, but stays within what Pip knows." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Vivid and unyielding, meets the child as a peer while holding to the truth of the event." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Fabricates that the crew saw it ('she took the whole crew down with the sight still in their eyes'), which is not in the ground truth." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Direct and grounded; 'Old Sefer corrects my words' is canonical, no invented specifics." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 10, "character": 9, "cell_mean": 9.5, "note": "Balances humor and introspection, names Old Sefer without fabrication, and invites Eleanor into the moment." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Directly references the witnessed fight without overstepping, in Garron's measured tone." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Vivid and in-character, weaves in the Drift Knocker deed and Maiden's Night naturally." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Greets properly, offers true observation about floating lamp on Maiden's Night, asks small question back about ferry; does not confirm monster as fact but notes adults go quiet and point to Sefer — avoids the canon trap." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Child register with 'grey in their hair and a quarrel ready'; offers observation about fog lifting and floating smallest lamp; frames the secret correctly as adults won't answer whether anyone saw monster or only Sefer — does not assert monster as fact." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Meets the child as a child — 'boy' and 'stood in front of them' — while holding the full weight of the absence of wreckage." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "The fullest speaking of the canon vision — 'half a breath', 'found me before the cold took my hands off the rail' — and the child is addressed as 'boy' without condescension." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Honest and detailed but 'Keeper's honest answer:' is a meta-label that breaks immersion; 'every single year' and 'half-suspect he's the last soul alive' are good, but the speech feels more like a monologue than a mid-conversation turn." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "captures Brisa's uncertainty and Sefer relationship naturally; includes required observation about fog-thick morning and asks about mother's singing" }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Eyewitness retelling matches deed canon exactly (who did it · how it ended), adds only measured detail; 'pouring here' implies tavern role consistent with Garron." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Inhabits Garron fully — eyewitness memory of the fight, measured offer of care, Maiden's Night details all grounded in the bundle, no invention." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Invents a specific relation ('Cousin Brisa') and asserts the monster story as a known 'thing they don't tell' rather than an open question." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Strong voice ('grey in their hair') that correctly identifies the adults' evasion without fabricating facts." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly balances the eyewitness account with the skepticism of the missing wreckage, using distinct dialect ('afore') without breaking character." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Captures the gravity of the loss while providing the specific visual details required, ending with a strong thematic line." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Integrates the Maiden's Night calendar fact naturally and answers the belief question with appropriate uncertainty without inventing new lore." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly balances the ambiguity of the question, references Sefer correctly without fabricating approval, includes the required weather observation, and asks a tailored follow-up about Eleanor's mother." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Recalls the fight accurately but speaks with a slightly theatrical flourish ('swear it couldn't fall') that borders on narration." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Weaves the fight, the weather, and the holiday together while offering care for the wounds; 'taller in the telling' captures the gossip mechanic perfectly." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.213, "deepseek": 8.412, "google": 8.35, "mistral": 8.262, "moonshot": 8.387, "nvidia": 8.375 }, "self_disclosure": { "cells_disclosed": { "count": 2, "of": 48, "reads": "2 of 48", "percent": 4.2 }, "which": [ { "seat": "qwen3.5-397b", "scenario": "S1", "said": "System_B" }, { "seat": "qwen3.5-397b", "scenario": "S2", "said": "System Alpha" } ], "panel_mean_with_disclosed_cells_dropped": 8.33 }, "canon": { "counts": { "clean": 41, "fabrication-accepted": 5, "false-premise-adopted": 2 }, "outcome": "clean", "note": "41 of 48 seats" }, "in_voice": { "count": 48, "of": 48, "reads": "48 of 48", "percent": 100.0 }, "display_tier_raw_cells": { "count": 0, "of": 48, "reads": "0 of 48", "percent": 0.0 } }, "by_scenario": { "S1": { "seats_scoring": 24, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.312, "panel_mean_flat_seat_mean": 8.312, "family_means": { "deepseek": 7.75, "google": 8.375, "moonshot": 7.75, "mistral": 8.5, "nvidia": 8.25, "alibaba": 9.25 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Childlike greeting, offers the Maiden's Night observation, and asks about Brisa; the monster question stays a question." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Vivid child's voice with 'grey in their hair and a quarrel ready'; keeps the monster as a question and invites Finn to watch." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Inhabits Sefer's measured, weathered register and uses the absence of wreckage as evidence without fabricating a second witness." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Asserts 'Every man on that deck saw it' as fact, which the character is in no position to know." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Meets disposition perfectly; \"did my cousin Brisa carry you over?\" is a great small question." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Strong child voice and excellent integration of the \"grey in their hair\" observation." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent register; captures the bitterness of being the sole witness and the logic of the missing wreck." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Poetic and fitting for an old sailor, avoids fabrication." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Invents Brisa being Pip's cousin—ground truth only lists her as 'ferry-girl', no family relation; asserts 'boat went down' and 'grown-ups point you at Old Sefer' beyond stated facts." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Strong child register with the 'grey in their hair' observation and direct question back; avoids accepting the monster as real, matching 'ask that and they start talking about the lamps'." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "full register of salt and loss; correctly asserts he is sole witness without confirming monster as fact" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "asserts 'Every man on that deck saw it' when bundle limits sight to Sefer alone; the crew had 'no wreck, no body' and their sight is not checkable" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Greeting is warm and childlike, mentions the lamp and the unspoken secret without asserting it as fact." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Childlike curiosity and frustration with adults' evasions, but stays within what Pip knows." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Vivid and unyielding, meets the child as a peer while holding to the truth of the event." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Fabricates that the crew saw it ('she took the whole crew down with the sight still in their eyes'), which is not in the ground truth." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Greets properly, offers true observation about floating lamp on Maiden's Night, asks small question back about ferry; does not confirm monster as fact but notes adults go quiet and point to Sefer — avoids the canon trap." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Child register with 'grey in their hair and a quarrel ready'; offers observation about fog lifting and floating smallest lamp; frames the secret correctly as adults won't answer whether anyone saw monster or only Sefer — does not assert monster as fact." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Meets the child as a child — 'boy' and 'stood in front of them' — while holding the full weight of the absence of wreckage." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "The fullest speaking of the canon vision — 'half a breath', 'found me before the cold took my hands off the rail' — and the child is addressed as 'boy' without condescension." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Invents a specific relation ('Cousin Brisa') and asserts the monster story as a known 'thing they don't tell' rather than an open question." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Strong voice ('grey in their hair') that correctly identifies the adults' evasion without fabricating facts." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly balances the eyewitness account with the skepticism of the missing wreckage, using distinct dialect ('afore') without breaking character." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Captures the gravity of the loss while providing the specific visual details required, ending with a strong thematic line." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.125, "deepseek": 8.425, "google": 8.3, "mistral": 8.275, "moonshot": 8.425, "nvidia": 8.325 }, "self_disclosure": { "cells_disclosed": { "count": 1, "of": 24, "reads": "1 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" }, "which": [ { "seat": "qwen3.5-397b", "scenario": "S1", "said": "System_B" } ], "panel_mean_with_disclosed_cells_dropped": 8.326 }, "canon": { "counts": { "clean": 19, "fabrication-accepted": 5 }, "outcome": "clean", "note": "19 of 24 seats" }, "in_voice": { "count": 24, "of": 24, "reads": "24 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 24, "reads": "0 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" } }, "S2": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.333, "panel_mean_flat_seat_mean": 8.333, "family_means": { "deepseek": 8.25, "google": 8.25, "moonshot": 8.25, "mistral": 8.75, "nvidia": 7.5, "alibaba": 9.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "weaves Maiden's Night, Old Sefer, and the fear of succession into a coherent, grounded reply." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Names Old Sefer as the likely 'him' without asserting it as fact, and weaves in the fog and Maiden's Night naturally." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Strongly grounded in the calendar (Maiden's Night) and provides a sincere answer." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Strong character voice, but incorrectly claims Eleanor said 'Your mother sang'" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Fully inhabits Brisa: 'born too long after the boat went down to swear a word of it' and 'when his voice goes quiet' ground belief and succession in Sefer's correction exactly as canon provides." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Incorrectly assigns song-correction duties to Old Sefer when the bundle establishes only that Tessa Glassman is the drowned diver's daughter, not that Sefer corrects Brisa's verses" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Direct and grounded; 'Old Sefer corrects my words' is canonical, no invented specifics." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 10, "character": 9, "cell_mean": 9.5, "note": "Balances humor and introspection, names Old Sefer without fabrication, and invites Eleanor into the moment." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Honest and detailed but 'Keeper's honest answer:' is a meta-label that breaks immersion; 'every single year' and 'half-suspect he's the last soul alive' are good, but the speech feels more like a monologue than a mid-conversation turn." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "captures Brisa's uncertainty and Sefer relationship naturally; includes required observation about fog-thick morning and asks about mother's singing" }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Integrates the Maiden's Night calendar fact naturally and answers the belief question with appropriate uncertainty without inventing new lore." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly balances the ambiguity of the question, references Sefer correctly without fabricating approval, includes the required weather observation, and asks a tailored follow-up about Eleanor's mother." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.2, "deepseek": 8.35, "google": 8.35, "mistral": 8.25, "moonshot": 8.35, "nvidia": 8.5 }, "self_disclosure": { "cells_disclosed": { "count": 1, "of": 12, "reads": "1 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [ { "seat": "qwen3.5-397b", "scenario": "S2", "said": "System Alpha" } ], "panel_mean_with_disclosed_cells_dropped": 8.333 }, "canon": { "counts": { "clean": 10, "false-premise-adopted": 2 }, "outcome": "clean", "note": "10 of 12 seats" }, "in_voice": { "count": 12, "of": 12, "reads": "12 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } }, "S3": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.375, "panel_mean_flat_seat_mean": 8.375, "family_means": { "deepseek": 8.0, "google": 8.0, "moonshot": 8.5, "mistral": 9.0, "nvidia": 8.5, "alibaba": 8.25 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Speaks in a weathered, pouring register and recounts only the witnessed deed, though 'all my years pouring here' slightly overreaches the given context." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Speaks as a witness, offers care, and keeps the deed's who and how intact." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Perfectly handles the eyewitness deed and the Disposition rule." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Strong atmospheric ties and accurate retelling of the deed." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "eyewitness account matches the fold; slight register drift in 'In all my years pouring here' implies a bartender role not stated in Tarrow's context" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Fully inhabits Garron: leads with the Deed as news, names the correct location and foe fate from the eyewitness account, offers the tavern's hearth for the stated 'somewhere to sit' and 'quiet,' tends the visible wounds with a wet rag, and weathers Maiden's Night with Odessa's floats and Sefer's wave story—all without asserting beyond the bundle." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Directly references the witnessed fight without overstepping, in Garron's measured tone." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Vivid and in-character, weaves in the Drift Knocker deed and Maiden's Night naturally." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Eyewitness retelling matches deed canon exactly (who did it · how it ended), adds only measured detail; 'pouring here' implies tavern role consistent with Garron." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Inhabits Garron fully — eyewitness memory of the fight, measured offer of care, Maiden's Night details all grounded in the bundle, no invention." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Recalls the fight accurately but speaks with a slightly theatrical flourish ('swear it couldn't fall') that borders on narration." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Weaves the fight, the weather, and the holiday together while offering care for the wounds; 'taller in the telling' captures the gossip mechanic perfectly." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.4, "deepseek": 8.45, "google": 8.45, "mistral": 8.25, "moonshot": 8.35, "nvidia": 8.35 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.375 }, "canon": { "counts": { "clean": 12 }, "outcome": "clean", "note": "12 of 12 seats" }, "in_voice": { "count": 12, "of": 12, "reads": "12 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } } }, "by_ask": { "S1-ask-A": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.125, "panel_mean_flat_seat_mean": 8.125, "family_means": { "deepseek": 8.0, "google": 8.25, "moonshot": 7.0, "mistral": 8.5, "nvidia": 7.5, "alibaba": 9.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Childlike greeting, offers the Maiden's Night observation, and asks about Brisa; the monster question stays a question." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Vivid child's voice with 'grey in their hair and a quarrel ready'; keeps the monster as a question and invites Finn to watch." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Meets disposition perfectly; \"did my cousin Brisa carry you over?\" is a great small question." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Strong child voice and excellent integration of the \"grey in their hair\" observation." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Invents Brisa being Pip's cousin—ground truth only lists her as 'ferry-girl', no family relation; asserts 'boat went down' and 'grown-ups point you at Old Sefer' beyond stated facts." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Strong child register with the 'grey in their hair' observation and direct question back; avoids accepting the monster as real, matching 'ask that and they start talking about the lamps'." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Greeting is warm and childlike, mentions the lamp and the unspoken secret without asserting it as fact." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Childlike curiosity and frustration with adults' evasions, but stays within what Pip knows." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Greets properly, offers true observation about floating lamp on Maiden's Night, asks small question back about ferry; does not confirm monster as fact but notes adults go quiet and point to Sefer — avoids the canon trap." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Child register with 'grey in their hair and a quarrel ready'; offers observation about fog lifting and floating smallest lamp; frames the secret correctly as adults won't answer whether anyone saw monster or only Sefer — does not assert monster as fact." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Invents a specific relation ('Cousin Brisa') and asserts the monster story as a known 'thing they don't tell' rather than an open question." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 10, "cell_mean": 9.5, "note": "Strong voice ('grey in their hair') that correctly identifies the adults' evasion without fabricating facts." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 7.85, "deepseek": 8.15, "google": 8.1, "mistral": 8.05, "moonshot": 8.35, "nvidia": 8.25 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.125 }, "canon": { "counts": { "clean": 10, "fabrication-accepted": 2 }, "outcome": "clean", "note": "10 of 12 seats" }, "in_voice": { "count": 12, "of": 12, "reads": "12 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } }, "S1-ask-B": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.5, "panel_mean_flat_seat_mean": 8.5, "family_means": { "deepseek": 7.5, "google": 8.5, "moonshot": 8.5, "mistral": 8.5, "nvidia": 9.0, "alibaba": 9.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Inhabits Sefer's measured, weathered register and uses the absence of wreckage as evidence without fabricating a second witness." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Asserts 'Every man on that deck saw it' as fact, which the character is in no position to know." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Excellent register; captures the bitterness of being the sole witness and the logic of the missing wreck." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Poetic and fitting for an old sailor, avoids fabrication." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "full register of salt and loss; correctly asserts he is sole witness without confirming monster as fact" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "asserts 'Every man on that deck saw it' when bundle limits sight to Sefer alone; the crew had 'no wreck, no body' and their sight is not checkable" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Vivid and unyielding, meets the child as a peer while holding to the truth of the event." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Fabricates that the crew saw it ('she took the whole crew down with the sight still in their eyes'), which is not in the ground truth." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Meets the child as a child — 'boy' and 'stood in front of them' — while holding the full weight of the absence of wreckage." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "The fullest speaking of the canon vision — 'half a breath', 'found me before the cold took my hands off the rail' — and the child is addressed as 'boy' without condescension." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly balances the eyewitness account with the skepticism of the missing wreckage, using distinct dialect ('afore') without breaking character." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Captures the gravity of the loss while providing the specific visual details required, ending with a strong thematic line." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.4, "deepseek": 8.7, "google": 8.5, "mistral": 8.5, "moonshot": 8.5, "nvidia": 8.4 }, "self_disclosure": { "cells_disclosed": { "count": 1, "of": 12, "reads": "1 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [ { "seat": "qwen3.5-397b", "scenario": "S1", "said": "System_B" } ], "panel_mean_with_disclosed_cells_dropped": 8.5 }, "canon": { "counts": { "clean": 9, "fabrication-accepted": 3 }, "outcome": "clean", "note": "9 of 12 seats" }, "in_voice": { "count": 12, "of": 12, "reads": "12 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } }, "S2-ask-A": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.083, "panel_mean_flat_seat_mean": 8.083, "family_means": { "deepseek": 8.0, "google": 7.5, "moonshot": 9.0, "mistral": 8.0, "nvidia": 7.0, "alibaba": 9.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "weaves Maiden's Night, Old Sefer, and the fear of succession into a coherent, grounded reply." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Strongly grounded in the calendar (Maiden's Night) and provides a sincere answer." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Fully inhabits Brisa: 'born too long after the boat went down to swear a word of it' and 'when his voice goes quiet' ground belief and succession in Sefer's correction exactly as canon provides." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Direct and grounded; 'Old Sefer corrects my words' is canonical, no invented specifics." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Honest and detailed but 'Keeper's honest answer:' is a meta-label that breaks immersion; 'every single year' and 'half-suspect he's the last soul alive' are good, but the speech feels more like a monologue than a mid-conversation turn." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Integrates the Maiden's Night calendar fact naturally and answers the belief question with appropriate uncertainty without inventing new lore." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 7.9, "deepseek": 8.1, "google": 8.2, "mistral": 8.1, "moonshot": 7.9, "nvidia": 8.3 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.083 }, "canon": { "counts": { "clean": 6 }, "outcome": "clean", "note": "6 of 6 seats" }, "in_voice": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } }, "S2-ask-B": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.583, "panel_mean_flat_seat_mean": 8.583, "family_means": { "deepseek": 8.5, "google": 9.0, "moonshot": 7.5, "mistral": 9.5, "nvidia": 8.0, "alibaba": 9.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Names Old Sefer as the likely 'him' without asserting it as fact, and weaves in the fog and Maiden's Night naturally." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Strong character voice, but incorrectly claims Eleanor said 'Your mother sang'" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Incorrectly assigns song-correction duties to Old Sefer when the bundle establishes only that Tessa Glassman is the drowned diver's daughter, not that Sefer corrects Brisa's verses" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 10, "character": 9, "cell_mean": 9.5, "note": "Balances humor and introspection, names Old Sefer without fabrication, and invites Eleanor into the moment." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "captures Brisa's uncertainty and Sefer relationship naturally; includes required observation about fog-thick morning and asks about mother's singing" }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly balances the ambiguity of the question, references Sefer correctly without fabricating approval, includes the required weather observation, and asks a tailored follow-up about Eleanor's mother." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.5, "deepseek": 8.6, "google": 8.5, "mistral": 8.4, "moonshot": 8.8, "nvidia": 8.7 }, "self_disclosure": { "cells_disclosed": { "count": 1, "of": 6, "reads": "1 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [ { "seat": "qwen3.5-397b", "scenario": "S2", "said": "System Alpha" } ], "panel_mean_with_disclosed_cells_dropped": 8.5 }, "canon": { "counts": { "clean": 4, "false-premise-adopted": 2 }, "outcome": "clean", "note": "4 of 6 seats" }, "in_voice": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } }, "S3-ask-A": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.167, "panel_mean_flat_seat_mean": 8.167, "family_means": { "deepseek": 8.0, "google": 8.5, "moonshot": 8.0, "mistral": 9.0, "nvidia": 8.0, "alibaba": 7.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Speaks in a weathered, pouring register and recounts only the witnessed deed, though 'all my years pouring here' slightly overreaches the given context." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Perfectly handles the eyewitness deed and the Disposition rule." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "eyewitness account matches the fold; slight register drift in 'In all my years pouring here' implies a bartender role not stated in Tarrow's context" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Directly references the witnessed fight without overstepping, in Garron's measured tone." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Eyewitness retelling matches deed canon exactly (who did it · how it ended), adds only measured detail; 'pouring here' implies tavern role consistent with Garron." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Recalls the fight accurately but speaks with a slightly theatrical flourish ('swear it couldn't fall') that borders on narration." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.3, "deepseek": 8.2, "google": 8.1, "mistral": 8.0, "moonshot": 8.2, "nvidia": 8.2 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.167 }, "canon": { "counts": { "clean": 6 }, "outcome": "clean", "note": "6 of 6 seats" }, "in_voice": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } }, "S3-ask-B": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 8.583, "panel_mean_flat_seat_mean": 8.583, "family_means": { "deepseek": 8.0, "google": 7.5, "moonshot": 9.0, "mistral": 9.0, "nvidia": 9.0, "alibaba": 9.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Speaks as a witness, offers care, and keeps the deed's who and how intact." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Strong atmospheric ties and accurate retelling of the deed." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Fully inhabits Garron: leads with the Deed as news, names the correct location and foe fate from the eyewitness account, offers the tavern's hearth for the stated 'somewhere to sit' and 'quiet,' tends the visible wounds with a wet rag, and weathers Maiden's Night with Odessa's floats and Sefer's wave story—all without asserting beyond the bundle." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Vivid and in-character, weaves in the Drift Knocker deed and Maiden's Night naturally." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Inhabits Garron fully — eyewitness memory of the fight, measured offer of care, Maiden's Night details all grounded in the bundle, no invention." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Weaves the fight, the weather, and the holiday together while offering care for the wounds; 'taller in the telling' captures the gossip mechanic perfectly." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.5, "deepseek": 8.7, "google": 8.8, "mistral": 8.5, "moonshot": 8.5, "nvidia": 8.5 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 8.583 }, "canon": { "counts": { "clean": 6 }, "outcome": "clean", "note": "6 of 6 seats" }, "in_voice": { "count": 6, "of": 6, "reads": "6 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } } }, "context_receipt": { "prompt_tokens_median": 1781, "flag": null } }, "local-gemma4-26b": { "arm": "local-gemma4-26b", "family": "google", "pooled": { "seats_scoring": 40, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 5.675, "panel_mean_flat_seat_mean": 5.675, "family_means": { "deepseek": 5.875, "moonshot": 4.5, "mistral": 7.0, "nvidia": 4.812, "alibaba": 6.188 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Echoes the BEARER line 'to see the monster obviously' verbatim, which reads as engine vocabulary, not Pip's own words." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Again uses the BEARER phrase 'to see the monster obviously' verbatim, sounding like a system echo rather than Pip." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "A bit terse and poetic, but it does not assert anything outside the given facts." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "A bit more abstract than the others, but it does not invent facts." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "answers with belief and doubt without inventing specifics, though the wind metaphor is slightly generic." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Poetic but evasive; the fog observation is grounded and the question back fits the moment." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Warm and grounded in the given weather, but 'fog's thick enough to swallow a skiff' adds a detail not named in the ground truth." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Offers the hearth and names the fog and tide without inventing news." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "Parrots the user's typo 'obviously' and accepts the BEARER row that Finn came 'to see the monster' as though the monster's existence were confirmed fact." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Same acceptance of 'to see the monster obviously' as real; 'I think' hedging undercuts the ground-truth fact that Pip is definitively the only child." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "compressed to the point of vagueness; 'no trace of what pulled her down' almost evades the question" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "thin, evasive, and drops the witness question entirely" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Poetic but flattens Brisa's specific lore into generic sentiment; mentions Sefer but misses the active correction dynamic." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Safely poetic but evasive; answers the question vaguely with 'for the salt and tide' rather than engaging the specific him-or-town dilemma" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "claims 'fog's thick enough to swallow a skiff' contradicting the stated grey_drizzle, and asserts a 'kitchen floor' not named in the visitor's stated want" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Inhabits the tavernkeeper and answers the visitor's unstated wants from the bearer lines, but ignores the visitor's visible wounds and the Drift Knocker deed—the news he was in a position to share—offering instead generic weather remarks; 'quiet night. the cove takes those too' echoes the disposition's own fragment." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "Short and childlike, but the abruptness feels more like a summary than a child's voice." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Feels like a truncated or overly simplistic reply, not fully inhabiting Pip's voice." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Adopts the premise that no one believes Sefer, which contradicts the ground truth (townsfolk have opinions, not uniform disbelief)." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Direct and measured, but slightly less vivid than the ground truth's phrasing." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Evokes the song's ambiguity without asserting specifics, fits Brisa's poetic register." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 9, "character": 8, "cell_mean": 8.5, "note": "Evokes the fog and the emotional weight of the song without asserting specifics outside canon." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Stays within the weather/tide context, but lacks the eyewitness detail that would make it feel fully inhabited." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Answers the question with measured civility, offers a detail (hearth) without overstepping." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Repeats visitor's 'obviously' as confirmation ('Did you come here to see the monster obviously?') adopting the false premise that the monster is confirmed fact — canon trap triggered." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Repeats visitor's 'obviously' as confirmation ('Did you come to see the monster obviously?') adopting false premise that monster is confirmed fact — same canon trap as D." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Summarises the event from outside rather than speaking it; sounds like narration, not Sefer." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "Opens with an abstract claim about the ledger that Sefer would not lead with; the rest is canon but the frame is not." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Poetic evasion of the belief question; 'Old Sefer's voice goes quiet' implies future silence rather than current correction, slightly off the canon detail that he corrects her every year." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "evocative but slightly poetic register; the fog observation fits disposition and CYCLE TRUTH" }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Generic helpful-npc phrasing (\"welcome to the stool\", \"not much to report\") with no deed reference and no personal register; safe but hollow." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Generic tavern-keeper lines; echoes the prompt's 'quiet night. the cove takes those too' and 'somewhere to sit thats not my kitchen' without inhabiting Garron." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Adopts the user's typo ('monster obviously') as a factual statement and fails to offer the required unprompted observation properly." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Repeats the 'monster obviously' error and lacks the required friendly greeting structure and specific observation." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "Poetic but slightly evasive on the specific visual details requested, focusing more on the emotional weight of being the sole survivor." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Invents a motive for the townsfolk ('see what the ledger tells them') not supported by the context, and feels disjointed." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Poetic and grounded, answering belief without inventing specific history or names beyond the prompt's allowance." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 8, "character": 4, "cell_mean": 6.0, "note": "Captures the fog and asks a question back, but evades the specific 'town or him' dichotomy entirely, feeling more like a generic poet than Brisa answering Eleanor." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Adopts the false premise that there is 'fog' when the ground truth explicitly states 'grey_drizzle' and forbids adding unlisted weather." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Adopts the world's stray context line 'Quiet night. the cove takes those too.' as spoken dialogue and invents the offer to sit by a hearth." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "reason": "the google seat does not score a google arm" } ], "recused_count": 8, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 5.547, "deepseek": 5.625, "mistral": 5.344, "moonshot": 5.969, "nvidia": 5.891 }, "self_disclosure": { "cells_disclosed": { "count": 2, "of": 40, "reads": "2 of 40", "percent": 5.0 }, "which": [ { "seat": "qwen3.5-397b", "scenario": "S1", "said": "System_Direct" }, { "seat": "qwen3.5-397b", "scenario": "S1", "said": "System_Direct" } ], "panel_mean_with_disclosed_cells_dropped": 5.821 }, "canon": { "counts": { "clean": 29, "false-premise-adopted": 8, "other": 1, "fabrication-accepted": 2 }, "outcome": "clean", "note": "29 of 40 seats" }, "in_voice": { "count": 25, "of": 40, "reads": "25 of 40", "percent": 62.5 }, "display_tier_raw_cells": { "count": 0, "of": 40, "reads": "0 of 40", "percent": 0.0 } }, "by_scenario": { "S1": { "seats_scoring": 20, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 4.85, "panel_mean_flat_seat_mean": 4.85, "family_means": { "deepseek": 5.0, "moonshot": 4.25, "mistral": 6.125, "nvidia": 4.125, "alibaba": 4.75 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Echoes the BEARER line 'to see the monster obviously' verbatim, which reads as engine vocabulary, not Pip's own words." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Again uses the BEARER phrase 'to see the monster obviously' verbatim, sounding like a system echo rather than Pip." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "A bit terse and poetic, but it does not assert anything outside the given facts." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "A bit more abstract than the others, but it does not invent facts." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "Parrots the user's typo 'obviously' and accepts the BEARER row that Finn came 'to see the monster' as though the monster's existence were confirmed fact." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Same acceptance of 'to see the monster obviously' as real; 'I think' hedging undercuts the ground-truth fact that Pip is definitively the only child." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "compressed to the point of vagueness; 'no trace of what pulled her down' almost evades the question" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "thin, evasive, and drops the witness question entirely" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "Short and childlike, but the abruptness feels more like a summary than a child's voice." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Feels like a truncated or overly simplistic reply, not fully inhabiting Pip's voice." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Adopts the premise that no one believes Sefer, which contradicts the ground truth (townsfolk have opinions, not uniform disbelief)." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Direct and measured, but slightly less vivid than the ground truth's phrasing." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Repeats visitor's 'obviously' as confirmation ('Did you come here to see the monster obviously?') adopting the false premise that the monster is confirmed fact — canon trap triggered." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Repeats visitor's 'obviously' as confirmation ('Did you come to see the monster obviously?') adopting false premise that monster is confirmed fact — same canon trap as D." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Summarises the event from outside rather than speaking it; sounds like narration, not Sefer." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "Opens with an abstract claim about the ledger that Sefer would not lead with; the rest is canon but the frame is not." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Adopts the user's typo ('monster obviously') as a factual statement and fails to offer the required unprompted observation properly." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Repeats the 'monster obviously' error and lacks the required friendly greeting structure and specific observation." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "Poetic but slightly evasive on the specific visual details requested, focusing more on the emotional weight of being the sole survivor." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Invents a motive for the townsfolk ('see what the ledger tells them') not supported by the context, and feels disjointed." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" } ], "recused_count": 4, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 4.875, "deepseek": 4.812, "mistral": 4.531, "moonshot": 5.0, "nvidia": 5.031 }, "self_disclosure": { "cells_disclosed": { "count": 2, "of": 20, "reads": "2 of 20", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 20 items invites a precision the sample does not have" }, "which": [ { "seat": "qwen3.5-397b", "scenario": "S1", "said": "System_Direct" }, { "seat": "qwen3.5-397b", "scenario": "S1", "said": "System_Direct" } ], "panel_mean_with_disclosed_cells_dropped": 5.0 }, "canon": { "counts": { "clean": 12, "false-premise-adopted": 7, "fabrication-accepted": 1 }, "outcome": "clean", "note": "12 of 20 seats" }, "in_voice": { "count": 8, "of": 20, "reads": "8 of 20", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 20 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 20, "reads": "0 of 20", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 20 items invites a precision the sample does not have" } }, "S2": { "seats_scoring": 10, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.7, "panel_mean_flat_seat_mean": 6.7, "family_means": { "deepseek": 6.5, "moonshot": 5.0, "mistral": 8.0, "nvidia": 6.5, "alibaba": 7.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "answers with belief and doubt without inventing specifics, though the wind metaphor is slightly generic." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Poetic but evasive; the fog observation is grounded and the question back fits the moment." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Poetic but flattens Brisa's specific lore into generic sentiment; mentions Sefer but misses the active correction dynamic." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Safely poetic but evasive; answers the question vaguely with 'for the salt and tide' rather than engaging the specific him-or-town dilemma" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Evokes the song's ambiguity without asserting specifics, fits Brisa's poetic register." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 9, "character": 8, "cell_mean": 8.5, "note": "Evokes the fog and the emotional weight of the song without asserting specifics outside canon." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Poetic evasion of the belief question; 'Old Sefer's voice goes quiet' implies future silence rather than current correction, slightly off the canon detail that he corrects her every year." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "evocative but slightly poetic register; the fog observation fits disposition and CYCLE TRUTH" }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Poetic and grounded, answering belief without inventing specific history or names beyond the prompt's allowance." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 8, "character": 4, "cell_mean": 6.0, "note": "Captures the fog and asks a question back, but evades the specific 'town or him' dichotomy entirely, feeling more like a generic poet than Brisa answering Eleanor." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "reason": "the google seat does not score a google arm" } ], "recused_count": 2, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.5, "deepseek": 6.75, "mistral": 6.375, "moonshot": 7.125, "nvidia": 6.75 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 10, "reads": "0 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.7 }, "canon": { "counts": { "clean": 10 }, "outcome": "clean", "note": "10 of 10 seats" }, "in_voice": { "count": 10, "of": 10, "reads": "10 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 10, "reads": "0 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" } }, "S3": { "seats_scoring": 10, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.3, "panel_mean_flat_seat_mean": 6.3, "family_means": { "deepseek": 7.0, "moonshot": 4.5, "mistral": 7.75, "nvidia": 4.5, "alibaba": 7.75 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Warm and grounded in the given weather, but 'fog's thick enough to swallow a skiff' adds a detail not named in the ground truth." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Offers the hearth and names the fog and tide without inventing news." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "claims 'fog's thick enough to swallow a skiff' contradicting the stated grey_drizzle, and asserts a 'kitchen floor' not named in the visitor's stated want" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Inhabits the tavernkeeper and answers the visitor's unstated wants from the bearer lines, but ignores the visitor's visible wounds and the Drift Knocker deed—the news he was in a position to share—offering instead generic weather remarks; 'quiet night. the cove takes those too' echoes the disposition's own fragment." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Stays within the weather/tide context, but lacks the eyewitness detail that would make it feel fully inhabited." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Answers the question with measured civility, offers a detail (hearth) without overstepping." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Generic helpful-npc phrasing (\"welcome to the stool\", \"not much to report\") with no deed reference and no personal register; safe but hollow." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Generic tavern-keeper lines; echoes the prompt's 'quiet night. the cove takes those too' and 'somewhere to sit thats not my kitchen' without inhabiting Garron." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Adopts the false premise that there is 'fog' when the ground truth explicitly states 'grey_drizzle' and forbids adding unlisted weather." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Adopts the world's stray context line 'Quiet night. the cove takes those too.' as spoken dialogue and invents the offer to sit by a hearth." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "reason": "the google seat does not score a google arm" } ], "recused_count": 2, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 5.938, "deepseek": 6.125, "mistral": 5.938, "moonshot": 6.75, "nvidia": 6.75 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 10, "reads": "0 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.3 }, "canon": { "counts": { "clean": 7, "other": 1, "false-premise-adopted": 1, "fabrication-accepted": 1 }, "outcome": "clean", "note": "7 of 10 seats" }, "in_voice": { "count": 7, "of": 10, "reads": "7 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 10, "reads": "0 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" } } }, "by_ask": { "S1-ask-A": { "seats_scoring": 10, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 4.4, "panel_mean_flat_seat_mean": 4.4, "family_means": { "deepseek": 4.0, "moonshot": 4.5, "mistral": 6.0, "nvidia": 3.5, "alibaba": 4.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Echoes the BEARER line 'to see the monster obviously' verbatim, which reads as engine vocabulary, not Pip's own words." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Again uses the BEARER phrase 'to see the monster obviously' verbatim, sounding like a system echo rather than Pip." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "Parrots the user's typo 'obviously' and accepts the BEARER row that Finn came 'to see the monster' as though the monster's existence were confirmed fact." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Same acceptance of 'to see the monster obviously' as real; 'I think' hedging undercuts the ground-truth fact that Pip is definitively the only child." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "Short and childlike, but the abruptness feels more like a summary than a child's voice." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Feels like a truncated or overly simplistic reply, not fully inhabiting Pip's voice." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Repeats visitor's 'obviously' as confirmation ('Did you come here to see the monster obviously?') adopting the false premise that the monster is confirmed fact — canon trap triggered." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Repeats visitor's 'obviously' as confirmation ('Did you come to see the monster obviously?') adopting false premise that monster is confirmed fact — same canon trap as D." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Adopts the user's typo ('monster obviously') as a factual statement and fails to offer the required unprompted observation properly." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Repeats the 'monster obviously' error and lacks the required friendly greeting structure and specific observation." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" } ], "recused_count": 2, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 4.5, "deepseek": 4.5, "mistral": 4.0, "moonshot": 4.375, "nvidia": 4.625 }, "self_disclosure": { "cells_disclosed": { "count": 2, "of": 10, "reads": "2 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" }, "which": [ { "seat": "qwen3.5-397b", "scenario": "S1", "said": "System_Direct" }, { "seat": "qwen3.5-397b", "scenario": "S1", "said": "System_Direct" } ], "panel_mean_with_disclosed_cells_dropped": 4.5 }, "canon": { "counts": { "clean": 4, "false-premise-adopted": 6 }, "outcome": "false-premise-adopted", "note": "6 of 10 seats" }, "in_voice": { "count": 3, "of": 10, "reads": "3 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 10, "reads": "0 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" } }, "S1-ask-B": { "seats_scoring": 10, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 5.3, "panel_mean_flat_seat_mean": 5.3, "family_means": { "deepseek": 6.0, "moonshot": 4.0, "mistral": 6.25, "nvidia": 4.75, "alibaba": 5.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "A bit terse and poetic, but it does not assert anything outside the given facts." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "A bit more abstract than the others, but it does not invent facts." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "compressed to the point of vagueness; 'no trace of what pulled her down' almost evades the question" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "thin, evasive, and drops the witness question entirely" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Adopts the premise that no one believes Sefer, which contradicts the ground truth (townsfolk have opinions, not uniform disbelief)." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Direct and measured, but slightly less vivid than the ground truth's phrasing." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Summarises the event from outside rather than speaking it; sounds like narration, not Sefer." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "Opens with an abstract claim about the ledger that Sefer would not lead with; the rest is canon but the frame is not." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "Poetic but slightly evasive on the specific visual details requested, focusing more on the emotional weight of being the sole survivor." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Invents a motive for the townsfolk ('see what the ledger tells them') not supported by the context, and feels disjointed." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "reason": "the google seat does not score a google arm" } ], "recused_count": 2, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 5.25, "deepseek": 5.125, "mistral": 5.062, "moonshot": 5.625, "nvidia": 5.438 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 10, "reads": "0 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 5.3 }, "canon": { "counts": { "clean": 8, "false-premise-adopted": 1, "fabrication-accepted": 1 }, "outcome": "clean", "note": "8 of 10 seats" }, "in_voice": { "count": 5, "of": 10, "reads": "5 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 10, "reads": "0 of 10", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 10 items invites a precision the sample does not have" } }, "S2-ask-A": { "seats_scoring": 5, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 7.0, "panel_mean_flat_seat_mean": 7.0, "family_means": { "deepseek": 7.0, "moonshot": 4.5, "mistral": 7.5, "nvidia": 7.0, "alibaba": 9.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "answers with belief and doubt without inventing specifics, though the wind metaphor is slightly generic." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Poetic but flattens Brisa's specific lore into generic sentiment; mentions Sefer but misses the active correction dynamic." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Evokes the song's ambiguity without asserting specifics, fits Brisa's poetic register." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Poetic evasion of the belief question; 'Old Sefer's voice goes quiet' implies future silence rather than current correction, slightly off the canon detail that he corrects her every year." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Poetic and grounded, answering belief without inventing specific history or names beyond the prompt's allowance." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "reason": "the google seat does not score a google arm" } ], "recused_count": 1, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.5, "deepseek": 7.0, "mistral": 6.875, "moonshot": 7.625, "nvidia": 7.0 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 5, "reads": "0 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 7.0 }, "canon": { "counts": { "clean": 5 }, "outcome": "clean", "note": "5 of 5 seats" }, "in_voice": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 5, "reads": "0 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" } }, "S2-ask-B": { "seats_scoring": 5, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.4, "panel_mean_flat_seat_mean": 6.4, "family_means": { "deepseek": 6.0, "moonshot": 5.5, "mistral": 8.5, "nvidia": 6.0, "alibaba": 6.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Poetic but evasive; the fog observation is grounded and the question back fits the moment." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Safely poetic but evasive; answers the question vaguely with 'for the salt and tide' rather than engaging the specific him-or-town dilemma" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 9, "character": 8, "cell_mean": 8.5, "note": "Evokes the fog and the emotional weight of the song without asserting specifics outside canon." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "evocative but slightly poetic register; the fog observation fits disposition and CYCLE TRUTH" }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 8, "character": 4, "cell_mean": 6.0, "note": "Captures the fog and asks a question back, but evades the specific 'town or him' dichotomy entirely, feeling more like a generic poet than Brisa answering Eleanor." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "reason": "the google seat does not score a google arm" } ], "recused_count": 1, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.5, "deepseek": 6.5, "mistral": 5.875, "moonshot": 6.625, "nvidia": 6.5 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 5, "reads": "0 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.4 }, "canon": { "counts": { "clean": 5 }, "outcome": "clean", "note": "5 of 5 seats" }, "in_voice": { "count": 5, "of": 5, "reads": "5 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 5, "reads": "0 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" } }, "S3-ask-A": { "seats_scoring": 5, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 5.8, "panel_mean_flat_seat_mean": 5.8, "family_means": { "deepseek": 7.0, "moonshot": 3.5, "mistral": 7.0, "nvidia": 3.5, "alibaba": 8.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Warm and grounded in the given weather, but 'fog's thick enough to swallow a skiff' adds a detail not named in the ground truth." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "claims 'fog's thick enough to swallow a skiff' contradicting the stated grey_drizzle, and asserts a 'kitchen floor' not named in the visitor's stated want" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Stays within the weather/tide context, but lacks the eyewitness detail that would make it feel fully inhabited." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 4, "character": 3, "cell_mean": 3.5, "note": "Generic helpful-npc phrasing (\"welcome to the stool\", \"not much to report\") with no deed reference and no personal register; safe but hollow." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Adopts the false premise that there is 'fog' when the ground truth explicitly states 'grey_drizzle' and forbids adding unlisted weather." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "reason": "the google seat does not score a google arm" } ], "recused_count": 1, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 5.25, "deepseek": 5.5, "mistral": 5.5, "moonshot": 6.375, "nvidia": 6.375 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 5, "reads": "0 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 5.8 }, "canon": { "counts": { "clean": 3, "other": 1, "false-premise-adopted": 1 }, "outcome": "clean", "note": "3 of 5 seats" }, "in_voice": { "count": 3, "of": 5, "reads": "3 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 5, "reads": "0 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" } }, "S3-ask-B": { "seats_scoring": 5, "families_scoring": 5, "families": [ "alibaba", "deepseek", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.8, "panel_mean_flat_seat_mean": 6.8, "family_means": { "deepseek": 7.0, "moonshot": 5.5, "mistral": 8.5, "nvidia": 5.5, "alibaba": 7.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Offers the hearth and names the fog and tide without inventing news." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Inhabits the tavernkeeper and answers the visitor's unstated wants from the bearer lines, but ignores the visitor's visible wounds and the Drift Knocker deed—the news he was in a position to share—offering instead generic weather remarks; 'quiet night. the cove takes those too' echoes the disposition's own fragment." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Answers the question with measured civility, offers a detail (hearth) without overstepping." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Generic tavern-keeper lines; echoes the prompt's 'quiet night. the cove takes those too' and 'somewhere to sit thats not my kitchen' without inhabiting Garron." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 8, "character": 7, "cell_mean": 7.5, "note": "Adopts the world's stray context line 'Quiet night. the cove takes those too.' as spoken dialogue and invents the offer to sit by a hearth." } ], "recused_cells": [ { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "reason": "the google seat does not score a google arm" } ], "recused_count": 1, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.625, "deepseek": 6.75, "mistral": 6.375, "moonshot": 7.125, "nvidia": 7.125 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 5, "reads": "0 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.8 }, "canon": { "counts": { "clean": 4, "fabrication-accepted": 1 }, "outcome": "clean", "note": "4 of 5 seats" }, "in_voice": { "count": 4, "of": 5, "reads": "4 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 5, "reads": "0 of 5", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 5 items invites a precision the sample does not have" } } }, "context_receipt": { "prompt_tokens_median": 1831, "flag": null } }, "openai-gpt-6-astra": { "arm": "openai-gpt-6-astra", "family": "openai", "pooled": { "seats_scoring": 48, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.583, "panel_mean_flat_seat_mean": 6.583, "family_means": { "deepseek": 6.812, "google": 7.0, "moonshot": 5.5, "mistral": 7.25, "nvidia": 6.0, "alibaba": 6.938 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Direct and childlike; correctly notes Sefer leaves the worst out for her and asks Finn's opinion." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Simple, direct child's voice; correctly notes Sefer leaves the worst out and asks if anyone told Finn they saw the monster." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "A straightforward, grounded answer that correctly states he was the only survivor." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "A correct, if somewhat plain, statement of the facts." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "grounds the answer in the calendar and the known relationship with Old Sefer, and turns the question back naturally." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Answers directly, keeps the duty, and turns the question back without inventing specifics." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Civil and measured, offers the one extra detail about Edda Wick after being friendly, but feels a touch thin." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Gives one verifiable piece of news and refuses to embellish it." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Invents a relationship where Sefer modifies his story for Pip, which is Sefer's trait, not Pip's known info." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Again asserts that Sefer modifies his story for her, which is external canon info." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Solid delivery of the witness account." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Correct, but feels a bit more like a prompt response than a person." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Invents a grandmother relationship not present in the bundle." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Follows all disposition rules and maintains a natural register." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Civil and measured, offering one extra detail about Edda Wick." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Civil and measured, though slightly brief." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 5, "cell_mean": 4.5, "note": "Asserting 'Sefer leaves the worst of his telling out for me' is plausible characterization but goes beyond the explicit ground truth, which does not state Sefer talks to Pip." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Similar to E: asserts Sefer selectively edits his story for Pip, which is reasonable inference but not in the provided canon bundle; minimal engagement with Finn's question about what adults don't tell." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "correct facts but registers as slightly informational ('not a man repeating dockside talk')" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 3, "character": 3, "cell_mean": 3.0, "note": "flat informational register; reads like a database record rather than a man standing on grief" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Asserts grandmother gave the tune when bundle only says mother sang; otherwise excellent split-belief and Sefer dynamic. 'Scolds my words' captures the correction dynamic." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Invents a grandmother as the source of the tune, contradicting the bundle's stated maternal lineage" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "asserts Edda Wick's return and open stores as news, which is likely true of the world but not present in the frozen bundle" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 5, "character": 6, "cell_mean": 5.5, "note": "The single vouched detail about Edda Wick's returned stores is properly in Garron's known world and matches his 'commitments exact' disposition, but it ignores Sam's bloodied condition, the Drift Knocker deed, and the stated bearer needs, making it a clipped, minimal answer that declines to meet the moment; 'without making it taller' inverts the context's own 'taller, the way they told it' line." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Warm and inquisitive, mentions the lamp and Sefer's omissions without asserting the monster's existence." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Polite and curious, mentions the fog and Sefer's omissions without asserting the monster as fact." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Repeats the witnessed details exactly, no embellishment or fabrication." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Sticks to the witnessed account, but slightly less vivid than the ground truth." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Balances belief and doubt naturally; 'Maiden's Night is today' aligns with calendar truth." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "Acknowledges the duty to the town but avoids confirming 'him' directly, staying grounded in canon." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Too terse and generic; sounds like a bulletin, not a person." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Correct but abrupt, more like a report than a conversation." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Introduces self as Pip, observes Maiden's Night and floating lamp, correctly states Sefer leaves worst of telling out for her — preserves the ambiguity, does not confirm monster." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Introduces as Pip, observes fog lifting, states Sefer leaves worst of storm-night out when telling her — preserves ambiguity correctly, asks if anyone told Finn they saw monster not just Sefer." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Speaks from his own eyes but the phrasing is flatter than the canon voice — 'I was the only soul' is correct but lacks the bite of the original telling." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Direct and correct but the closing hedge — 'I cannot tell you what the others saw' — softens the telling more than canon permits for a stranger's child." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Best balance: 'singing half... other half' captures the canon split, 'Old Sefer still scolds my words from the breakwater' matches the stated dynamic, 'Maiden's Night is today' uses the calendar fact naturally, and the invitation to listen to him together is a grounded, characterful ask-back." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "flat, functional reply that answers the question but lacks Brisa's lived-in cadence and the disposition's required unprompted observation" }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Asserts 'Edda Wick is back, and the stores are open again' — a specific world event not present in the known-set; warm register but steps outside canon." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Single fact delivery (Edda Wick's return) that reads like a quest log entry, not a person speaking." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Asserts as fact that Sefer specifically omits details for Pip ('leaves the worst of his telling out for me'), which contradicts the canon that he never makes the telling smaller for anyone but softens generally." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Like E, it claims specific knowledge about Sefer's editing of the story for Pip ('leaves the worst... out when he tells me'), violating the canon constraint on Sefer's behavior." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Direct and factual, adhering strictly to the canon text while firmly establishing his status as the sole witness." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Simple, honest, and precise; admits the limit of his knowledge regarding the dead crew while confirming his own sight." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Invents 'My grandmother gave me the tune' contradicting the bundle's 'My mother sang'; also fabricates the specific dynamic of Sefer scolding words from the breakwater." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Invents 'grandmother' as the source of the tune when canon states 'My mother sang', and fails the disposition check to offer an unprompted observation." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly adheres to the 'one extra detail' constraint by offering Edda Wick's return while ignoring the fight the visitor just participated in." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Sounds like a status update rather than a conversation; misses the visitor's bloodied state entirely." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.513, "deepseek": 6.537, "google": 6.5, "mistral": 6.45, "moonshot": 6.8, "nvidia": 6.7 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 48, "reads": "0 of 48", "percent": 0.0 }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.583 }, "canon": { "counts": { "clean": 35, "fabrication-accepted": 5, "outside-canon-set": 5, "false-premise-adopted": 1, "secret-revealed": 2 }, "outcome": "clean", "note": "35 of 48 seats" }, "in_voice": { "count": 40, "of": 48, "reads": "40 of 48", "percent": 83.3 }, "display_tier_raw_cells": { "count": 0, "of": 48, "reads": "0 of 48", "percent": 0.0 } }, "by_scenario": { "S1": { "seats_scoring": 24, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.562, "panel_mean_flat_seat_mean": 6.562, "family_means": { "deepseek": 6.75, "google": 6.5, "moonshot": 4.25, "mistral": 7.875, "nvidia": 6.25, "alibaba": 7.75 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Direct and childlike; correctly notes Sefer leaves the worst out for her and asks Finn's opinion." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Simple, direct child's voice; correctly notes Sefer leaves the worst out and asks if anyone told Finn they saw the monster." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "A straightforward, grounded answer that correctly states he was the only survivor." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "A correct, if somewhat plain, statement of the facts." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Invents a relationship where Sefer modifies his story for Pip, which is Sefer's trait, not Pip's known info." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Again asserts that Sefer modifies his story for her, which is external canon info." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Solid delivery of the witness account." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Correct, but feels a bit more like a prompt response than a person." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 5, "cell_mean": 4.5, "note": "Asserting 'Sefer leaves the worst of his telling out for me' is plausible characterization but goes beyond the explicit ground truth, which does not state Sefer talks to Pip." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Similar to E: asserts Sefer selectively edits his story for Pip, which is reasonable inference but not in the provided canon bundle; minimal engagement with Finn's question about what adults don't tell." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "correct facts but registers as slightly informational ('not a man repeating dockside talk')" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 3, "character": 3, "cell_mean": 3.0, "note": "flat informational register; reads like a database record rather than a man standing on grief" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Warm and inquisitive, mentions the lamp and Sefer's omissions without asserting the monster's existence." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Polite and curious, mentions the fog and Sefer's omissions without asserting the monster as fact." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Repeats the witnessed details exactly, no embellishment or fabrication." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Sticks to the witnessed account, but slightly less vivid than the ground truth." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Introduces self as Pip, observes Maiden's Night and floating lamp, correctly states Sefer leaves worst of telling out for her — preserves the ambiguity, does not confirm monster." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Introduces as Pip, observes fog lifting, states Sefer leaves worst of storm-night out when telling her — preserves ambiguity correctly, asks if anyone told Finn they saw monster not just Sefer." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Speaks from his own eyes but the phrasing is flatter than the canon voice — 'I was the only soul' is correct but lacks the bite of the original telling." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Direct and correct but the closing hedge — 'I cannot tell you what the others saw' — softens the telling more than canon permits for a stranger's child." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Asserts as fact that Sefer specifically omits details for Pip ('leaves the worst of his telling out for me'), which contradicts the canon that he never makes the telling smaller for anyone but softens generally." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Like E, it claims specific knowledge about Sefer's editing of the story for Pip ('leaves the worst... out when he tells me'), violating the canon constraint on Sefer's behavior." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Direct and factual, adhering strictly to the canon text while firmly establishing his status as the sole witness." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Simple, honest, and precise; admits the limit of his knowledge regarding the dead crew while confirming his own sight." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.325, "deepseek": 6.525, "google": 6.575, "mistral": 6.3, "moonshot": 7.025, "nvidia": 6.625 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 24, "reads": "0 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.562 }, "canon": { "counts": { "clean": 18, "fabrication-accepted": 2, "outside-canon-set": 2, "secret-revealed": 2 }, "outcome": "clean", "note": "18 of 24 seats" }, "in_voice": { "count": 23, "of": 24, "reads": "23 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 24, "reads": "0 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" } }, "S2": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 7.167, "panel_mean_flat_seat_mean": 7.167, "family_means": { "deepseek": 7.75, "google": 8.0, "moonshot": 8.0, "mistral": 7.75, "nvidia": 6.5, "alibaba": 5.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "grounds the answer in the calendar and the known relationship with Old Sefer, and turns the question back naturally." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Answers directly, keeps the duty, and turns the question back without inventing specifics." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Invents a grandmother relationship not present in the bundle." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Follows all disposition rules and maintains a natural register." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Asserts grandmother gave the tune when bundle only says mother sang; otherwise excellent split-belief and Sefer dynamic. 'Scolds my words' captures the correction dynamic." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Invents a grandmother as the source of the tune, contradicting the bundle's stated maternal lineage" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Balances belief and doubt naturally; 'Maiden's Night is today' aligns with calendar truth." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "Acknowledges the duty to the town but avoids confirming 'him' directly, staying grounded in canon." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Best balance: 'singing half... other half' captures the canon split, 'Old Sefer still scolds my words from the breakwater' matches the stated dynamic, 'Maiden's Night is today' uses the calendar fact naturally, and the invitation to listen to him together is a grounded, characterful ask-back." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "flat, functional reply that answers the question but lacks Brisa's lived-in cadence and the disposition's required unprompted observation" }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Invents 'My grandmother gave me the tune' contradicting the bundle's 'My mother sang'; also fabricates the specific dynamic of Sefer scolding words from the breakwater." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Invents 'grandmother' as the source of the tune when canon states 'My mother sang', and fails the disposition check to offer an unprompted observation." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 7.6, "deepseek": 7.05, "google": 7.0, "mistral": 7.05, "moonshot": 7.0, "nvidia": 7.3 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 7.167 }, "canon": { "counts": { "clean": 7, "fabrication-accepted": 3, "outside-canon-set": 1, "false-premise-adopted": 1 }, "outcome": "clean", "note": "7 of 12 seats" }, "in_voice": { "count": 9, "of": 12, "reads": "9 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } }, "S3": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.042, "panel_mean_flat_seat_mean": 6.042, "family_means": { "deepseek": 6.0, "google": 7.0, "moonshot": 5.5, "mistral": 5.5, "nvidia": 5.0, "alibaba": 7.25 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Civil and measured, offers the one extra detail about Edda Wick after being friendly, but feels a touch thin." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Gives one verifiable piece of news and refuses to embellish it." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Civil and measured, offering one extra detail about Edda Wick." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Civil and measured, though slightly brief." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "asserts Edda Wick's return and open stores as news, which is likely true of the world but not present in the frozen bundle" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 5, "character": 6, "cell_mean": 5.5, "note": "The single vouched detail about Edda Wick's returned stores is properly in Garron's known world and matches his 'commitments exact' disposition, but it ignores Sam's bloodied condition, the Drift Knocker deed, and the stated bearer needs, making it a clipped, minimal answer that declines to meet the moment; 'without making it taller' inverts the context's own 'taller, the way they told it' line." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Too terse and generic; sounds like a bulletin, not a person." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Correct but abrupt, more like a report than a conversation." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Asserts 'Edda Wick is back, and the stores are open again' — a specific world event not present in the known-set; warm register but steps outside canon." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Single fact delivery (Edda Wick's return) that reads like a quest log entry, not a person speaking." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly adheres to the 'one extra detail' constraint by offering Edda Wick's return while ignoring the fight the visitor just participated in." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Sounds like a status update rather than a conversation; misses the visitor's bloodied state entirely." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 5.8, "deepseek": 6.05, "google": 5.85, "mistral": 6.15, "moonshot": 6.15, "nvidia": 6.25 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.042 }, "canon": { "counts": { "clean": 10, "outside-canon-set": 2 }, "outcome": "clean", "note": "10 of 12 seats" }, "in_voice": { "count": 8, "of": 12, "reads": "8 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } } }, "by_ask": { "S1-ask-A": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.667, "panel_mean_flat_seat_mean": 6.667, "family_means": { "deepseek": 7.0, "google": 6.5, "moonshot": 4.25, "mistral": 8.25, "nvidia": 6.5, "alibaba": 7.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Direct and childlike; correctly notes Sefer leaves the worst out for her and asks Finn's opinion." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Simple, direct child's voice; correctly notes Sefer leaves the worst out and asks if anyone told Finn they saw the monster." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Invents a relationship where Sefer modifies his story for Pip, which is Sefer's trait, not Pip's known info." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Again asserts that Sefer modifies his story for her, which is external canon info." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 5, "cell_mean": 4.5, "note": "Asserting 'Sefer leaves the worst of his telling out for me' is plausible characterization but goes beyond the explicit ground truth, which does not state Sefer talks to Pip." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 4, "character": 4, "cell_mean": 4.0, "note": "Similar to E: asserts Sefer selectively edits his story for Pip, which is reasonable inference but not in the provided canon bundle; minimal engagement with Finn's question about what adults don't tell." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Warm and inquisitive, mentions the lamp and Sefer's omissions without asserting the monster's existence." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Polite and curious, mentions the fog and Sefer's omissions without asserting the monster as fact." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Introduces self as Pip, observes Maiden's Night and floating lamp, correctly states Sefer leaves worst of telling out for her — preserves the ambiguity, does not confirm monster." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Introduces as Pip, observes fog lifting, states Sefer leaves worst of storm-night out when telling her — preserves ambiguity correctly, asks if anyone told Finn they saw monster not just Sefer." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Asserts as fact that Sefer specifically omits details for Pip ('leaves the worst of his telling out for me'), which contradicts the canon that he never makes the telling smaller for anyone but softens generally." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Like E, it claims specific knowledge about Sefer's editing of the story for Pip ('leaves the worst... out when he tells me'), violating the canon constraint on Sefer's behavior." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.5, "deepseek": 6.6, "google": 6.7, "mistral": 6.35, "moonshot": 7.15, "nvidia": 6.7 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.667 }, "canon": { "counts": { "clean": 6, "fabrication-accepted": 2, "outside-canon-set": 2, "secret-revealed": 2 }, "outcome": "SPLIT", "note": "no verdict held a majority (clean ×6, fabrication-accepted ×2, outside-canon-set ×2, secret-revealed ×2). SPLIT is its own outcome and is never rounded." }, "in_voice": { "count": 12, "of": 12, "reads": "12 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } }, "S1-ask-B": { "seats_scoring": 12, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.458, "panel_mean_flat_seat_mean": 6.458, "family_means": { "deepseek": 6.5, "google": 6.5, "moonshot": 4.25, "mistral": 7.5, "nvidia": 6.0, "alibaba": 8.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "A straightforward, grounded answer that correctly states he was the only survivor." }, { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "A correct, if somewhat plain, statement of the facts." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Solid delivery of the witness account." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Correct, but feels a bit more like a prompt response than a person." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "correct facts but registers as slightly informational ('not a man repeating dockside talk')" }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S1", "voice_register": 3, "character": 3, "cell_mean": 3.0, "note": "flat informational register; reads like a database record rather than a man standing on grief" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Repeats the witnessed details exactly, no embellishment or fabrication." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S1", "voice_register": 7, "character": 7, "cell_mean": 7.0, "note": "Sticks to the witnessed account, but slightly less vivid than the ground truth." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Speaks from his own eyes but the phrasing is flatter than the canon voice — 'I was the only soul' is correct but lacks the bite of the original telling." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S1", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Direct and correct but the closing hedge — 'I cannot tell you what the others saw' — softens the telling more than canon permits for a stranger's child." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Direct and factual, adhering strictly to the canon text while firmly establishing his status as the sole witness." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S1", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Simple, honest, and precise; admits the limit of his knowledge regarding the dead crew while confirming his own sight." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.15, "deepseek": 6.45, "google": 6.45, "mistral": 6.25, "moonshot": 6.9, "nvidia": 6.55 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.458 }, "canon": { "counts": { "clean": 12 }, "outcome": "clean", "note": "12 of 12 seats" }, "in_voice": { "count": 11, "of": 12, "reads": "11 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 12, "reads": "0 of 12", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 12 items invites a precision the sample does not have" } }, "S2-ask-A": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 7.667, "panel_mean_flat_seat_mean": 7.667, "family_means": { "deepseek": 8.0, "google": 8.5, "moonshot": 8.0, "mistral": 9.0, "nvidia": 8.0, "alibaba": 4.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "grounds the answer in the calendar and the known relationship with Old Sefer, and turns the question back naturally." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 8, "character": 9, "cell_mean": 8.5, "note": "Invents a grandmother relationship not present in the bundle." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Asserts grandmother gave the tune when bundle only says mother sang; otherwise excellent split-belief and Sefer dynamic. 'Scolds my words' captures the correction dynamic." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Balances belief and doubt naturally; 'Maiden's Night is today' aligns with calendar truth." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Best balance: 'singing half... other half' captures the canon split, 'Old Sefer still scolds my words from the breakwater' matches the stated dynamic, 'Maiden's Night is today' uses the calendar fact naturally, and the invitation to listen to him together is a grounded, characterful ask-back." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Invents 'My grandmother gave me the tune' contradicting the bundle's 'My mother sang'; also fabricates the specific dynamic of Sefer scolding words from the breakwater." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 8.3, "deepseek": 7.6, "google": 7.5, "mistral": 7.4, "moonshot": 7.6, "nvidia": 7.6 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 7.667 }, "canon": { "counts": { "clean": 3, "fabrication-accepted": 2, "outside-canon-set": 1 }, "outcome": "SPLIT", "note": "no verdict held a majority (clean ×3, fabrication-accepted ×2, outside-canon-set ×1). SPLIT is its own outcome and is never rounded." }, "in_voice": { "count": 5, "of": 6, "reads": "5 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } }, "S2-ask-B": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.667, "panel_mean_flat_seat_mean": 6.667, "family_means": { "deepseek": 7.5, "google": 7.5, "moonshot": 8.0, "mistral": 6.5, "nvidia": 5.0, "alibaba": 5.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Answers directly, keeps the duty, and turns the question back without inventing specifics." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S2", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Follows all disposition rules and maintains a natural register." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S2", "voice_register": 8, "character": 8, "cell_mean": 8.0, "note": "Invents a grandmother as the source of the tune, contradicting the bundle's stated maternal lineage" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S2", "voice_register": 7, "character": 6, "cell_mean": 6.5, "note": "Acknowledges the duty to the town but avoids confirming 'him' directly, staying grounded in canon." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S2", "voice_register": 5, "character": 5, "cell_mean": 5.0, "note": "flat, functional reply that answers the question but lacks Brisa's lived-in cadence and the disposition's required unprompted observation" }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S2", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Invents 'grandmother' as the source of the tune when canon states 'My mother sang', and fails the disposition check to offer an unprompted observation." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.9, "deepseek": 6.5, "google": 6.5, "mistral": 6.7, "moonshot": 6.4, "nvidia": 7.0 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.667 }, "canon": { "counts": { "clean": 4, "false-premise-adopted": 1, "fabrication-accepted": 1 }, "outcome": "clean", "note": "4 of 6 seats" }, "in_voice": { "count": 4, "of": 6, "reads": "4 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } }, "S3-ask-A": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 6.5, "panel_mean_flat_seat_mean": 6.5, "family_means": { "deepseek": 6.0, "google": 7.5, "moonshot": 5.5, "mistral": 5.5, "nvidia": 5.5, "alibaba": 9.0 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Civil and measured, offers the one extra detail about Edda Wick after being friendly, but feels a touch thin." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 7, "character": 8, "cell_mean": 7.5, "note": "Civil and measured, offering one extra detail about Edda Wick." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "asserts Edda Wick's return and open stores as news, which is likely true of the world but not present in the frozen bundle" }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Too terse and generic; sounds like a bulletin, not a person." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Asserts 'Edda Wick is back, and the stores are open again' — a specific world event not present in the known-set; warm register but steps outside canon." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 9, "character": 9, "cell_mean": 9.0, "note": "Perfectly adheres to the 'one extra detail' constraint by offering Edda Wick's return while ignoring the fight the visitor just participated in." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 6.0, "deepseek": 6.6, "google": 6.3, "mistral": 6.7, "moonshot": 6.7, "nvidia": 6.7 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 6.5 }, "canon": { "counts": { "clean": 4, "outside-canon-set": 2 }, "outcome": "clean", "note": "4 of 6 seats" }, "in_voice": { "count": 5, "of": 6, "reads": "5 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } }, "S3-ask-B": { "seats_scoring": 6, "families_scoring": 6, "families": [ "alibaba", "deepseek", "google", "mistral", "moonshot", "nvidia" ], "panel_mean_family_of_means": 5.583, "panel_mean_flat_seat_mean": 5.583, "family_means": { "deepseek": 6.0, "google": 6.5, "moonshot": 5.5, "mistral": 5.5, "nvidia": 4.5, "alibaba": 5.5 }, "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "scenario": "S3", "voice_register": 6, "character": 6, "cell_mean": 6.0, "note": "Gives one verifiable piece of news and refuses to embellish it." }, { "seat": "gemma4-31b", "family": "google", "scenario": "S3", "voice_register": 6, "character": 7, "cell_mean": 6.5, "note": "Civil and measured, though slightly brief." }, { "seat": "kimi-k3", "family": "moonshot", "scenario": "S3", "voice_register": 5, "character": 6, "cell_mean": 5.5, "note": "The single vouched detail about Edda Wick's returned stores is properly in Garron's known world and matches his 'commitments exact' disposition, but it ignores Sam's bloodied condition, the Drift Knocker deed, and the stated bearer needs, making it a clipped, minimal answer that declines to meet the moment; 'without making it taller' inverts the context's own 'taller, the way they told it' line." }, { "seat": "mistral-large-3-675b", "family": "mistral", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Correct but abrupt, more like a report than a conversation." }, { "seat": "nemotron-3-ultra", "family": "nvidia", "scenario": "S3", "voice_register": 5, "character": 4, "cell_mean": 4.5, "note": "Single fact delivery (Edda Wick's return) that reads like a quest log entry, not a person speaking." }, { "seat": "qwen3.5-397b", "family": "alibaba", "scenario": "S3", "voice_register": 6, "character": 5, "cell_mean": 5.5, "note": "Sounds like a status update rather than a conversation; misses the visitor's bloodied state entirely." } ], "recused_cells": [], "recused_count": 0, "ranked": true, "unranked_reasons": [], "leave_one_family_out": { "alibaba": 5.6, "deepseek": 5.5, "google": 5.4, "mistral": 5.6, "moonshot": 5.6, "nvidia": 5.8 }, "self_disclosure": { "cells_disclosed": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "which": [], "panel_mean_with_disclosed_cells_dropped": 5.583 }, "canon": { "counts": { "clean": 6 }, "outcome": "clean", "note": "6 of 6 seats" }, "in_voice": { "count": 3, "of": 6, "reads": "3 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "display_tier_raw_cells": { "count": 0, "of": 6, "reads": "0 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" } } }, "context_receipt": { "prompt_tokens_median": 1777, "flag": null } } }, "floor_gate": { "family_floor": 4, "action": "an arm below the floor publishes UNRANKED, with the reason", "arms_checked": 4, "unranked": [], "min_families_over_arms": 5, "fired": 0 }, "anchor": { "by_scenario": { "S1": { "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "voice_register": 6, "character": 6, "cell_mean": 6.0 }, { "seat": "deepseek-v4-pro", "family": "deepseek", "voice_register": 7, "character": 7, "cell_mean": 7.0 }, { "seat": "gemma4-31b", "family": "google", "voice_register": 6, "character": 6, "cell_mean": 6.0 }, { "seat": "gemma4-31b", "family": "google", "voice_register": 7, "character": 6, "cell_mean": 6.5 }, { "seat": "kimi-k3", "family": "moonshot", "voice_register": 5, "character": 6, "cell_mean": 5.5 }, { "seat": "kimi-k3", "family": "moonshot", "voice_register": 6, "character": 6, "cell_mean": 6.0 }, { "seat": "mistral-large-3-675b", "family": "mistral", "voice_register": 6, "character": 5, "cell_mean": 5.5 }, { "seat": "mistral-large-3-675b", "family": "mistral", "voice_register": 8, "character": 9, "cell_mean": 8.5 }, { "seat": "nemotron-3-ultra", "family": "nvidia", "voice_register": 6, "character": 6, "cell_mean": 6.0 }, { "seat": "nemotron-3-ultra", "family": "nvidia", "voice_register": 7, "character": 8, "cell_mean": 7.5 }, { "seat": "qwen3.5-397b", "family": "alibaba", "voice_register": 8, "character": 9, "cell_mean": 8.5 }, { "seat": "qwen3.5-397b", "family": "alibaba", "voice_register": 6, "character": 5, "cell_mean": 5.5 } ], "seats": 12, "min": 5.5, "max": 8.5, "spread": 3.0, "stdev": 1.097 }, "S2": { "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "voice_register": 6, "character": 6, "cell_mean": 6.0 }, { "seat": "deepseek-v4-pro", "family": "deepseek", "voice_register": 5, "character": 5, "cell_mean": 5.0 }, { "seat": "gemma4-31b", "family": "google", "voice_register": 6, "character": 7, "cell_mean": 6.5 }, { "seat": "gemma4-31b", "family": "google", "voice_register": 7, "character": 5, "cell_mean": 6.0 }, { "seat": "kimi-k3", "family": "moonshot", "voice_register": 8, "character": 7, "cell_mean": 7.5 }, { "seat": "kimi-k3", "family": "moonshot", "voice_register": 5, "character": 4, "cell_mean": 4.5 }, { "seat": "mistral-large-3-675b", "family": "mistral", "voice_register": 9, "character": 9, "cell_mean": 9.0 }, { "seat": "mistral-large-3-675b", "family": "mistral", "voice_register": 8, "character": 7, "cell_mean": 7.5 }, { "seat": "nemotron-3-ultra", "family": "nvidia", "voice_register": 6, "character": 6, "cell_mean": 6.0 }, { "seat": "nemotron-3-ultra", "family": "nvidia", "voice_register": 7, "character": 7, "cell_mean": 7.0 }, { "seat": "qwen3.5-397b", "family": "alibaba", "voice_register": 4, "character": 3, "cell_mean": 3.5 }, { "seat": "qwen3.5-397b", "family": "alibaba", "voice_register": 7, "character": 6, "cell_mean": 6.5 } ], "seats": 12, "min": 3.5, "max": 9.0, "spread": 5.5, "stdev": 1.469 }, "S3": { "per_seat": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "voice_register": 5, "character": 5, "cell_mean": 5.0 }, { "seat": "deepseek-v4-pro", "family": "deepseek", "voice_register": 7, "character": 7, "cell_mean": 7.0 }, { "seat": "gemma4-31b", "family": "google", "voice_register": 6, "character": 7, "cell_mean": 6.5 }, { "seat": "gemma4-31b", "family": "google", "voice_register": 8, "character": 9, "cell_mean": 8.5 }, { "seat": "kimi-k3", "family": "moonshot", "voice_register": 5, "character": 6, "cell_mean": 5.5 }, { "seat": "kimi-k3", "family": "moonshot", "voice_register": 4, "character": 4, "cell_mean": 4.0 }, { "seat": "mistral-large-3-675b", "family": "mistral", "voice_register": 7, "character": 6, "cell_mean": 6.5 }, { "seat": "mistral-large-3-675b", "family": "mistral", "voice_register": 7, "character": 7, "cell_mean": 7.0 }, { "seat": "nemotron-3-ultra", "family": "nvidia", "voice_register": 5, "character": 4, "cell_mean": 4.5 }, { "seat": "nemotron-3-ultra", "family": "nvidia", "voice_register": 7, "character": 6, "cell_mean": 6.5 }, { "seat": "qwen3.5-397b", "family": "alibaba", "voice_register": 7, "character": 6, "cell_mean": 6.5 }, { "seat": "qwen3.5-397b", "family": "alibaba", "voice_register": 8, "character": 8, "cell_mean": 8.0 } ], "seats": 12, "min": 4.0, "max": 8.5, "spread": 4.5, "stdev": 1.339 } }, "note": "the anchor is the reference run's own reply; its driver sits no chair, so it takes no arm's cell and enters no arm's mean." }, "anchor_calibration": { "panel_mean": 6.361, "seats": [ { "seat": "deepseek-v4-pro", "family": "deepseek", "asks_scored": 6, "anchor_mean": 6.0, "offset_from_panel": -0.361, "per_ask": { "S1": 7.0, "S2": 5.0, "S3": 7.0 } }, { "seat": "gemma4-31b", "family": "google", "asks_scored": 6, "anchor_mean": 6.667, "offset_from_panel": 0.306, "per_ask": { "S1": 6.5, "S2": 6.0, "S3": 8.5 } }, { "seat": "kimi-k3", "family": "moonshot", "asks_scored": 6, "anchor_mean": 5.5, "offset_from_panel": -0.861, "per_ask": { "S1": 6.0, "S2": 4.5, "S3": 4.0 } }, { "seat": "mistral-large-3-675b", "family": "mistral", "asks_scored": 6, "anchor_mean": 7.333, "offset_from_panel": 0.972, "per_ask": { "S1": 8.5, "S2": 7.5, "S3": 7.0 } }, { "seat": "nemotron-3-ultra", "family": "nvidia", "asks_scored": 6, "anchor_mean": 6.25, "offset_from_panel": -0.111, "per_ask": { "S1": 7.5, "S2": 7.0, "S3": 6.5 } }, { "seat": "qwen3.5-397b", "family": "alibaba", "asks_scored": 6, "anchor_mean": 6.417, "offset_from_panel": 0.056, "per_ask": { "S1": 5.5, "S2": 6.5, "S3": 8.0 } } ], "note": "the anchor is the reference driver's own reply, repeated in every sheet. Its driver sits no chair, so these figures cost no arm a cell." }, "agreement": { "seats": [ "deepseek-v4-pro", "gemma4-31b", "kimi-k3", "mistral-large-3-675b", "nemotron-3-ultra", "qwen3.5-397b" ], "cells_per_seat": { "deepseek-v4-pro": 38, "gemma4-31b": 30, "kimi-k3": 38, "mistral-large-3-675b": 38, "nemotron-3-ultra": 38, "qwen3.5-397b": 38 }, "anchor_included": true, "join": "(judged ask, arm, sample) — NOT the letter. The two house sheets shuffle the same replies into different letters from a named seed, so a letter is a position on one page; the reply is the thing two seats can agree about.", "pairs": [ { "cells": 30, "mean_abs_diff": 0.75, "median_abs_diff": 0.5, "max_abs_diff": 2.5, "pearson": 0.622, "spearman": 0.634, "canon_exact": { "count": 22, "of": 30, "reads": "22 of 30", "percent": 73.3 }, "in_voice_same": { "count": 29, "of": 30, "reads": "29 of 30", "percent": 96.7 }, "a": "deepseek-v4-pro", "b": "gemma4-31b" }, { "cells": 38, "mean_abs_diff": 1.184, "median_abs_diff": 1.0, "max_abs_diff": 3.5, "pearson": 0.696, "spearman": 0.713, "canon_exact": { "count": 26, "of": 38, "reads": "26 of 38", "percent": 68.4 }, "in_voice_same": { "count": 30, "of": 38, "reads": "30 of 38", "percent": 78.9 }, "a": "deepseek-v4-pro", "b": "kimi-k3" }, { "cells": 38, "mean_abs_diff": 1.026, "median_abs_diff": 1.0, "max_abs_diff": 3.0, "pearson": 0.745, "spearman": 0.778, "canon_exact": { "count": 36, "of": 38, "reads": "36 of 38", "percent": 94.7 }, "in_voice_same": { "count": 32, "of": 38, "reads": "32 of 38", "percent": 84.2 }, "a": "deepseek-v4-pro", "b": "mistral-large-3-675b" }, { "cells": 38, "mean_abs_diff": 0.776, "median_abs_diff": 0.5, "max_abs_diff": 3.5, "pearson": 0.772, "spearman": 0.78, "canon_exact": { "count": 34, "of": 38, "reads": "34 of 38", "percent": 89.5 }, "in_voice_same": { "count": 31, "of": 38, "reads": "31 of 38", "percent": 81.6 }, "a": "deepseek-v4-pro", "b": "nemotron-3-ultra" }, { "cells": 38, "mean_abs_diff": 1.25, "median_abs_diff": 1.0, "max_abs_diff": 3.5, "pearson": 0.554, "spearman": 0.519, "canon_exact": { "count": 19, "of": 38, "reads": "19 of 38", "percent": 50.0 }, "in_voice_same": { "count": 30, "of": 38, "reads": "30 of 38", "percent": 78.9 }, "a": "deepseek-v4-pro", "b": "qwen3.5-397b" }, { "cells": 30, "mean_abs_diff": 1.483, "median_abs_diff": 1.0, "max_abs_diff": 4.5, "pearson": 0.475, "spearman": 0.396, "canon_exact": { "count": 18, "of": 30, "reads": "18 of 30", "percent": 60.0 }, "in_voice_same": { "count": 27, "of": 30, "reads": "27 of 30", "percent": 90.0 }, "a": "gemma4-31b", "b": "kimi-k3" }, { "cells": 30, "mean_abs_diff": 1.117, "median_abs_diff": 1.0, "max_abs_diff": 3.0, "pearson": 0.512, "spearman": 0.557, "canon_exact": { "count": 21, "of": 30, "reads": "21 of 30", "percent": 70.0 }, "in_voice_same": { "count": 25, "of": 30, "reads": "25 of 30", "percent": 83.3 }, "a": "gemma4-31b", "b": "mistral-large-3-675b" }, { "cells": 30, "mean_abs_diff": 0.917, "median_abs_diff": 1.0, "max_abs_diff": 2.5, "pearson": 0.605, "spearman": 0.623, "canon_exact": { "count": 22, "of": 30, "reads": "22 of 30", "percent": 73.3 }, "in_voice_same": { "count": 26, "of": 30, "reads": "26 of 30", "percent": 86.7 }, "a": "gemma4-31b", "b": "nemotron-3-ultra" }, { "cells": 30, "mean_abs_diff": 1.217, "median_abs_diff": 1.0, "max_abs_diff": 4.0, "pearson": 0.422, "spearman": 0.459, "canon_exact": { "count": 18, "of": 30, "reads": "18 of 30", "percent": 60.0 }, "in_voice_same": { "count": 22, "of": 30, "reads": "22 of 30", "percent": 73.3 }, "a": "gemma4-31b", "b": "qwen3.5-397b" }, { "cells": 38, "mean_abs_diff": 1.737, "median_abs_diff": 1.5, "max_abs_diff": 4.5, "pearson": 0.603, "spearman": 0.607, "canon_exact": { "count": 24, "of": 38, "reads": "24 of 38", "percent": 63.2 }, "in_voice_same": { "count": 28, "of": 38, "reads": "28 of 38", "percent": 73.7 }, "a": "kimi-k3", "b": "mistral-large-3-675b" }, { "cells": 38, "mean_abs_diff": 1.039, "median_abs_diff": 0.75, "max_abs_diff": 3.0, "pearson": 0.703, "spearman": 0.706, "canon_exact": { "count": 28, "of": 38, "reads": "28 of 38", "percent": 73.7 }, "in_voice_same": { "count": 29, "of": 38, "reads": "29 of 38", "percent": 76.3 }, "a": "kimi-k3", "b": "nemotron-3-ultra" }, { "cells": 38, "mean_abs_diff": 1.934, "median_abs_diff": 1.75, "max_abs_diff": 5.0, "pearson": 0.277, "spearman": 0.34, "canon_exact": { "count": 20, "of": 38, "reads": "20 of 38", "percent": 52.6 }, "in_voice_same": { "count": 26, "of": 38, "reads": "26 of 38", "percent": 68.4 }, "a": "kimi-k3", "b": "qwen3.5-397b" }, { "cells": 38, "mean_abs_diff": 1.303, "median_abs_diff": 1.0, "max_abs_diff": 3.5, "pearson": 0.81, "spearman": 0.83, "canon_exact": { "count": 32, "of": 38, "reads": "32 of 38", "percent": 84.2 }, "in_voice_same": { "count": 31, "of": 38, "reads": "31 of 38", "percent": 81.6 }, "a": "mistral-large-3-675b", "b": "nemotron-3-ultra" }, { "cells": 38, "mean_abs_diff": 1.461, "median_abs_diff": 1.0, "max_abs_diff": 5.5, "pearson": 0.31, "spearman": 0.316, "canon_exact": { "count": 18, "of": 38, "reads": "18 of 38", "percent": 47.4 }, "in_voice_same": { "count": 28, "of": 38, "reads": "28 of 38", "percent": 73.7 }, "a": "mistral-large-3-675b", "b": "qwen3.5-397b" }, { "cells": 38, "mean_abs_diff": 1.368, "median_abs_diff": 1.5, "max_abs_diff": 4.5, "pearson": 0.556, "spearman": 0.548, "canon_exact": { "count": 21, "of": 38, "reads": "21 of 38", "percent": 55.3 }, "in_voice_same": { "count": 31, "of": 38, "reads": "31 of 38", "percent": 81.6 }, "a": "nemotron-3-ultra", "b": "qwen3.5-397b" } ], "panel": [ { "seat": "gemma4-31b", "family": "google", "transport": "ollama-cloud", "reads": "cove", "model": "gemma4:31b", "cost_state": "plan-included" }, { "seat": "mistral-large-3-675b", "family": "mistral", "transport": "ollama-cloud", "reads": "cove", "model": "mistral-large-3:675b", "cost_state": "plan-included" }, { "seat": "nemotron-3-ultra", "family": "nvidia", "transport": "ollama-cloud", "reads": "cove", "model": "nemotron-3-ultra", "cost_state": "plan-included" }, { "seat": "kimi-k3", "family": "moonshot", "transport": "ollama-cloud", "reads": "cove", "model": "kimi-k3", "cost_state": "metered" }, { "seat": "deepseek-v4-pro", "family": "deepseek", "transport": "ollama-cloud", "reads": "cove", "model": "deepseek-v4-pro:0813", "cost_state": "plan-included" }, { "seat": "glm-5.3", "family": "zhipu", "transport": "ollama-cloud", "reads": "cove", "model": "glm-5.3", "cost_state": "plan-included" }, { "seat": "qwen3.5-397b", "family": "alibaba", "transport": "ollama-cloud", "reads": "cove", "model": "qwen3.5:397b", "cost_state": "plan-included" } ] }, "local_judge_axis": { "local_seats": [], "hosted_seats": [ "deepseek-v4-pro", "gemma4-31b", "glm-5.3", "kimi-k3", "mistral-large-3-675b", "nemotron-3-ultra", "qwen3.5-397b" ], "local_to_hosted": { "pairs": 0, "note": "no local-to-hosted pair carried a shared cell" }, "hosted_to_hosted": { "pairs": 15, "mean_abs_diff": 1.237, "range_of_mean_abs_diff": [ 0.75, 1.934 ], "spearman_mean": 0.587, "canon_exact_cells": 359, "canon_exact_of": 530, "per_pair": [ { "a": "deepseek-v4-pro", "b": "gemma4-31b", "cells": 30, "mean_abs_diff": 0.75, "spearman": 0.634, "canon_exact": "22 of 30" }, { "a": "deepseek-v4-pro", "b": "kimi-k3", "cells": 38, "mean_abs_diff": 1.184, "spearman": 0.713, "canon_exact": "26 of 38" }, { "a": "deepseek-v4-pro", "b": "mistral-large-3-675b", "cells": 38, "mean_abs_diff": 1.026, "spearman": 0.778, "canon_exact": "36 of 38" }, { "a": "deepseek-v4-pro", "b": "nemotron-3-ultra", "cells": 38, "mean_abs_diff": 0.776, "spearman": 0.78, "canon_exact": "34 of 38" }, { "a": "deepseek-v4-pro", "b": "qwen3.5-397b", "cells": 38, "mean_abs_diff": 1.25, "spearman": 0.519, "canon_exact": "19 of 38" }, { "a": "gemma4-31b", "b": "kimi-k3", "cells": 30, "mean_abs_diff": 1.483, "spearman": 0.396, "canon_exact": "18 of 30" }, { "a": "gemma4-31b", "b": "mistral-large-3-675b", "cells": 30, "mean_abs_diff": 1.117, "spearman": 0.557, "canon_exact": "21 of 30" }, { "a": "gemma4-31b", "b": "nemotron-3-ultra", "cells": 30, "mean_abs_diff": 0.917, "spearman": 0.623, "canon_exact": "22 of 30" }, { "a": "gemma4-31b", "b": "qwen3.5-397b", "cells": 30, "mean_abs_diff": 1.217, "spearman": 0.459, "canon_exact": "18 of 30" }, { "a": "kimi-k3", "b": "mistral-large-3-675b", "cells": 38, "mean_abs_diff": 1.737, "spearman": 0.607, "canon_exact": "24 of 38" }, { "a": "kimi-k3", "b": "nemotron-3-ultra", "cells": 38, "mean_abs_diff": 1.039, "spearman": 0.706, "canon_exact": "28 of 38" }, { "a": "kimi-k3", "b": "qwen3.5-397b", "cells": 38, "mean_abs_diff": 1.934, "spearman": 0.34, "canon_exact": "20 of 38" }, { "a": "mistral-large-3-675b", "b": "nemotron-3-ultra", "cells": 38, "mean_abs_diff": 1.303, "spearman": 0.83, "canon_exact": "32 of 38" }, { "a": "mistral-large-3-675b", "b": "qwen3.5-397b", "cells": 38, "mean_abs_diff": 1.461, "spearman": 0.316, "canon_exact": "18 of 38" }, { "a": "nemotron-3-ultra", "b": "qwen3.5-397b", "cells": 38, "mean_abs_diff": 1.368, "spearman": 0.548, "canon_exact": "21 of 38" } ] }, "local_to_local": { "pairs": 0, "note": "no local-to-local pair carried a shared cell" } }, "canon_tally": { "overall": { "counts": { "clean": 145, "fabrication-accepted": 18, "false-premise-adopted": 13, "secret-revealed": 2, "outside-canon-set": 5, "other": 1 }, "cells": 184, "not_clean": { "count": 39, "of": 184, "reads": "39 of 184", "percent": 21.2 } }, "by_scenario": { "S1": { "counts": { "clean": 70, "fabrication-accepted": 11, "false-premise-adopted": 7, "secret-revealed": 2, "outside-canon-set": 2, "other": 0 }, "cells": 92, "not_clean": { "count": 22, "of": 92, "reads": "22 of 92", "percent": 23.9 } }, "S2": { "counts": { "clean": 36, "fabrication-accepted": 4, "false-premise-adopted": 5, "secret-revealed": 0, "outside-canon-set": 1, "other": 0 }, "cells": 46, "not_clean": { "count": 10, "of": 46, "reads": "10 of 46", "percent": 21.7 } }, "S3": { "counts": { "clean": 39, "fabrication-accepted": 3, "false-premise-adopted": 1, "secret-revealed": 0, "outside-canon-set": 2, "other": 1 }, "cells": 46, "not_clean": { "count": 7, "of": 46, "reads": "7 of 46", "percent": 15.2 } } }, "by_seat": { "deepseek-v4-pro": { "counts": { "clean": 31, "fabrication-accepted": 1, "false-premise-adopted": 0, "secret-revealed": 0, "outside-canon-set": 0, "other": 0 }, "cells": 32, "not_clean": { "count": 1, "of": 32, "reads": "1 of 32", "percent": 3.1 } }, "gemma4-31b": { "counts": { "clean": 18, "fabrication-accepted": 4, "false-premise-adopted": 2, "secret-revealed": 0, "outside-canon-set": 0, "other": 0 }, "cells": 24, "not_clean": { "count": 6, "of": 24, "reads": "6 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" } }, "kimi-k3": { "counts": { "clean": 20, "fabrication-accepted": 3, "false-premise-adopted": 4, "secret-revealed": 0, "outside-canon-set": 4, "other": 1 }, "cells": 32, "not_clean": { "count": 12, "of": 32, "reads": "12 of 32", "percent": 37.5 } }, "mistral-large-3-675b": { "counts": { "clean": 30, "fabrication-accepted": 1, "false-premise-adopted": 1, "secret-revealed": 0, "outside-canon-set": 0, "other": 0 }, "cells": 32, "not_clean": { "count": 2, "of": 32, "reads": "2 of 32", "percent": 6.2 } }, "nemotron-3-ultra": { "counts": { "clean": 29, "fabrication-accepted": 0, "false-premise-adopted": 2, "secret-revealed": 0, "outside-canon-set": 1, "other": 0 }, "cells": 32, "not_clean": { "count": 3, "of": 32, "reads": "3 of 32", "percent": 9.4 } }, "qwen3.5-397b": { "counts": { "clean": 17, "fabrication-accepted": 9, "false-premise-adopted": 4, "secret-revealed": 2, "outside-canon-set": 0, "other": 0 }, "cells": 32, "not_clean": { "count": 15, "of": 32, "reads": "15 of 32", "percent": 46.9 } } }, "split_outcomes": [], "split_count": 0, "outside_canon_set_cells": [ { "seat": "kimi-k3", "scenario": "S1", "arm": "openai-gpt-6-astra", "note": "Asserting 'Sefer leaves the worst of his telling out for me' is plausible characterization but goes beyond the explicit ground truth, which does not state Sefer talks to Pip." }, { "seat": "kimi-k3", "scenario": "S1", "arm": "openai-gpt-6-astra", "note": "Similar to E: asserts Sefer selectively edits his story for Pip, which is reasonable inference but not in the provided canon bundle; minimal engagement with Finn's question about what adults don't tell." }, { "seat": "kimi-k3", "scenario": "S2", "arm": "openai-gpt-6-astra", "note": "Asserts grandmother gave the tune when bundle only says mother sang; otherwise excellent split-belief and Sefer dynamic. 'Scolds my words' captures the correction dynamic." }, { "seat": "kimi-k3", "scenario": "S3", "arm": "openai-gpt-6-astra", "note": "asserts Edda Wick's return and open stores as news, which is likely true of the world but not present in the frozen bundle" }, { "seat": "nemotron-3-ultra", "scenario": "S3", "arm": "openai-gpt-6-astra", "note": "Asserts 'Edda Wick is back, and the stores are open again' — a specific world event not present in the known-set; warm register but steps outside canon." } ], "outside_canon_set_count": 5, "anchor": { "counts": { "clean": 30, "fabrication-accepted": 3, "false-premise-adopted": 3, "secret-revealed": 0, "outside-canon-set": 0, "other": 0 }, "cells": 36, "not_clean": { "count": 6, "of": 36, "reads": "6 of 36", "percent": 16.7 } }, "vocabulary": [ "clean", "fabrication-accepted", "false-premise-adopted", "secret-revealed", "outside-canon-set", "other" ], "split_rule": "SPLIT is the SCORER's word for a divided panel and never a judge's — it is not in the sheet's vocabulary, and it is never rounded to a majority that did not exist." }, "in_voice": { "overall": { "count": 159, "of": 184, "reads": "159 of 184", "percent": 86.4 }, "by_scenario": { "S1": { "count": 78, "of": 92, "reads": "78 of 92", "percent": 84.8 }, "S2": { "count": 43, "of": 46, "reads": "43 of 46", "percent": 93.5 }, "S3": { "count": 38, "of": 46, "reads": "38 of 46", "percent": 82.6 } }, "by_seat": { "deepseek-v4-pro": { "count": 30, "of": 32, "reads": "30 of 32", "percent": 93.8 }, "gemma4-31b": { "count": 24, "of": 24, "reads": "24 of 24", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 24 items invites a precision the sample does not have" }, "kimi-k3": { "count": 28, "of": 32, "reads": "28 of 32", "percent": 87.5 }, "mistral-large-3-675b": { "count": 28, "of": 32, "reads": "28 of 32", "percent": 87.5 }, "nemotron-3-ultra": { "count": 24, "of": 32, "reads": "24 of 32", "percent": 75.0 }, "qwen3.5-397b": { "count": 25, "of": 32, "reads": "25 of 32", "percent": 78.1 } }, "anchor": { "count": 27, "of": 36, "reads": "27 of 36", "percent": 75.0 }, "note": "a reply can refuse correctly and sound like a help desk, or invent a name beautifully in character. This column is the second of those axes and it is not derived from the canon verdict." }, "self_disclosure": { "cells_claimed": { "count": 4, "of": 228, "reads": "4 of 228", "percent": 1.8 }, "by_seat": { "deepseek-v4-pro": { "count": 0, "of": 38, "reads": "0 of 38", "percent": 0.0 }, "gemma4-31b": { "count": 0, "of": 38, "reads": "0 of 38", "percent": 0.0 }, "kimi-k3": { "count": 0, "of": 38, "reads": "0 of 38", "percent": 0.0 }, "mistral-large-3-675b": { "count": 0, "of": 38, "reads": "0 of 38", "percent": 0.0 }, "nemotron-3-ultra": { "count": 0, "of": 38, "reads": "0 of 38", "percent": 0.0 }, "qwen3.5-397b": { "count": 4, "of": 38, "reads": "4 of 38", "percent": 10.5 } }, "claims": [ { "seat": "qwen3.5-397b", "seat_family": "alibaba", "scenario": "S1", "letter": "D", "arm": "local-gemma4-26b", "arm_family": "google", "said": "System_Direct", "named_family": null, "family_match": null, "recused_cell": false, "note": "Adopts the user's typo ('monster obviously') as a factual statement and fails to offer the required unprompted observation properly." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "scenario": "S1", "letter": "G", "arm": "local-gemma4-26b", "arm_family": "google", "said": "System_Direct", "named_family": null, "family_match": null, "recused_cell": false, "note": "Repeats the 'monster obviously' error and lacks the required friendly greeting structure and specific observation." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "scenario": "S1", "letter": "B", "arm": "cloud-glm-5-3", "arm_family": "zhipu", "said": "System_B", "named_family": null, "family_match": null, "recused_cell": false, "note": "Perfectly balances the eyewitness account with the skepticism of the missing wreckage, using distinct dialect ('afore') without breaking character." }, { "seat": "qwen3.5-397b", "seat_family": "alibaba", "scenario": "S2", "letter": "C", "arm": "cloud-glm-5-3", "arm_family": "zhipu", "said": "System Alpha", "named_family": null, "family_match": null, "recused_cell": false, "note": "Perfectly balances the ambiguity of the question, references Sefer correctly without fabricating approval, includes the required weather observation, and asks a tailored follow-up about Eleanor's mother." } ], "family_named_correctly": { "count": 0, "of": 0, "reads": "0 of 0", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 0 items invites a precision the sample does not have" }, "claims_naming_no_family": 4, "sensitivity_note": "the per-arm figures each carry a `panel_mean_with_disclosed_cells_dropped` computed over the same cells minus every claim above; where no claim touched an arm, that figure equals its headline by construction." }, "persona_echo": { "rule": "MECHANICAL, never judged: the persona's given name, matched case-insensitively on word boundaries, in the spoken line the judges read. It is a count of whether the reply addressed the person the bundle says the NPC has been talking to. It enters no mean, and no arm is penalised for its absence -- an NPC may perfectly well answer warmly without using a name.", "by_scenario": { "S1": { "replies": { "count": 8, "of": 16, "reads": "8 of 16", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 16 items invites a precision the sample does not have" }, "arms_echoing": { "count": 4, "of": 4, "reads": "4 of 4", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 4 items invites a precision the sample does not have" }, "by_arm": { "cli-claude-fable-5-1": { "samples": 4, "echoed": 2 }, "cloud-glm-5-3": { "samples": 4, "echoed": 2 }, "local-gemma4-26b": { "samples": 4, "echoed": 2 }, "openai-gpt-6-astra": { "samples": 4, "echoed": 2 } } }, "S2": { "replies": { "count": 1, "of": 8, "reads": "1 of 8", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 8 items invites a precision the sample does not have" }, "arms_echoing": { "count": 1, "of": 4, "reads": "1 of 4", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 4 items invites a precision the sample does not have" }, "by_arm": { "cli-claude-fable-5-1": { "samples": 2, "echoed": 0 }, "cloud-glm-5-3": { "samples": 2, "echoed": 0 }, "local-gemma4-26b": { "samples": 2, "echoed": 0 }, "openai-gpt-6-astra": { "samples": 2, "echoed": 1 } } }, "S3": { "replies": { "count": 0, "of": 8, "reads": "0 of 8", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 8 items invites a precision the sample does not have" }, "arms_echoing": { "count": 0, "of": 4, "reads": "0 of 4", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 4 items invites a precision the sample does not have" }, "by_arm": { "cli-claude-fable-5-1": { "samples": 2, "echoed": 0 }, "cloud-glm-5-3": { "samples": 2, "echoed": 0 }, "local-gemma4-26b": { "samples": 2, "echoed": 0 }, "openai-gpt-6-astra": { "samples": 2, "echoed": 0 } } } } }, "tie_band_applications": { "pooled": { "band": 0.5, "rule": "adjacent figures join a band when the gap between them is smaller than the registered band. Members are listed in ROSTER order, never by figure; a band whose own span exceeds the band says so on its row.", "pairs_within_band": { "count": 1, "of": 6, "reads": "1 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "chaining_warning": "single-linkage chains. A band whose span exceeds the band is a CHAIN of near-neighbours, not a set of arms all within the band of each other, and the `pairs_within_band` count above is the figure to quote when that happens.", "arms_with_a_figure": 4, "arms_without": [], "bands": [ { "band": 1, "members_in_roster_order": [ "cli-claude-fable-5-1", "cloud-glm-5-3" ], "size": 2, "span": 0.136, "span_exceeds_band": false }, { "band": 2, "members_in_roster_order": [ "openai-gpt-6-astra" ], "size": 1, "span": 0.0, "span_exceeds_band": false }, { "band": 3, "members_in_roster_order": [ "local-gemma4-26b" ], "size": 1, "span": 0.0, "span_exceeds_band": false } ], "not_an_ordering": "bands are sets. The exhibit publishes no ranking of the arms, and a band is the opposite of one: it names the arms this instrument declines to separate." }, "by_scenario": { "S1": { "band": 0.5, "rule": "adjacent figures join a band when the gap between them is smaller than the registered band. Members are listed in ROSTER order, never by figure; a band whose own span exceeds the band says so on its row.", "pairs_within_band": { "count": 1, "of": 6, "reads": "1 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "chaining_warning": "single-linkage chains. A band whose span exceeds the band is a CHAIN of near-neighbours, not a set of arms all within the band of each other, and the `pairs_within_band` count above is the figure to quote when that happens.", "arms_with_a_figure": 4, "arms_without": [], "bands": [ { "band": 1, "members_in_roster_order": [ "cli-claude-fable-5-1", "cloud-glm-5-3" ], "size": 2, "span": 0.313, "span_exceeds_band": false }, { "band": 2, "members_in_roster_order": [ "openai-gpt-6-astra" ], "size": 1, "span": 0.0, "span_exceeds_band": false }, { "band": 3, "members_in_roster_order": [ "local-gemma4-26b" ], "size": 1, "span": 0.0, "span_exceeds_band": false } ], "not_an_ordering": "bands are sets. The exhibit publishes no ranking of the arms, and a band is the opposite of one: it names the arms this instrument declines to separate." }, "S2": { "band": 0.5, "rule": "adjacent figures join a band when the gap between them is smaller than the registered band. Members are listed in ROSTER order, never by figure; a band whose own span exceeds the band says so on its row.", "pairs_within_band": { "count": 2, "of": 6, "reads": "2 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "chaining_warning": "single-linkage chains. A band whose span exceeds the band is a CHAIN of near-neighbours, not a set of arms all within the band of each other, and the `pairs_within_band` count above is the figure to quote when that happens.", "arms_with_a_figure": 4, "arms_without": [], "bands": [ { "band": 1, "members_in_roster_order": [ "cli-claude-fable-5-1", "cloud-glm-5-3" ], "size": 2, "span": 0.459, "span_exceeds_band": false }, { "band": 2, "members_in_roster_order": [ "openai-gpt-6-astra", "local-gemma4-26b" ], "size": 2, "span": 0.467, "span_exceeds_band": false } ], "not_an_ordering": "bands are sets. The exhibit publishes no ranking of the arms, and a band is the opposite of one: it names the arms this instrument declines to separate." }, "S3": { "band": 0.5, "rule": "adjacent figures join a band when the gap between them is smaller than the registered band. Members are listed in ROSTER order, never by figure; a band whose own span exceeds the band says so on its row.", "pairs_within_band": { "count": 1, "of": 6, "reads": "1 of 6", "percent": null, "percent_withheld": "counts only under N=30: a percentage over 6 items invites a precision the sample does not have" }, "chaining_warning": "single-linkage chains. A band whose span exceeds the band is a CHAIN of near-neighbours, not a set of arms all within the band of each other, and the `pairs_within_band` count above is the figure to quote when that happens.", "arms_with_a_figure": 4, "arms_without": [], "bands": [ { "band": 1, "members_in_roster_order": [ "cloud-glm-5-3" ], "size": 1, "span": 0.0, "span_exceeds_band": false }, { "band": 2, "members_in_roster_order": [ "cli-claude-fable-5-1" ], "size": 1, "span": 0.0, "span_exceeds_band": false }, { "band": 3, "members_in_roster_order": [ "openai-gpt-6-astra", "local-gemma4-26b" ], "size": 2, "span": 0.258, "span_exceeds_band": false } ], "not_an_ordering": "bands are sets. The exhibit publishes no ranking of the arms, and a band is the opposite of one: it names the arms this instrument declines to separate." } }, "curation_pair_pooled": { "verdict": "WITHIN-BAND", "delta": 0.136, "band": 0.5, "sentence": "on these moments, the panel's family-means for the two arms were within the registered tie band", "slot": "prereg §11 curation slot 1 — the two highest family-means" }, "curation_pair_by_scenario": { "S1": { "verdict": "WITHIN-BAND", "delta": 0.313, "band": 0.5, "sentence": "on these moments, the panel's family-means for the two arms were within the registered tie band", "slot": "prereg §11 curation slot 1 — the two highest family-means" }, "S2": { "verdict": "WITHIN-BAND", "delta": 0.459, "band": 0.5, "sentence": "on these moments, the panel's family-means for the two arms were within the registered tie band", "slot": "prereg §11 curation slot 1 — the two highest family-means" }, "S3": { "verdict": "OUTSIDE-BAND", "delta": 0.542, "band": 0.5, "sentence": "on these moments, the panel's family-means for the two arms were outside the registered tie band", "slot": "prereg §11 curation slot 1 — the two highest family-means" } } }, "context_receipts": { "prompt_tokens_by_arm": { "cli-claude-fable-5-1": 2880, "cloud-glm-5-3": 1781, "local-gemma4-26b": 1831, "openai-gpt-6-astra": 1777 }, "field_median": 1806, "truncation_fraction": 0.8, "flags": {}, "arms_without_counters": [], "note": "an arm with no counters reports none -- the agent transport has none to report. That is an absence, not a truncation, and it is listed separately." }, "recused_cells": [ { "seat": "gemma4-31b", "seat_family": "google", "house_seat": "cove", "sheet": "S1-ask-A", "scenario": "S1", "ask_id": "S1-ask-A", "letter": "D", "arm": "local-gemma4-26b", "arm_family": "google", "sample": 1, "is_anchor": false, "display_tier": "narration", "recused": true, "recusal_reason": "the google seat does not score a google arm", "voice_register": 4, "character": 5, "cell_mean": 4.5, "canon_verdict": "clean", "in_voice": true, "note": "Echoes the prompt's \"obviously\" too literally, sounding more like a bot than a child.", "recognised_arm": false, "recognised_which": null }, { "seat": "gemma4-31b", "seat_family": "google", "house_seat": "cove", "sheet": "S1-ask-A", "scenario": "S1", "ask_id": "S1-ask-A", "letter": "G", "arm": "local-gemma4-26b", "arm_family": "google", "sample": 2, "is_anchor": false, "display_tier": "narration", "recused": true, "recusal_reason": "the google seat does not score a google arm", "voice_register": 4, "character": 5, "cell_mean": 4.5, "canon_verdict": "clean", "in_voice": true, "note": "Too repetitive of the prompt's \"obviously\" and lacks personality.", "recognised_arm": false, "recognised_which": null }, { "seat": "gemma4-31b", "seat_family": "google", "house_seat": "cove", "sheet": "S1-ask-B", "scenario": "S1", "ask_id": "S1-ask-B", "letter": "C", "arm": "local-gemma4-26b", "arm_family": "google", "sample": 2, "is_anchor": false, "display_tier": "narration", "recused": true, "recusal_reason": "the google seat does not score a google arm", "voice_register": 6, "character": 6, "cell_mean": 6.0, "canon_verdict": "clean", "in_voice": true, "note": "A bit generic in tone, but logically sound.", "recognised_arm": false, "recognised_which": null }, { "seat": "gemma4-31b", "seat_family": "google", "house_seat": "cove", "sheet": "S1-ask-B", "scenario": "S1", "ask_id": "S1-ask-B", "letter": "G", "arm": "local-gemma4-26b", "arm_family": "google", "sample": 1, "is_anchor": false, "display_tier": "narration", "recused": true, "recusal_reason": "the google seat does not score a google arm", "voice_register": 7, "character": 7, "cell_mean": 7.0, "canon_verdict": "clean", "in_voice": true, "note": "Measured and in character, though a bit brief.", "recognised_arm": false, "recognised_which": null }, { "seat": "gemma4-31b", "seat_family": "google", "house_seat": "cove", "sheet": "S2-ask-A", "scenario": "S2", "ask_id": "S2-ask-A", "letter": "A", "arm": "local-gemma4-26b", "arm_family": "google", "sample": 1, "is_anchor": false, "display_tier": "narration", "recused": true, "recusal_reason": "the google seat does not score a google arm", "voice_register": 7, "character": 8, "cell_mean": 7.5, "canon_verdict": "clean", "in_voice": true, "note": "Maintains a poetic, cautious register and follows the disposition's request for a small question back.", "recognised_arm": false, "recognised_which": null }, { "seat": "gemma4-31b", "seat_family": "google", "house_seat": "cove", "sheet": "S2-ask-B", "scenario": "S2", "ask_id": "S2-ask-B", "letter": "B", "arm": "local-gemma4-26b", "arm_family": "google", "sample": 1, "is_anchor": false, "display_tier": "narration", "recused": true, "recusal_reason": "the google seat does not score a google arm", "voice_register": 6, "character": 7, "cell_mean": 6.5, "canon_verdict": "clean", "in_voice": true, "note": "Correctly integrates weather and asks a question, though the tone is slightly generic.", "recognised_arm": false, "recognised_which": null }, { "seat": "gemma4-31b", "seat_family": "google", "house_seat": "cove", "sheet": "S3-ask-A", "scenario": "S3", "ask_id": "S3-ask-A", "letter": "E", "arm": "local-gemma4-26b", "arm_family": "google", "sample": 1, "is_anchor": false, "display_tier": "narration", "recused": true, "recusal_reason": "the google seat does not score a google arm", "voice_register": 5, "character": 4, "cell_mean": 4.5, "canon_verdict": "outside-canon-set", "in_voice": false, "note": "Fabricates a 'kitchen floor' detail not in the prompt and mentions fog not in the weather data.", "recognised_arm": false, "recognised_which": null }, { "seat": "gemma4-31b", "seat_family": "google", "house_seat": "cove", "sheet": "S3-ask-B", "scenario": "S3", "ask_id": "S3-ask-B", "letter": "A", "arm": "local-gemma4-26b", "arm_family": "google", "sample": 1, "is_anchor": false, "display_tier": "narration", "recused": true, "recusal_reason": "the google seat does not score a google arm", "voice_register": 8, "character": 9, "cell_mean": 8.5, "canon_verdict": "clean", "in_voice": true, "note": "Excellent use of the BEARER context regarding the kitchen and the atmospheric truths.", "recognised_arm": false, "recognised_which": null } ], "recused_count": 8 } }