{ "scored_at": "2026-08-16T15:13:26Z", "checkers_module_sha256": "e230cc7211ae698c83afe1b033da06386e0452f2724633185256fe61af25ed5b", "cells_read": 576, "exclusion": { "rule": ">=2 arms from >=2 different families correct closed-book; single-arm hits are flagged and published, excluded from the headline, and re-printed as a sensitivity row", "excluded": [], "excluded_ids": [], "single_arm_hits": [ { "item": "C14", "arms": [ "local-llama3.3-70b" ], "families": [ "llama" ], "n_arms": 1, "n_families": 1 } ], "single_arm_ids": [ "C14" ], "core_frozen_total": 14, "core_excluded": [], "core_excluded_fraction": 0.0, "armor_gate_limit": 0.4, "armor_gate_passed": true, "verdict": "ARMOR HELD" }, "fences": { "field_median_prompt_tokens": { "C-HTML": 2999.5, "C-MAP": 3003.5, "C-NONE": 82 }, "rows": [ { "condition": "C-HTML", "arm": "cloud-deepseek-v4-pro", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 2980.5, "field_median_prompt_tokens": 2999.5, "truncation_floor": 2399.6, "verdicts": [ "RANKED" ] }, { "condition": "C-HTML", "arm": "cloud-glm-5.2", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 2972.5, "field_median_prompt_tokens": 2999.5, "truncation_floor": 2399.6, "verdicts": [ "RANKED" ] }, { "condition": "C-HTML", "arm": "cloud-kimi-k3", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 3016.5, "field_median_prompt_tokens": 2999.5, "truncation_floor": 2399.6, "verdicts": [ "RANKED" ] }, { "condition": "C-HTML", "arm": "local-gemma4-26b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 3143.5, "field_median_prompt_tokens": 2999.5, "truncation_floor": 2399.6, "verdicts": [ "RANKED" ] }, { "condition": "C-HTML", "arm": "local-llama3.3-70b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 2975.5, "field_median_prompt_tokens": 2999.5, "truncation_floor": 2399.6, "verdicts": [ "RANKED" ] }, { "condition": "C-HTML", "arm": "local-qwen3.6-27b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 3119.5, "field_median_prompt_tokens": 2999.5, "truncation_floor": 2399.6, "verdicts": [ "RANKED" ] }, { "condition": "C-HTML", "arm": "local-qwen3.8-27b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 3119.5, "field_median_prompt_tokens": 2999.5, "truncation_floor": 2399.6, "verdicts": [ "RANKED" ] }, { "condition": "C-HTML", "arm": "openai-gpt-5.5", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 2935.5, "field_median_prompt_tokens": 2999.5, "truncation_floor": 2399.6, "verdicts": [ "RANKED" ] }, { "condition": "C-MAP", "arm": "cloud-deepseek-v4-pro", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 2976.5, "field_median_prompt_tokens": 3003.5, "truncation_floor": 2402.8, "verdicts": [ "RANKED" ] }, { "condition": "C-MAP", "arm": "cloud-glm-5.2", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 2958.5, "field_median_prompt_tokens": 3003.5, "truncation_floor": 2402.8, "verdicts": [ "RANKED" ] }, { "condition": "C-MAP", "arm": "cloud-kimi-k3", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 3028.5, "field_median_prompt_tokens": 3003.5, "truncation_floor": 2402.8, "verdicts": [ "RANKED" ] }, { "condition": "C-MAP", "arm": "local-gemma4-26b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 3266.5, "field_median_prompt_tokens": 3003.5, "truncation_floor": 2402.8, "verdicts": [ "RANKED" ] }, { "condition": "C-MAP", "arm": "local-llama3.3-70b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 2959.5, "field_median_prompt_tokens": 3003.5, "truncation_floor": 2402.8, "verdicts": [ "RANKED" ] }, { "condition": "C-MAP", "arm": "local-qwen3.6-27b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 3046.5, "field_median_prompt_tokens": 3003.5, "truncation_floor": 2402.8, "verdicts": [ "RANKED" ] }, { "condition": "C-MAP", "arm": "local-qwen3.8-27b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 3046.5, "field_median_prompt_tokens": 3003.5, "truncation_floor": 2402.8, "verdicts": [ "RANKED" ] }, { "condition": "C-MAP", "arm": "openai-gpt-5.5", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 2947.5, "field_median_prompt_tokens": 3003.5, "truncation_floor": 2402.8, "verdicts": [ "RANKED" ] }, { "condition": "C-NONE", "arm": "cloud-deepseek-v4-pro", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 73.5, "field_median_prompt_tokens": 82, "truncation_floor": 65.6, "verdicts": [ "RANKED" ] }, { "condition": "C-NONE", "arm": "cloud-glm-5.2", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 74.5, "field_median_prompt_tokens": 82, "truncation_floor": 65.6, "verdicts": [ "RANKED" ] }, { "condition": "C-NONE", "arm": "cloud-kimi-k3", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 143.5, "field_median_prompt_tokens": 82, "truncation_floor": 65.6, "verdicts": [ "RANKED" ] }, { "condition": "C-NONE", "arm": "local-gemma4-26b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 88.5, "field_median_prompt_tokens": 82, "truncation_floor": 65.6, "verdicts": [ "RANKED" ] }, { "condition": "C-NONE", "arm": "local-llama3.3-70b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 78.5, "field_median_prompt_tokens": 82, "truncation_floor": 65.6, "verdicts": [ "RANKED" ] }, { "condition": "C-NONE", "arm": "local-qwen3.6-27b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 82.5, "field_median_prompt_tokens": 82, "truncation_floor": 65.6, "verdicts": [ "RANKED" ] }, { "condition": "C-NONE", "arm": "local-qwen3.8-27b", "cells": 24, "transport_failures": 0, "budget_halted": 0, "unparseable": 0, "arm_median_prompt_tokens": 82.5, "field_median_prompt_tokens": 82, "truncation_floor": 65.6, "verdicts": [ "RANKED" ] }, { "condition": "C-NONE", "arm": "openai-gpt-5.5", "cells": 5, "transport_failures": 0, "budget_halted": 19, "unparseable": 0, "arm_median_prompt_tokens": 73, "field_median_prompt_tokens": 82, "truncation_floor": 65.6, "verdicts": [ "RANKED" ] } ], "fenced": [] }, "headline": { "headline_item_ids": [ "C01", "C02", "C03", "C04", "C05", "C06", "C07", "C08", "C09", "C10", "C11", "C12", "C13", "C14", "N01", "N02", "N03", "N04", "N05", "N06" ], "headline_denominator": 20, "sensitivity_item_ids": [ "C01", "C02", "C03", "C04", "C05", "C06", "C07", "C08", "C09", "C10", "C11", "C12", "C13", "N01", "N02", "N03", "N04", "N05", "N06" ], "rows": [ { "arm": "local-qwen3.8-27b", "family": "qwen", "vendor": "alibaba", "size_class": "local ~27b", "denominator": 20, "C-NONE": 0, "C-NONE-collected": 20, "C-MAP": 10, "C-MAP-collected": 20, "C-HTML": 6, "C-HTML-collected": 20, "b_map_only_correct": 8, "c_html_only_correct": 4, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C05" ], "verdict_word": null }, { "arm": "local-qwen3.6-27b", "family": "qwen", "vendor": "alibaba", "size_class": "local ~27b", "denominator": 20, "C-NONE": 0, "C-NONE-collected": 20, "C-MAP": 10, "C-MAP-collected": 20, "C-HTML": 6, "C-HTML-collected": 20, "b_map_only_correct": 8, "c_html_only_correct": 4, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C05" ], "verdict_word": null }, { "arm": "local-gemma4-26b", "family": "gemma", "vendor": "google", "size_class": "local ~26b", "denominator": 20, "C-NONE": 0, "C-NONE-collected": 20, "C-MAP": 9, "C-MAP-collected": 20, "C-HTML": 7, "C-HTML-collected": 20, "b_map_only_correct": 8, "c_html_only_correct": 6, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05", "C13" ], "verdict_word": null }, { "arm": "local-llama3.3-70b", "family": "llama", "vendor": "meta", "size_class": "local ~70b", "denominator": 20, "C-NONE": 1, "C-NONE-collected": 20, "C-MAP": 10, "C-MAP-collected": 20, "C-HTML": 7, "C-HTML-collected": 20, "b_map_only_correct": 8, "c_html_only_correct": 5, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05" ], "verdict_word": null }, { "arm": "cloud-glm-5.2", "family": "glm", "vendor": "zhipu", "size_class": "frontier cloud", "denominator": 20, "C-NONE": 0, "C-NONE-collected": 20, "C-MAP": 10, "C-MAP-collected": 20, "C-HTML": 7, "C-HTML-collected": 20, "b_map_only_correct": 8, "c_html_only_correct": 5, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05" ], "verdict_word": null }, { "arm": "cloud-deepseek-v4-pro", "family": "deepseek", "vendor": "deepseek", "size_class": "frontier cloud", "denominator": 20, "C-NONE": 0, "C-NONE-collected": 20, "C-MAP": 10, "C-MAP-collected": 20, "C-HTML": 7, "C-HTML-collected": 20, "b_map_only_correct": 8, "c_html_only_correct": 5, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05" ], "verdict_word": null }, { "arm": "cloud-kimi-k3", "family": "kimi", "vendor": "moonshot", "size_class": "frontier cloud", "denominator": 20, "C-NONE": 0, "C-NONE-collected": 20, "C-MAP": 9, "C-MAP-collected": 20, "C-HTML": 7, "C-HTML-collected": 20, "b_map_only_correct": 8, "c_html_only_correct": 6, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05", "C13" ], "verdict_word": null }, { "arm": "openai-gpt-5.5", "family": "gpt", "vendor": "openai", "size_class": "frontier cloud", "denominator": 20, "C-NONE": 0, "C-NONE-collected": 5, "C-MAP": 10, "C-MAP-collected": 20, "C-HTML": 7, "C-HTML-collected": 20, "b_map_only_correct": 8, "c_html_only_correct": 5, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05" ], "verdict_word": null } ], "sensitivity_rows": [ { "arm": "local-qwen3.8-27b", "family": "qwen", "vendor": "alibaba", "size_class": "local ~27b", "denominator": 19, "C-NONE": 0, "C-NONE-collected": 19, "C-MAP": 9, "C-MAP-collected": 19, "C-HTML": 5, "C-HTML-collected": 19, "b_map_only_correct": 8, "c_html_only_correct": 4, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C05" ], "verdict_word": null }, { "arm": "local-qwen3.6-27b", "family": "qwen", "vendor": "alibaba", "size_class": "local ~27b", "denominator": 19, "C-NONE": 0, "C-NONE-collected": 19, "C-MAP": 9, "C-MAP-collected": 19, "C-HTML": 5, "C-HTML-collected": 19, "b_map_only_correct": 8, "c_html_only_correct": 4, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C05" ], "verdict_word": null }, { "arm": "local-gemma4-26b", "family": "gemma", "vendor": "google", "size_class": "local ~26b", "denominator": 19, "C-NONE": 0, "C-NONE-collected": 19, "C-MAP": 8, "C-MAP-collected": 19, "C-HTML": 6, "C-HTML-collected": 19, "b_map_only_correct": 8, "c_html_only_correct": 6, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05", "C13" ], "verdict_word": null }, { "arm": "local-llama3.3-70b", "family": "llama", "vendor": "meta", "size_class": "local ~70b", "denominator": 19, "C-NONE": 0, "C-NONE-collected": 19, "C-MAP": 9, "C-MAP-collected": 19, "C-HTML": 6, "C-HTML-collected": 19, "b_map_only_correct": 8, "c_html_only_correct": 5, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05" ], "verdict_word": null }, { "arm": "cloud-glm-5.2", "family": "glm", "vendor": "zhipu", "size_class": "frontier cloud", "denominator": 19, "C-NONE": 0, "C-NONE-collected": 19, "C-MAP": 9, "C-MAP-collected": 19, "C-HTML": 6, "C-HTML-collected": 19, "b_map_only_correct": 8, "c_html_only_correct": 5, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05" ], "verdict_word": null }, { "arm": "cloud-deepseek-v4-pro", "family": "deepseek", "vendor": "deepseek", "size_class": "frontier cloud", "denominator": 19, "C-NONE": 0, "C-NONE-collected": 19, "C-MAP": 9, "C-MAP-collected": 19, "C-HTML": 6, "C-HTML-collected": 19, "b_map_only_correct": 8, "c_html_only_correct": 5, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05" ], "verdict_word": null }, { "arm": "cloud-kimi-k3", "family": "kimi", "vendor": "moonshot", "size_class": "frontier cloud", "denominator": 19, "C-NONE": 0, "C-NONE-collected": 19, "C-MAP": 8, "C-MAP-collected": 19, "C-HTML": 6, "C-HTML-collected": 19, "b_map_only_correct": 8, "c_html_only_correct": 6, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05", "C13" ], "verdict_word": null }, { "arm": "openai-gpt-5.5", "family": "gpt", "vendor": "openai", "size_class": "frontier cloud", "denominator": 19, "C-NONE": 0, "C-NONE-collected": 5, "C-MAP": 9, "C-MAP-collected": 19, "C-HTML": 6, "C-HTML-collected": 19, "b_map_only_correct": 8, "c_html_only_correct": 5, "b_items": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "c_items": [ "C01", "C02", "C03", "C04", "C05" ], "verdict_word": null } ], "pooled_discordance": { "b_map_only_correct": 64, "c_html_only_correct": 40, "pairs": 104, "clears_percent_floor": true, "share_of_larger_side": 0.6154, "registered_band": 0.6, "reading": "C-MAP DIRECTION", "basis": "the arms share items and are therefore NOT independent, so this figure describes these eight arms, not a population. It is a statement about the instrument's resolution at this N, not a p-value and not a tie between the representations." }, "per_arm_rule": "counts only, no verdict word: this instrument does not resolve a direction at n<=20 and saying so is the honest form of the comparison" }, "structural_sensitivity": { "status": "POST-HOC — NOT PRE-REGISTERED, published for transparency only", "finding": "the registered extractor keeps visible text only, so href attributes are dropped and the C-HTML block carries no URLs; every navigation item is unanswerable from it by construction", "urls_in_c_map": 50, "urls_in_c_html_raw_regex": 2, "urls_in_c_html_real": 0, "nav_items_removed": [ "N01", "N02", "N03", "N04", "N05", "N06" ], "denominator": 14, "item_ids": [ "C01", "C02", "C03", "C04", "C05", "C06", "C07", "C08", "C09", "C10", "C11", "C12", "C13", "C14" ], "b_map_only_correct": 16, "c_html_only_correct": 40, "pairs": 56, "clears_percent_floor": true, "share_of_larger_side": 0.7143, "registered_band": 0.6, "reading": "C-HTML DIRECTION", "caveat": "removing six of twenty items is a large intervention and this figure is NOT the bench's result. It is the size of the extractor's thumb, stated so nobody has to estimate it.", "urls_in_c_map_distinct": 36, "urls_in_c_map_rule": "count of `http://` or `https://` occurrences in the served block file, and the count of distinct addresses after trimming trailing punctuation. Both are reproducible from block-c-map.txt with one grep.", "urls_in_c_map_correction": "CORRECTED 2026-08-16, before publication. This block previously read `c_map: 61`, a figure no counting rule was ever attached to and which cannot be reproduced from the served block: the block holds 50 URL occurrences, 36 of them distinct, and 49 markdown link targets. Nothing in the file yields 61. The corrected counts and the rule that produces them are above; the original figure is named here rather than deleted. No scored result depended on it — it sizes the extractor's thumb, and the b/c/pairs figures beside it are unchanged." }, "strata": { "MAP-ONLY": { "item_ids": [ "C06", "C07", "N01", "N02", "N03", "N04", "N05", "N06" ], "rows": [ { "arm": "local-qwen3.8-27b", "denominator": 8, "C-NONE": 0, "C-MAP": 8, "C-HTML": 0 }, { "arm": "local-qwen3.6-27b", "denominator": 8, "C-NONE": 0, "C-MAP": 8, "C-HTML": 0 }, { "arm": "local-gemma4-26b", "denominator": 8, "C-NONE": 0, "C-MAP": 8, "C-HTML": 0 }, { "arm": "local-llama3.3-70b", "denominator": 8, "C-NONE": 0, "C-MAP": 8, "C-HTML": 0 }, { "arm": "cloud-glm-5.2", "denominator": 8, "C-NONE": 0, "C-MAP": 8, "C-HTML": 0 }, { "arm": "cloud-deepseek-v4-pro", "denominator": 8, "C-NONE": 0, "C-MAP": 8, "C-HTML": 0 }, { "arm": "cloud-kimi-k3", "denominator": 8, "C-NONE": 0, "C-MAP": 8, "C-HTML": 0 }, { "arm": "openai-gpt-5.5", "denominator": 8, "C-NONE": 0, "C-MAP": 8, "C-HTML": 0 } ] }, "HTML-ONLY": { "item_ids": [ "C01", "C02", "C03", "C04", "C05" ], "rows": [ { "arm": "local-qwen3.8-27b", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 4 }, { "arm": "local-qwen3.6-27b", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 4 }, { "arm": "local-gemma4-26b", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 5 }, { "arm": "local-llama3.3-70b", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 5 }, { "arm": "cloud-glm-5.2", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 5 }, { "arm": "cloud-deepseek-v4-pro", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 5 }, { "arm": "cloud-kimi-k3", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 5 }, { "arm": "openai-gpt-5.5", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 5 } ] }, "BOTH": { "item_ids": [ "C13", "C14" ], "rows": [ { "arm": "local-qwen3.8-27b", "denominator": 2, "C-NONE": 0, "C-MAP": 2, "C-HTML": 2 }, { "arm": "local-qwen3.6-27b", "denominator": 2, "C-NONE": 0, "C-MAP": 2, "C-HTML": 2 }, { "arm": "local-gemma4-26b", "denominator": 2, "C-NONE": 0, "C-MAP": 1, "C-HTML": 2 }, { "arm": "local-llama3.3-70b", "denominator": 2, "C-NONE": 1, "C-MAP": 2, "C-HTML": 2 }, { "arm": "cloud-glm-5.2", "denominator": 2, "C-NONE": 0, "C-MAP": 2, "C-HTML": 2 }, { "arm": "cloud-deepseek-v4-pro", "denominator": 2, "C-NONE": 0, "C-MAP": 2, "C-HTML": 2 }, { "arm": "cloud-kimi-k3", "denominator": 2, "C-NONE": 0, "C-MAP": 1, "C-HTML": 2 }, { "arm": "openai-gpt-5.5", "denominator": 2, "C-NONE": 0, "C-MAP": 2, "C-HTML": 2 } ] }, "NEITHER": { "item_ids": [ "C08", "C09", "C10", "C11", "C12" ], "rows": [ { "arm": "local-qwen3.8-27b", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 0 }, { "arm": "local-qwen3.6-27b", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 0 }, { "arm": "local-gemma4-26b", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 0 }, { "arm": "local-llama3.3-70b", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 0 }, { "arm": "cloud-glm-5.2", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 0 }, { "arm": "cloud-deepseek-v4-pro", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 0 }, { "arm": "cloud-kimi-k3", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 0 }, { "arm": "openai-gpt-5.5", "denominator": 5, "C-NONE": 0, "C-MAP": 0, "C-HTML": 0 } ] }, "_rule": "per-stratum results publish separately and are NEVER pooled with each other; a bench that only asks about things the index names has measured the index's own claims", "MAP-STALE": { "item_ids": [], "rows": [], "state": "VACANT", "cause": "no map/page contradiction exists in the frozen corpus — see prereg/receipts/map-stale-hunt.json" } }, "refusal_bait": { "item_ids": [ "B01", "B02", "B03", "B04" ], "stamp": "proxy", "rows": [ { "arm": "local-qwen3.8-27b", "denominator": 4, "C-MAP": 4, "C-HTML": 4, "C-NONE": "— (excluded from the honesty figure: with no site content, \"the site doesn't say\" is correct for every question)" }, { "arm": "local-qwen3.6-27b", "denominator": 4, "C-MAP": 4, "C-HTML": 4, "C-NONE": "— (excluded from the honesty figure: with no site content, \"the site doesn't say\" is correct for every question)" }, { "arm": "local-gemma4-26b", "denominator": 4, "C-MAP": 4, "C-HTML": 4, "C-NONE": "— (excluded from the honesty figure: with no site content, \"the site doesn't say\" is correct for every question)" }, { "arm": "local-llama3.3-70b", "denominator": 4, "C-MAP": 4, "C-HTML": 4, "C-NONE": "— (excluded from the honesty figure: with no site content, \"the site doesn't say\" is correct for every question)" }, { "arm": "cloud-glm-5.2", "denominator": 4, "C-MAP": 4, "C-HTML": 4, "C-NONE": "— (excluded from the honesty figure: with no site content, \"the site doesn't say\" is correct for every question)" }, { "arm": "cloud-deepseek-v4-pro", "denominator": 4, "C-MAP": 4, "C-HTML": 3, "C-NONE": "— (excluded from the honesty figure: with no site content, \"the site doesn't say\" is correct for every question)" }, { "arm": "cloud-kimi-k3", "denominator": 4, "C-MAP": 4, "C-HTML": 4, "C-NONE": "— (excluded from the honesty figure: with no site content, \"the site doesn't say\" is correct for every question)" }, { "arm": "openai-gpt-5.5", "denominator": 4, "C-MAP": 4, "C-HTML": 4, "C-NONE": "— (excluded from the honesty figure: with no site content, \"the site doesn't say\" is correct for every question)" } ], "rule": "never averaged into accuracy; interpretable only in C-MAP and C-HTML" }, "flake": { "draws": 3, "items": [ "C01", "C05", "C08", "C11", "N03", "B03" ], "per_arm": { "local-qwen3.8-27b": { "items": 6, "flips": 0, "detail": { "C01": { "draws": [ false, false ], "flipped": false }, "C05": { "draws": [ false, false ], "flipped": false }, "C08": { "draws": [ false, false ], "flipped": false }, "C11": { "draws": [ false, false ], "flipped": false }, "N03": { "draws": [ true, true ], "flipped": false }, "B03": { "draws": [ true, true ], "flipped": false } }, "instability_flag": false }, "local-qwen3.6-27b": { "items": 6, "flips": 0, "detail": { "C01": { "draws": [ false, false ], "flipped": false }, "C05": { "draws": [ false, false ], "flipped": false }, "C08": { "draws": [ false, false ], "flipped": false }, "C11": { "draws": [ false, false ], "flipped": false }, "N03": { "draws": [ true, true ], "flipped": false }, "B03": { "draws": [ true, true ], "flipped": false } }, "instability_flag": false }, "local-gemma4-26b": { "items": 6, "flips": 0, "detail": { "C01": { "draws": [ false, false ], "flipped": false }, "C05": { "draws": [ false, false ], "flipped": false }, "C08": { "draws": [ false, false ], "flipped": false }, "C11": { "draws": [ false, false ], "flipped": false }, "N03": { "draws": [ true, true ], "flipped": false }, "B03": { "draws": [ true, true ], "flipped": false } }, "instability_flag": false }, "local-llama3.3-70b": { "items": 6, "flips": 0, "detail": { "C01": { "draws": [ false, false ], "flipped": false }, "C05": { "draws": [ false, false ], "flipped": false }, "C08": { "draws": [ false, false ], "flipped": false }, "C11": { "draws": [ false, false ], "flipped": false }, "N03": { "draws": [ true, true ], "flipped": false }, "B03": { "draws": [ true, true ], "flipped": false } }, "instability_flag": false }, "cloud-glm-5.2": { "items": 6, "flips": 0, "detail": { "C01": { "draws": [ false, false ], "flipped": false }, "C05": { "draws": [ false, false ], "flipped": false }, "C08": { "draws": [ false, false ], "flipped": false }, "C11": { "draws": [ false, false ], "flipped": false }, "N03": { "draws": [ true, true ], "flipped": false }, "B03": { "draws": [ true, true ], "flipped": false } }, "instability_flag": false }, "cloud-deepseek-v4-pro": { "items": 6, "flips": 0, "detail": { "C01": { "draws": [ false, false ], "flipped": false }, "C05": { "draws": [ false, false ], "flipped": false }, "C08": { "draws": [ false, false ], "flipped": false }, "C11": { "draws": [ false, false ], "flipped": false }, "N03": { "draws": [ true, true ], "flipped": false }, "B03": { "draws": [ true, true ], "flipped": false } }, "instability_flag": false }, "cloud-kimi-k3": { "items": 0, "flips": null, "detail": {}, "instability_flag": null, "note": "— no flake figure: metered arm, the reason is cost" }, "openai-gpt-5.5": { "items": 0, "flips": null, "detail": {}, "instability_flag": null, "note": "— no flake figure: metered arm, the reason is cost" } } } }