{ "schema": "s2s-bench-v1", "exhibit": "chair-trials", "published_utc": "2026-08-13", "status": "PUBLISHED — live at research.strata2signal.com.", "licence": "CC BY 4.0", "attribution": "strata→signal research, research.strata2signal.com", "hardware": "One 96G VRAM workstation, which also answered live requests for three of our apps throughout these trials — two public, one serving playtests for humans and agents alike. Contention is measured in both directions and published with the figures rather than assumed away.", "dataset": "chair-trials-c2-assistant", "table": "C2", "what_this_is": "Twenty everyday tasks — constraint writing, faithful summarization, JSON extraction, arithmetic, tone, shell and regex, finding a planted fact at 8k and 24k tokens of context, and four false-premise questions where the honest answer is a correction. Eight posture rows, two repeats each, conjunctive: an item scores only if both repeats pass.", "verdict_discipline": "DESCRIPTIVE - no floors, no winner, no ordering by margin (PLAN v2)", "primary_metric": "conjunctive pass over 2 repeats, out of 20 frozen items", "columns": [ "model", "posture", "/20", "exact /13", "proxy /7", "repeat disagreement", "failure ledger" ], "counting_rules": { "conjunctive": "Two repeats per item at temperature zero; an item scores 1 only if BOTH repeats pass. The disagreement column is the count of items whose two repeats disagreed — published rather than smoothed.", "two_halves_never_added": "Thirteen items are checked EXACTLY (a deterministic predicate over the answer: a sentence count, a parsed field, a number within a tolerance). Seven are checked by PROXY (marker- and length-based heuristics). The /20 is the registered CONJUNCTIVE ITEM COUNT and the two subtotals are its composition, not two scores summed: what the registration bars is blending them into one QUALITY figure, because a proxy check is evidence about shape and an exact check is evidence about correctness, and averaging those two would claim a precision neither has. The subtotals are published side by side so a reader can weight them differently than we did.", "no_floors": "Descriptive: no floors, no winner, no ordering by margin. Differences under two items read TIED by pre-registration.", "unmeasurable": "A row whose response-failure rate exceeds 10% of its calls is UNMEASURABLE and its score is not admissible as a score, however the arithmetic came out.", "long_context": "The two long-context items vary the PROMPT length (8k and 24k tokens) while num_ctx stays 32 768 everywhere. Observed prompt_eval_count rides on every repeat so a reader can check the tier label against what the model actually read." }, "item_matrix": { "c2-tone-1": { "item_id": "c2-tone-1", "category": "tone-rewrite", "proxy": true, "passed_by": [], "failed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "passes": 0, "of_rows": 9, "counts_of_n": "0/9" }, "c2-hon-3": { "item_id": "c2-hon-3", "category": "false-premise-honesty", "proxy": true, "passed_by": [ "gemma4:12b · think:false" ], "failed_by": [ "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "passes": 1, "of_rows": 9, "counts_of_n": "1/9" }, "c2-cw-2": { "item_id": "c2-cw-2", "category": "constraint-writing", "proxy": false, "passed_by": [ "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true" ], "failed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "passes": 2, "of_rows": 9, "counts_of_n": "2/9" }, "c2-sum-1": { "item_id": "c2-sum-1", "category": "faithful-summary", "proxy": true, "passed_by": [ "gemma4:26b · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true" ], "failed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "passes": 3, "of_rows": 9, "counts_of_n": "3/9" }, "c2-sum-2": { "item_id": "c2-sum-2", "category": "faithful-summary", "proxy": true, "passed_by": [ "gemma4:26b · think:false", "kimi-k3 · think:false", "qwen3.6:27b · think:false" ], "failed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:true", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false" ], "passes": 3, "of_rows": 9, "counts_of_n": "3/9" }, "c2-cw-4": { "item_id": "c2-cw-4", "category": "constraint-writing", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true" ], "failed_by": [ "gemma4:26b · think:true", "kimi-k3 · think:false", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "passes": 4, "of_rows": 9, "counts_of_n": "4/9" }, "c2-hon-1": { "item_id": "c2-hon-1", "category": "false-premise-honesty", "proxy": true, "passed_by": [ "gemma4:26b · think:true", "muse-glimmer:30b-q8_0-dflash · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false" ], "passes": 4, "of_rows": 9, "counts_of_n": "4/9" }, "c2-json-2": { "item_id": "c2-json-2", "category": "json-extraction", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "qwen3.6:27b · think:false" ], "failed_by": [ "gemma4:26b · think:false", "gemma4:26b · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false" ], "passes": 5, "of_rows": 9, "counts_of_n": "5/9" }, "c2-re-1": { "item_id": "c2-re-1", "category": "shell-regex", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "qwen3.6:27b · think:false" ], "failed_by": [ "gemma4:26b · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false" ], "passes": 6, "of_rows": 9, "counts_of_n": "6/9" }, "c2-num-2": { "item_id": "c2-num-2", "category": "arithmetic-date-unit", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [ "gemma4:26b · think:true", "qwen3.5:27b · think:false" ], "passes": 7, "of_rows": 9, "counts_of_n": "7/9" }, "c2-cw-1": { "item_id": "c2-cw-1", "category": "constraint-writing", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [ "gemma4:26b · think:true" ], "passes": 8, "of_rows": 9, "counts_of_n": "8/9" }, "c2-num-3": { "item_id": "c2-num-3", "category": "arithmetic-date-unit", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [ "gemma4:26b · think:true" ], "passes": 8, "of_rows": 9, "counts_of_n": "8/9" }, "c2-ctx-24k": { "item_id": "c2-ctx-24k", "category": "longctx-find-the-fact", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [], "passes": 9, "of_rows": 9, "counts_of_n": "9/9" }, "c2-ctx-8k": { "item_id": "c2-ctx-8k", "category": "longctx-find-the-fact", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [], "passes": 9, "of_rows": 9, "counts_of_n": "9/9" }, "c2-cw-3": { "item_id": "c2-cw-3", "category": "constraint-writing", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [], "passes": 9, "of_rows": 9, "counts_of_n": "9/9" }, "c2-hon-2": { "item_id": "c2-hon-2", "category": "false-premise-honesty", "proxy": true, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [], "passes": 9, "of_rows": 9, "counts_of_n": "9/9" }, "c2-hon-4": { "item_id": "c2-hon-4", "category": "false-premise-honesty", "proxy": true, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [], "passes": 9, "of_rows": 9, "counts_of_n": "9/9" }, "c2-json-1": { "item_id": "c2-json-1", "category": "json-extraction", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [], "passes": 9, "of_rows": 9, "counts_of_n": "9/9" }, "c2-num-1": { "item_id": "c2-num-1", "category": "arithmetic-date-unit", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [], "passes": 9, "of_rows": 9, "counts_of_n": "9/9" }, "c2-sh-1": { "item_id": "c2-sh-1", "category": "shell-regex", "proxy": false, "passed_by": [ "gemma4:12b · think:false", "gemma4:26b · think:false", "gemma4:26b · think:true", "kimi-k3 · think:false", "muse-glimmer:30b-q8_0-dflash · think:false", "muse-glimmer:30b-q8_0-dflash · think:true", "nemotron-3.5-lightning:30b-a3b · think:false", "qwen3.5:27b · think:false", "qwen3.6:27b · think:false" ], "failed_by": [], "passes": 9, "of_rows": 9, "counts_of_n": "9/9" } }, "long_context": { "items": [ { "item_id": "c2-ctx-24k", "n_observations": 18, "prompt_eval_count_min": 22804, "prompt_eval_count_max": 23352 }, { "item_id": "c2-ctx-8k", "n_observations": 18, "prompt_eval_count_min": 7851, "prompt_eval_count_max": 8040 } ], "display": "7 851–8 040 and 22 804–23 352", "rule": "min and max of prompt_eval_count as the daemon counted it, over all eight rows and both repeats of each long-context item. Within a row the two repeats are identical, so the spread is tokenizer difference between models rather than run-to-run variance — and every call ran at num_ctx 32 768." }, "golden": { "artifact": "assistant-c2.json", "sha256": "dd63efa94bdebc878e1dc1c1065913892d8eb7cf16026642ff9fd0bae470d50c", "frozen_count": 20, "drafted_count": 22, "exact_items": 13, "proxy_items": 7, "quota_frozen": { "constraint-writing": 4, "faithful-summary": 2, "json-extraction": 2, "arithmetic-date-unit": 3, "tone-rewrite": 1, "shell-regex": 2, "longctx-find-the-fact": 2, "false-premise-honesty": 4 }, "protocol": { "temperature": 0, "top_p": 1, "repeats": 2, "pass_rule": "conjunctive - an item scores 1 only if BOTH repeats pass", "num_ctx": 32768, "num_predict": 1536, "timeout_s": 240, "num_predict_think_true": 4096, "timeout_s_think_true": 600, "postures": "think:false for every model; think:true ADDITIONALLY for muse-glimmer:30b-q8_0-dflash and gemma4:26b only (PLAN v2)", "long_context_note": "the 8k and 24k items vary PROMPT length only; options.num_ctx is 32768 on every call in this leg" }, "source": { "title": "Foster's Complete Hoyle, 1914 (Project Gutenberg #53881)", "sha256": "7927555a91c2f6f5c6c03b84eca313531389d92dbaf5ae3c5a23d025bccf2567", "note": "public domain in the US; PG boilerplate stripped. Passage and haystack fixtures are rebuilt by c2_build_fixtures.py, which refuses to run against any other sha.", "planted_facts": "each long-context haystack carries exactly one invented sentence, recorded in c2_fixtures/MANIFEST.json" }, "published_in_this_kit": "assistant-c2.json — all twenty frozen items with their prompts and expectations; CHECKERS.md — what each checker actually tests, exact or proxy" }, "limits": [ "Twenty items is a probe, not a suite: a single item's pass or fail moves a row by a twentieth, which is why the tie rule is two items wide and the item matrix ships.", "Proxy checkers measure shape. Seven of the twenty scores are therefore weaker evidence than the other thirteen, and the two subtotals are published apart so a reader can weight them differently than we did.", "Thinking postures were run for two arms only; every other row is think:false, and the stopwatch leg carries the thinking story for the rest." ], "provenance": { "artifacts": [ { "role": "the scored record every figure here is copied out of", "artifact": "c2-scores.json", "sha256": "cef6da831f770f2fab11413845d1488791a1ba01a37d7b67538427bfa4f33d05" }, { "role": "the frozen item set, published whole in this kit", "artifact": "assistant-c2.json", "sha256": "dd63efa94bdebc878e1dc1c1065913892d8eb7cf16026642ff9fd0bae470d50c" }, { "role": "the checkers — the only judges in this leg", "artifact": "c2_checkers.py", "sha256": "a9ebbd91503b936f3427cf7e6410bbacb45bb1574f750a8b45be9ae03481727a" }, { "role": "pre-registration, committed before the first scored call", "artifact": "PREREG-C2.md", "sha256": "5445775d0024b6bc165ff965fd06ba25b13457ef2b6c7bc2faaeb71621d19e0c" }, { "role": "the runner", "artifact": "c2_run.py", "sha256": "99cf3230b6cab4f99877c8dc36708a514e28afb69df186570ea87942b784e23c" }, { "role": "the scorer", "artifact": "c2_score.py", "sha256": "b4c7ce5684bfa9800598a2c443a21078b195c48ede3c26a899ad3ff8ed82f70c" } ], "checker_sha256": "a9ebbd91503b936f3427cf7e6410bbacb45bb1574f750a8b45be9ae03481727a", "scored_at": "2026-08-13T04:57:39.527567+00:00", "no_paths": "Artifacts are named, never located. No box names, no addresses, no filesystem paths." }, "rows": [ { "model": "gemma4:12b", "model_display": "gemma4:12b", "posture": "think:false", "outcome": "RANKED", "outcome_scorer_label": "DESCRIPTIVE", "outcome_why": "admissible. This leg has no floors and no winner: RANKED here means the row's numbers were produced under the contract, never that the row was placed above another one", "score_20": 15, "of": 20, "exact": { "hits": 12, "of": 13, "counts_of_n": "12/13" }, "proxy": { "hits": 3, "of": 7, "counts_of_n": "3/7" }, "calls": 40, "response_failures": 0, "done_reason_length": 0, "repeat_disagreements": 0, "repeat_disagreement_items": [], "failure_ledger": "0/40 response failures", "items": [ { "item_id": "c2-ctx-24k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23073, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23073 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23073, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23073 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-ctx-8k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7944, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7944 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7944, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7944 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-1", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 156, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 133, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-2", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "word_count: 32 words, want 30", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 84, "answer_chars": 177, "checks": [ { "name": "word_count", "ok": false, "detail": "32 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_count: 32 words, want 30", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 84, "answer_chars": 181, "checks": [ { "name": "word_count", "ok": false, "detail": "32 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-cw-3", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 105, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 70, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-4", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 61, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": true, "detail": "'e' count: 0" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 56, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": true, "detail": "'e' count: 0" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-1", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 49, "answer_chars": 331, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 49, "answer_chars": 343, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-2", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 41, "answer_chars": 807, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['invented', '1852', 'died', '19th century', 'anachronis']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 41, "answer_chars": 807, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['invented', '1852', 'died', '19th century', 'anachronis']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-3", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 443, "answer_chars": 290, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 443, "answer_chars": 290, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-4", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 47, "answer_chars": 2668, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not an si', 'joke', 'humorous']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 47, "answer_chars": 2668, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not an si', 'joke', 'humorous']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-1", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 613, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 613, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-2", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 708, "answer_chars": 180, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": true, "detail": "got 24, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 708, "answer_chars": 180, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": true, "detail": "got 24, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-1", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 884, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 855, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-2", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 93, "answer_chars": 1432, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 93, "answer_chars": 1432, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-3", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 80, "answer_chars": 1109, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 80, "answer_chars": 1109, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-re-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 132, "answer_chars": 33, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^[^ ]+ - [A-Z]+ [^ ]+ 5\\d{2} \\d+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^[^ ]+ - [A-Z]+ [^ ]+ 5\\d{2} \\d+$", "matched": [ 2, 3, 6, 8, 10 ] } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 132, "answer_chars": 31, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^[^ ]+ - [A-Z]+ \\S+ 5\\d{2} \\d+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^[^ ]+ - [A-Z]+ \\S+ 5\\d{2} \\d+$", "matched": [ 2, 3, 6, 8, 10 ] } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sh-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 106, "answer_chars": 52, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, 'NR>1 && $3==\"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, 'NR>1 && $3==\"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 106, "answer_chars": 52, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, 'NR>1 && $3==\"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, 'NR>1 && $3==\"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-1", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "coverage: missing groups: [['double six', 'double-six']]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 544, "answer_chars": 360, "checks": [ { "name": "word_cap", "ok": true, "detail": "57 words, cap 60" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['double six', 'double-six']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_cap: 61 words, cap 60", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 544, "answer_chars": 388, "checks": [ { "name": "word_cap", "ok": false, "detail": "61 words, cap 60" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['double six', 'double-six']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-sum-2", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "word_cap: 73 words, cap 70", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 620, "answer_chars": 400, "checks": [ { "name": "word_cap", "ok": false, "detail": "73 words, cap 70" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['bar']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_cap: 73 words, cap 70", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 620, "answer_chars": 399, "checks": [ { "name": "word_cap", "ok": false, "detail": "73 words, cap 70" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['bar']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-tone-1", "category": "tone-rewrite", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_banned_phrases: hit: ['do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 140, "answer_chars": 540, "checks": [ { "name": "word_band", "ok": true, "detail": "91 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['apolog', 'understand', 'appreciate', 'thank you']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_banned_phrases: hit: ['do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 140, "answer_chars": 540, "checks": [ { "name": "word_band", "ok": true, "detail": "90 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['apolog', 'understand', 'appreciate', 'thank you']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 } ] }, { "model": "gemma4:26b", "model_display": "gemma4:26b", "posture": "think:false", "outcome": "RANKED", "outcome_scorer_label": "DESCRIPTIVE", "outcome_why": "admissible. This leg has no floors and no winner: RANKED here means the row's numbers were produced under the contract, never that the row was placed above another one", "score_20": 15, "of": 20, "exact": { "hits": 11, "of": 13, "counts_of_n": "11/13" }, "proxy": { "hits": 4, "of": 7, "counts_of_n": "4/7" }, "calls": 40, "response_failures": 0, "done_reason_length": 0, "repeat_disagreements": 1, "repeat_disagreement_items": [ "c2-cw-2" ], "failure_ledger": "0/40 response failures", "items": [ { "item_id": "c2-ctx-24k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23073, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23073 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23073, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23073 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-ctx-8k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7944, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7944 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7944, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7944 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-1", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 159, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 159, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-2", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 84, "answer_chars": 188, "checks": [ { "name": "word_count", "ok": true, "detail": "30 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_count: 29 words, want 30", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 84, "answer_chars": 181, "checks": [ { "name": "word_count", "ok": false, "detail": "29 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2, "disagreed": true }, { "item_id": "c2-cw-3", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 112, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 115, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-4", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 68, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": true, "detail": "'e' count: 0" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 64, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": true, "detail": "'e' count: 0" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-1", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 49, "answer_chars": 817, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 49, "answer_chars": 817, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-2", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 41, "answer_chars": 918, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['1852', 'died', '19th century']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 41, "answer_chars": 918, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['1852', 'died', '19th century']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-3", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 443, "answer_chars": 244, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 443, "answer_chars": 244, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-4", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 47, "answer_chars": 2895, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not the si', 'joke', 'humorous']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 47, "answer_chars": 2895, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not the si', 'joke', 'humorous']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-1", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 613, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 613, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-2", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "field:highest_point_number: got 6, want 24 (integer)", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 708, "answer_chars": 179, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": false, "detail": "got 6, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": false, "reason": "field:highest_point_number: got 6, want 24 (integer)", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 708, "answer_chars": 179, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": false, "detail": "got 6, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-num-1", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 910, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 896, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-2", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 93, "answer_chars": 1415, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 93, "answer_chars": 1415, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-3", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 80, "answer_chars": 1099, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 80, "answer_chars": 1099, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-re-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 132, "answer_chars": 33, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^[^ ]+ - [A-Z]+ [^ ]+ 5\\d{2} \\d+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^[^ ]+ - [A-Z]+ [^ ]+ 5\\d{2} \\d+$", "matched": [ 2, 3, 6, 8, 10 ] } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 132, "answer_chars": 33, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^[^ ]+ - [A-Z]+ [^ ]+ 5\\d{2} \\d+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^[^ ]+ - [A-Z]+ [^ ]+ 5\\d{2} \\d+$", "matched": [ 2, 3, 6, 8, 10 ] } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sh-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 106, "answer_chars": 67, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, '$3 == \"brass\" {count++} END {print count+0}' inventory.csv" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, '$3 == \"brass\" {count++} END {print count+0}' inventory.csv", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 106, "answer_chars": 67, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, '$3 == \"brass\" {count++} END {print count+0}' inventory.csv" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, '$3 == \"brass\" {count++} END {print count+0}' inventory.csv", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-1", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 544, "answer_chars": 372, "checks": [ { "name": "word_cap", "ok": true, "detail": "54 words, cap 60" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 544, "answer_chars": 354, "checks": [ { "name": "word_cap", "ok": true, "detail": "54 words, cap 60" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-2", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 620, "answer_chars": 353, "checks": [ { "name": "word_cap", "ok": true, "detail": "66 words, cap 70" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 620, "answer_chars": 352, "checks": [ { "name": "word_cap", "ok": true, "detail": "66 words, cap 70" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-tone-1", "category": "tone-rewrite", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_banned_phrases: hit: ['do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 140, "answer_chars": 478, "checks": [ { "name": "word_band", "ok": true, "detail": "87 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'apolog']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_banned_phrases: hit: ['do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 140, "answer_chars": 475, "checks": [ { "name": "word_band", "ok": true, "detail": "88 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'apolog']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 } ] }, { "model": "gemma4:26b", "model_display": "gemma4:26b +think", "posture": "think:true", "outcome": "UNMEASURABLE", "outcome_scorer_label": "UNMEASURABLE", "outcome_why": "response failures 14/40 exceed the pre-registered 10% ceiling", "score_20": 9, "of": 20, "exact": { "hits": 6, "of": 13, "counts_of_n": "6/13" }, "proxy": { "hits": 3, "of": 7, "counts_of_n": "3/7" }, "calls": 40, "response_failures": 14, "done_reason_length": 16, "repeat_disagreements": 1, "repeat_disagreement_items": [ "c2-json-2" ], "failure_ledger": "score not admissible · 14/40 response failures · 16/40 length-stopped, 8 items", "items": [ { "item_id": "c2-ctx-24k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23076, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23076 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23076, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23076 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-ctx-8k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7947, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7947 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7947, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7947 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-1", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 114, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 114, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-cw-2", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 87, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 87, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-cw-3", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 102, "answer_chars": 59, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 102, "answer_chars": 59, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-4", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 100, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 100, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-1", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 52, "answer_chars": 359, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'no mention', 'not a recognized']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 52, "answer_chars": 359, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'no mention', 'not a recognized']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-2", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 44, "answer_chars": 663, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['fictional', 'invented', 'appears to be a', '1852']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 44, "answer_chars": 663, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['fictional', 'invented', 'appears to be a', '1852']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-3", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 446, "answer_chars": 201, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 446, "answer_chars": 201, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-4", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 50, "answer_chars": 2065, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not the si', 'joke', 'incorrect']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 50, "answer_chars": 2065, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not the si', 'joke', 'incorrect']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-1", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 616, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 616, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-2", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "field:highest_point_number: got 6, want 24 (integer)", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 711, "answer_chars": 151, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": false, "detail": "got 6, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 711, "answer_chars": 180, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": true, "detail": "got 24, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": false, "attempts": 2, "disagreed": true }, { "item_id": "c2-num-1", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 116, "answer_chars": 968, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 116, "answer_chars": 968, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-2", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 96, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 96, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-num-3", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "answer_line_present: ANSWER line: None", "response_failure": false, "done_reason": "length", "prompt_eval_count": 83, "answer_chars": 911, "checks": [ { "name": "answer_line_present", "ok": false, "detail": "ANSWER line: None" }, { "name": "value", "ok": false, "detail": "got None, want 6.69 +-0.005" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "answer_line_present: ANSWER line: None", "response_failure": false, "done_reason": "length", "prompt_eval_count": 83, "answer_chars": 911, "checks": [ { "name": "answer_line_present", "ok": false, "detail": "ANSWER line: None" }, { "name": "value", "ok": false, "detail": "got None, want 6.69 +-0.005" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-re-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 135, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 135, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-sh-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 109, "answer_chars": 45, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, '$3 == \"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, '$3 == \"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 109, "answer_chars": 45, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, '$3 == \"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, '$3 == \"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-1", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 547, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 547, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-sum-2", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 623, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "response: empty answer", "response_failure": true, "done_reason": "length", "prompt_eval_count": 623, "answer_chars": 0, "checks": [ { "name": "response", "ok": false, "detail": "empty answer" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-tone-1", "category": "tone-rewrite", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_banned_phrases: hit: ['do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 143, "answer_chars": 500, "checks": [ { "name": "word_band", "ok": true, "detail": "84 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'apolog', 'understand', 'thank you']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_banned_phrases: hit: ['do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 143, "answer_chars": 500, "checks": [ { "name": "word_band", "ok": true, "detail": "84 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'apolog', 'understand', 'thank you']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 } ] }, { "model": "muse-glimmer:30b-q8_0-dflash", "model_display": "muse-glimmer:30b-q8_0-dflash", "posture": "think:false", "outcome": "RANKED", "outcome_scorer_label": "DESCRIPTIVE", "outcome_why": "admissible. This leg has no floors and no winner: RANKED here means the row's numbers were produced under the contract, never that the row was placed above another one", "score_20": 17, "of": 20, "exact": { "hits": 13, "of": 13, "counts_of_n": "13/13" }, "proxy": { "hits": 4, "of": 7, "counts_of_n": "4/7" }, "calls": 40, "response_failures": 0, "done_reason_length": 0, "repeat_disagreements": 1, "repeat_disagreement_items": [ "c2-sum-2" ], "failure_ledger": "0/40 response failures", "items": [ { "item_id": "c2-ctx-24k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 22804, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 22804 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 22804, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 22804 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-ctx-8k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7851, "answer_chars": 104, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7851 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7851, "answer_chars": 104, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7851 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-1", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 126, "answer_chars": 286, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 126, "answer_chars": 286, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-2", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 98, "answer_chars": 179, "checks": [ { "name": "word_count", "ok": true, "detail": "30 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 98, "answer_chars": 179, "checks": [ { "name": "word_count", "ok": true, "detail": "30 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-3", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 149, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 137, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-4", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 73, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": true, "detail": "'e' count: 0" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 73, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": true, "detail": "'e' count: 0" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-1", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 69, "answer_chars": 918, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'not mentioned', 'there is no', 'no chapter']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 69, "answer_chars": 918, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'not mentioned', 'there is no', 'no chapter']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-2", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 60, "answer_chars": 797, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['there is no', '1852', 'died', 'anachronis']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 60, "answer_chars": 797, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['there is no', '1852', 'died', 'anachronis']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-3", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 464, "answer_chars": 299, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'no mention']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 464, "answer_chars": 320, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'no mention']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-4", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 69, "answer_chars": 1299, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: [\"that's not\", 'meter per second', 'meters per second', 'm/s', 'not an si']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 69, "answer_chars": 1299, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: [\"that's not\", 'meter per second', 'meters per second', 'm/s', 'not an si']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-1", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 624, "answer_chars": 81, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 624, "answer_chars": 81, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-2", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 708, "answer_chars": 142, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": true, "detail": "got 24, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 708, "answer_chars": 142, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": true, "detail": "got 24, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-1", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 127, "answer_chars": 160, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 127, "answer_chars": 172, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.5'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-2", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 104, "answer_chars": 25, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 104, "answer_chars": 25, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-3", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 96, "answer_chars": 12, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 96, "answer_chars": 131, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-re-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 145, "answer_chars": 27, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^\\S+ - \\S+ \\S+ 5\\d{2} \\S+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^\\S+ - \\S+ \\S+ 5\\d{2} \\S+$", "matched": [ 2, 3, 6, 8, 10 ] } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 145, "answer_chars": 27, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^\\S+ - \\S+ \\S+ 5\\d{2} \\S+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^\\S+ - \\S+ \\S+ 5\\d{2} \\S+$", "matched": [ 2, 3, 6, 8, 10 ] } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sh-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 122, "answer_chars": 63, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, 'NR>1 && $3==\"brass\"{c++} END{print c+0}' inventory.csv" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, 'NR>1 && $3==\"brass\"{c++} END{print c+0}' inventory.csv", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 122, "answer_chars": 63, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, 'NR>1 && $3==\"brass\"{c++} END{print c+0}' inventory.csv" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, 'NR>1 && $3==\"brass\"{c++} END{print c+0}' inventory.csv", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-1", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 558, "answer_chars": 351, "checks": [ { "name": "word_cap", "ok": true, "detail": "60 words, cap 60" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 558, "answer_chars": 346, "checks": [ { "name": "word_cap", "ok": true, "detail": "60 words, cap 60" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-2", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 630, "answer_chars": 370, "checks": [ { "name": "word_cap", "ok": true, "detail": "69 words, cap 70" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "coverage: missing groups: [['24', 'twenty-four']]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 630, "answer_chars": 347, "checks": [ { "name": "word_cap", "ok": true, "detail": "65 words, cap 70" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['24', 'twenty-four']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2, "disagreed": true }, { "item_id": "c2-tone-1", "category": "tone-rewrite", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_banned_phrases: hit: ['policy is policy', 'do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 149, "answer_chars": 435, "checks": [ { "name": "word_band", "ok": true, "detail": "80 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['policy is policy', 'do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'apolog', 'understand', 'thank you']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_banned_phrases: hit: ['policy is policy', 'do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 149, "answer_chars": 435, "checks": [ { "name": "word_band", "ok": true, "detail": "80 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['policy is policy', 'do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'apolog', 'understand', 'thank you']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 } ] }, { "model": "muse-glimmer:30b-q8_0-dflash", "model_display": "muse-glimmer:30b-q8_0-dflash +think", "posture": "think:true", "outcome": "RANKED", "outcome_scorer_label": "DESCRIPTIVE", "outcome_why": "admissible. This leg has no floors and no winner: RANKED here means the row's numbers were produced under the contract, never that the row was placed above another one", "score_20": 16, "of": 20, "exact": { "hits": 13, "of": 13, "counts_of_n": "13/13" }, "proxy": { "hits": 3, "of": 7, "counts_of_n": "3/7" }, "calls": 40, "response_failures": 0, "done_reason_length": 0, "repeat_disagreements": 1, "repeat_disagreement_items": [ "c2-sum-2" ], "failure_ledger": "0/40 response failures", "items": [ { "item_id": "c2-ctx-24k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 22804, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 22804 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 22804, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 22804 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-ctx-8k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7851, "answer_chars": 90, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7851 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 7851, "answer_chars": 90, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 7851 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-1", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 126, "answer_chars": 284, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 126, "answer_chars": 284, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-2", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 98, "answer_chars": 173, "checks": [ { "name": "word_count", "ok": true, "detail": "30 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 98, "answer_chars": 173, "checks": [ { "name": "word_count", "ok": true, "detail": "30 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-3", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 152, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 152, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-4", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 95, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": true, "detail": "'e' count: 0" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 113, "answer_chars": 95, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": true, "detail": "'e' count: 0" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-1", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 69, "answer_chars": 1689, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 69, "answer_chars": 1689, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-2", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 60, "answer_chars": 848, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['no such', 'there is no', 'invented', '1852', 'died']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 60, "answer_chars": 848, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['no such', 'there is no', 'invented', '1852', 'died']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-3", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 464, "answer_chars": 281, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not mention', 'no such']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 464, "answer_chars": 276, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-4", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 69, "answer_chars": 1016, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: [\"that's not\", 'metre per second', 'm/s', 'not an si', 'not correct']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 69, "answer_chars": 1016, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: [\"that's not\", 'metre per second', 'm/s', 'not an si', 'not correct']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-1", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 624, "answer_chars": 81, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 624, "answer_chars": 81, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-2", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 708, "answer_chars": 142, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": true, "detail": "got 24, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 708, "answer_chars": 142, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": true, "detail": "got 24, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-1", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 127, "answer_chars": 151, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.5'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 127, "answer_chars": 151, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.5'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-2", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 104, "answer_chars": 279, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 104, "answer_chars": 279, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-3", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 96, "answer_chars": 149, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 96, "answer_chars": 149, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-re-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 145, "answer_chars": 27, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^\\S+ - \\S+ \\S+ 5\\d{2} \\S+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^\\S+ - \\S+ \\S+ 5\\d{2} \\S+$", "matched": [ 2, 3, 6, 8, 10 ] } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 145, "answer_chars": 27, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^\\S+ - \\S+ \\S+ 5\\d{2} \\S+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^\\S+ - \\S+ \\S+ 5\\d{2} \\S+$", "matched": [ 2, 3, 6, 8, 10 ] } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sh-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 122, "answer_chars": 63, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, 'NR>1 && $3==\"brass\"{c++} END{print c+0}' inventory.csv" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, 'NR>1 && $3==\"brass\"{c++} END{print c+0}' inventory.csv", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 122, "answer_chars": 63, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, 'NR>1 && $3==\"brass\"{c++} END{print c+0}' inventory.csv" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, 'NR>1 && $3==\"brass\"{c++} END{print c+0}' inventory.csv", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-1", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 558, "answer_chars": 341, "checks": [ { "name": "word_cap", "ok": true, "detail": "59 words, cap 60" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 558, "answer_chars": 335, "checks": [ { "name": "word_cap", "ok": true, "detail": "59 words, cap 60" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-2", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 630, "answer_chars": 344, "checks": [ { "name": "word_cap", "ok": true, "detail": "64 words, cap 70" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "coverage: missing groups: [['24', 'twenty-four']]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 630, "answer_chars": 360, "checks": [ { "name": "word_cap", "ok": true, "detail": "66 words, cap 70" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['24', 'twenty-four']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2, "disagreed": true }, { "item_id": "c2-tone-1", "category": "tone-rewrite", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_banned_phrases: hit: ['policy is policy', 'do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 149, "answer_chars": 377, "checks": [ { "name": "word_band", "ok": true, "detail": "67 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['policy is policy', 'do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'understand', 'appreciate', 'thank you']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_banned_phrases: hit: ['policy is policy', 'do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 149, "answer_chars": 377, "checks": [ { "name": "word_band", "ok": true, "detail": "67 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['policy is policy', 'do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'understand', 'appreciate', 'thank you']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 } ] }, { "model": "nemotron-3.5-lightning:30b-a3b", "model_display": "nemotron-3.5-lightning:30b-a3b", "posture": "think:false", "outcome": "RANKED", "outcome_scorer_label": "DESCRIPTIVE", "outcome_why": "admissible. This leg has no floors and no winner: RANKED here means the row's numbers were produced under the contract, never that the row was placed above another one", "score_20": 11, "of": 20, "exact": { "hits": 9, "of": 13, "counts_of_n": "9/13" }, "proxy": { "hits": 2, "of": 7, "counts_of_n": "2/7" }, "calls": 40, "response_failures": 0, "done_reason_length": 0, "repeat_disagreements": 0, "repeat_disagreement_items": [], "failure_ledger": "0/40 response failures", "items": [ { "item_id": "c2-ctx-24k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23352, "answer_chars": 16, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23352 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23352, "answer_chars": 16, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23352 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-ctx-8k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 8024, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 8024 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 8024, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 8024 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-1", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 163, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 170, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-2", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "word_count: 23 words, want 30", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 84, "answer_chars": 130, "checks": [ { "name": "word_count", "ok": false, "detail": "23 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_count: 23 words, want 30", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 84, "answer_chars": 130, "checks": [ { "name": "word_count", "ok": false, "detail": "23 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-cw-3", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 58, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 58, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-4", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_letter_e: 'e' count: 6", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 63, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": false, "detail": "'e' count: 6" }, { "name": "required_word", "ok": false, "detail": "'night' present: False" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_letter_e: 'e' count: 6", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 63, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": false, "detail": "'e' count: 6" }, { "name": "required_word", "ok": false, "detail": "'night' present: False" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-1", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 54, "answer_chars": 1004, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['the kestrel gambit involves']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 54, "answer_chars": 1004, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['the kestrel gambit involves']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-2", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 46, "answer_chars": 2792, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['1852', 'died', '19th century']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 46, "answer_chars": 2792, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['1852', 'died', '19th century']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-3", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 454, "answer_chars": 360, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms', 'must be re-spotted']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "corrected_or_abstained: markers hit: none", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 454, "answer_chars": 360, "checks": [ { "name": "corrected_or_abstained", "ok": false, "detail": "markers hit: none" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms', 'must be re-spotted']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-4", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 53, "answer_chars": 3175, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meter per second', 'meters per second', 'm/s', 'not an si', 'humorous']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 53, "answer_chars": 3175, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meter per second', 'meters per second', 'm/s', 'not an si', 'humorous']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-1", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 632, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 632, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-2", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "field:highest_point_number: got 6, want 24 (integer)", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 700, "answer_chars": 151, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": false, "detail": "got 6, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": false, "reason": "field:highest_point_number: got 6, want 24 (integer)", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 700, "answer_chars": 151, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": false, "detail": "got 6, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-num-1", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 117, "answer_chars": 577, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 117, "answer_chars": 577, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-2", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 94, "answer_chars": 810, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 94, "answer_chars": 810, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-3", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 84, "answer_chars": 331, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 84, "answer_chars": 331, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-re-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "matched_lines: got [], want [2, 3, 6, 8, 10]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 134, "answer_chars": 33, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^.* - .* (HTTP/1\\.\\d|) 5\\d{2} .*$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": false, "detail": "got [], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^.* - .* (HTTP/1\\.\\d|) 5\\d{2} .*$", "matched": [] } }, { "repeat": 2, "pass": false, "reason": "matched_lines: got [], want [2, 3, 6, 8, 10]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 134, "answer_chars": 33, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^.* - .* (HTTP/1\\.\\d|) 5\\d{2} .*$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": false, "detail": "got [], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^.* - .* (HTTP/1\\.\\d|) 5\\d{2} .*$", "matched": [] } } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-sh-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 107, "answer_chars": 45, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F',' '$3==\"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F',' '$3==\"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 107, "answer_chars": 45, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F',' '$3==\"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F',' '$3==\"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-1", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "coverage: missing groups: [['double six', 'double-six']]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 566, "answer_chars": 326, "checks": [ { "name": "word_cap", "ok": true, "detail": "54 words, cap 60" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['double six', 'double-six']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "coverage: missing groups: [['double six', 'double-six']]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 566, "answer_chars": 326, "checks": [ { "name": "word_cap", "ok": true, "detail": "54 words, cap 60" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['double six', 'double-six']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-sum-2", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "word_cap: 84 words, cap 70", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 622, "answer_chars": 447, "checks": [ { "name": "word_cap", "ok": false, "detail": "84 words, cap 70" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['24', 'twenty-four']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_cap: 82 words, cap 70", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 622, "answer_chars": 425, "checks": [ { "name": "word_cap", "ok": false, "detail": "82 words, cap 70" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-tone-1", "category": "tone-rewrite", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "word_band: 52 words, band 60-140", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 141, "answer_chars": 292, "checks": [ { "name": "word_band", "ok": false, "detail": "52 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['policy is policy', 'do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'understand', 'appreciate']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_band: 52 words, band 60-140", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 141, "answer_chars": 292, "checks": [ { "name": "word_band", "ok": false, "detail": "52 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['policy is policy', 'do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'understand', 'appreciate']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 } ] }, { "model": "qwen3.5:27b", "model_display": "qwen3.5:27b", "posture": "think:false", "outcome": "RANKED", "outcome_scorer_label": "DESCRIPTIVE", "outcome_why": "admissible. This leg has no floors and no winner: RANKED here means the row's numbers were produced under the contract, never that the row was placed above another one", "score_20": 11, "of": 20, "exact": { "hits": 8, "of": 13, "counts_of_n": "8/13" }, "proxy": { "hits": 3, "of": 7, "counts_of_n": "3/7" }, "calls": 40, "response_failures": 0, "done_reason_length": 2, "repeat_disagreements": 0, "repeat_disagreement_items": [], "failure_ledger": "0/40 response failures · 2/40 length-stopped, 1 item", "items": [ { "item_id": "c2-ctx-24k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23352, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23352 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23352, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23352 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-ctx-8k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 8040, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 8040 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 8040, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 8040 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-1", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 267, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 267, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-2", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "word_count: 54 words, want 30", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 83, "answer_chars": 303, "checks": [ { "name": "word_count", "ok": false, "detail": "54 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_count: 54 words, want 30", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 83, "answer_chars": 303, "checks": [ { "name": "word_count", "ok": false, "detail": "54 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-cw-3", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 118, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 118, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-4", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_letter_e: 'e' count: 3", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 103, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": false, "detail": "'e' count: 3" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_letter_e: 'e' count: 3", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 103, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": false, "detail": "'e' count: 3" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-1", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 49, "answer_chars": 1245, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'does not appear', 'does not exist', 'no such', 'there is no']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 49, "answer_chars": 1245, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'does not appear', 'does not exist', 'no such', 'there is no']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-2", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 41, "answer_chars": 1237, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['there is no', 'no record', 'invented', '1852', 'died']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 41, "answer_chars": 1237, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['there is no', 'no record', 'invented', '1852', 'died']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-3", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 453, "answer_chars": 301, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 453, "answer_chars": 301, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-4", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 47, "answer_chars": 2495, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not an si', 'joke', 'humorous']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 47, "answer_chars": 2495, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not an si', 'joke', 'humorous']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-1", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 620, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 620, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-2", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "field:highest_point_number: got 6, want 24 (integer)", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 700, "answer_chars": 143, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": false, "detail": "got 6, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": false, "reason": "field:highest_point_number: got 6, want 24 (integer)", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 700, "answer_chars": 143, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": false, "detail": "got 6, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-num-1", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 969, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 969, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-2", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "answer_line_present: ANSWER line: None", "response_failure": false, "done_reason": "length", "prompt_eval_count": 92, "answer_chars": 4265, "checks": [ { "name": "answer_line_present", "ok": false, "detail": "ANSWER line: None" }, { "name": "date", "ok": false, "detail": "want 2026-04-12 in None" }, { "name": "weekday", "ok": false, "detail": "want sunday in None" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "answer_line_present: ANSWER line: None", "response_failure": false, "done_reason": "length", "prompt_eval_count": 92, "answer_chars": 4265, "checks": [ { "name": "answer_line_present", "ok": false, "detail": "ANSWER line: None" }, { "name": "date", "ok": false, "detail": "want 2026-04-12 in None" }, { "name": "weekday", "ok": false, "detail": "want sunday in None" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-num-3", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 81, "answer_chars": 1055, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 81, "answer_chars": 1055, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-re-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "matched_lines: got [], want [2, 3, 6, 8, 10]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 129, "answer_chars": 34, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^\\S+ - \\w+ /\\d* [0-9]* 5\\d{2} \\d+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": false, "detail": "got [], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^\\S+ - \\w+ /\\d* [0-9]* 5\\d{2} \\d+$", "matched": [] } }, { "repeat": 2, "pass": false, "reason": "matched_lines: got [], want [2, 3, 6, 8, 10]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 129, "answer_chars": 34, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^\\S+ - \\w+ /\\d* [0-9]* 5\\d{2} \\d+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": false, "detail": "got [], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^\\S+ - \\w+ /\\d* [0-9]* 5\\d{2} \\d+$", "matched": [] } } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-sh-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 102, "answer_chars": 53, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F',' 'NR>1 && $3==\"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F',' 'NR>1 && $3==\"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 102, "answer_chars": 53, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F',' 'NR>1 && $3==\"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F',' 'NR>1 && $3==\"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-1", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "word_cap: 67 words, cap 60", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 554, "answer_chars": 421, "checks": [ { "name": "word_cap", "ok": false, "detail": "67 words, cap 60" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['double six', 'double-six']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_cap: 67 words, cap 60", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 554, "answer_chars": 421, "checks": [ { "name": "word_cap", "ok": false, "detail": "67 words, cap 60" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['double six', 'double-six']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-sum-2", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "word_cap: 75 words, cap 70", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 622, "answer_chars": 479, "checks": [ { "name": "word_cap", "ok": false, "detail": "75 words, cap 70" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_cap: 75 words, cap 70", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 622, "answer_chars": 479, "checks": [ { "name": "word_cap", "ok": false, "detail": "75 words, cap 70" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-tone-1", "category": "tone-rewrite", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_banned_phrases: hit: ['policy is policy', 'do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 141, "answer_chars": 612, "checks": [ { "name": "word_band", "ok": true, "detail": "99 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['policy is policy', 'do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'understand', 'thank you']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_banned_phrases: hit: ['policy is policy', 'do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 141, "answer_chars": 612, "checks": [ { "name": "word_band", "ok": true, "detail": "99 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['policy is policy', 'do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['sorry', 'understand', 'thank you']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 } ] }, { "model": "qwen3.6:27b", "model_display": "qwen3.6:27b", "posture": "think:false", "outcome": "RANKED", "outcome_scorer_label": "DESCRIPTIVE", "outcome_why": "admissible. This leg has no floors and no winner: RANKED here means the row's numbers were produced under the contract, never that the row was placed above another one", "score_20": 15, "of": 20, "exact": { "hits": 11, "of": 13, "counts_of_n": "11/13" }, "proxy": { "hits": 4, "of": 7, "counts_of_n": "4/7" }, "calls": 40, "response_failures": 0, "done_reason_length": 0, "repeat_disagreements": 0, "repeat_disagreement_items": [], "failure_ledger": "0/40 response failures", "items": [ { "item_id": "c2-ctx-24k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23352, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23352 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 23352, "answer_chars": 84, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 23352 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-ctx-8k", "category": "longctx-find-the-fact", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 8040, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 8040 } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 8040, "answer_chars": 71, "checks": [ { "name": "planted_fact_recovered", "ok": true, "detail": "missing: none" } ], "diagnostic": { "prompt_eval_count": 8040 } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-1", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 202, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 202, "checks": [ { "name": "sentence_count", "ok": true, "detail": "4 sentences" }, { "name": "every_sentence_starts_with_B", "ok": true, "detail": "offenders: none" }, { "name": "banned_word_absent", "ok": true, "detail": "'dog' present: False" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-2", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "word_count: 34 words, want 30", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 83, "answer_chars": 207, "checks": [ { "name": "word_count", "ok": false, "detail": "34 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "word_count: 34 words, want 30", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 83, "answer_chars": 207, "checks": [ { "name": "word_count", "ok": false, "detail": "34 words, want 30" }, { "name": "required_words_once", "ok": true, "detail": "counts {'lantern': 1, 'tide': 1, 'ledger': 1}" }, { "name": "no_commas", "ok": true, "detail": "banned chars present: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-cw-3", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 121, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 99, "answer_chars": 121, "checks": [ { "name": "line_count", "ok": true, "detail": "3 non-blank lines" }, { "name": "bullet_prefix", "ok": true, "detail": "offenders: none" }, { "name": "words_per_line", "ok": true, "detail": "over-long: none" }, { "name": "no_digits", "ok": true, "detail": "digits present: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-cw-4", "category": "constraint-writing", "proxy": false, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_letter_e: 'e' count: 2", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 55, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": false, "detail": "'e' count: 2" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_letter_e: 'e' count: 2", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 97, "answer_chars": 55, "checks": [ { "name": "sentence_count", "ok": true, "detail": "2 sentences" }, { "name": "no_letter_e", "ok": false, "detail": "'e' count: 2" }, { "name": "required_word", "ok": true, "detail": "'night' present: True" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-1", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 49, "answer_chars": 1562, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'does not exist', 'no such', 'there is no', 'no chapter']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 49, "answer_chars": 1562, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain', 'does not exist', 'no such', 'there is no', 'no chapter']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-2", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 41, "answer_chars": 1345, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['there is no', 'no record', '1852', 'died', 'anachronis']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 41, "answer_chars": 1345, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['there is no', 'no record', '1852', 'died', 'anachronis']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-hon-3", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 453, "answer_chars": 262, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_fabrication: fabrication markers hit: ['after three consecutive carroms']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 453, "answer_chars": 262, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['does not contain']" }, { "name": "no_fabrication", "ok": false, "detail": "fabrication markers hit: ['after three consecutive carroms']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-hon-4", "category": "false-premise-honesty", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 47, "answer_chars": 2408, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not an si', 'not the si', 'joke']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 47, "answer_chars": 2408, "checks": [ { "name": "corrected_or_abstained", "ok": true, "detail": "markers hit: ['meters per second', 'm/s', 'not an si', 'not the si', 'joke']" }, { "name": "no_fabrication", "ok": true, "detail": "fabrication markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-1", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 620, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 620, "answer_chars": 98, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set'], want ['pieces', 'pips_blank_suit', 'pips_six_suit', 'standard_set']" }, { "name": "field:standard_set", "ok": true, "detail": "got 'double six', accepted ['double six', 'double-six', 'the double six']" }, { "name": "field:pieces", "ok": true, "detail": "got 28, want 28 (integer)" }, { "name": "field:pips_blank_suit", "ok": true, "detail": "got 21, want 21 (integer)" }, { "name": "field:pips_six_suit", "ok": true, "detail": "got 63, want 63 (integer)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-json-2", "category": "json-extraction", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 700, "answer_chars": 180, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": true, "detail": "got 24, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 700, "answer_chars": 180, "checks": [ { "name": "strict_parse", "ok": true, "detail": "parsed as an object" }, { "name": "key_set", "ok": true, "detail": "got ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player'], want ['dice_per_player', 'highest_point_number', 'home_table_points', 'men_per_player']" }, { "name": "field:men_per_player", "ok": true, "detail": "got 15, want 15 (integer)" }, { "name": "field:dice_per_player", "ok": true, "detail": "got 2, want 2 (integer)" }, { "name": "field:highest_point_number", "ok": true, "detail": "got 24, want 24 (integer)" }, { "name": "field:home_table_points", "ok": true, "detail": "got ['Ace', 'Deuce', 'Trey', 'Four', 'Five', 'Six'], want ['ace', 'deuce', 'trey', 'four', 'five', 'six'] (ordered)" } ], "diagnostic": { "leak_stripped_parsed": true, "note": "leak-stripped parse is diagnostic only; the verdict binds on strict parse" } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-1", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 960, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 111, "answer_chars": 960, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '71.50'" }, { "name": "value", "ok": true, "detail": "got 71.5, want 71.5" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-2", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 92, "answer_chars": 2141, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 92, "answer_chars": 2141, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '2026-04-12 Sunday'" }, { "name": "date", "ok": true, "detail": "want 2026-04-12 in '2026-04-12 Sunday'" }, { "name": "weekday", "ok": true, "detail": "want sunday in '2026-04-12 Sunday'" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-num-3", "category": "arithmetic-date-unit", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 81, "answer_chars": 952, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 81, "answer_chars": 952, "checks": [ { "name": "answer_line_present", "ok": true, "detail": "ANSWER line: '6.69'" }, { "name": "value", "ok": true, "detail": "got 6.69, want 6.69 +-0.005" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-re-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 129, "answer_chars": 26, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^\\S+ - \\S+ \\S+ 5\\d{2} \\d+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^\\S+ - \\S+ \\S+ 5\\d{2} \\d+$", "matched": [ 2, 3, 6, 8, 10 ] } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 129, "answer_chars": 26, "checks": [ { "name": "pattern_extracted", "ok": true, "detail": "^\\S+ - \\S+ \\S+ 5\\d{2} \\d+$" }, { "name": "compiles_and_runs", "ok": true, "detail": "None" }, { "name": "matched_lines", "ok": true, "detail": "got [2, 3, 6, 8, 10], want [2, 3, 6, 8, 10]" } ], "diagnostic": { "pattern": "^\\S+ - \\S+ \\S+ 5\\d{2} \\d+$", "matched": [ 2, 3, 6, 8, 10 ] } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sh-1", "category": "shell-regex", "proxy": false, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 102, "answer_chars": 51, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, 'NR>1 && $3==\"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, 'NR>1 && $3==\"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 102, "answer_chars": 51, "checks": [ { "name": "command_extracted", "ok": true, "detail": "awk -F, 'NR>1 && $3==\"brass\"' inventory.csv | wc -l" }, { "name": "executed", "ok": true, "detail": "refused: []" }, { "name": "exit_status", "ok": true, "detail": "rc=0 stderr=''" }, { "name": "stdout", "ok": true, "detail": "got '4', want '4'" } ], "diagnostic": { "command": "awk -F, 'NR>1 && $3==\"brass\"' inventory.csv | wc -l", "run": { "executed": true, "refused": [], "stdout": "4\n", "stderr": "", "returncode": 0, "note": "" } } } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-sum-1", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "coverage: missing groups: [['double six', 'double-six']]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 554, "answer_chars": 370, "checks": [ { "name": "word_cap", "ok": true, "detail": "57 words, cap 60" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['double six', 'double-six']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "coverage: missing groups: [['double six', 'double-six']]", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 554, "answer_chars": 370, "checks": [ { "name": "word_cap", "ok": true, "detail": "57 words, cap 60" }, { "name": "coverage", "ok": false, "detail": "missing groups: [['double six', 'double-six']]" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 }, { "item_id": "c2-sum-2", "category": "faithful-summary", "proxy": true, "repeats": [ { "repeat": 1, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 622, "answer_chars": 362, "checks": [ { "name": "word_cap", "ok": true, "detail": "65 words, cap 70" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} }, { "repeat": 2, "pass": true, "reason": "all checks passed", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 622, "answer_chars": 362, "checks": [ { "name": "word_cap", "ok": true, "detail": "65 words, cap 70" }, { "name": "coverage", "ok": true, "detail": "missing groups: none" }, { "name": "no_fabrication", "ok": true, "detail": "markers hit: none" } ], "diagnostic": {} } ], "conjunctive_pass": true, "attempts": 2 }, { "item_id": "c2-tone-1", "category": "tone-rewrite", "proxy": true, "repeats": [ { "repeat": 1, "pass": false, "reason": "no_banned_phrases: hit: ['do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 141, "answer_chars": 782, "checks": [ { "name": "word_band", "ok": true, "detail": "130 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['apolog', 'understand', 'appreciate', 'regret']" } ], "diagnostic": {} }, { "repeat": 2, "pass": false, "reason": "no_banned_phrases: hit: ['do not reply']", "response_failure": false, "done_reason": "stop", "prompt_eval_count": 141, "answer_chars": 782, "checks": [ { "name": "word_band", "ok": true, "detail": "130 words, band 60-140" }, { "name": "facts_preserved", "ok": true, "detail": "missing: none" }, { "name": "no_banned_phrases", "ok": false, "detail": "hit: ['do not reply']" }, { "name": "no_banned_chars", "ok": true, "detail": "hit: none" }, { "name": "no_invented_concession", "ok": true, "detail": "hit: none" }, { "name": "register_marker", "ok": true, "detail": "hit: ['apolog', 'understand', 'appreciate', 'regret']" } ], "diagnostic": {} } ], "conjunctive_pass": false, "attempts": 2 } ] } ], "n_rows": 8, "contamination_caveat": "Published 2026-08-13. Models with a later training cutoff may have seen these sets. We author fresh sets each cycle; this one is not a standard, it is our kit, yours to reuse.", "not_a_standard": "Our own trials, our own hardware, for our own chairs. Not first independent numbers, and not a benchmark." }