{
  "schema": "s2s-bench-v1",
  "exhibit": "outside-judges",
  "published_utc": "2026-08-13",
  "status": "PUBLISHED — live at research.strata2signal.com.",
  "licence": "CC BY 4.0",
  "attribution": "strata→signal research, research.strata2signal.com",
  "hardware": "One 96G VRAM workstation, which also answered live requests for three of our apps throughout these trials — two public, one serving playtests for humans and agents alike. Contention is measured in both directions and published with the figures rather than assumed away.",
  "dataset": "outside-judges-counting-rules",
  "what_this_is": "Every counting rule on the page, in one file. Each one is copied out of the companion that uses it rather than restated here, so a rule cannot drift between the two places it appears.",
  "shared_vocabulary": {
    "CARRIES": "a candidate's audition reply parsed and validated against the round's schema after at most the one registered format-reminder retry.",
    "NOT-CARRIED": "answers exist and do not hold the format. A round is carried only WHOLE, so a judge malformed on any batch is excluded from that round entirely and the fact publishes with the failing batch names.",
    "NOT-AUDITIONED": "never called, with the rule that stopped the queue on the row. Distinct from NOT-CARRIED, which means it was called and failed.",
    "SKIPPED": "that vendor already holds a seat. Never two seated models from one vendor.",
    "NOT-RUN": "never run, and the reason always rides the row. Never a blank.",
    "NOT SCORED": "the seat instrument computed this row's floors and withholds its verdict, because no seat verdict is emitted over an unprobed run. The counts print; the verdict does not.",
    "PASS": "on the seat instrument: this candidate cleared both floors under a fully probed protocol. Admissible — never placed above another row, because the instrument is floors-and-counts and orders nothing.",
    "SPLIT": "a tie in a categorical round is its own published outcome, kept and never rounded to a majority (the tie rule the voice addendum registered after its original round — its id there, G5).",
    "DESCRIPTIVE": "these rounds describe; they do not rank. More vendors do not upgrade a descriptive leg into a ranked one — they widen the error bar the reader is entitled to see.",
    "counts_of_n": "every figure is a count against its denominator. Intervals ride beside counts as labelled companions and gate nothing.",
    "em_dash": "a figure we do not hold. It is never a zero, and 0.00 never means unmeasured — the bill uses both, for different facts.",
    "upper_bound": "every metered cost is priced at the uncached input rate, so it over-states and never under-states — with one exception, named on the row it affects."
  },
  "data_boundary": "What left the box for judging was model outputs over public-domain text and published game canon — never user data, never a character's private canon, never telemetry. The cloud judges saw the sealed batches published in this kit's terms and nothing else: no letter map, no model name, no arm roster, no other judge's answer, no part of a key file.",
  "judge_dating": "Cloud judges are versionless hosted services. Every cloud verdict in this kit was produced 2026-08-12/13 and is dated rather than version-pinned; it may not reproduce against a later checkpoint of the same tag. The local instruments and scorers these judges audit are sha-frozen, and the asymmetry is stated rather than hidden.",
  "tables": [
    {
      "table": "the audition",
      "companion": "audition.json",
      "what_this_is": "Every candidate in the registered judging queue, in queue order: the four that auditioned with their raw first responses verbatim, and the six that never did with the rule that stopped them. A candidate that carried has already ANSWERED the audition batch — it is a real batch of a real round, not a rehearsal — and its full round resumes past it.",
      "counting_rules": {
        "carriage": "a round is carried only WHOLE: a judge missing or malformed on any batch of a round is excluded from that round entirely and the fact publishes with the failing batch names. A round scored over 44 of 45 batches is not the round that was registered.",
        "recomputed_from_disk": "carriage is recomputed by the scorer from the verdict files, with the driver's own validators, and cross-checked against the driver's status file. The files are the authority; a disagreement is a finding.",
        "attempts": "calls to the endpoint for the audition batch, including the one registered format-reminder retry. 1 means it held the schema first try.",
        "verdict_objects": "a count of JUDGMENTS, not of comparisons. The judged population does not move: 270 comparisons, 36 tasks, 36 replies and 18 abstention replies, unchanged.",
        "em_dash": "a figure we do not hold. A bench model that was never called has no attempt count and no batch — that is an em dash, never a zero."
      },
      "limits": [
        "Four auditions is not a survey of frontier models against strict schemas. It is four.",
        "The audition is one batch of one round. A candidate that carries it has shown it can hold this verdict format on six replies, which is why carriage is then recomputed per round over every batch rather than inferred from the audition.",
        "All four carried, so this table has no failure row. The cascade is published anyway, because the next run of this protocol may need it and a bench nobody can see is a bench nobody can check."
      ]
    },
    {
      "table": "what five vendors said about the same comparisons",
      "companion": "panel-vendors.json",
      "what_this_is": "Every sealed round of the chair trials and the voice addendum, scored once per vendor. The same twelve items, six arms, two samples, 270 comparisons, 36 tasks, 36 replies and 18 abstention replies as the published rounds — the same batch files, byte for byte. What the extension adds is more readers of them.",
      "counting_rules": {
        "win_rate": "overall pairwise win-rate over all items, with a cluster bootstrap over items (effective N = 9 items) as its interval. A swap-discordant pair scores 0.5/0.5 rather than being discarded.",
        "population_unchanged": "the judged POPULATION does not move: 270 comparisons, 36 tasks, 36 replies, 18 abstention replies. Only the number of readers moves.",
        "per_vendor": "the same published scorer run over one vendor's seats.",
        "combined_all": "the same scorer over every carrying seat, with each verdict's family relabelled to its vendor.",
        "original_panel": "a verbatim call to the published scorer: the four-judge numbers, unchanged and untouched by this extension.",
        "structurally_unmovable": "the published scorer and the committed recount both walk KEY files, and a cloud seat has no key directory — so neither can see a cloud verdict at all, and the pre-registered 270 / 540 / 2160 / 432 / 144 cannot move. Not because we were careful: because there is nothing for them to read.",
        "em_dash": "a figure we do not hold. Never a zero."
      },
      "limits": [
        "One page, one shuffle per cloud judge. Each cloud judge is paired with ONE original seat, so its agreement with the other three carries that seat's presentation with it.",
        "Small N rides every rate here exactly as it rides the rounds being extended: nine effective items, two samples, three characters.",
        "One of the six judges' families authored the probe questions, this page, and the exhibits it audits. That is precisely why the other five exist.",
        "First-attempt only: production's corrective-retry ladder is not simulated."
      ]
    },
    {
      "table": "the inter-vendor agreement matrix",
      "companion": "agreement.json",
      "what_this_is": "Five vendors' judges scoring identical blinded comparisons, with the disagreements kept. For each round: every arm's rate under every vendor, the spread across the five, and all ten vendor pairs with their mean and maximum per-arm difference and their rank correlation.",
      "counting_rules": {
        "spread": "max rate minus min rate across the five vendors, on one arm, in one round — computed before rounding, so a spread cell can differ from subtracting the printed rates by 0.001.",
        "mean_abs_diff": "mean over arms of |rate(vendor A) − rate(vendor B)|.",
        "rank_spearman": "Spearman rank correlation over the six arms. Six arms is a very small basis for a rank statistic and it is a companion to the differences, never a substitute.",
        "divergence_leads": "divergence between vendors is a finding; it leads, it is not averaged away",
        "legB_panel_units": "the legB-panel figures are 0–10 dimension means, not rates, so its spreads and differences are in scale points and are NOT comparable with the rate rounds' figures."
      },
      "limits": [
        "Six arms and nine effective items. Every statistic here inherits that, and a rank correlation over six points moves a long way on one swapped pair.",
        "Each cloud judge read one original seat's shuffle, so a cross-vendor difference carries that seat's presentation with it.",
        "Agreement that is TOO good would be a fact about the replies being easy to separate, not about the panel being right. It is reported, not celebrated."
      ]
    },
    {
      "table": "the gauntlet",
      "companion": "gauntlet.json",
      "what_this_is": "Three candidates on the house's frozen seat instrument: 43 claim-cases against a human-verified key, three repeats each, twin floors frozen since July. Two of the three are the models that judged the chair trials and wrote this page.",
      "counting_rules": {
        "kills_denominator": "27 — an unmeasured kill case stays in the denominator.",
        "keeps_denominator": "16 — same rule.",
        "self_consistency": "unanimous cases / cases with at least one measurable repeat.",
        "long_rationales": "answered calls whose `why` exceeded the contract's maxLength. Recorded, never counted as failures: a verdict is not less of a verdict for a long rationale.",
        "latency": "reported and NON-BINDING in this run — cloud responses carry no load counter, so warm-equivalent is uncomputable and the agent-harness rows have no wire timings at all. Empty, never zero.",
        "em_dash": "a figure we do not hold. Never a zero."
      },
      "limits": [
        "A candidate clearing both floors on 43 cases is one instrument's verdict on one synthetic-adjacent population, not a general claim about the model.",
        "Two of the three candidates were judged by the same family that wrote this addendum, the questions and the article.",
        "The Claude arm's thinking posture is a confound, and it is the biggest one in the arm — stated first rather than last.",
        "This instrument cannot decide whether any candidate takes a production chair. Seating is a separate, measured product decision."
      ]
    },
    {
      "table": "the bill",
      "companion": "bill.json",
      "what_this_is": "Every duty this night bought, on both sides of the table: four cloud judges re-judging five sealed rounds, and the one metered model's three candidate legs besides. Published because nobody else does, and because \"we can't afford outside judges\" should now require a footnote.",
      "counting_rules": {
        "calls": "one request on the wire, retries included.",
        "tokens": "the endpoint's own prompt_eval_count and eval_count, summed over the calls that banked them. Never estimated, never modelled.",
        "cost": "tokens x the uncached price list, to four decimals; displayed to the cent.",
        "em_dash": "a figure we do not hold — an unmetered transport with no counters. It is never a zero, and 0.00 never means unmeasured."
      },
      "limits": [
        "These are our prices on our key on one night. A different plan, a different endpoint or a different day prices the same work differently.",
        "The gauntlet's four pre-run probe calls were charged and banked no counters, so the candidate-side figure is short by four small calls.",
        "The agent-harness duties are real work with a real cost that this instrument cannot see. Nothing here should be read as their being free."
      ]
    },
    {
      "table": "the verdict sets",
      "companion": "verdict-sets.json",
      "what_this_is": "Every judge's answer sheet from the cloud judging extension, as answered, paired with the original seat whose sealed pages it read. Five rounds, four seat pairs, 2592 cloud verdict objects over 228 batches, with the four original seats' sheets beside them so the pairs can be compared ref for ref.",
      "counting_rules": {
        "verdict_object": "one judgment. A c4-pairwise object is one (comparison, order); a c4-distinctness object is one task carrying three letter assignments; a legB-panel object is one letter scored on six dimensions; a c4-canon or legB-abstention object is one reply adjudicated.",
        "carriage": "a round is carried only whole. Every sheet in this file belongs to a round that judge carried; a round a judge dropped would be absent entirely and named in audition.json.",
        "seat_pair_agreement": "agreement over the slots the two sheets share, compared the way that round registers — one categorical field, the whole object, or per-dimension mean absolute difference for the one scale round. Raw agreement between two readers; never a quality measure."
      },
      "limits": [
        "Letters only. Without the map these sheets cannot be decoded, which is the point and also the limit.",
        "A seat pair is one cloud judge and one original judge, so a pair's agreement carries that seat's presentation with it.",
        "The other two original seats on the legB-panel round (a six-seat round) have no cloud partner and are not published here; the round's full scored output is in panel-vendors.json."
      ]
    }
  ],
  "contamination_caveat": "Published 2026-08-13. Models with a later training cutoff may have seen these sets. We author fresh sets each cycle; this one is not a standard, it is our kit, yours to reuse.",
  "not_a_standard": "Our own trials, our own judges, our own instruments. Not first independent numbers, and not a benchmark."
}