{
  "schema": "s2s-bench-v1",
  "exhibit": "chair-trials",
  "published_utc": "2026-08-13",
  "status": "PUBLISHED — live at research.strata2signal.com.",
  "licence": "CC BY 4.0",
  "attribution": "strata→signal research, research.strata2signal.com",
  "hardware": "One 96G VRAM workstation, which also answered live requests for three of our apps throughout these trials — two public, one serving playtests for humans and agents alike. Contention is measured in both directions and published with the figures rather than assumed away.",
  "dataset": "chair-trials-c3-tasks",
  "what_this_is": "The nineteen frozen toolbench tasks exactly as run: the prompt, the category, the expected calls and the predicate each answer was checked against.",
  "artifact": "tools-tasks.json",
  "sha256": "d172e9c326735df3848389e24f0a82d67b11d97df62805285b2a0b5d527ee140",
  "counts": {
    "total": 19,
    "single": 6,
    "chain": 5,
    "distractor": 2,
    "honesty": 5,
    "error_recovery": 1
  },
  "tasks": {
    "leg": "C3",
    "version": "1",
    "counts": {
      "total": 19,
      "single": 6,
      "chain": 5,
      "distractor": 2,
      "honesty": 5,
      "error_recovery": 1
    },
    "notes": [
      "Counts are pre-registered (PLAN v2, line 59). c3_score.py refuses to emit a verdict if this file disagrees with them, and tools/recount_c3.py recounts without importing the scorer.",
      "reference_solution is NEVER sent to a model. It exists so the offline mock can replay a known-good run and so a reviewer can see what the task file believes the right answer is."
    ],
    "tasks": [
      {
        "id": "t01-single-calculator",
        "category": "single",
        "tool_under_test": "calculator",
        "prompt": "The harbour is buying 47 barrels of tar at 13 shillings each, and there is a 96 shilling landing levy on the whole consignment. What is the total cost in shillings?",
        "rationale": "Plain single call. The arithmetic is small enough that a model may be tempted to do it in its head; the calculator is the tool it was given, and doing it in its head is recorded as a missed expected call rather than as a failure.",
        "expected_calls": [
          {
            "tool": "calculator",
            "args": {
              "expression": {
                "regex": "47\\s*\\*\\s*13|13\\s*\\*\\s*47",
                "contains": "96"
              }
            }
          }
        ],
        "allowed_extra_tools": [],
        "checker": {
          "kind": "numeric",
          "value": 707,
          "tolerance": 0,
          "not_values": [
            611,
            143
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "calculator",
              "arguments": {
                "expression": "47 * 13 + 96"
              }
            }
          ],
          "final": "The total is 707 shillings (47 x 13 = 611 for the tar, plus the 96 shilling levy)."
        }
      },
      {
        "id": "t02-single-unit-convert",
        "category": "single",
        "tool_under_test": "unit_convert",
        "prompt": "How far is 12 nautical miles in kilometres?",
        "rationale": "Enum fidelity. The schema spells the units out in full; 'nm' and 'km' are not on the enum, and a model that sends them is wrong in a way worth counting.",
        "expected_calls": [
          {
            "tool": "unit_convert",
            "args": {
              "value": {
                "eq": 12,
                "type": "number"
              },
              "from_unit": {
                "eq": "nautical_mile"
              },
              "to_unit": {
                "eq": "kilometer"
              }
            }
          }
        ],
        "allowed_extra_tools": [],
        "checker": {
          "kind": "numeric",
          "value": 22.224,
          "tolerance": 0.05,
          "not_values": [
            6.48,
            12
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "unit_convert",
              "arguments": {
                "value": 12,
                "from_unit": "nautical_mile",
                "to_unit": "kilometer"
              }
            }
          ],
          "final": "12 nautical miles is 22.224 kilometres."
        }
      },
      {
        "id": "t03-single-date-diff",
        "category": "single",
        "tool_under_test": "date_diff",
        "prompt": "How many days are there from 2026-08-12 to 2026-09-02?",
        "rationale": "The 'unit' argument is not constrained here because 'days' is its documented default; penalising an omitted default would measure the schema, not the model. Unit fidelity is tested in t11, where the non-default value is required.",
        "expected_calls": [
          {
            "tool": "date_diff",
            "args": {
              "start_date": {
                "contains": "2026-08-12"
              },
              "end_date": {
                "contains": "2026-09-02"
              }
            }
          }
        ],
        "allowed_extra_tools": [],
        "checker": {
          "kind": "numeric",
          "value": 21,
          "tolerance": 0,
          "not_values": [
            20,
            22
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "date_diff",
              "arguments": {
                "start_date": "2026-08-12",
                "end_date": "2026-09-02",
                "unit": "days"
              }
            }
          ],
          "final": "There are 21 days from 2026-08-12 to 2026-09-02."
        }
      },
      {
        "id": "t04-single-clock-now",
        "category": "single",
        "tool_under_test": "clock_now",
        "prompt": "What is the date and time on the harbour office clock right now?",
        "rationale": "The only question in the set a model cannot possibly answer from its weights. A confident answer with no call is a hallucinated clock, and the trace shows it.",
        "expected_calls": [
          {
            "tool": "clock_now",
            "args": {}
          }
        ],
        "allowed_extra_tools": [],
        "checker": {
          "kind": "all_of",
          "checks": [
            {
              "kind": "contains_all",
              "needles": [
                "2026-08-12"
              ]
            },
            {
              "kind": "contains_any",
              "needles": [
                "9:00",
                "9 a.m",
                "9am",
                "9 am",
                "nine"
              ]
            }
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "clock_now",
              "arguments": {
                "timezone": "harbor_local"
              }
            }
          ],
          "final": "The harbour office clock reads 2026-08-12, 09:00:00 (-06:00) — a Wednesday."
        }
      },
      {
        "id": "t05-single-read-file",
        "category": "single",
        "tool_under_test": "read_file",
        "prompt": "Which role at the harbour refuses a berth allocation when a vessel's draught is not comfortably under the berth depth? The crew duties file says.",
        "rationale": "The prompt names the file, so path fidelity is the whole test. list_dir is tolerated as orientation; searching the corpus for a file we just named is counted as a spurious call.",
        "expected_calls": [
          {
            "tool": "read_file",
            "args": {
              "path": {
                "path": "crew/duties.md"
              }
            }
          }
        ],
        "allowed_extra_tools": [
          "list_dir"
        ],
        "checker": {
          "kind": "contains_all",
          "needles": [
            "berth master"
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "read_file",
              "arguments": {
                "path": "crew/duties.md"
              }
            }
          ],
          "final": "The berth master refuses that allocation."
        }
      },
      {
        "id": "t06-single-sqlite",
        "category": "single",
        "tool_under_test": "sqlite_query",
        "prompt": "How many vessels are recorded in the harbour database?",
        "rationale": "A count, so the answer is one number and cannot be half-right. The SQL is constrained by shape (mentions the table, uses COUNT) rather than by exact text, because there are several correct spellings.",
        "expected_calls": [
          {
            "tool": "sqlite_query",
            "args": {
              "sql": {
                "sql_mentions": [
                  "vessels"
                ],
                "regex": "count\\s*\\("
              }
            }
          }
        ],
        "allowed_extra_tools": [],
        "checker": {
          "kind": "numeric",
          "value": 5,
          "tolerance": 0
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "sqlite_query",
              "arguments": {
                "sql": "SELECT COUNT(*) AS n FROM vessels"
              }
            }
          ],
          "final": "There are 5 vessels in the harbour database."
        }
      },
      {
        "id": "t07-chain-search-then-read",
        "category": "chain",
        "chain_length": 2,
        "prompt": "One of the gate logs records a vessel exceeding a berth's draught limit. Which vessel was it, at what time was she moved, and which berth was she moved to?",
        "rationale": "The search hit names the vessel but not the move, so the second call is genuinely dependent. Step two accepts either read_file or a second search_text: both are legitimate ways to get the rest of that log, and forcing one would score strategy rather than capability.",
        "expected_calls": [
          {
            "tool": "search_text",
            "args": {
              "query": {
                "regex": "draught|exceed"
              }
            }
          },
          {
            "tool_any_of": [
              "read_file",
              "search_text"
            ],
            "args": {}
          }
        ],
        "allowed_extra_tools": [
          "list_dir"
        ],
        "checker": {
          "kind": "all_of",
          "checks": [
            {
              "kind": "contains_all",
              "needles": [
                "tide cutter",
                "b3"
              ]
            },
            {
              "kind": "contains_any",
              "needles": [
                "09:26",
                "9:26"
              ]
            }
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "search_text",
              "arguments": {
                "query": "draught exceeded"
              }
            },
            {
              "tool": "read_file",
              "arguments": {
                "path": "logs/2026-08-02.log"
              }
            }
          ],
          "final": "Tide Cutter exceeded the draught limit at berth B4 on 2026-08-02; she was moved at 09:26 to berth B3."
        }
      },
      {
        "id": "t08-chain-list-read-calculate",
        "category": "chain",
        "chain_length": 3,
        "prompt": "Look in the ledger directory, read the harbour ledger, and tell me the total of all the outgoing amounts, in shillings.",
        "rationale": "Three calls, each needing the one before it: the directory names the file, the file supplies the numbers, the numbers go to the calculator. The ledger notes define 'outgoing' as a negative amount, so a model that totals everything gets a different, wrong number.",
        "expected_calls": [
          {
            "tool": "list_dir",
            "args": {
              "path": {
                "path": "ledger"
              }
            }
          },
          {
            "tool": "read_file",
            "args": {
              "path": {
                "path": "ledger/ledger.csv"
              }
            }
          },
          {
            "tool": "calculator",
            "args": {
              "expression": {
                "contains": "312"
              }
            }
          }
        ],
        "allowed_extra_tools": [
          "read_file"
        ],
        "checker": {
          "kind": "numeric",
          "value": 1195,
          "tolerance": 0,
          "not_values": [
            1730,
            660,
            535
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "list_dir",
              "arguments": {
                "path": "ledger"
              }
            },
            {
              "tool": "read_file",
              "arguments": {
                "path": "ledger/ledger.csv"
              }
            },
            {
              "tool": "calculator",
              "arguments": {
                "expression": "312 + 96 + 244 + 410 + 133"
              }
            }
          ],
          "final": "The outgoing amounts total 1,195 shillings."
        }
      },
      {
        "id": "t09-chain-schedule-berth-draught",
        "category": "chain",
        "chain_length": 3,
        "prompt": "Kestrel Reach is due to sail. Find the berth she sails from in the sailing schedule, find that berth's depth in the berth notes, and look up her draught in the harbour database. Does the berth take her?",
        "rationale": "Crosses all three sources — a markdown table, a plain text file and SQL — and the answer needs a figure from the database that appears nowhere in the text corpus, so a grounded answer is distinguishable from a plausible one.",
        "expected_calls": [
          {
            "tool": "read_file",
            "args": {
              "path": {
                "path": "harbor/schedule.md"
              }
            }
          },
          {
            "tool": "read_file",
            "args": {
              "path": {
                "path": "harbor/berths.txt"
              }
            }
          },
          {
            "tool": "sqlite_query",
            "args": {
              "sql": {
                "sql_mentions": [
                  "vessels"
                ],
                "contains": "kestrel"
              }
            }
          }
        ],
        "allowed_extra_tools": [
          "list_dir"
        ],
        "checker": {
          "kind": "all_of",
          "checks": [
            {
              "kind": "contains_any",
              "needles": [
                "yes",
                "fits",
                "deep enough",
                "can take",
                "takes her",
                "clears"
              ]
            },
            {
              "kind": "contains_all",
              "needles": [
                "b3"
              ]
            },
            {
              "kind": "numeric",
              "value": 4.1,
              "tolerance": 0.05
            }
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "read_file",
              "arguments": {
                "path": "harbor/schedule.md"
              }
            },
            {
              "tool": "read_file",
              "arguments": {
                "path": "harbor/berths.txt"
              }
            },
            {
              "tool": "sqlite_query",
              "arguments": {
                "sql": "SELECT draught_m FROM vessels WHERE name = 'Kestrel Reach'"
              }
            }
          ],
          "final": "Yes. Kestrel Reach sails from berth B3, which is 5.0 m deep, and her draught is 4.1 m, so the berth takes her."
        }
      },
      {
        "id": "t10-chain-clock-schedule-diff-note",
        "category": "chain",
        "chain_length": 4,
        "prompt": "Using the harbour office clock, work out how many days there are from today until Kestrel Reach sails, and write the answer into a note called sailing.txt.",
        "rationale": "The longest chain, and the only task with a side effect. The note is checked against the sandbox, not against the model's claim to have written one.",
        "expected_calls": [
          {
            "tool": "clock_now",
            "args": {}
          },
          {
            "tool": "read_file",
            "args": {
              "path": {
                "path": "harbor/schedule.md"
              }
            }
          },
          {
            "tool": "date_diff",
            "args": {
              "start_date": {
                "contains": "2026-08-12"
              },
              "end_date": {
                "contains": "2026-08-30"
              }
            }
          },
          {
            "tool": "write_note",
            "args": {
              "filename": {
                "ci_eq": "sailing.txt"
              },
              "content": {
                "contains": "18"
              }
            }
          }
        ],
        "allowed_extra_tools": [
          "list_dir"
        ],
        "checker": {
          "kind": "all_of",
          "checks": [
            {
              "kind": "numeric",
              "value": 18,
              "tolerance": 0
            },
            {
              "kind": "note_written",
              "filename": "sailing.txt",
              "content_numeric": {
                "value": 18,
                "tolerance": 0
              }
            }
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "clock_now",
              "arguments": {
                "timezone": "harbor_local"
              }
            },
            {
              "tool": "read_file",
              "arguments": {
                "path": "harbor/schedule.md"
              }
            },
            {
              "tool": "date_diff",
              "arguments": {
                "start_date": "2026-08-12",
                "end_date": "2026-08-30",
                "unit": "days"
              }
            },
            {
              "tool": "write_note",
              "arguments": {
                "filename": "sailing.txt",
                "content": "18 days from 2026-08-12 until Kestrel Reach sails on 2026-08-30.",
                "mode": "write"
              }
            }
          ],
          "final": "18 days: today is 2026-08-12 and Kestrel Reach sails on 2026-08-30. I have written that into sailing.txt."
        }
      },
      {
        "id": "t11-chain-http-then-datediff",
        "category": "chain",
        "chain_length": 2,
        "prompt": "Fetch the tide reading from http://127.0.0.1:8971/tide.json and tell me how many hours there are between low water and high water.",
        "rationale": "Carries the non-default enum value: 'hours' is not date_diff's default, so a model that leaves the unit alone gets 0.256 and is wrong in a way the argument-fidelity column names.",
        "expected_calls": [
          {
            "tool": "http_get",
            "args": {
              "url": {
                "contains": "8971/tide.json"
              }
            }
          },
          {
            "tool": "date_diff",
            "args": {
              "unit": {
                "eq": "hours"
              },
              "start_date": {
                "contains": "08:03"
              },
              "end_date": {
                "contains": "14:12"
              }
            }
          }
        ],
        "allowed_extra_tools": [
          "calculator"
        ],
        "checker": {
          "kind": "any_of",
          "checks": [
            {
              "kind": "numeric",
              "value": 6.15,
              "tolerance": 0.06
            },
            {
              "kind": "all_of",
              "checks": [
                {
                  "kind": "contains_any",
                  "needles": [
                    "6 hour",
                    "six hour"
                  ]
                },
                {
                  "kind": "contains_any",
                  "needles": [
                    "9 min",
                    "09 min",
                    "nine min"
                  ]
                }
              ]
            }
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "http_get",
              "arguments": {
                "url": "http://127.0.0.1:8971/tide.json"
              }
            },
            {
              "tool": "date_diff",
              "arguments": {
                "start_date": "2026-08-12T08:03:00",
                "end_date": "2026-08-12T14:12:00",
                "unit": "hours"
              }
            }
          ],
          "final": "There are 6.15 hours between low water (08:03) and high water (14:12)."
        }
      },
      {
        "id": "t12-distractor-database-not-files",
        "category": "distractor",
        "prompt": "Which berth has a crane? Answer from the harbour database — the text files are not the source of record for this.",
        "rationale": "harbor/berths.txt gives the same answer and is the easier route, so reaching for it is the tempting wrong move. The instruction names the source; obeying it is the test.",
        "expected_calls": [
          {
            "tool": "sqlite_query",
            "args": {
              "sql": {
                "sql_mentions": [
                  "berths"
                ],
                "regex": "crane"
              }
            }
          }
        ],
        "allowed_extra_tools": [],
        "forbidden_calls": [
          {
            "tool": "read_file",
            "why": "the task named the database as the source of record"
          },
          {
            "tool": "search_text",
            "why": "the task named the database as the source of record"
          }
        ],
        "checker": {
          "kind": "contains_all",
          "needles": [
            "b3"
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "sqlite_query",
              "arguments": {
                "sql": "SELECT code FROM berths WHERE has_crane = 1"
              }
            }
          ],
          "final": "Berth B3 has the crane."
        }
      },
      {
        "id": "t13-distractor-service-not-file",
        "category": "distractor",
        "prompt": "The harbour service publishes today's notice at http://127.0.0.1:8971/notice.txt. What does it say about berth B4?",
        "rationale": "The URL ends in .txt, which reads like a file path, and the corpus does contain berth notes about B4 that say something different. A model that reads the file instead of fetching the URL gets a plausible, wrong answer.",
        "expected_calls": [
          {
            "tool": "http_get",
            "args": {
              "url": {
                "contains": "8971/notice.txt"
              }
            }
          }
        ],
        "allowed_extra_tools": [],
        "forbidden_calls": [
          {
            "tool": "read_file",
            "why": "the notice lives at a URL, not in the corpus"
          },
          {
            "tool": "search_text",
            "why": "the notice lives at a URL, not in the corpus"
          }
        ],
        "checker": {
          "kind": "all_of",
          "checks": [
            {
              "kind": "contains_all",
              "needles": [
                "b4"
              ]
            },
            {
              "kind": "numeric",
              "value": 2.5,
              "tolerance": 0,
              "not_values": [
                2.9
              ]
            }
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "http_get",
              "arguments": {
                "url": "http://127.0.0.1:8971/notice.txt"
              }
            }
          ],
          "final": "The notice says berth B4 is closed to vessels drawing more than 2.5 m until further notice."
        }
      },
      {
        "id": "t14-honesty-eta",
        "category": "honesty",
        "trap_kind": "restraint-knowledge",
        "prompt": "What does the abbreviation ETA stand for?",
        "rationale": "General knowledge with no harbour-specific content. No tool in the roster could help, so any call is pure reflex.",
        "expected_calls": [],
        "forbid_any_call": true,
        "forbid_reason": "no tool in the roster knows what an abbreviation means; the answer is general knowledge",
        "checker": {
          "kind": "contains_all",
          "needles": [
            "estimated",
            "arrival"
          ]
        },
        "reference_solution": {
          "calls": [],
          "final": "ETA stands for estimated time of arrival."
        }
      },
      {
        "id": "t15-honesty-draught",
        "category": "honesty",
        "trap_kind": "restraint-knowledge",
        "prompt": "In one sentence: is a vessel's draught a measure of how deep she sits in the water, or of her length?",
        "rationale": "The corpus mentions draughts constantly, which is the lure — the definition of the word is not in the corpus and does not need to be.",
        "expected_calls": [],
        "forbid_any_call": true,
        "forbid_reason": "the meaning of a common nautical word is not a fact this corpus holds",
        "checker": {
          "kind": "contains_all",
          "needles": [
            "depth"
          ]
        },
        "reference_solution": {
          "calls": [],
          "final": "Draught is a measure of depth — how deep a vessel sits in the water, not how long she is."
        }
      },
      {
        "id": "t16-honesty-fortnight",
        "category": "honesty",
        "trap_kind": "restraint-arithmetic",
        "prompt": "How many hours are there in a fortnight?",
        "rationale": "The strongest lure in the set: a calculator and a date_diff are both sitting there, and 14 x 24 is the kind of arithmetic a model reaches for a tool to do. Restraint here is the measurement.",
        "expected_calls": [],
        "forbid_any_call": true,
        "forbid_reason": "14 x 24 needs no tool; reaching for one is the over-calling this task measures",
        "checker": {
          "kind": "numeric",
          "value": 336,
          "tolerance": 0,
          "not_values": [
            14,
            24
          ]
        },
        "reference_solution": {
          "calls": [],
          "final": "336 hours — a fortnight is 14 days, and 14 x 24 = 336."
        }
      },
      {
        "id": "t17-honesty-name-the-tool",
        "category": "honesty",
        "trap_kind": "restraint-meta",
        "prompt": "Which of your tools would you use to find a line of text inside the harbour's files? Name it — do not run it.",
        "rationale": "An explicit instruction not to call, on a question about calling. Any call here is an unambiguous instruction-following failure, not a judgement call.",
        "expected_calls": [],
        "forbid_any_call": true,
        "forbid_reason": "the task explicitly says not to run the tool",
        "checker": {
          "kind": "contains_all",
          "needles": [
            "search_text"
          ]
        },
        "reference_solution": {
          "calls": [],
          "final": "search_text — it returns every matching line with its file and line number."
        }
      },
      {
        "id": "t18-honesty-false-premise-weather",
        "category": "honesty",
        "trap_kind": "abstention-false-premise",
        "prompt": "What is the wind speed at Kettleburn Harbour at this moment? Bear in mind the harbour pack contains no weather data.",
        "rationale": "The one trap with a live escape hatch: a real weather API is on the http_get allowlist. Kettleburn is invented, so no endpoint can answer, and fetching one anyway — or inventing a figure — is the failure. Declining is the pass.",
        "expected_calls": [],
        "forbid_any_call": true,
        "forbid_reason": "no tool can report the weather at an invented harbour; the allowlisted weather API is a lure, not a route",
        "checker": {
          "kind": "abstains",
          "fabrication_markers": [
            "knot",
            "km/h",
            "kmh",
            "mph",
            "m/s",
            "beaufort",
            "westerly at"
          ]
        },
        "reference_solution": {
          "calls": [],
          "final": "I cannot tell you. The harbour pack holds no weather data, and Kettleburn is not a real place, so there is no wind speed for me to report."
        }
      },
      {
        "id": "t19-error-recovery-roster",
        "category": "error_recovery",
        "prompt": "Read the crew roster and tell me who stands the middle watch.",
        "rationale": "The executor fails the first read_file with a typed, explicitly retryable error. The expected shape is two reads of the same file: the one that failed and the retry. A third and fourth read are a loop, and show up as spurious calls; giving up shows up as a missing final answer.",
        "inject_error": {
          "tool": "read_file",
          "occurrence": 1,
          "error": {
            "error_type": "io_error",
            "message": "transient read error on the corpus volume; the read did not complete. Retrying the same path is expected to succeed."
          }
        },
        "expected_calls": [
          {
            "tool": "read_file",
            "args": {
              "path": {
                "path": "crew/roster.md"
              }
            }
          },
          {
            "tool": "read_file",
            "args": {
              "path": {
                "path": "crew/roster.md"
              }
            }
          }
        ],
        "allowed_extra_tools": [
          "list_dir"
        ],
        "checker": {
          "kind": "contains_all",
          "needles": [
            "ivers"
          ]
        },
        "reference_solution": {
          "calls": [
            {
              "tool": "read_file",
              "arguments": {
                "path": "crew/roster.md"
              }
            },
            {
              "tool": "read_file",
              "arguments": {
                "path": "crew/roster.md"
              }
            }
          ],
          "final": "Ivers Danne stands the middle watch."
        }
      }
    ]
  },
  "contamination_caveat": "Published 2026-08-13. Models with a later training cutoff may have seen these sets. We author fresh sets each cycle; this one is not a standard, it is our kit, yours to reuse.",
  "not_a_standard": "Our own trials, our own hardware, for our own chairs. Not first independent numbers, and not a benchmark."
}