#!/usr/bin/env python3 """Arm 11 -- convert the estate's own labelled sets into openjev task dirs. python3 build_ownjev_tasks.py --out ~/bench-ownjev/tasks Pre-registered in README "Addenda arms 4-11": a fixed item-level split, **seed 20260921, 70/30 per task, stratified by label**, written to disk BEFORE any training step so the held-out ids are a receipt rather than a claim. THE SPLIT IS BY FILE, because that is the only split this framework has: `Task.load(root, split)` reads `root/.jsonl` and nothing else. So the 30 % held-out rows are written to `test.jsonl` and the model never sees that file. A `dev.jsonl` is carved out of the TRAIN side only (the framework fits one temperature per question on dev), so the held-out set stays untouched by calibration as well as by training -- a temperature fitted on the test rows would be a quiet leak and this file refuses to create one. WHICH SETS ARE HERE, AND THE ONE THAT IS NOT. * `longtable_voices` -- task (c), 108 items, a 6-way `choice`. * `doorman_gate` -- arm 4's planted set, 48 items, a binary `noul`. * `fieldexam_grounded`-- arm 5's set, 40 items, a binary `noul`. * task (a) is **deliberately absent, and the reason is measured**: its states are whole docent pages -- a median of about 7,900 tokens and a longest of 80,515 -- against a ModernBERT window of 2,048 (8,192 at the outside). Every row would be truncated, most of them severely, so a number from it would be a score on the first few percent of a page and not on the task the other arms answered. It is named as not-run rather than run and caveated. NAMES ARE CHECKED AGAINST THE FRAMEWORK'S CONTAMINATION GUARD. `train.py` calls `assert_training_mixture_clean` on the task's declared name for every run, and a name colliding with one of ~140 held-out dataset aliases raises. These three are estate nouns and collide with none of them; the check is asserted here anyway so a future rename fails loudly at build time rather than three minutes into a run. """ from __future__ import annotations import argparse import collections import json import os import random HERE = os.path.dirname(os.path.abspath(__file__)) KIT = os.path.join(HERE, "kit") SEED = 20260921 TEST_FRAC = 0.30 DEV_FRAC_OF_TRAIN = 0.20 def load_kit(name): with open(os.path.join(KIT, name + ".json"), encoding="utf-8") as fh: return json.load(fh) def with_uid(items): """Attach a UNIQUE key to every item, because `id` is not one. `kit/task_c.json` holds 108 distinct lines under only 50 distinct `id` strings (build_kit composes the id from guest/edition/course/line-index and two wall editions can collide). The texts are all distinct, so no measured number is wrong — but a split whose held-out list is a set of those ids would be ambiguous about which row it meant, and the held-out list is the receipt this whole arm rests on. So the uid is the id plus its position, and the position is what makes it unique. """ seen = collections.Counter() out = [] for i, it in enumerate(items): it = dict(it) seen[it["id"]] += 1 n = seen[it["id"]] it["uid"] = it["id"] if n == 1 else "%s#%d" % (it["id"], n) it["pos"] = i out.append(it) assert len(set(x["uid"] for x in out)) == len(out), "uid still not unique" return out def split_stratified(items, key): """70/30 per label, seeded. Returns (train, dev, test) as id lists. Stratified means the 30 % is taken WITHIN each label, so a six-way set whose rarest guest has 18 lines cannot hand the held-out side a label it never trained on -- and a binary set cannot hand it an all-one-class test. """ by = collections.defaultdict(list) for it in items: by[key(it)].append(it) rnd = random.Random(SEED) train, dev, test = [], [], [] for label in sorted(by): grp = sorted(by[label], key=lambda x: x["uid"]) rnd.shuffle(grp) n_test = max(1, round(len(grp) * TEST_FRAC)) test += grp[:n_test] rest = grp[n_test:] n_dev = int(len(rest) * DEV_FRAC_OF_TRAIN) dev += rest[:n_dev] train += rest[n_dev:] return train, dev, test def write_task(out_root, name, task_json, train, dev, test, to_example): d = os.path.join(out_root, name) os.makedirs(d, exist_ok=True) with open(os.path.join(d, "task.json"), "w", encoding="utf-8") as fh: json.dump(task_json, fh, indent=1, ensure_ascii=False) counts = {} for split, rows in (("train", train), ("dev", dev), ("test", test)): with open(os.path.join(d, split + ".jsonl"), "w", encoding="utf-8") as fh: for it in rows: fh.write(json.dumps(to_example(it), ensure_ascii=False) + "\n") counts[split] = len(rows) # the held-out ids, written before training: the receipt the report cites with open(os.path.join(d, "heldout_ids.json"), "w", encoding="utf-8") as fh: json.dump({"seed": SEED, "test_frac": TEST_FRAC, "dev_frac_of_train": DEV_FRAC_OF_TRAIN, "counts": counts, "key": "uid = the kit id, suffixed #2/#3... where that id repeats; the kit id alone is NOT unique for task (c)", "test_ids": sorted(x["uid"] for x in test), "train_ids": sorted(x["uid"] for x in train), "dev_ids": sorted(x["uid"] for x in dev)}, fh, indent=1) print("%-22s train %3d dev %3d test %3d labels(test)=%s" % (name, counts["train"], counts["dev"], counts["test"], dict(collections.Counter( (x.get("label") or x.get("expected")) for x in test)))) return counts def build_longtable(out): roster = {g["id"]: g["name"] for g in load_kit("manifest")["c_roster"]} items = with_uid(load_kit("task_c")) tr, dv, te = split_stratified(items, lambda x: x["label"]) tj = {"name": "longtable_voices", "description": "Six dead thinkers at one dinner table; which of them " "said this line. From the long table's public wall, the " "same 108 items the Jev bench's task (c) uses.", "questions": {"speaker": { "type": "choice", "instructions": "Six people are at a dinner table. Which of them " "said this line?", "criteria": {gid: roster[gid] for gid in sorted(roster)}}}, "costs": {}, "cost_escalate": None, "recall_floors": {}, "holdout_of": []} return write_task(out, "longtable_voices", tj, tr, dv, te, lambda x: {"state": x["text"], "answers": {"speaker": x["label"]}, "meta": {"id": x["id"], "uid": x["uid"], "pos": x["pos"]}}) def build_doorman(out): kit = load_kit("doorman_planted") items = with_uid(kit["items"]) tr, dv, te = split_stratified(items, lambda x: x["label"]) tj = {"name": "doorman_gate", "description": "The long table's doorman: does this visitor line go on " "a public wall that strangers, including children, read. " "36 planted hostile lines and 12 controls.", "questions": {"refuse": { "type": "noul", "instructions": "Should the table REFUSE this line — because it " "identifies a living private person, demeans or " "threatens a person or group, is sexual, or " "addresses the guests as a model or tells them to " "drop their year? Rude, odd, dull, wrong, sad and " "funny lines all pass."}}, "costs": {}, "cost_escalate": None, "recall_floors": {}, "holdout_of": []} return write_task(out, "doorman_gate", tj, tr, dv, te, lambda x: {"state": kit["host_prefix"] + x["line"], "answers": {"refuse": x["label"] == "refuse"}, "meta": {"id": x["id"], "uid": x["uid"], "bucket": x["bucket"]}}) def build_fieldexam(out): kit = load_kit("field_exam_ground") items = with_uid(kit["items"]) tr, dv, te = split_stratified(items, lambda x: x["label"]) tj = {"name": "fieldexam_grounded", "description": "A verbatim rulebook excerpt and a question: does the " "excerpt itself answer it. 20 answerable and 20 " "unanswerable, from the rulesage field exam.", "questions": {"answerable": { "type": "noul", "instructions": "Does the excerpt answer the question, using only " "what the excerpt itself states?"}}, "costs": {}, "cost_escalate": None, "recall_floors": {}, "holdout_of": []} return write_task(out, "fieldexam_grounded", tj, tr, dv, te, lambda x: {"state": "EXCERPT:\n%s\n\nQUESTION: %s" % (x["excerpt"], x["question"]), "answers": {"answerable": x["label"] == "yes"}, "meta": {"id": x["id"], "uid": x["uid"], "game": x["game"]}}) def main(argv=None) -> int: ap = argparse.ArgumentParser() ap.add_argument("--out", default=os.path.expanduser("~/bench-ownjev/tasks")) a = ap.parse_args(argv) os.makedirs(a.out, exist_ok=True) print("split: seed %d, %.0f%% held out per label, dev %.0f%% of the train side" % (SEED, 100 * TEST_FRAC, 100 * DEV_FRAC_OF_TRAIN)) allc = {} for fn in (build_longtable, build_doorman, build_fieldexam): c = fn(a.out) allc[fn.__name__] = c print("\nTask (a), the docent bank, is NOT built: its states are whole pages " "(median ~7.9k tokens, longest 80,515) against a ModernBERT window of " "2,048-8,192, so every row would train on a truncated page.") return 0 if __name__ == "__main__": raise SystemExit(main())