{
 "arm": "laya-cpu",
 "task": "kperm",
 "rows": 108,
 "summarise": {
  "n": 108,
  "n_reached": 108,
  "n_labelled": 108,
  "refused": 0,
  "refused_by_reason": {},
  "correct": 19,
  "errors": 0,
  "brier": 1.3583993800925926,
  "reliability": [
   {
    "lo": 0.0,
    "hi": 0.1,
    "n": 0,
    "sum_p": 0.0,
    "correct": 0,
    "mean_p": null,
    "accuracy": null,
    "gap": null
   },
   {
    "lo": 0.1,
    "hi": 0.2,
    "n": 0,
    "sum_p": 0.0,
    "correct": 0,
    "mean_p": null,
    "accuracy": null,
    "gap": null
   },
   {
    "lo": 0.2,
    "hi": 0.3,
    "n": 0,
    "sum_p": 0.0,
    "correct": 0,
    "mean_p": null,
    "accuracy": null,
    "gap": null
   },
   {
    "lo": 0.3,
    "hi": 0.4,
    "n": 4,
    "sum_p": 1.4176,
    "correct": 1,
    "mean_p": 0.3544,
    "accuracy": 0.25,
    "gap": 0.10439999999999999
   },
   {
    "lo": 0.4,
    "hi": 0.5,
    "n": 9,
    "sum_p": 4.019699999999999,
    "correct": 5,
    "mean_p": 0.44663333333333327,
    "accuracy": 0.5555555555555556,
    "gap": -0.10892222222222231
   },
   {
    "lo": 0.5,
    "hi": 0.6,
    "n": 16,
    "sum_p": 8.7934,
    "correct": 3,
    "mean_p": 0.5495875,
    "accuracy": 0.1875,
    "gap": 0.3620875
   },
   {
    "lo": 0.6,
    "hi": 0.7,
    "n": 9,
    "sum_p": 5.8871,
    "correct": 1,
    "mean_p": 0.6541222222222223,
    "accuracy": 0.1111111111111111,
    "gap": 0.5430111111111111
   },
   {
    "lo": 0.7,
    "hi": 0.8,
    "n": 14,
    "sum_p": 10.3598,
    "correct": 1,
    "mean_p": 0.7399857142857142,
    "accuracy": 0.07142857142857142,
    "gap": 0.6685571428571428
   },
   {
    "lo": 0.8,
    "hi": 0.9,
    "n": 18,
    "sum_p": 15.4662,
    "correct": 2,
    "mean_p": 0.8592333333333334,
    "accuracy": 0.1111111111111111,
    "gap": 0.7481222222222224
   },
   {
    "lo": 0.9,
    "hi": 1.0,
    "n": 38,
    "sum_p": 36.19070000000001,
    "correct": 6,
    "mean_p": 0.9523868421052634,
    "accuracy": 0.15789473684210525,
    "gap": 0.7944921052631582
   }
  ],
  "median_seconds": 0.26511428141384386,
  "p95_seconds": 0.6090155803297724,
  "prompt_tokens_total": 23887,
  "eval_tokens_total": 0,
  "unparsed": 0,
  "floored_rows": 0,
  "letters_floored_total": 0,
  "accuracy": 0.17592592592592593
 },
 "right": 19,
 "scored": 108,
 "wilson95": [
  0.11561862287239963,
  0.25849540693210094
 ],
 "hash_matched": 108,
 "hash_no_twin": 0,
 "argmax_disagrees": 0,
 "latency_published": false,
 "started": "2026-10-01T07:14:09Z",
 "ended": "2026-10-01T07:18:18Z",
 "meta": {
  "arm": "laya-cpu",
  "label": "LAYA-1",
  "model": "convaiinnovations/laya@55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",
  "server": "laya-serve (laya 0.3.22), its own CLI, 127.0.0.1:8032 inside the network-less sandbox; one question per request; run_kev.kev_post unedited",
  "readout": "Laya Router (laya-serve): one [MASK] option marker per option, softmax over the markers; noul = the two options false/true with the criteria's descriptions; the Router picks the checkpoint per request (english or multilingual; typed-decisions only on request)",
  "box": "this laptop",
  "processor": "Intel(R) Core(TM) Ultra 9 290HX Plus",
  "cpu_affinity": [
   0,
   1,
   2,
   3,
   4,
   5,
   6,
   7
  ],
  "os": {
   "pretty_name": "Ubuntu 26.04.1 LTS",
   "id": "ubuntu",
   "version_id": "26.04",
   "kernel": "7.0.0-34-generic",
   "arch": "x86_64",
   "source": "/etc/os-release"
  },
  "versions": {
   "python": "3.13.14",
   "laya": "0.3.22",
   "torch": "2.14.1+cpu",
   "transformers": "5.18.0",
   "tokenizers": "0.23.2",
   "huggingface-hub": "1.33.0",
   "safetensors": "0.8.0",
   "numpy": "2.5.3",
   "fastapi": "0.142.2",
   "starlette": "1.7.0",
   "uvicorn": "0.54.0",
   "pydantic": "2.13.5",
   "accelerate": null,
   "hf-xet": "1.6.0",
   "openvino": null,
   "openvino-telemetry": null,
   "onnxruntime": null,
   "opentelemetry-api": "1.45.0",
   "dists": [
    "Jinja2==3.1.6",
    "MarkupSafe==3.0.3",
    "PyYAML==6.0.3",
    "Pygments==2.21.0",
    "annotated-doc==0.0.5",
    "annotated-types==0.8.0",
    "anyio==4.15.1",
    "certifi==2026.7.22",
    "click==8.5.0",
    "fastapi==0.142.2",
    "filelock==4.0.7",
    "fsspec==2026.9.0",
    "h11==0.16.0",
    "hf-xet==1.6.0",
    "httpcore==1.0.9",
    "httpx==0.28.1",
    "huggingface_hub==1.33.0",
    "idna==3.20",
    "laya==0.3.22",
    "markdown-it-py==4.2.0",
    "mdurl==0.1.2",
    "mpmath==1.3.0",
    "networkx==3.7",
    "numpy==2.5.3",
    "opentelemetry-api==1.45.0",
    "packaging==26.3",
    "pydantic==2.13.5",
    "pydantic_core==2.46.5",
    "python-multipart==0.0.32",
    "regex==2026.9.29",
    "rich==15.0.0",
    "safetensors==0.8.0",
    "setuptools==84.0.0",
    "shellingham==1.5.4",
    "starlette==1.7.0",
    "sympy==1.14.0",
    "tokenizers==0.23.2",
    "torch==2.14.1+cpu",
    "tqdm==4.70.1",
    "transformers==5.18.0",
    "typer==0.27.2",
    "typing-inspection==0.4.4",
    "typing_extensions==4.16.0",
    "uvicorn==0.54.0"
   ]
  },
  "switches": {
   "CUDA_VISIBLE_DEVICES": "",
   "HF_HUB_OFFLINE": "1",
   "TRANSFORMERS_OFFLINE": "1",
   "HF_HUB_DISABLE_TELEMETRY": "1",
   "ORT_DISABLE_TELEMETRY": "1",
   "VLLM_NO_USAGE_STATS": "1",
   "GRADIO_ANALYTICS_ENABLED": "False",
   "DO_NOT_TRACK": null,
   "OMP_NUM_THREADS": "8",
   "MKL_NUM_THREADS": "8",
   "TOKENIZERS_PARALLELISM": "false",
   "USE_TF": "0",
   "PYTHONDONTWRITEBYTECODE": "1",
   "VON_DEVICE": "cpu",
   "LAYA_DEVICE": "cpu",
   "LAYA_REVISION": "reviewed",
   "LAYA_MODELS": "english,multilingual",
   "LAYA_PRELOAD": "1",
   "LAYA_THREADS": "8",
   "LAYA_AUTO_TASK": null,
   "HF_HOME": "/workshop/bench-archive/plans-2026-10-01/deciders-von-laya/hf"
  },
  "health": {
   "status": "ok",
   "loaded": [
    "english",
    "multilingual"
   ],
   "revisions": {
    "english": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",
    "multilingual": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"
   },
   "device": "cpu",
   "device_is_preference": false,
   "checkpoint_devices": {
    "english": "cpu",
    "multilingual": "cpu"
   },
   "cpu_fallbacks": {
    "english": {
     "count": 0,
     "last_reason": null
    },
    "multilingual": {
     "count": 0,
     "last_reason": null
    }
   }
  },
  "models": {
   "unavailable": "HTTP Error 404: Not Found"
  },
  "server_ready_s": 7.0,
  "seed": 20260921,
  "smoke": false,
  "preflight": {
   "kc": {
    "bodies": 108,
    "equal_to_twin": 108,
    "skipped_inherited_overlong": 0
   },
   "kd": {
    "bodies": 48,
    "equal_to_twin": 48,
    "skipped_inherited_overlong": 0
   },
   "kf": {
    "bodies": 40,
    "equal_to_twin": 40,
    "skipped_inherited_overlong": 0
   },
   "kj": {
    "bodies": 119,
    "equal_to_twin": 119,
    "skipped_inherited_overlong": 0
   },
   "ka": {
    "bodies": 53,
    "equal_to_twin": 53,
    "skipped_inherited_overlong": 10
   },
   "kperm": {
    "items": 108,
    "order0_equal_to_kev9b": 108,
    "orders_per_item": 6
   },
   "kh": {
    "items": 48,
    "builder": "task_kc's six-way builder; S0 rebuilt with it 108/108 equal"
   }
  },
  "server_vmhwm_mb": 3984.1
 },
 "kperm_moved": 52
}
