{ "D": { "raw": { "ok": 0, "n": 364, "compare": { "C|long-write|L16|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 2765d409cb99, vllm 2611 sha 5adb3c45e0dc" ] }, "C|long-write|L16|r0|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 28dac3b7e7fd, vllm 2611 sha 32ca1bc1813b" ] }, "C|long-write|L16|r0|q10": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha e34b7aabca30, vllm 2611 sha e680df5b65b4" ] }, "C|long-write|L16|r0|q11": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha b1d72366fa72, vllm 2611 sha 27fe34694798" ] }, "C|long-write|L16|r0|q12": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 12673d687bdd, vllm 2611 sha af44e4c810b6" ] }, "C|long-write|L16|r0|q13": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 34f522e53584, vllm 2611 sha b34620cfd007" ] }, "C|long-write|L16|r0|q14": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha da15a1b687b0, vllm 2611 sha ee70d2e551ff" ] }, "C|long-write|L16|r0|q15": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha edf7ace752f1, vllm 2611 sha dd3111911138" ] }, "C|long-write|L16|r0|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 6f23f8e1b5f0, vllm 2611 sha a76269dfe48a" ] }, "C|long-write|L16|r0|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 514d1c54535c, vllm 2611 sha b903335e6dd7" ] }, "C|long-write|L16|r0|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha f14914e902c9, vllm 2611 sha 0dc161ddc39a" ] }, "C|long-write|L16|r0|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 94003b1daeba, vllm 2611 sha 49083d3d5c06" ] }, "C|long-write|L16|r0|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 03c1cb7573fd, vllm 2611 sha f67cd59fc73f" ] }, "C|long-write|L16|r0|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha d8360099bbae, vllm 2611 sha 1b00744ae3cb" ] }, "C|long-write|L16|r0|q8": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha e50100181ae3, vllm 2611 sha 0e61ca7e41e5" ] }, "C|long-write|L16|r0|q9": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 28e07890e4ca, vllm 2611 sha d86472665652" ] }, "C|long-write|L16|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 0e6d6bb41e44, vllm 2611 sha ccf44a0b045a" ] }, "C|long-write|L16|r1|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 3cb203d7c3de, vllm 2611 sha fb5a66a839c7" ] }, "C|long-write|L16|r1|q10": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 8c2b95d68638, vllm 2611 sha b53b20c301f1" ] }, "C|long-write|L16|r1|q11": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 0bb8dd09fa6f, vllm 2611 sha 2cc9e538eb8b" ] }, "C|long-write|L16|r1|q12": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha c6c103898075, vllm 2611 sha ab73bf846b28" ] }, "C|long-write|L16|r1|q13": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 193ae67e7423, vllm 2611 sha 647271ea160d" ] }, "C|long-write|L16|r1|q14": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 673874f81406, vllm 2611 sha 099b63344cac" ] }, "C|long-write|L16|r1|q15": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 44dda585dfb1, vllm 2611 sha 5c2e6b2013c9" ] }, "C|long-write|L16|r1|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha ccfb50bb729a, vllm 2611 sha fd95137db2d6" ] }, "C|long-write|L16|r1|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 4bba1914e7c0, vllm 2611 sha c62d98b01acf" ] }, "C|long-write|L16|r1|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha d161690ee46e, vllm 2611 sha 92175d229aa7" ] }, "C|long-write|L16|r1|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 68a0dab178cf, vllm 2611 sha 4765e739a032" ] }, "C|long-write|L16|r1|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 3fd95015496c, vllm 2611 sha b0952363cb3a" ] }, "C|long-write|L16|r1|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 0764eabec26a, vllm 2611 sha 181d72dbaf44" ] }, "C|long-write|L16|r1|q8": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha febf532e675c, vllm 2611 sha 744162774cd7" ] }, "C|long-write|L16|r1|q9": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 3332c96601ca, vllm 2611 sha e93994105e11" ] }, "C|long-write|L16|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1d63a94fe876, vllm 2611 sha 202c377b4da7" ] }, "C|long-write|L16|r2|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha e4e1fa7e6c90, vllm 2611 sha e00c0e593319" ] }, "C|long-write|L16|r2|q10": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1316eef59ba1, vllm 2611 sha 7d8b559d3807" ] }, "C|long-write|L16|r2|q11": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 3cf4799fa5fc, vllm 2611 sha b9a5f93743e3" ] }, "C|long-write|L16|r2|q12": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha f25b8b82a2e5, vllm 2611 sha 3a739a8ed3c4" ] }, "C|long-write|L16|r2|q13": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1b2419c22458, vllm 2611 sha deb1a635d0ed" ] }, "C|long-write|L16|r2|q14": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha ff3292f579af, vllm 2611 sha 1f798fca402f" ] }, "C|long-write|L16|r2|q15": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha d272c86e3bf6, vllm 2611 sha 3cd5caeb6baf" ] }, "C|long-write|L16|r2|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha ccbba06b7c77, vllm 2611 sha 42252cf0f8c4" ] }, "C|long-write|L16|r2|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 05ae80bf0ba1, vllm 2611 sha 9a33dd3b4a6d" ] }, "C|long-write|L16|r2|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha c72af9449510, vllm 2611 sha 83ffdf2f9596" ] }, "C|long-write|L16|r2|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 90cbfc6ff878, vllm 2611 sha 759a47054545" ] }, "C|long-write|L16|r2|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha b729b85711cd, vllm 2611 sha fce7f9416815" ] }, "C|long-write|L16|r2|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 158aabb034c6, vllm 2611 sha 2fd7082571c5" ] }, "C|long-write|L16|r2|q8": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha ab9ad693f6e7, vllm 2611 sha e6f765010104" ] }, "C|long-write|L16|r2|q9": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 850305c64b91, vllm 2611 sha 3014470a8efd" ] }, "C|long-write|L16|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 2f3d749f36f0, vllm 2611 sha 164bfd1edd17" ] }, "C|long-write|L16|r3|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha ef97a815f794, vllm 2611 sha 6d58417a6a4e" ] }, "C|long-write|L16|r3|q10": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 641e4010351f, vllm 2611 sha c592382237c8" ] }, "C|long-write|L16|r3|q11": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha b0f3311ede89, vllm 2611 sha 1ea234507dab" ] }, "C|long-write|L16|r3|q12": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1aa82ae7b150, vllm 2611 sha 29ee384bf75a" ] }, "C|long-write|L16|r3|q13": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha e07d7ffca7d8, vllm 2611 sha 72867f3ccdaa" ] }, "C|long-write|L16|r3|q14": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha b0cc85d91950, vllm 2611 sha b9a998aa71d7" ] }, "C|long-write|L16|r3|q15": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha a32bb8b54738, vllm 2611 sha 9398df8c1fb0" ] }, "C|long-write|L16|r3|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 65bc7045f3b7, vllm 2611 sha 3963b5b82835" ] }, "C|long-write|L16|r3|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 263406e3b49f, vllm 2611 sha 127a02c167e6" ] }, "C|long-write|L16|r3|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 3f60a752f87c, vllm 2611 sha a2975dbcb3d7" ] }, "C|long-write|L16|r3|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1f72b2d450d8, vllm 2611 sha 5e84032079c3" ] }, "C|long-write|L16|r3|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha cc0ed2119a3a, vllm 2611 sha 2be3ebfb273a" ] }, "C|long-write|L16|r3|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1397c18b3ace, vllm 2611 sha f6d776fdf13c" ] }, "C|long-write|L16|r3|q8": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 238875c90339, vllm 2611 sha 10be2ee6014c" ] }, "C|long-write|L16|r3|q9": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha ba2446dfffc9, vllm 2611 sha 482329467023" ] }, "C|long-write|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha d74b2a70f22c, vllm 2611 sha dbb767705d05" ] }, "C|long-write|L1|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha dd0ebb6da71c, vllm 2611 sha 598c80d63e79" ] }, "C|long-write|L1|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha dca9ef559253, vllm 2611 sha a68e5a39e108" ] }, "C|long-write|L1|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 8733d6172ace, vllm 2611 sha 0fa914f3e5a1" ] }, "C|long-write|L1|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 06ef6f4388d2, vllm 2611 sha 26dad5647587" ] }, "C|long-write|L1|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha d57beb1c8c10, vllm 2611 sha 5c12152013b8" ] }, "C|long-write|L1|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha cb886a53a875, vllm 2611 sha 992d690647f6" ] }, "C|long-write|L1|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 4f2ef65d5f0e, vllm 2611 sha 79df059f674f" ] }, "C|long-write|L1|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 56a9f0a0557a, vllm 2611 sha 5566af903f39" ] }, "C|long-write|L1|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 2ff343d2bb5e, vllm 2611 sha 08384b945272" ] }, "C|long-write|L1|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 6e8118bed7f1, vllm 2611 sha 26e5718974b8" ] }, "C|long-write|L2|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha e39a45732f8c, vllm 2611 sha 21f508de0104" ] }, "C|long-write|L2|r0|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 83e96c2dc961, vllm 2611 sha 89be919409b9" ] }, "C|long-write|L2|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha f121d4603274, vllm 2611 sha b7c9e5d77ccc" ] }, "C|long-write|L2|r10|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 82b5c5eb2e96, vllm 2611 sha f7207aa6d720" ] }, "C|long-write|L2|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha ba8a7e301a68, vllm 2611 sha a03c213c5ec2" ] }, "C|long-write|L2|r1|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 095696c6e81e, vllm 2611 sha 95a7bd35a15f" ] }, "C|long-write|L2|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha d209d518ff9b, vllm 2611 sha 75f0e783fa54" ] }, "C|long-write|L2|r2|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha bea1d3e0cd11, vllm 2611 sha 182c4218514a" ] }, "C|long-write|L2|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 02e9badf953c, vllm 2611 sha b46dc44dcb80" ] }, "C|long-write|L2|r3|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 3f8d7e446def, vllm 2611 sha 88779696e595" ] }, "C|long-write|L2|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 16a00f590ba8, vllm 2611 sha 24816df3f67c" ] }, "C|long-write|L2|r4|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha b7a11a5b4724, vllm 2611 sha ac76b81989ec" ] }, "C|long-write|L2|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha bea3708dc92b, vllm 2611 sha e3d1bd55e36a" ] }, "C|long-write|L2|r5|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha cbe704d42d2b, vllm 2611 sha 0c994cc5cd55" ] }, "C|long-write|L2|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1d7c4eafefb7, vllm 2611 sha 927d6b9d6d6c" ] }, "C|long-write|L2|r6|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 516b49513807, vllm 2611 sha 7aafcc435bc6" ] }, "C|long-write|L2|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 28ad4506e5cd, vllm 2611 sha 266d1126755a" ] }, "C|long-write|L2|r7|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 9ebbb180ccbe, vllm 2611 sha a4c13049b307" ] }, "C|long-write|L2|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha a660e3438c45, vllm 2611 sha 20174058b096" ] }, "C|long-write|L2|r8|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha effd442b3498, vllm 2611 sha 30e18d6ea27f" ] }, "C|long-write|L2|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 113e7e8c06b4, vllm 2611 sha 8eb48bee7bd4" ] }, "C|long-write|L2|r9|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha ab277ac4949b, vllm 2611 sha 7f37df789f85" ] }, "C|long-write|L4|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 3fba6345e13d, vllm 2611 sha 5b99f90fc7d3" ] }, "C|long-write|L4|r0|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha c954bedb8abb, vllm 2611 sha 863e73ce7267" ] }, "C|long-write|L4|r0|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha e626ed7e6c02, vllm 2611 sha dcc879412176" ] }, "C|long-write|L4|r0|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 122b968febc5, vllm 2611 sha 87975f1b0e96" ] }, "C|long-write|L4|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha df98f4af79cd, vllm 2611 sha e61460401c6b" ] }, "C|long-write|L4|r1|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha bad8853c46d8, vllm 2611 sha b825b186685e" ] }, "C|long-write|L4|r1|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha bbd17a932cce, vllm 2611 sha 5ea92d8a4d64" ] }, "C|long-write|L4|r1|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 7ad27d5a20d5, vllm 2611 sha 3763d6db9b93" ] }, "C|long-write|L4|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha d8baa699c9cd, vllm 2611 sha ed09b70b5290" ] }, "C|long-write|L4|r2|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 290122be79da, vllm 2611 sha 56178b67c8a2" ] }, "C|long-write|L4|r2|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha b3fd19a3b009, vllm 2611 sha d54116355974" ] }, "C|long-write|L4|r2|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha de8596ba51cb, vllm 2611 sha d07cb7c2c922" ] }, "C|long-write|L4|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 46ba4da2500f, vllm 2611 sha e7113cf9baa8" ] }, "C|long-write|L4|r3|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 49369f0c8765, vllm 2611 sha 73126c39473a" ] }, "C|long-write|L4|r3|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 2b94fb2d6e83, vllm 2611 sha 3855ed2c6ecc" ] }, "C|long-write|L4|r3|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 65f77792e78b, vllm 2611 sha 6ed973579ff0" ] }, "C|long-write|L8|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 68b824edb390, vllm 2611 sha a0609d822558" ] }, "C|long-write|L8|r0|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 28558eb3dc09, vllm 2611 sha 48c54ee109db" ] }, "C|long-write|L8|r0|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 5f17bf4f1fcb, vllm 2611 sha 100af624468a" ] }, "C|long-write|L8|r0|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha c550fcf534db, vllm 2611 sha 86f2e9f4be26" ] }, "C|long-write|L8|r0|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 7fd06f1f8c37, vllm 2611 sha e01ad77d02ed" ] }, "C|long-write|L8|r0|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 9eee55b23fd6, vllm 2611 sha 07869dcdb584" ] }, "C|long-write|L8|r0|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 8a4eb8be502b, vllm 2611 sha 3fb0f440a059" ] }, "C|long-write|L8|r0|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 48c3deea404d, vllm 2611 sha 0f8a19c0d8df" ] }, "C|long-write|L8|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 3df729b83748, vllm 2611 sha c33291efd3a1" ] }, "C|long-write|L8|r1|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 2bf7890eb05c, vllm 2611 sha f8ed45292f8f" ] }, "C|long-write|L8|r1|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 5cd6075b35a2, vllm 2611 sha 2e8fe3889169" ] }, "C|long-write|L8|r1|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha ec368a1147bf, vllm 2611 sha d56d08f283ee" ] }, "C|long-write|L8|r1|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1483b36db2de, vllm 2611 sha cfb264112e09" ] }, "C|long-write|L8|r1|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 08dad71ce127, vllm 2611 sha b7f45135aa78" ] }, "C|long-write|L8|r1|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 7c05130fd278, vllm 2611 sha 72fc2a1c8c45" ] }, "C|long-write|L8|r1|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 97a79cc17857, vllm 2611 sha 7427e37839eb" ] }, "C|long-write|L8|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha d3d7b00c74e6, vllm 2611 sha eb2c81a8ba35" ] }, "C|long-write|L8|r2|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1e8f033afa4f, vllm 2611 sha 4f249d79e924" ] }, "C|long-write|L8|r2|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha f8aff20ee8e3, vllm 2611 sha 96eca1798b9d" ] }, "C|long-write|L8|r2|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha c382a4dedd75, vllm 2611 sha b0684895f4f4" ] }, "C|long-write|L8|r2|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 6605a3ac8e74, vllm 2611 sha 82bd507bc036" ] }, "C|long-write|L8|r2|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 7c247e1c7634, vllm 2611 sha 513cff568b5a" ] }, "C|long-write|L8|r2|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 455fca43e6a6, vllm 2611 sha dfe44b3b1b2b" ] }, "C|long-write|L8|r2|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha e8ce8f5f1724, vllm 2611 sha d61d0d54bd2b" ] }, "C|long-write|L8|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 8296fac40750, vllm 2611 sha 8cf718d76258" ] }, "C|long-write|L8|r3|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 579d4286f2cf, vllm 2611 sha 7becbee47383" ] }, "C|long-write|L8|r3|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha c29c86728b07, vllm 2611 sha df8673610b23" ] }, "C|long-write|L8|r3|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha bda647d28dd5, vllm 2611 sha 0f81b0af69e3" ] }, "C|long-write|L8|r3|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 26042455e4b5, vllm 2611 sha 9e5a07886c57" ] }, "C|long-write|L8|r3|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 1f86d51d602a, vllm 2611 sha d5a978e15811" ] }, "C|long-write|L8|r3|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha a21672ceb748, vllm 2611 sha 6550e9eaa740" ] }, "C|long-write|L8|r3|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 77a6f0f4d12b, vllm 2611 sha 3fa48cf0cb32" ] }, "C|series-p512|L16|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 10ecc501e21a, vllm 547 sha 6e98b5edb9df" ] }, "C|series-p512|L16|r0|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 0a1eb5dcf95c, vllm 547 sha 57d3788d1966" ] }, "C|series-p512|L16|r0|q10": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8ea405ded75d, vllm 547 sha 37770e088108" ] }, "C|series-p512|L16|r0|q11": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha dbe3bf8fbd4f, vllm 547 sha 4048dcc9bd35" ] }, "C|series-p512|L16|r0|q12": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha e07b5519e72e, vllm 547 sha 7f991bb44d78" ] }, "C|series-p512|L16|r0|q13": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 511d0e2a29ff, vllm 547 sha 86dafe93bd50" ] }, "C|series-p512|L16|r0|q14": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d0311631c945, vllm 547 sha 91df2eadcf97" ] }, "C|series-p512|L16|r0|q15": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha b1b0766f08f8, vllm 547 sha fd1b19fad8ce" ] }, "C|series-p512|L16|r0|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha a7f905a8744a, vllm 547 sha 733c037cac7e" ] }, "C|series-p512|L16|r0|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 1e57601c2ce2, vllm 547 sha 05a72942f006" ] }, "C|series-p512|L16|r0|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 188a522888ca, vllm 547 sha 3d401bb160a2" ] }, "C|series-p512|L16|r0|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8ef010693daf, vllm 547 sha 4f98ee8c3c97" ] }, "C|series-p512|L16|r0|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 501bdde20a46, vllm 547 sha 4de57e18e6f6" ] }, "C|series-p512|L16|r0|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 613754cad5b7, vllm 547 sha 0a4b85bb20aa" ] }, "C|series-p512|L16|r0|q8": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 17780b80d3ac, vllm 547 sha 56f9ad10a4d0" ] }, "C|series-p512|L16|r0|q9": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3a6cf1d77ad2, vllm 547 sha 651d15890f85" ] }, "C|series-p512|L16|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 7e2a3214bbca, vllm 547 sha a482b058705d" ] }, "C|series-p512|L16|r1|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 674e5a3fb067, vllm 547 sha 21a324ed7368" ] }, "C|series-p512|L16|r1|q10": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c9d69af5d21e, vllm 547 sha ca252eb83a66" ] }, "C|series-p512|L16|r1|q11": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 88dc5a94dcdb, vllm 547 sha f61882ff105c" ] }, "C|series-p512|L16|r1|q12": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 7aa78f08f817, vllm 547 sha 7c07b1e8811c" ] }, "C|series-p512|L16|r1|q13": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 321b3d754c9e, vllm 547 sha d6c4220f6246" ] }, "C|series-p512|L16|r1|q14": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 4727e24ed598, vllm 547 sha f6daa43c4c39" ] }, "C|series-p512|L16|r1|q15": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha eca63f915612, vllm 547 sha 41f7a4be7205" ] }, "C|series-p512|L16|r1|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha b15735624ca4, vllm 547 sha 34a153f12ea9" ] }, "C|series-p512|L16|r1|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 2c8709b6b12a, vllm 547 sha 926feea7b2db" ] }, "C|series-p512|L16|r1|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha e4ae478cd9d7, vllm 547 sha 43165db8e6a8" ] }, "C|series-p512|L16|r1|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 0739511e8a08, vllm 547 sha 3c4b3a3e158e" ] }, "C|series-p512|L16|r1|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha dd2ca4e8104a, vllm 547 sha 600265b2aa1c" ] }, "C|series-p512|L16|r1|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha da144a2bc2ac, vllm 547 sha 998c49bb39e4" ] }, "C|series-p512|L16|r1|q8": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha e37bf2c4483c, vllm 547 sha 76aa0e5c0100" ] }, "C|series-p512|L16|r1|q9": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3cb2a52a0b11, vllm 547 sha a3d811ff9a78" ] }, "C|series-p512|L16|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8d33f277423d, vllm 547 sha 2dd479bda3cb" ] }, "C|series-p512|L16|r2|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 936b81ed6aaf, vllm 547 sha a89781baf966" ] }, "C|series-p512|L16|r2|q10": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 00c700511c04, vllm 547 sha 7fee7fb45ae8" ] }, "C|series-p512|L16|r2|q11": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c256eba578c3, vllm 547 sha 87e891705a93" ] }, "C|series-p512|L16|r2|q12": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 15338942e209, vllm 547 sha 04b0e8af4e5a" ] }, "C|series-p512|L16|r2|q13": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha da2384f717bb, vllm 547 sha fc8ed57a407d" ] }, "C|series-p512|L16|r2|q14": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 6b0047515dfd, vllm 547 sha a5f8221f4468" ] }, "C|series-p512|L16|r2|q15": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 41e1b1b34993, vllm 547 sha dc0b361b64fa" ] }, "C|series-p512|L16|r2|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 0d98371b5e88, vllm 547 sha 9b56301a3ab7" ] }, "C|series-p512|L16|r2|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha ce104e54ac37, vllm 547 sha 0094c4b883cf" ] }, "C|series-p512|L16|r2|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 96eefeef7570, vllm 547 sha 2453794d9022" ] }, "C|series-p512|L16|r2|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c2a13a5aca10, vllm 547 sha 5bc42a5692cd" ] }, "C|series-p512|L16|r2|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha aa55e552dcd2, vllm 547 sha 888519dcf037" ] }, "C|series-p512|L16|r2|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 2819d87ca90c, vllm 547 sha 6583015ab480" ] }, "C|series-p512|L16|r2|q8": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c46fca3cca67, vllm 547 sha 492390e1eaab" ] }, "C|series-p512|L16|r2|q9": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d1929d7e2f1f, vllm 547 sha 16ed2b1f1abe" ] }, "C|series-p512|L16|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha efde309c3c8a, vllm 547 sha 91f302440504" ] }, "C|series-p512|L16|r3|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 45c085c246f3, vllm 547 sha bc585477666d" ] }, "C|series-p512|L16|r3|q10": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha af33a86b04d9, vllm 547 sha d9f1f3cf9437" ] }, "C|series-p512|L16|r3|q11": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 0d33e318d5fb, vllm 547 sha f97325f5c326" ] }, "C|series-p512|L16|r3|q12": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 56b45f32247d, vllm 547 sha dae5a7889254" ] }, "C|series-p512|L16|r3|q13": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 786081b738e6, vllm 547 sha 69221265996f" ] }, "C|series-p512|L16|r3|q14": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha bb748ba4543e, vllm 547 sha 7d796dab8942" ] }, "C|series-p512|L16|r3|q15": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d63a1102df62, vllm 547 sha 68c5528ffc78" ] }, "C|series-p512|L16|r3|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 450aaae1017f, vllm 547 sha ba60bfa50ff7" ] }, "C|series-p512|L16|r3|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 87e7eb995d1a, vllm 547 sha f08afd5f6b52" ] }, "C|series-p512|L16|r3|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 752e819a4dca, vllm 547 sha 16ee8537a59b" ] }, "C|series-p512|L16|r3|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 13788501dead, vllm 547 sha 9b402fcd6283" ] }, "C|series-p512|L16|r3|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 807a95180ed7, vllm 547 sha 1f67c3d4ad73" ] }, "C|series-p512|L16|r3|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c4383e82be71, vllm 547 sha 54e6753340b9" ] }, "C|series-p512|L16|r3|q8": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha f8f532bc8d87, vllm 547 sha 8d8d5fc63f63" ] }, "C|series-p512|L16|r3|q9": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha b5223fc3e3ef, vllm 547 sha 196ce76ca639" ] }, "C|series-p512|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d68532be7141, vllm 547 sha 7689031cf039" ] }, "C|series-p512|L1|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 15458720f044, vllm 547 sha 0356e328562a" ] }, "C|series-p512|L1|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3d49b90ed733, vllm 547 sha ad80622639bc" ] }, "C|series-p512|L1|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 6cbc99806bd8, vllm 547 sha 06693c8aa5d1" ] }, "C|series-p512|L1|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 522b2bf0740e, vllm 547 sha c2d8fa9e08c4" ] }, "C|series-p512|L1|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 5fbebd7dbf84, vllm 547 sha de38d84fd595" ] }, "C|series-p512|L1|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha df3767c1f8e3, vllm 547 sha b05104b373fa" ] }, "C|series-p512|L1|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha bac4baeed359, vllm 547 sha 8f65fbfe0ede" ] }, "C|series-p512|L1|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 69c1a9d2b981, vllm 547 sha db0e62e6e9ac" ] }, "C|series-p512|L1|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 19e85ae9b11b, vllm 547 sha b209509a82b8" ] }, "C|series-p512|L1|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 091a1bfd4a76, vllm 547 sha f5d755278f8b" ] }, "C|series-p512|L2|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 4bbca2da2894, vllm 547 sha 586ecd6722d8" ] }, "C|series-p512|L2|r0|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 7406b388ab4a, vllm 547 sha 0211cee9a1c6" ] }, "C|series-p512|L2|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 4602ce5cc40d, vllm 547 sha bd13b2e6c606" ] }, "C|series-p512|L2|r10|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha eb5c06567366, vllm 547 sha c0cd5219348f" ] }, "C|series-p512|L2|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 105c3e62701c, vllm 547 sha 86ed72828194" ] }, "C|series-p512|L2|r1|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha a5b3ace79da1, vllm 547 sha c22b5615257e" ] }, "C|series-p512|L2|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 1c6a455c8707, vllm 547 sha d8ce5d1841ce" ] }, "C|series-p512|L2|r2|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 14f3cf91a8e5, vllm 547 sha 252135de3872" ] }, "C|series-p512|L2|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 0c8a589b2289, vllm 547 sha 82df0b3a8ce5" ] }, "C|series-p512|L2|r3|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d808ba1f718f, vllm 547 sha 7cd03b7fc83d" ] }, "C|series-p512|L2|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha ec3c39410a04, vllm 547 sha 5aeaa5df6f11" ] }, "C|series-p512|L2|r4|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3a6e73a7873c, vllm 547 sha a5ce6dea692f" ] }, "C|series-p512|L2|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha b9c94d218e00, vllm 547 sha ae4c7f812755" ] }, "C|series-p512|L2|r5|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8173d884acdd, vllm 547 sha e21b2e04f137" ] }, "C|series-p512|L2|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha e8198df3901d, vllm 547 sha dfccbfedd147" ] }, "C|series-p512|L2|r6|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha cb5e8adc1f13, vllm 547 sha c52d692772ab" ] }, "C|series-p512|L2|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 86fddad30498, vllm 547 sha 5c399283cb27" ] }, "C|series-p512|L2|r7|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 5aeffe34ac53, vllm 547 sha 3bfd5da916fb" ] }, "C|series-p512|L2|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha bf324bb72d27, vllm 547 sha 476be4be8f93" ] }, "C|series-p512|L2|r8|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 1dde09652aa7, vllm 547 sha a463db0eb535" ] }, "C|series-p512|L2|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha f00bd8cf3f36, vllm 547 sha b723cfd4cea9" ] }, "C|series-p512|L2|r9|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 388533275d83, vllm 547 sha 7225a7991b64" ] }, "C|series-p512|L4|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 6d96a394e08b, vllm 547 sha 112b86868778" ] }, "C|series-p512|L4|r0|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8222007006e4, vllm 547 sha c558c08aacc2" ] }, "C|series-p512|L4|r0|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 75412a56f0bc, vllm 547 sha cf419a123c74" ] }, "C|series-p512|L4|r0|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 6aa478e45f56, vllm 547 sha 8a3047ba79a0" ] }, "C|series-p512|L4|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha fbafc70b3ab8, vllm 547 sha c2741fc01a54" ] }, "C|series-p512|L4|r1|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3baf4d114c9d, vllm 547 sha a9aa26b4c00c" ] }, "C|series-p512|L4|r1|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 09d42f39eeb0, vllm 547 sha a41bbad889b3" ] }, "C|series-p512|L4|r1|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8af11d26c3cd, vllm 547 sha 5cc5a5a6f919" ] }, "C|series-p512|L4|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d467ea35064a, vllm 547 sha b5c11840a6a9" ] }, "C|series-p512|L4|r2|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 2b06ecb3bc0e, vllm 547 sha b9d7a90e729d" ] }, "C|series-p512|L4|r2|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha ac821ed22750, vllm 547 sha bd36b6f92f1f" ] }, "C|series-p512|L4|r2|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 34ee8126a5f9, vllm 547 sha 0196b4227504" ] }, "C|series-p512|L4|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha ff1a5421a399, vllm 547 sha f0386d7b1e30" ] }, "C|series-p512|L4|r3|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3a6b53ce87dc, vllm 547 sha 54f4dd15e723" ] }, "C|series-p512|L4|r3|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha bed394482b26, vllm 547 sha 843b216c4314" ] }, "C|series-p512|L4|r3|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 496cd8a755cf, vllm 547 sha 2a795a175b93" ] }, "C|series-p512|L8|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3f7787ff3b58, vllm 547 sha 194fe6a53fcf" ] }, "C|series-p512|L8|r0|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d41bd43c7150, vllm 547 sha ae9755e14d26" ] }, "C|series-p512|L8|r0|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha a100c5bb9573, vllm 547 sha b27ca0b2c0f8" ] }, "C|series-p512|L8|r0|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha f19cde110aa8, vllm 547 sha 178e403c5f6f" ] }, "C|series-p512|L8|r0|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 14b24d6214e2, vllm 547 sha 839be6332c85" ] }, "C|series-p512|L8|r0|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 9ba9110f4d03, vllm 547 sha d2e2ab464852" ] }, "C|series-p512|L8|r0|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 680e4d95cfd6, vllm 547 sha 58932442ac3e" ] }, "C|series-p512|L8|r0|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d85a0cf6ee05, vllm 547 sha 4b301710c842" ] }, "C|series-p512|L8|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha ae1e478a7d5c, vllm 547 sha 3975a21e9fe6" ] }, "C|series-p512|L8|r1|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8ff2fad64f01, vllm 547 sha 174f499bddf6" ] }, "C|series-p512|L8|r1|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 97717963ec99, vllm 547 sha 77637988ce0c" ] }, "C|series-p512|L8|r1|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 287a4505780f, vllm 547 sha 9e220f25001d" ] }, "C|series-p512|L8|r1|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8a90fae9d57e, vllm 547 sha 1d78358e5539" ] }, "C|series-p512|L8|r1|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c65aac7ed005, vllm 547 sha 744938432dec" ] }, "C|series-p512|L8|r1|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 6d109da032c0, vllm 547 sha 98b60870d04f" ] }, "C|series-p512|L8|r1|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha ec95f6315a68, vllm 547 sha c50940070e13" ] }, "C|series-p512|L8|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 832cedcc41ad, vllm 547 sha 9966227f784b" ] }, "C|series-p512|L8|r2|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 972956447c4b, vllm 547 sha 7fe7353a898b" ] }, "C|series-p512|L8|r2|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha bcbf34377756, vllm 547 sha bd86affc9011" ] }, "C|series-p512|L8|r2|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha bf53cf62523a, vllm 547 sha fbfbab09063f" ] }, "C|series-p512|L8|r2|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha cc7762c7da51, vllm 547 sha 9d5ee101cc8b" ] }, "C|series-p512|L8|r2|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3b45204613ef, vllm 547 sha 043af82cf32d" ] }, "C|series-p512|L8|r2|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha a147ec61efbc, vllm 547 sha 08cc8a120047" ] }, "C|series-p512|L8|r2|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 0844fb02088f, vllm 547 sha 24d2e53da78a" ] }, "C|series-p512|L8|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 74f2531fe469, vllm 547 sha e776e34465e0" ] }, "C|series-p512|L8|r3|q1": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 56a78bce6f5e, vllm 547 sha 959f42dbd9b3" ] }, "C|series-p512|L8|r3|q2": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 399934bfcc39, vllm 547 sha 8d4404e5ce06" ] }, "C|series-p512|L8|r3|q3": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8d0c7c4f827a, vllm 547 sha 6d9e114653f3" ] }, "C|series-p512|L8|r3|q4": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha e6614055fc0a, vllm 547 sha 830db4e7a3c7" ] }, "C|series-p512|L8|r3|q5": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha cc9d94600861, vllm 547 sha d41eab3c6406" ] }, "C|series-p512|L8|r3|q6": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 9c44331f0a5a, vllm 547 sha c4a1d4da1fa2" ] }, "C|series-p512|L8|r3|q7": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 13d9f207f3dd, vllm 547 sha b2ce8cc829d1" ] }, "D-cal|long-prefill|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2553 IDs sha 0bcf3f054b79, vllm 2552 sha b5956da90088" ] }, "D-cal|long-write|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 5b7b6ebfed02, vllm 2611 sha 2523d52ab667" ] }, "D-cal|series-p512|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3cd6db379cd3, vllm 547 sha 7a2f07bcf0b3" ] }, "D-cal|short-chat|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 90 IDs sha 652a8a9c5ef2, vllm 89 sha f9770f23bfac" ] }, "D|long-prefill|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2553 IDs sha 3dd19636020a, vllm 2552 sha 5d817e0e22c0" ] }, "D|long-write|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 2612 IDs sha 39cfe63ff685, vllm 2611 sha 3ec9071b6ac9" ] }, "D|series-p512|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 6981bc03e76a, vllm 547 sha 9fd282927925" ] }, "D|short-chat|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 90 IDs sha 5609bb0ce9f2, vllm 89 sha 008af6ee9707" ] }, "S|series-p512|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha ae95a5547e1f, vllm 547 sha 4617dc2d0357" ] }, "S|series-p512|L1|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 561afc81f83e, vllm 547 sha 7cb87fd69732" ] }, "S|series-p512|L1|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha db3e12e3c81f, vllm 547 sha 359bbd6dc86e" ] }, "S|series-p512|L1|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 1bcfda15e3a3, vllm 547 sha bf7e7443f6d8" ] }, "S|series-p512|L1|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 7c6c7ece96f1, vllm 547 sha 5dfde218aadc" ] }, "S|series-p512|L1|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 2cc12734199b, vllm 547 sha b903c76e95f6" ] }, "S|series-p512|L1|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 83bb9cbd377e, vllm 547 sha e77519ae8668" ] }, "S|series-p512|L1|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha b3c2d5f5a661, vllm 547 sha 3d05440cfbb5" ] }, "S|series-p512|L1|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha f8ff95edd13a, vllm 547 sha 2379d57bd315" ] }, "S|series-p512|L1|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 82943ba158a8, vllm 547 sha e1f71ee7b1c5" ] }, "S|series-p512|L1|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha b0a5a5c0ff44, vllm 547 sha 4c12de1feda5" ] }, "TT-a|series-p512|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 9312d90ff659, vllm 547 sha 697995c3a55c" ] }, "TT-a|series-p512|L1|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c8695c8a3f2e, vllm 547 sha ddbb9aece393" ] }, "TT-a|series-p512|L1|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 0f80bf575b4f, vllm 547 sha 4d5887cb76eb" ] }, "TT-a|series-p512|L1|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha a136ad249fc4, vllm 547 sha 9cb901fb19e7" ] }, "TT-a|series-p512|L1|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3b3071df63e3, vllm 547 sha e3d38eb8345e" ] }, "TT-a|series-p512|L1|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 5e6147c27392, vllm 547 sha cf04c954d4ef" ] }, "TT-a|series-p512|L1|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha e164c7158196, vllm 547 sha 51a4f900e580" ] }, "TT-a|series-p512|L1|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha f8ee4f581629, vllm 547 sha 37ec1f979e62" ] }, "TT-a|series-p512|L1|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 368eaeb26a47, vllm 547 sha fc0ceeb9fcc5" ] }, "TT-a|series-p512|L1|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 420c6632c58d, vllm 547 sha 6d7bd601fb91" ] }, "TT-a|series-p512|L1|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha a29c0b70bee0, vllm 547 sha 60c46aa8dad6" ] }, "TT-b|series-p512|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 70effcabfb0b, vllm 547 sha 1d3edf96657d" ] }, "TT-b|series-p512|L1|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c6e820e26236, vllm 547 sha e66a96e69cd7" ] }, "TT-b|series-p512|L1|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c3b3f6686fcc, vllm 547 sha dbc9f094441d" ] }, "TT-b|series-p512|L1|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 934190c5ca64, vllm 547 sha 930038d4ae68" ] }, "TT-b|series-p512|L1|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 3ac19e91a9d3, vllm 547 sha e8cb14861b5a" ] }, "TT-b|series-p512|L1|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 080994f40444, vllm 547 sha db50e9f08918" ] }, "TT-b|series-p512|L1|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha e99f5df0d815, vllm 547 sha 1c37def9102c" ] }, "TT-b|series-p512|L1|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 628d37126de4, vllm 547 sha b10d48329694" ] }, "TT-b|series-p512|L1|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha a27aa760c571, vllm 547 sha f4faba007e6b" ] }, "TT-b|series-p512|L1|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha e349f9cbd612, vllm 547 sha 24fea9f5aa63" ] }, "TT-b|series-p512|L1|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 34c16f8d0de9, vllm 547 sha 07f98c3e5525" ] }, "TT-c|series-p512|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha e55ffc8fcab2, vllm 547 sha 7588a4bb3d55" ] }, "TT-c|series-p512|L1|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha edd5a9b2b222, vllm 547 sha aa87e9a2857e" ] }, "TT-c|series-p512|L1|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 2dfccf0de2b4, vllm 547 sha 995e7cda77e3" ] }, "TT-c|series-p512|L1|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 840b5c598619, vllm 547 sha def6d2ed6c8d" ] }, "TT-c|series-p512|L1|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 49a4bbb7c04e, vllm 547 sha 90208faad0fb" ] }, "TT-c|series-p512|L1|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 61c1b17eaae7, vllm 547 sha a67bab26df16" ] }, "TT-c|series-p512|L1|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha a11bd7e035f1, vllm 547 sha 2e0ce975a777" ] }, "TT-c|series-p512|L1|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 0b454ca72db9, vllm 547 sha ac6e8c0c7502" ] }, "TT-c|series-p512|L1|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 4e443bca92c5, vllm 547 sha d908efcb6ef5" ] }, "TT-c|series-p512|L1|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 5d51acccd18b, vllm 547 sha 49ed526a0baa" ] }, "TT-c|series-p512|L1|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8d569ec71bd0, vllm 547 sha 796cea219ea8" ] }, "TT-d|series-p512|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 8842eebdb1bf, vllm 547 sha c4626c6dace9" ] }, "TT-d|series-p512|L1|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 9cb45a371604, vllm 547 sha d030f41da7f1" ] }, "TT-d|series-p512|L1|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 4f64fcda71ff, vllm 547 sha 567210289295" ] }, "TT-d|series-p512|L1|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 94a597bef300, vllm 547 sha 5acb8c77f499" ] }, "TT-d|series-p512|L1|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha da040409af9d, vllm 547 sha c4a61c353fef" ] }, "TT-d|series-p512|L1|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 370c1390d119, vllm 547 sha 0ee832e109fc" ] }, "TT-d|series-p512|L1|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 1c94708ae1e4, vllm 547 sha 757956b9e240" ] }, "TT-d|series-p512|L1|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 4aeffb0dbd3c, vllm 547 sha 0ee5b3a99ae8" ] }, "TT-d|series-p512|L1|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d1131cdda634, vllm 547 sha 6a2c93c0eb64" ] }, "TT-d|series-p512|L1|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha f9cca817842d, vllm 547 sha abbf0a4d0583" ] }, "TT-d|series-p512|L1|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 142b701cee28, vllm 547 sha f2249cf02627" ] }, "TT-e|series-p512|L1|r0|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 6c5ae4b71b81, vllm 547 sha 6e98d30601cb" ] }, "TT-e|series-p512|L1|r10|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha de3ca74645c4, vllm 547 sha 3d17f68cfe72" ] }, "TT-e|series-p512|L1|r1|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 02d50a7f7336, vllm 547 sha d046c3577e0d" ] }, "TT-e|series-p512|L1|r2|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha c2994ab72484, vllm 547 sha 4686361f7b01" ] }, "TT-e|series-p512|L1|r3|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha cfe64e1f94c6, vllm 547 sha c308c584ce15" ] }, "TT-e|series-p512|L1|r4|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha f177aa0facf7, vllm 547 sha 5dbc8afabf5f" ] }, "TT-e|series-p512|L1|r5|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha d09a04155925, vllm 547 sha 1c945e9845ae" ] }, "TT-e|series-p512|L1|r6|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 888b9399d271, vllm 547 sha db279711d9ee" ] }, "TT-e|series-p512|L1|r7|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 32d8dd32bc9c, vllm 547 sha 998fae104cf9" ] }, "TT-e|series-p512|L1|r8|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 5e4ec307b79c, vllm 547 sha 0dc47a1c4e65" ] }, "TT-e|series-p512|L1|r9|q0": { "verdict": "broken", "count": null, "problems": [ "BOS is not first exactly once on vllm", "llama-server 548 IDs sha 7933dbf9c015, vllm 547 sha e531255c049c" ] } } }, "render": { "series-p512": { "verdict": "differs", "base": "vllm /tokenize messages", "counts": { "vllm /tokenize messages": 548, "ollama /api/chat think:false": 548, "ollama /v1/chat/completions reasoning_effort:none": 548, "llama-server /apply-template": 558 }, "diffs": [ { "a": "vllm /tokenize messages", "b": "llama-server /apply-template", "first_index": 1, "a_window": { "from": 0, "ids": [ 2, 105, 2364, 107, 51181, 236743, 236800, 236810, 236828, 236832 ], "pieces": [ "", "<|turn>", "user", "\n", "Ticket", " ", "3", "5", "8", "7" ], "pieces_from": "the engine's detokenizer" }, "b_window": { "from": 0, "ids": [ 2, 236820, 236909, 548, 236779, 3041, 111038, 2364, 107, 51181 ], "pieces": [ "", "<", "|", "im", "_", "start", "|>", "user", "\n", "Ticket" ], "pieces_from": "the engine's detokenizer" }, "a_count": 548, "b_count": 558 } ], "shas": { "vllm /tokenize messages": "6981bc03e76a70e41c728865c10f114c9782c718e34ee8e0061123d17cb7338f", "ollama /api/chat think:false": "6981bc03e76a70e41c728865c10f114c9782c718e34ee8e0061123d17cb7338f", "ollama /v1/chat/completions reasoning_effort:none": "6981bc03e76a70e41c728865c10f114c9782c718e34ee8e0061123d17cb7338f", "llama-server /apply-template": "4ddb5768dcdd6fbca1c2618c8aa4989382e8e98feac784f31a8628fbb446b3f1" }, "refused": false, "rule": "SPEC \u03946, R-P1 (reported, never refused): each engine's chat render of a prompt as token IDs -- vLLM POST /tokenize with messages; Ollama /api/chat and /v1/chat/completions with _debug_render_only, then its runner's /tokenize; llama-server /apply-template then /tokenize -- identical IDs, or the first differing index printed with 8 decoded tokens either side and both counts" }, "long-write": { "verdict": "differs", "base": "vllm /tokenize messages", "counts": { "vllm /tokenize messages": 2612, "ollama /api/chat think:false": 2612, "ollama /v1/chat/completions reasoning_effort:none": 2612, "llama-server /apply-template": 2622 }, "diffs": [ { "a": "vllm /tokenize messages", "b": "llama-server /apply-template", "first_index": 1, "a_window": { "from": 0, "ids": [ 2, 105, 2364, 107, 51181, 236743, 236810, 236771, 236778, 236770 ], "pieces": [ "", "<|turn>", "user", "\n", "Ticket", " ", "5", "0", "2", "1" ], "pieces_from": "the engine's detokenizer" }, "b_window": { "from": 0, "ids": [ 2, 236820, 236909, 548, 236779, 3041, 111038, 2364, 107, 51181 ], "pieces": [ "", "<", "|", "im", "_", "start", "|>", "user", "\n", "Ticket" ], "pieces_from": "the engine's detokenizer" }, "a_count": 2612, "b_count": 2622 } ], "shas": { "vllm /tokenize messages": "39cfe63ff685b8371451a8617f2894ec6f64729d8e521bd84b7986464a226686", "ollama /api/chat think:false": "39cfe63ff685b8371451a8617f2894ec6f64729d8e521bd84b7986464a226686", "ollama /v1/chat/completions reasoning_effort:none": "39cfe63ff685b8371451a8617f2894ec6f64729d8e521bd84b7986464a226686", "llama-server /apply-template": "47e540481058523c01ed0e593bc681bfa859cd57266fea1ec6720085d89d681e" }, "refused": false, "rule": "SPEC \u03946, R-P1 (reported, never refused): each engine's chat render of a prompt as token IDs -- vLLM POST /tokenize with messages; Ollama /api/chat and /v1/chat/completions with _debug_render_only, then its runner's /tokenize; llama-server /apply-template then /tokenize -- identical IDs, or the first differing index printed with 8 decoded tokens either side and both counts" }, "long-prefill": { "verdict": "differs", "base": "vllm /tokenize messages", "counts": { "vllm /tokenize messages": 2553, "ollama /api/chat think:false": 2553, "ollama /v1/chat/completions reasoning_effort:none": 2553, "llama-server /apply-template": 2564 }, "diffs": [ { "a": "vllm /tokenize messages", "b": "llama-server /apply-template", "first_index": 1, "a_window": { "from": 0, "ids": [ 2, 105, 2364, 107, 51181, 236743, 236800, 236770, 236800, 236771 ], "pieces": [ "", "<|turn>", "user", "\n", "Ticket", " ", "3", "1", "3", "0" ], "pieces_from": "the engine's detokenizer" }, "b_window": { "from": 0, "ids": [ 2, 236820, 236909, 548, 236779, 3041, 111038, 2364, 107, 51181 ], "pieces": [ "", "<", "|", "im", "_", "start", "|>", "user", "\n", "Ticket" ], "pieces_from": "the engine's detokenizer" }, "a_count": 2553, "b_count": 2564 } ], "shas": { "vllm /tokenize messages": "3dd19636020a80ff11feb43f4f82b49c00364c2e596ec780be886b33d837416a", "ollama /api/chat think:false": "3dd19636020a80ff11feb43f4f82b49c00364c2e596ec780be886b33d837416a", "ollama /v1/chat/completions reasoning_effort:none": "3dd19636020a80ff11feb43f4f82b49c00364c2e596ec780be886b33d837416a", "llama-server /apply-template": "b9a78f18c81ec9315dd8c4f0c7a0859e2cef9d6e3d16756f276a26a721529f2b" }, "refused": false, "rule": "SPEC \u03946, R-P1 (reported, never refused): each engine's chat render of a prompt as token IDs -- vLLM POST /tokenize with messages; Ollama /api/chat and /v1/chat/completions with _debug_render_only, then its runner's /tokenize; llama-server /apply-template then /tokenize -- identical IDs, or the first differing index printed with 8 decoded tokens either side and both counts" }, "short-chat": { "verdict": "differs", "base": "vllm /tokenize messages", "counts": { "vllm /tokenize messages": 90, "ollama /api/chat think:false": 90, "ollama /v1/chat/completions reasoning_effort:none": 90, "llama-server /apply-template": 100 }, "diffs": [ { "a": "vllm /tokenize messages", "b": "llama-server /apply-template", "first_index": 1, "a_window": { "from": 0, "ids": [ 2, 105, 2364, 107, 51181, 236743, 236810, 236770, 236800, 236810 ], "pieces": [ "", "<|turn>", "user", "\n", "Ticket", " ", "5", "1", "3", "5" ], "pieces_from": "the engine's detokenizer" }, "b_window": { "from": 0, "ids": [ 2, 236820, 236909, 548, 236779, 3041, 111038, 2364, 107, 51181 ], "pieces": [ "", "<", "|", "im", "_", "start", "|>", "user", "\n", "Ticket" ], "pieces_from": "the engine's detokenizer" }, "a_count": 90, "b_count": 100 } ], "shas": { "vllm /tokenize messages": "5609bb0ce9f22dbbc0298ddfb79fb047eb3a61bc99ce1f555d27906af28c341d", "ollama /api/chat think:false": "5609bb0ce9f22dbbc0298ddfb79fb047eb3a61bc99ce1f555d27906af28c341d", "ollama /v1/chat/completions reasoning_effort:none": "5609bb0ce9f22dbbc0298ddfb79fb047eb3a61bc99ce1f555d27906af28c341d", "llama-server /apply-template": "3cfcf1e03a83166352e531258580ff28e9072c691fa7ea80ba704f9e93e01243" }, "refused": false, "rule": "SPEC \u03946, R-P1 (reported, never refused): each engine's chat render of a prompt as token IDs -- vLLM POST /tokenize with messages; Ollama /api/chat and /v1/chat/completions with _debug_render_only, then its runner's /tokenize; llama-server /apply-template then /tokenize -- identical IDs, or the first differing index printed with 8 decoded tokens either side and both counts" } }, "frame": { "verdict": "refused", "proofs": [ { "prompt": "series-p512", "verdict": "refused", "first_index": 0, "chat_count": 548, "composed_count": 547, "sentence": "REFUSED: the frozen frame is not the template for series-p512: chat IDs and composed IDs differ at index 0" }, { "prompt": "long-write", "verdict": "refused", "first_index": 0, "chat_count": 2612, "composed_count": 2611, "sentence": "REFUSED: the frozen frame is not the template for long-write: chat IDs and composed IDs differ at index 0" }, { "prompt": "long-prefill", "verdict": "refused", "first_index": 0, "chat_count": 2553, "composed_count": 2552, "sentence": "REFUSED: the frozen frame is not the template for long-prefill: chat IDs and composed IDs differ at index 0" }, { "prompt": "short-chat", "verdict": "refused", "first_index": 0, "chat_count": 90, "composed_count": 89, "sentence": "REFUSED: the frozen frame is not the template for short-chat: chat IDs and composed IDs differ at index 0" } ], "rule": "R-F1: vLLM's chat render of each prompt equals the composed string, ID for ID" }, "rules": { "raw": "SPEC \u03946, R-P2 (a gate): every string an arm sends on the raw path tokenizes to the same IDs (count and sha256 of the list) on every engine the arm uses, with BOS first exactly once; otherwise that prompt is REFUSED for that engine pair and carries no scored row", "render": "SPEC \u03946, R-P1 (reported, never refused): each engine's chat render of a prompt as token IDs -- vLLM POST /tokenize with messages; Ollama /api/chat and /v1/chat/completions with _debug_render_only, then its runner's /tokenize; llama-server /apply-template then /tokenize -- identical IDs, or the first differing index printed with 8 decoded tokens either side and both counts" } }, "os": { "pretty_name": "Ubuntu 26.04 LTS", "id": "ubuntu", "version_id": "26.04", "kernel": "7.0.0-31-generic", "arch": "x86_64", "source": "/etc/os-release" }, "withheld": [] }