{ "schema_version": "1.0", "page_kind": "article", "pack_kind": "generate", "slug": "two-hours-on-battery", "title": "Two Hours at 12 tok/s — On Battery", "dek": "Take the graphics card away and the 30-billion-parameter model answers faster than the 9-billion one — on all three machines.", "published": "2026-08-31", "series": [ "bench" ], "licence": "CC BY 4.0", "status": "pending-judge", "pack_note": "no judge is seated, so every chip is held", "notice": "items with published:false are held and are not this page's published words", "source": { "url": "https://research.strata2signal.com/two-hours-on-battery/", "md_url": "https://research.strata2signal.com/two-hours-on-battery/index.md", "md_sha": "50c2a17537523c032137b0e5f1ca95ce8cd43b3d53b436d2910e452607540c4e", "html_sha": "ba9e1e2de70460638e51329184745c1d1104de793eb69b1b07dd6f34f308d616", "anchors_sha": "380014870c1db072021a1b96f276411b4de1fd352a83df62960785344cfab10f" }, "short": { "paragraph": "Take the graphics card away and the 30-billion-parameter model answers faster than the 9-billion one - on all three machines.", "from_dek": true, "counts": { "words": 5578, "minutes": 25, "tables": 9, "kit": true }, "bullets": [ { "text": "the mini PC on a consumer 1500 VA UPS will run a 26 B model at ~12 tok/s", "figure": "~12 tok/s", "cite": "two-hours-at-12-tok-s-the-test-that-named-the-page", "quote": "The mini node is the inversion of that design: it cannot answer fast, but it can answer for hours on a battery the size of a shoebox, at a speed - ~12 tok/s - that reads faster than most people do.", "span": [ 7588, 7598 ] }, { "text": "the server is 1.70×", "figure": "1.70×", "cite": "read-this-before-the-table-cpu-speed-is-a-memory-bandwidth", "quote": "We measured that ceiling directly with a STREAM triad (a[i] = b[i] + scalarc[i] over a 2,289 MiB working set, far larger than any cache on any of these boxes; gcc -O3 -march=native -fopenmp; best of 9 timed reps): box best triad worst spread the server (128G ECC) 100.7 GB/s 100.4 GB/s 0.3% Rig B laptop 59.3 GB/s 49.2 GB/s 20.5% the mini (32G) 51.1 GB/s 50.9 GB/s 0.5% The server is 1.70× the laptop's bandwidth and 1.97× the mini's; the laptop is 1.16× the mini.", "span": [ 10647, 10655 ] }, { "text": "the 12B dense model agreed 99.5%", "figure": "12B dense model agreed 99.5%", "cite": "the-judgment-column-same-sealed-exams", "quote": "The 12B dense model agreed 99.5%, matching the production judge's own false-positive rate (2 of 329 clean items); the 9B: 99.2%.", "span": [ 24837, 24868 ] } ] }, "sections": [ { "id": "a-few-words-this-page-leans-on", "heading": "A few words this page leans on", "level": 2, "span": [ 1013, 2318 ], "chunks": [ [ 1013, 2318 ] ], "chars": 1305, "digest": "this section defines technical terms used throughout the page, including token, parameter, prefill, decode, dense / MoE, Q4, and thread.", "digest_skipped": null }, { "id": "two-hours-at-12-tok-s-the-test-that-named-the-page", "heading": "Two hours at 12 tok/s — the test that named the page", "level": 2, "span": [ 2318, 7703 ], "chunks": [ [ 2318, 7703 ] ], "chars": 5385, "digest": "a mini PC on a 1500 VA UPS runs a 26 B model at ~12 tokens/second for about two hours on battery. the test showed the model generating full ~1,400-token answers continuously at full speed on battery.", "digest_skipped": null }, { "id": "the-question-our-gpu-benches-never-ask", "heading": "The question our GPU benches never ask", "level": 2, "span": [ 7703, 9581 ], "chunks": [ [ 7703, 9581 ] ], "chars": 1878, "digest": null, "digest_skipped": null }, { "id": "read-this-before-the-table-cpu-speed-is-a-memory-bandwidth", "heading": "Read this before the table: CPU speed is a memory-bandwidth story", "level": 2, "span": [ 9581, 11643 ], "chunks": [ [ 9581, 11643 ] ], "chars": 2062, "digest": null, "digest_skipped": null }, { "id": "the-one-table-to-remember", "heading": "The one table to remember", "level": 2, "span": [ 11643, 15166 ], "chunks": [ [ 11643, 15166 ] ], "chars": 3523, "digest": "the table shows decode speed for various models on the server, Rig B, and the mini. it notes that the 30-billion-parameter MoE decodes faster than the 9-billion dense model on every machine.", "digest_skipped": null }, { "id": "the-wait-prefill-is-the-real-cpu-tax", "heading": "The wait: prefill is the real CPU tax", "level": 2, "span": [ 15166, 18405 ], "chunks": [ [ 15166, 18405 ] ], "chars": 3239, "digest": "prefill is the time before the first token when the model reads the prompt. more cores buy prefill, not decode, and cached-prefix first tokens are between 99–712 ms.", "digest_skipped": null }, { "id": "set-the-thread-count-the-one-free-lever-and-the-exception", "heading": "Set the thread count — the one free lever, and the exception that proves the rule", "level": 2, "span": [ 18405, 24132 ], "chunks": [ [ 18405, 24132 ] ], "chars": 5727, "digest": "tuning the thread count can increase dense model speeds, but for MoE models, increasing threads can make them slower. the 30B MoE on the mini collapses to 1.27 tok/s at t=16.", "digest_skipped": null }, { "id": "the-judgment-column-same-sealed-exams", "heading": "The judgment column: same sealed exams", "level": 2, "span": [ 24132, 26316 ], "chunks": [ [ 24132, 26316 ] ], "chars": 2184, "digest": "all six models passed the field exam, scoring 57 to 59 out of 60. the models also sat a moderation exam where they all landed within a 5-item band on 374 items.", "digest_skipped": null }, { "id": "what-this-buys-the-house-and-maybe-yours", "heading": "What this buys the house (and maybe yours)", "level": 2, "span": [ 26316, 27573 ], "chunks": [ [ 26316, 27573 ] ], "chars": 1257, "digest": "a 26–30B MoE on the 32G mini provides a real emergency tier. the article also notes that the embedder never needed the GPU and concurrency barely helps.", "digest_skipped": null }, { "id": "what-we-did-not-measure", "heading": "What we did NOT measure", "level": 2, "span": [ 27573, 28332 ], "chunks": [ [ 27573, 28332 ] ], "chars": 759, "digest": "the article notes that cross-machine ratios are configuration-inclusive and that cpufreq governors differed. it also excludes non-Q4 quantizations, sustained thermals, and Rig B's GPU.", "digest_skipped": null }, { "id": "what-to-take-with-you", "heading": "What to take with you", "level": 2, "span": [ 28332, 29647 ], "chunks": [ [ 28332, 29647 ] ], "chars": 1315, "digest": "on CPU decode, sparsity is the strategy. prefill is the CPU tax, and more cores buy prefill, not decode. a 32G mini PC is a real fallback inference node.", "digest_skipped": null }, { "id": "how-to-check-our-work", "heading": "How to check our work", "level": 2, "span": [ 29647, 31557 ], "chunks": [ [ 29647, 31557 ] ], "chars": 1910, "digest": null, "digest_skipped": "credits" }, { "id": "who-ran-this-and-thanks", "heading": "Who ran this, and thanks", "level": 2, "span": [ 31557, 32644 ], "chunks": [ [ 31557, 32644 ] ], "chars": 1087, "digest": null, "digest_skipped": "credits" }, { "id": "the-rest-of-the-seminar", "heading": "The rest of the seminar", "level": 2, "span": [ 32644, 34785 ], "chunks": [ [ 32644, 34785 ] ], "chars": 2141, "digest": null, "digest_skipped": "credits" } ], "chips": [ { "id": "c-98cb1028", "q": "how do tokens and parameters affect model performance?", "a": "Tokens are the chunks a model reads and writes, roughly three-quarters of a word. Parameters are learned numbers within a model; more parameters require more movement for every token, which is a central factor in performance.", "cites": [ "a-few-words-this-page-leans-on" ], "published": false, "publish_state": "held", "quote": "", "quote_cite": "" }, { "id": "c-b12cf018", "q": "how long can a mini pc run a 26b model on battery?", "a": "A mid-range mini PC using a 1500 VA UPS can run a 26 B model at approximately 12 tokens per second for about two hours on battery, while remaining ready to answer for seven and a half hours when idle.", "cites": [ "two-hours-at-12-tok-s-the-test-that-named-the-page" ], "published": false, "publish_state": "held", "quote": "", "quote_cite": "" }, { "id": "c-32bb8616", "q": "what happens to performance when using cpu instead of gpu?", "a": "The article explores what happens if every GPU is busy or gone, testing whether language models can still answer using only CPU-only inference across various machines to determine if a usable fallback exists.", "cites": [ "the-question-our-gpu-benches-never-ask" ], "published": false, "publish_state": "held", "quote": "", "quote_cite": "" }, { "id": "c-6b9565dd", "q": "how does memory bandwidth limit cpu decode speed?", "a": "On a CPU, decoding a token requires streaming the model's working weights from RAM to the processor once per token. Consequently, decode speed is capped by how fast the machine can move bytes rather than processor intelligence.", "cites": [ "read-this-before-the-table-cpu-speed-is-a-memory-bandwidth" ], "published": false, "publish_state": "held", "quote": "", "quote_cite": "" }, { "id": "c-aa9a92c1", "q": "what determines the time spent during the prefill phase?", "a": "Prefill is the process where a model reads a prompt before answering. This phase is heavily influenced by the number of cores, as prefill parallelizes across cores, whereas decode speed is primarily limited by memory bandwidth.", "cites": [ "the-wait-prefill-is-the-real-cpu-tax" ], "published": false, "publish_state": "held", "quote": "", "quote_cite": "" }, { "id": "c-f1d54b84", "q": "what are the main takeaways regarding cpu inference?", "a": "Sparsity is a key strategy because MoE models out-decode dense models. Additionally, prefill is a significant CPU tax, and a 32G mini PC serves as a viable fallback inference node for running 30B MoE models.", "cites": [ "what-to-take-with-you" ], "published": false, "publish_state": "held", "quote": "", "quote_cite": "" } ], "related": [ { "slug": "two-used-3080s-priced", "why": "shares ground with § What we did NOT measure · § What this page does not measure" }, { "slug": "outside-judges", "why": "shares ground with § The judgment column: same sealed exams · § The gauntlet: our judges, examined" }, { "slug": "one-3090-against-one-3090-ti", "why": "shares ground with § What we did NOT measure · § What this page does not measure" } ], "thanks": "## Who ran this, and thanks {#who-ran-this-and-thanks}\n\nThe measurements were run by a small human team with a fleet of AI agents doing the fetching,\nscripting and cross-checking; the humans set the method, ran the harness, and signed every number.\nThe thing that surprised us most was not the headline — sparse-beats-dense on a bandwidth-bound\nmachine is, in hindsight, arithmetic — but the MoE's thread cliff: the same lever that makes a\ndense model 1.8× faster makes a mixture-of-experts collapse to a tenth of its speed. We went in\nexpecting one right answer per box and found one right answer per *model*.\n\nThanks are owed to the people who made the pieces we merely measured: the teams behind\n`qwen3.5`, `gemma4`, `phi4`, `mistral-small`, `nemotron`, and `llama3.3`, whose weights are the\nwhole subject here; and to the authors of Ollama and llama.cpp — including for the thread-count\nquirk we spent a section chasing, which is a two-line heuristic doing its honest best on hardware\nit did not expect. Open weights and an open runtime are what let a bench like this exist at all.", "kit": { "url": "https://research.strata2signal.com/two-hours-on-battery/data/", "licence": "CC BY 4.0", "files": [ "README.md", "NOTES.md", "counting-rules.md", "prereg.md", "analysis/dispersion.md", "analysis/bandwidth.md", "analysis/cause.md", "receipts/triad-cpu-server.txt", "receipts/triad-cpu-laptop.txt", "receipts/triad-cpu-mini.txt", "receipts/coretype-cpu-laptop.txt", "receipts/coretype-cpu-mini.txt", "quality/field-exam.txt", "quality/field-exam.json", "quality/classifier-agreement.json", "quality/classifier-gates.json", "runs/cpu-laptop-m1.json", "runs/cpu-laptop-m2.json", "runs/cpu-laptop-m3.json", "runs/cpu-laptop-m4.json", "runs/cpu-laptop-m5.json", "runs/cpu-laptop-m6.json", "runs/cpu-laptop-x1.json", "runs/cpu-mini-m1.json", "runs/cpu-mini-m2.json", "runs/cpu-mini-m3.json", "runs/cpu-mini-m4.json", "runs/cpu-mini-m5.json", "runs/cpu-mini-m6-memfit.json", "runs/cpu-mini-m6.json", "runs/cpu-mini-x1.json", "runs/cpu-server-m1.CONTAMINATED.json", "runs/cpu-server-m1.json", "runs/cpu-server-m2.json", "runs/cpu-server-m3.json", "runs/cpu-server-m4.json", "runs/cpu-server-m5-c1.json", "runs/cpu-server-m5-c2.json", "runs/cpu-server-m5.json", "runs/cpu-server-m6.json", "runs/cpu-server-x1.json", "runs/cpu-server-x2.json", "runs/firstpass-gemma4-26b.json", "runs/fresh-gemma4-26b-rerun.json", "runs/fresh-gemma4-26b.json", "runs/fresh-qwen3.5-9b.json", "runs/threadsweep-cpu-laptop-m1.json", "runs/threadsweep-cpu-mini-m1.json", "supplement-runs/default4-cpu-mini-m1.json", "supplement-runs/default4-cpu-mini-m2.json", "supplement-runs/default4-cpu-mini-m3.json", "supplement-runs/default4-cpu-mini-m4.json", "supplement-runs/default4-cpu-mini-m5.json", "supplement-runs/default4-cpu-mini-m6.json", "supplement-runs/sweep-cpu-laptop-m1-stability.json", "supplement-runs/sweep-cpu-laptop-m1.json", "supplement-runs/sweep-cpu-mini-m1-stability.json", "supplement-runs/sweep-cpu-mini-m1.json", "supplement-runs/sweep-cpu-mini-m6-moe.json", "supplement-runs/tuned-cpu-laptop-m1.json", "supplement-runs/tuned-cpu-laptop-m2.json", "supplement-runs/tuned-cpu-laptop-m3.json", "supplement-runs/tuned-cpu-laptop-m4.json", "supplement-runs/tuned-cpu-laptop-m5.json", "supplement-runs/tuned-cpu-laptop-m6.json", "supplement-runs/tuned-cpu-mini-m1.json", "supplement-runs/tuned-cpu-mini-m2.json", "supplement-runs/tuned-cpu-mini-m3.json", "supplement-runs/tuned-cpu-mini-m4.json", "supplement-runs/tuned-cpu-mini-m5.json", "supplement-runs/tuned-cpu-mini-m6.json" ] }, "seat_class": { "writer": "gemma-class", "judge": null, "writer_runtime": "vllm" }, "bench": { "bullets_written": 3, "bullets_kept": 3, "digests_written": 11, "digests_kept": 9, "chips_written": 8, "chips_kept": 6, "chips_grounded": 0, "chips_published": 0, "dropped_by": { "new_noun": 3, "figure": 0, "length": 0, "cite": 0, "judge": 0, "quote": 0, "directive": 0, "redaction": 1, "profanity": 0 }, "new_noun_tokens_checked": [ "1", "1.27", "1.70×", "12", "12B", "1500", "16", "24B", "26", "26-30B", "26b", "30-billion-parameter", "30B", "32G", "374", "400-token", "5-item", "57", "59", "60", "9-billion", "99-712", "99.5%", "B", "B's", "CPU", "CPU-only", "GPU", "MoE", "PC", "Q4", "RAM", "Rig", "UPS", "VA", "non-Q4" ], "new_noun_tokens_withheld": 3, "figure_definition": "v3", "drops": [ { "kind": "digest", "reason": "new_noun", "item": null, "item_chars": 4, "withheld": "a string naming something the article does not", "detail": "" }, { "kind": "digest", "reason": "redaction", "item": null, "item_chars": 158, "withheld": "a string carrying a fenced literal", "detail": "" }, { "kind": "chip", "reason": "new_noun", "item": null, "item_chars": 4, "withheld": "a string naming something the article does not", "detail": "" }, { "kind": "chip", "reason": "new_noun", "item": null, "item_chars": 5, "withheld": "a string naming something the article does not", "detail": "" } ] }, "generated_utc": "2026-09-09T01:26:11Z" }