#!/usr/bin/env python3 """DV-5's stop guard and card sampler on benchbox, read from OUTSIDE the bencher (its own user unit). I-2's guard_i2.py (the house's GPU-leg guard: L-2, A-1, I-2) with the unit names and paths renamed, and the UPS rules of benchbox's kit added, as the speed leg's brief states them. It trips, records the trip first, then stops every dv-speed bench unit, and never lets a bench "record and continue": the UPS (NUT `pr1500`, read every second): * `ups.status` reads OB (on battery) -> stop * `ups.status` reads LB (low battery) -> stop * the UPS is unreadable on 3 reads in a row -> stop * `battery.runtime` reads under 300 s on 3 reads in a row -> stop * `ups.load` reads 80 % or more (the house's GPU-leg rule) -> stop the card (sampled at 2 Hz by its UUID, read from the environment, never written): * the core reads 83 C or more -> stop * the sampler exits -> stop the disk: free bytes on / under 15 GiB -> stop DV_CARD= DV_DIR= python3 -B guard_speed.py --tag # until stopped It REFUSES to start unless every phone-home switch is set and DO_NOT_TRACK is absent (A1.9's rule: the guard carries the switches too). It touches guard/guard.heartbeat on every clean read; the unit body and the runner refuse to start an arm on a heartbeat older than 10 s. Writes receipts-speed/-telemetry.jsonl (the card) and receipts-speed/-guard.jsonl (its own reads). Neither file carries the UUID, a bus or a box address. """ from __future__ import annotations import argparse import json import os import subprocess import threading import time DV_DIR = os.environ.get("DV_DIR", os.path.expanduser("/workshop/bench-dv-speed")) HEARTBEAT = os.path.join(DV_DIR, "guard", "guard.heartbeat") REC_DIR = os.path.join(DV_DIR, "receipts-speed") UPS = "pr1500@localhost" UPS_LOAD_MAX = 80.0 RUNTIME_MIN_S = 300.0 STRAIGHT = 3 CORE_MAX_C = 83 FREE_MIN = 15 * 2 ** 30 ARMS = ("speed-von-cpu", "speed-laya-cpu", "speed-von-gpu", "speed-laya-gpu") REQUIRED = {"ORT_DISABLE_TELEMETRY": "1", "HF_HUB_DISABLE_TELEMETRY": "1", "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "VLLM_NO_USAGE_STATS": "1", "GRADIO_ANALYTICS_ENABLED": "False"} FIELDS = ("power.draw", "power.draw.instant", "temperature.gpu", "memory.used", "utilization.gpu", "clocks.sm", "pstate") def utc() -> str: return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) def ups() -> dict: r = subprocess.run(["upsc", UPS], capture_output=True, text=True, timeout=5) kv = dict(line.split(": ", 1) for line in r.stdout.splitlines() if ": " in line) if r.returncode != 0 or "ups.status" not in kv: raise RuntimeError("upsc rc %d, no ups.status" % r.returncode) num = lambda k: float(kv[k]) if k in kv else None # noqa: E731 return {"ups_status": kv["ups.status"], "battery_runtime_s": num("battery.runtime"), "battery_charge_pct": num("battery.charge"), "ups_load_pct": num("ups.load"), "ups_realpower_w": num("ups.realpower")} def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--tag", required=True) a = ap.parse_args() uuid = os.environ["DV_CARD"] missing = {k: os.environ.get(k) for k, v in REQUIRED.items() if os.environ.get(k) != v} if missing or "DO_NOT_TRACK" in os.environ: raise SystemExit("REFUSED: phone-home switches wrong in the guard: %s, DO_NOT_TRACK %s" % (missing, "set" if "DO_NOT_TRACK" in os.environ else "absent")) os.makedirs(REC_DIR, exist_ok=True) os.makedirs(os.path.dirname(HEARTBEAT), exist_ok=True) units = ["dv-speed-%s-%s.service" % (a.tag, arm) for arm in ARMS] tel = open(os.path.join(REC_DIR, "%s-telemetry.jsonl" % a.tag), "a", buffering=1) log = open(os.path.join(REC_DIR, "%s-guard.jsonl" % a.tag), "a", buffering=1) last = {} smi = subprocess.Popen(["nvidia-smi", "-i", uuid, "--query-gpu=" + ",".join(FIELDS), "--format=csv,noheader,nounits", "-lms", "500"], stdout=subprocess.PIPE, text=True) def sampler() -> None: for line in smi.stdout: c = [x.strip() for x in line.split(",")] if len(c) != len(FIELDS): continue num = lambda s: float(s) if s.replace(".", "", 1).isdigit() else None # noqa: E731 rec = {"t": time.time(), "power_w": num(c[0]), "power_instant_w": num(c[1]), "temp_c": num(c[2]), "mem_mib": num(c[3]), "util_pct": num(c[4]), "sm_mhz": num(c[5]), "pstate": c[6]} last.update(rec) tel.write(json.dumps(rec) + "\n") threading.Thread(target=sampler, daemon=True).start() log.write(json.dumps({"utc": utc(), "event": "start", "tag": a.tag, "units_it_stops": units, "limits": {"ups_status_stops": ["OB", "LB"], "ups_unreadable_straight": STRAIGHT, "battery_runtime_min_s": RUNTIME_MIN_S, "runtime_straight": STRAIGHT, "ups_load_max_pct": UPS_LOAD_MAX, "core_max_c": CORE_MAX_C, "free_min_bytes": FREE_MIN}, "env": {k: os.environ.get(k) for k in list(REQUIRED) + ["DO_NOT_TRACK"]}}) + "\n") blind = low_runtime = 0 while True: rec = {"t": time.time(), "utc": utc()} try: rec.update(ups()) blind = 0 except Exception as exc: # noqa: BLE001 -- recorded; an unreadable UPS is itself a stop rec["ups_error"] = "%s: %s" % (type(exc).__name__, exc) blind += 1 rt = rec.get("battery_runtime_s") low_runtime = (low_runtime + 1) if (rt is not None and rt < RUNTIME_MIN_S) else 0 st = os.statvfs("/") rec["free_bytes"] = st.f_bavail * st.f_frsize rec["gpu_temp_c"] = last.get("temp_c") rec["gpu_mem_mib"] = last.get("mem_mib") rec["gpu_power_w"] = last.get("power_w") try: apps = subprocess.run(["nvidia-smi", "-i", uuid, "--query-compute-apps=pid", "--format=csv,noheader"], capture_output=True, text=True, timeout=5).stdout rec["compute_apps"] = len([x for x in apps.splitlines() if x.strip()]) pos = subprocess.run(["nvidia-smi", "-i", uuid, "--query-gpu=power.limit,persistence_mode", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5).stdout lim, per = [x.strip() for x in pos.strip().split(",")] rec["power_limit_w"], rec["persistence"] = float(lim), per except Exception as exc: # noqa: BLE001 -- recorded, not a trip rec["card_read_error"] = "%s: %s" % (type(exc).__name__, exc) status = (rec.get("ups_status") or "").split() why = [] if "OB" in status: why.append("the UPS is on battery (%s)" % rec["ups_status"]) if "LB" in status: why.append("the UPS reads low battery (%s)" % rec["ups_status"]) if blind >= STRAIGHT: why.append("the UPS unreadable on %d reads in a row" % blind) if low_runtime >= STRAIGHT: why.append("the UPS runtime under %.0f s on %d reads in a row (%s s)" % (RUNTIME_MIN_S, low_runtime, rt)) if rec.get("ups_load_pct") is not None and rec["ups_load_pct"] >= UPS_LOAD_MAX: why.append("UPS load %.0f %% >= %.0f %%" % (rec["ups_load_pct"], UPS_LOAD_MAX)) if rec["gpu_temp_c"] is not None and rec["gpu_temp_c"] >= CORE_MAX_C: why.append("card core %.0f C >= %d C" % (rec["gpu_temp_c"], CORE_MAX_C)) if rec["free_bytes"] < FREE_MIN: why.append("free disk %d B under 15 GiB" % rec["free_bytes"]) if smi.poll() is not None: why.append("the card sampler exited (%s)" % smi.returncode) if why: rec["trip"] = why log.write(json.dumps(rec) + "\n") stop = subprocess.run(["systemctl", "--user", "stop", *units], capture_output=True, text=True) log.write(json.dumps({"utc": utc(), "event": "stopped the bench units", "units": units, "rc": stop.returncode, "stderr": stop.stderr.strip()[:300]}) + "\n") smi.terminate() raise SystemExit(3) log.write(json.dumps(rec) + "\n") with open(HEARTBEAT, "w") as fh: fh.write(rec["utc"] + "\n") time.sleep(1.0) if __name__ == "__main__": main()