"""Fetch Von and Laya at their pinned revisions into this bench's own HF cache, then hash every file. Run with HF_HOME=/hf HF_HUB_DISABLE_TELEMETRY=1. Writes receipts/weights.sha256 and receipts/weights-pins.json. Never fetches the card's images or logs (assets/, eval/, run.log). """ import hashlib, json, os, sys, time, urllib.request from huggingface_hub import snapshot_download, HfApi B = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PINS = { "wfzyx/von": "411c44401cccddd792f341edfe033ea834557d13", "convaiinnovations/laya": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851", } SKIP = ("assets/", "eval/", "run.log", ".gitattributes") api = HfApi() out = {} lines = [] for repo, rev in PINS.items(): info = api.model_info(repo, revision=rev, files_metadata=True) assert info.sha == rev, (repo, info.sha, rev) want = {s.rfilename: (s.lfs.sha256 if s.lfs else None, s.size) for s in info.siblings if not s.rfilename.startswith(SKIP) and s.rfilename not in SKIP} t0 = time.time() path = snapshot_download(repo, revision=rev, allow_patterns=sorted(want)) took = time.time() - t0 files = {} for rel, (lfs, size) in sorted(want.items()): p = os.path.join(path, rel) h = hashlib.sha256() with open(p, "rb") as fh: for chunk in iter(lambda: fh.read(1 << 22), b""): h.update(chunk) got = h.hexdigest() n = os.path.getsize(p) ok_lfs = (lfs is None) or (got == lfs) ok_size = (size is None) or (n == size) if not (ok_lfs and ok_size): sys.exit("STOP: %s %s sha/size mismatch: %s %s vs lfs %s size %s" % (repo, rel, got, n, lfs, size)) files[rel] = {"sha256": got, "bytes": n, "equals_hf_lfs_oid": lfs is not None} lines.append("%s %s@%s/%s" % (got, repo, rev[:8], rel)) out[repo] = {"revision": rev, "snapshot": path, "last_modified": str(info.last_modified), "fetch_seconds": round(took, 1), "files": files} print("FETCHED", repo, rev, len(files), "files in %.1f s" % took, flush=True) os.makedirs(os.path.join(B, "receipts"), exist_ok=True) with open(os.path.join(B, "receipts", "weights-pins.json"), "w") as fh: json.dump(out, fh, indent=1); fh.write("\n") with open(os.path.join(B, "receipts", "weights.sha256"), "w") as fh: fh.write("\n".join(lines) + "\n") print("DONE", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))