In this tutorial, we work with Underdog Saluki 27B, a 7.9 GB build of Qwen3.8-27B that its makers say keeps tool calling intact at about two bits per weight and even edges out the full 54 GB model on their own 120-task benchmark. Saluki is built on ISTA-DASLab’s GSQ-RCO quantization of the same model, so we put both files side by side in stock llama.cpp. We start with the files themselves: a tensor-by-tensor diff of the two GGUF headers, then a byte-level comparison that shows whether Saluki is a remix of ISTA’s bits or a changed model. We then serve each file on one GPU, read its memory and speed, and score both on the same seeded 120 tasks from the Berkeley Function Calling Leaderboard, using BFCL’s own checker with strict and lenient parsing.
Copy CodeCopiedUse a different Browser
import os
import re
import ast
import sys
import json
import math
import copy
import enum
import glob
import time
import types
import random
import shutil
import struct
import hashlib
import tarfile
import platform
import traceback
import subprocess
import collections
import urllib.request
import urllib.error
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held) rows; held is True, False or None (not decided in this run)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1)
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, held))
return held
banner("1. Platform, pins and helpers")
SYSTEM, MACHINE = platform.system(), platform.machine()
ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None
assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac"
WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work"))
for d in ("models", "llama", "bfcl", "runs"):
os.makedirs(os.path.join(WORK, d), exist_ok=True)
# Everything the notebook downloads is pinned by size and SHA-256.
LLAMA_TAG = "b11565" # llama.cpp release of 2026-10-12, commit f9a95aeb2
LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844,
"61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"),
"Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509,
"19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
MODELS = {
"Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c",
file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152,
sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"),
"ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440",
file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472,
sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"),
}
# BFCL v4 data and BFCL's own checker code, at one gorilla commit.
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/"
"berkeley-function-call-leaderboard/bfcl_eval/")
BFCL_FILES = {
"data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991",
"data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
"data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6",
"data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
"data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579",
"data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc",
"data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047",
"data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0",
"data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3",
"eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538",
"constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a",
"constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd",
"utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd",
"model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3",
}
# Underdog Bench's five kinds of tasks, 24 each; we draw our own seeded sample because its task list is not published.
KINDS = [("simple_python", "Call the one right function"), ("multiple", "Pick the right function from several"),
("irrelevance", "Know when no tool fits"), ("parallel", "Same function, several calls"),
("parallel_multiple", "Several different functions")]
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
# llama-server's own defaults are temperature 0.8 and top_k 40, so every request states its sampler.
GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0)
def sh(cmd, **kw):
return subprocess.run(cmd, capture_output=True, text=True, **kw)
def sha256_file(path, chunk=1 << 22):
h = hashlib.sha256()
with open(path, "rb") as f:
for b in iter(lambda: f.read(chunk), b""):
h.update(b)
return h.hexdigest()
def fetch(url, dest, size=None, sha=None, quiet=False):
"""Resumable download with curl, then a size and SHA-256 check. Returns seconds spent downloading."""
t0 = time.time()
if not (os.path.exists(dest) and (size is None or os.path.getsize(dest) == size)):
for attempt in range(5):
r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url])
if r.returncode == 0 and (size is None or os.path.getsize(dest) == size):
break
time.sleep(5 * (attempt + 1))
got = os.path.getsize(dest)
assert size is None or got == size, f"{os.path.basename(dest)}: {got} bytes, expected {size}"
if sha:
assert sha256_file(dest) == sha, f"{os.path.basename(dest)}: SHA-256 mismatch"
return time.time() - t0
print(f"platform : {SYSTEM} {MACHINE}, Python {platform.python_version()}, glibc {platform.libc_ver()[1] or 'n/a'}")
if SYSTEM == "Linux":
print("os :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip()
print("gpu :", gpu)
# The CUDA 12.8 build of llama.cpp ships without the CUDA runtime; Colab's image has it as pip wheels.
import importlib.util
if importlib.util.find_spec("nvidia.cuda_runtime") is None or importlib.util.find_spec("nvidia.cublas") is None:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "nvidia-cuda-runtime-cu12==12.9.79",
"nvidia-cublas-cu12==12.9.2.10"], check=True)
import nvidia.cuda_runtime, nvidia.cublas
CUDA_LIBS = [os.path.join(list(m.__path__)[0], "lib") for m in (nvidia.cuda_runtime, nvidia.cublas)]
else:
chip = sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30
print(f"gpu : {chip}, {mem:.0f} GB unified memory (Metal)")
CUDA_LIBS = []
HW = gpu.split(",")[0] if SYSTEM == "Linux" else chip
print(f"work dir : {WORK} ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)")
Everything the notebook touches is pinned before anything runs. The setup cell records the llama.cpp release, both model files and fourteen BFCL files by size and SHA-256, and it fixes the sampler for every request, because llama-server otherwise falls back to defaults we did not choose. It detects the platform, an NVIDIA GPU on Linux or an Apple-silicon Mac, and on Colab it ensures the CUDA 12 runtime libraries the prebuilt llama. cpp needs are importable.cpp needs are importable.
Copy CodeCopiedUse a different Browser
@section("2. Download llama.cpp b11565 and both 2-bit files, and check every byte against its pin")
def download_step():
global LLAMA_DIR, SERVER_BIN, SERVER_ENV
name, size, sha = LLAMA_ASSET
tar = os.path.join(WORK, "llama", name)
fetch(f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/{name}", tar, size, sha)
with tarfile.open(tar) as t:
t.extractall(os.path.join(WORK, "llama"), filter="data")
SERVER_BIN = glob.glob(os.path.join(WORK, "llama", "**", "llama-server"), recursive=True)[0]
LLAMA_DIR = os.path.dirname(SERVER_BIN)
for b in glob.glob(os.path.join(LLAMA_DIR, "llama-*")):
os.chmod(b, 0o755)
SERVER_ENV = dict(os.environ)
if CUDA_LIBS:
SERVER_ENV["LD_LIBRARY_PATH"] = ":".join(CUDA_LIBS + [LLAMA_DIR, SERVER_ENV.get("LD_LIBRARY_PATH", "")])
missing = [l for l in sh(["ldd", SERVER_BIN], env=SERVER_ENV).stdout.splitlines() if "not found" in l]
assert not missing, f"unresolved libraries: {missing}"
r = sh([SERVER_BIN, "--version"], env=SERVER_ENV)
ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
print(f"llama.cpp: {name} (SHA-256 checked) -> {ver}")
for rel, digest in BFCL_FILES.items():
dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
fetch(GORILLA + rel, dest, sha=digest)
print(f"BFCL v4: {len(BFCL_FILES)} files from gorilla@6ea5797 (5 task files, 4 answer files, 5 checker sources), all SHA-256 checked")
for tag, m in MODELS.items():
path = os.path.join(WORK, "models", m["file"])
had = os.path.exists(path) and os.path.getsize(path) == m["size"]
url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
dl = fetch(url, path, m["size"]) # size check now, hash below
t0 = time.time()
ok = sha256_file(path) == m["sha256"]
how = "already on disk" if had else f"downloaded in {dl:.0f} s ({m['size'] / dl / 1e6:.0f} MB/s)"
print(f"{tag:7s}: {m['file']} {m['size']:,} bytes, {how}; SHA-256 {'matches' if ok else 'DOES NOT MATCH'} "
f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
claim(f"{m['file']} is byte-identical to the file pinned at {m['repo']}@{m['rev'][:7]}", ok)
assert ok
return f"llama.cpp {LLAMA_TAG}; Saluki 7.90 GB and ISTA IQ2_XS 8.42 GB verified by SHA-256"
download_step()
We download the CUDA build of llama.cpp b11565, both 2-bit files and the BFCL data, and check every byte against its pin. The Hugging Face downloads are resumable, and the hashes are the Hub’s own LFS digests at a fixed revision, so a silently replaced file would stop the notebook rather than change its results. BFCL’s task files, answer files, and checker code all come from one gorilla commit. The cell prints how long the downloads took, which is the slowest part of the run on Colab.
Copy CodeCopiedUse a different Browser
GGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
30: ("BF16", 1, 2)} # name, values per block, bytes per block (ggml-common.h)
def read_gguf(path):
"""GGUF v3 header: metadata, tensor table (name, dims, type, offset) and where the tensor data starts."""
f = open(path, "rb")
u = lambda fmt: struct.unpack("<" + fmt, f.read(struct.calcsize(fmt)))[0]
s = lambda: f.read(u("Q")).decode("utf-8", "replace")
scalar = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
def val(t):
if t == 8:
return s()
if t == 9:
et, n = u("I"), u("Q")
return [val(et) for _ in range(n)]
return u(scalar[t])
assert f.read(4) == b"GGUF"
version, n_tensors, n_kv = u("I"), u("Q"), u("Q")
kv = {}
for _ in range(n_kv):
k = s()
kv[k] = val(u("I"))
tensors = []
for _ in range(n_tensors):
name = s()
dims = [u("Q") for _ in range(u("I"))]
ttype, off = u("I"), u("Q")
n = math.prod(dims)
_, per, nbytes = GGML_TYPES[ttype]
tensors.append(dict(name=name, dims=dims, type=GGML_TYPES[ttype][0], n=n, bytes=n // per * nbytes, off=off))
align = kv.get("general.alignment", 32)
data_start = (f.tell() + align - 1) // align * align
return kv, tensors, data_start
def tensor_class(name):
if not name.startswith("blk."):
return name.split(".")[0] # token_embd, output, output_norm
kind = name.split(".")[2]
if kind in ("attn_q", "attn_k", "attn_v", "attn_output"):
return "full-attention " + kind[5:]
if kind in ("attn_qkv", "attn_gate", "ssm_out", "ssm_conv1d"):
return "linear-attention " + kind
if kind in ("ssm_alpha", "ssm_beta"):
return "linear-attention gates (alpha, beta)"
if kind in ("ffn_gate", "ffn_up", "ffn_down"):
return kind
return "norms and small vectors"
@section("3. What Saluki changes: a tensor-by-tensor diff of the two GGUF headers")
def header_diff():
global HDR
HDR = {tag: read_gguf(os.path.join(WORK, "models", m["file"])) for tag, m in MODELS.items()}
(kv_s, t_s, start_s), (kv_i, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
assert [t["name"] for t in t_s] == [t["name"] for t in t_i] and [t["dims"] for t in t_s] == [t["dims"] for t in t_i]
n_params = sum(t["n"] for t in t_s)
print(f"both files: {len(t_s)} tensors with the same names and shapes, {n_params:,} weights, "
f"architecture '{kv_s['general.architecture']}', {kv_s['qwen35.block_count']} blocks")
for tag, (kv, ts, start) in HDR.items():
data = sum(t["bytes"] for t in ts)
size = MODELS[tag]["size"]
print(f" {tag:7s}: header {start:,} B + tensor data {data:,} B = {start + data:,} B "
f"({'equals' if start + data == size else 'DIFFERS FROM'} the file size); {8 * data / n_params:.3f} bits per weight")
claim(f"{tag}: header plus tensor bytes computed from the type table equals the file size exactly", start + data == size)
rows = collections.OrderedDict()
moves = collections.Counter()
up = down = 0
for a, b in zip(t_i, t_s):
r = rows.setdefault(tensor_class(a["name"]), dict(k=0, n=0, bi=0, bs=0))
r["k"] += 1; r["n"] += a["n"]; r["bi"] += a["bytes"]; r["bs"] += b["bytes"]
if a["type"] != b["type"]:
moves[(a["type"], b["type"])] += 1
up += b["bytes"] > a["bytes"]; down += b["bytes"] < a["bytes"]
print(f"\n{'tensor class':34s} {'count':>5s} {'weights':>9s} {'ISTA bpw':>9s} {'Saluki bpw':>10s} {'change MB':>10s}")
for c, r in sorted(rows.items(), key=lambda kv: -kv[1]["n"]):
if r["n"] < 1e6:
continue
print(f"{c:34s} {r['k']:5d} {r['n'] / 1e9:8.3f}B {8 * r['bi'] / r['n']:9.3f} {8 * r['bs'] / r['n']:10.3f} {(r['bs'] - r['bi']) / 1e6:+10.1f}")
changed = sum(moves.values())
print(f"\n{changed} of {len(t_s)} tensors change type: " + ", ".join(f"{n} {a}->{b}" for (a, b), n in moves.most_common()))
print(f" {down} get fewer bytes, {up} get more")
claim("every tensor whose type changes gets fewer bits; none gets more", up == 0 and down == changed)
untouched = [c for c, r in rows.items() if r["bi"] == r["bs"] and r["n"] >= 1e6]
print(f" classes with no change at all: {', '.join(sorted(untouched))}")
proj = lambda c: c.startswith(("full-attention", "linear-attention")) and "gates" not in c
claim("every attention and DeltaNet projection matrix keeps its type (only the small alpha/beta gates change)",
all(a["type"] == b["type"] for a, b in zip(t_i, t_s) if proj(tensor_class(a["name"]))))
keys = sorted(k for k in set(kv_s) | set(kv_i) if kv_s.get(k) != kv_i.get(k) and not k.startswith("tokenizer.ggml.t"))
print(f"\nmetadata keys that differ: {keys}")
print(f"Saluki's header still reads: general.name = {kv_s['general.name']!r}, quantized_by = {kv_s['general.quantized_by']!r}")
print(f"sampling stored in both headers: temp {kv_s['general.sampling.temp']}, top_k {kv_s['general.sampling.top_k']}, "
f"top_p {kv_s['general.sampling.top_p']:.2f} (these are Qwen's thinking-mode settings; step 5 shows what the server does with them)")
saved = (MODELS["ISTA"]["size"] - MODELS["Saluki"]["size"]) / 1e6
return f"{changed} of {len(t_s)} tensor types changed, all downgrades; {saved:.0f} MB smaller"
header_diff()
Reading the two GGUF headers in plain Python shows exactly what Saluki changed. Both files have the same 851 tensors with the same shapes, and the type table predicts each file’s size in bytes. Saluki changes the type of 146 tensors and every change cuts bits: most fall on the feed-forward down projections and the output layer, and the small DeltaNet gate vectors drop from BF16 to Q8_0. No attention or DeltaNet projection matrix changes, and the header still names ISTA DASLab as the quantiser.
Copy CodeCopiedUse a different Browser
@section("4. Byte-level provenance: are the unchanged tensors ISTA's own bytes?")
def provenance():
import numpy as np
(_, t_s, start_s), (_, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
ms = np.memmap(os.path.join(WORK, "models", MODELS["Saluki"]["file"]), dtype=np.uint8, mode="r")
mi = np.memmap(os.path.join(WORK, "models", MODELS["ISTA"]["file"]), dtype=np.uint8, mode="r")
q = dict(k=0, same=0, eq_bytes=0, bytes=0)
f32 = collections.defaultdict(lambda: [0, 0, []]) # kind -> [identical, differ, relative L2 changes]
for a, b in zip(t_i, t_s):
if a["type"] != b["type"]:
continue
x = mi[start_i + a["off"]: start_i + a["off"] + a["bytes"]]
y = ms[start_s + b["off"]: start_s + b["off"] + b["bytes"]]
if a["type"] == "F32":
kind = re.sub(r"blk\.\d+\.", "", a["name"])
u, v = x.view(np.float32).astype(np.float64), y.view(np.float32).astype(np.float64)
same = bool(np.array_equal(u, v))
f32[kind][0 if same else 1] += 1
if not same:
f32[kind][2].append(np.linalg.norm(u - v) / np.linalg.norm(u))
else:
eq = int(np.count_nonzero(x == y))
q["k"] += 1; q["same"] += eq == a["bytes"]; q["eq_bytes"] += eq; q["bytes"] += a["bytes"]
print(f"quantised matrices that keep their type: {q['same']} of {q['k']} byte-identical, but "
f"{100 * q['eq_bytes'] / q['bytes']:.1f}% of their {q['bytes'] / 1e9:.2f} GB of bytes are equal")
print(f"\nunquantised F32 vectors (never re-encoded, so any change means the weights changed):")
print(f" {'kind':28s} {'identical':>9s} {'changed':>8s} median relative change")
for kind, (same, diff, rel) in sorted(f32.items(), key=lambda kv: -kv[1][1]):
med = f"{sorted(rel)[len(rel) // 2]:.1e}" if rel else "-"
print(f" {kind:28s} {same:9d} {diff:8d} {med}")
changed = sum(v[1] for v in f32.values())
total = sum(v[0] + v[1] for v in f32.values())
claim("Saluki's unchanged-type tensors are byte-for-byte ISTA's (a pure type re-mix)", q["same"] == q["k"] and changed == 0)
claim("Saluki's F32 norm and gate vectors differ from ISTA's, so its weights were changed, not only re-encoded", changed > 0)
return (f"{q['same']}/{q['k']} same-type matrices byte-identical, {100 * q['eq_bytes'] / q['bytes']:.1f}% of their bytes equal; "
f"{changed}/{total} F32 vectors changed")
provenance()
Same types don’t mean the same weights, so we compare the bytes of the 705 tensors that keep their type. Only one of 352 quantized matrices is byte-identical, yet 98.4% of their bytes match, consistent with small edits to the underlying weights followed by re-quantizing on the same grids. The unquantized norm and gate vectors settle it: most changed by a median of a few parts in ten thousand. Saluki is a lightly changed model, not a re-encoding of ISTA’s file.
Copy CodeCopiedUse a different Browser
SERVER = {}
def start_server(tag):
"""One llama-server per model, one slot, all layers on the GPU, debug log kept for unparsed replies."""
stop_server()
log_path = os.path.join(WORK, "runs", f"server_{tag}.log")
cmd = [SERVER_BIN, "-m", os.path.join(WORK, "models", MODELS[tag]["file"]), "--host", "127.0.0.1", "--port", str(PORT),
"-c", str(CTX), "-np", "1", "-ngl", "99", "-fa", "on", "--jinja", "--reasoning-format", "deepseek", "-lv", "5"]
t0 = time.time()
proc = subprocess.Popen(cmd, stdout=open(log_path, "w"), stderr=subprocess.STDOUT, env=SERVER_ENV)
while True:
try:
if urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=5).status == 200:
break
except urllib.error.HTTPError as e:
assert e.code == 503, e
except urllib.error.URLError:
pass
assert proc.poll() is None, f"llama-server exited; see {log_path}"
assert time.time() - t0 < 900, "server did not become healthy in 15 minutes"
time.sleep(2)
SERVER.update(tag=tag, proc=proc, log=log_path, load_s=time.time() - t0)
return SERVER["load_s"]
def stop_server():
if SERVER.get("proc") and SERVER["proc"].poll() is None:
SERVER["proc"].terminate()
SERVER["proc"].wait(timeout=60)
SERVER.clear()
def api(path, body=None, timeout=1800):
req = urllib.request.Request(f"http://127.0.0.1:{PORT}{path}", None if body is None else json.dumps(body).encode(),
{"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read())
def memory_report():
"""Buffer sizes from the server's load log, plus what the GPU reports."""
text = open(SERVER["log"]).read()
out = {}
for label, what in (("weights", "model"), ("KV cache", "KV"), ("recurrent state", "RS"), ("compute", "compute")):
last = {} # llama.cpp sizes every buffer in a dry run first; keep the final value per device
for dev, mib in re.findall(rf"(\S+) +{what} buffer size = +([\d.]+) MiB", text):
last[dev] = float(mib)
out[label] = sum(last.values())
if ON_CUDA:
out["GPU in use (nvidia-smi)"] = float(sh(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"]).stdout.split()[0])
return out
@section("5. A hybrid model on one GPU: start the server, read its memory, and look at the prompt it builds")
def architecture_and_server():
kv = HDR["Saluki"][0]
L, every = kv["qwen35.block_count"], kv["qwen35.full_attention_interval"]
full = [i for i in range(L) if (i + 1) % every == 0]
heads_kv, dk = kv["qwen35.attention.head_count_kv"], kv["qwen35.attention.key_length"]
kv_tok = len(full) * heads_kv * dk * 2 * 2 # K and V, f16
conv = (kv["qwen35.ssm.conv_kernel"] - 1) * (kv["qwen35.ssm.inner_size"] + 2 * kv["qwen35.ssm.group_count"] * kv["qwen35.ssm.state_size"])
state = kv["qwen35.ssm.state_size"] * kv["qwen35.ssm.inner_size"]
rs = (L - len(full)) * (conv + state) * 4
print(f"{L} blocks: {L - len(full)} Gated DeltaNet (linear attention) and {len(full)} full-attention layers (3, 7, ..., {full[-1]})")
print(f"KV cache: {kv_tok // 1024} KiB per token for the {len(full)} attention layers; a model with attention in all "
f"{L} layers would need {L * heads_kv * dk * 4 // 1024} KiB")
print(f"recurrent state: {rs / 2**20:.1f} MiB per sequence, fixed, whatever the context length")
print(f"expected at -c {CTX}: KV {kv_tok * CTX / 2**20:.0f} MiB + state {rs / 2**20:.0f} MiB")
load_s = start_server("Saluki")
mem = memory_report()
print(f"\nllama-server up in {load_s:.0f} s (-c {CTX}, -np 1, -ngl 99, -fa on); buffers from its log:")
for k, v in mem.items():
print(f" {k:24s} {v:8.0f} MiB")
claim(f"the server's KV buffer at -c {CTX} matches 64 KiB per token within 2%",
abs(mem["KV cache"] - kv_tok * CTX / 2**20) <= 0.02 * kv_tok * CTX / 2**20)
SERVER_MEM["Saluki"] = mem
props = api("/props")
p = props["default_generation_settings"]["params"]
g = {k: kv[f"general.sampling.{k}"] for k in ("temp", "top_k", "top_p", "min_p")}
from_file = abs(p["temperature"] - g["temp"]) < 1e-6 and p["top_k"] == g["top_k"] and abs(p["top_p"] - g["top_p"]) < 1e-6
print(f"\nserver defaults: temperature {p['temperature']:.2f}, top_k {p['top_k']}, top_p {p['top_p']:.2f}, min_p {p['min_p']:.2f}"
f" -> {'taken from the GGUF header' if from_file else 'llama.cpp built-ins, not the header'}; the card suggests 0.6 for thinking,"
f" so every request below states its own sampler")
claim("llama-server takes its default sampler from the GGUF's general.sampling keys", from_file)
caps = props.get("chat_template_caps", {})
print("template supports: " + ", ".join(k.replace("supports_", "") for k, v in caps.items() if v))
msgs = [{"role": "user", "content": "What is the weather in Oslo?"}]
tool = [{"type": "function", "function": {"name": "get_weather", "description": "Current weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}]
for label, kw in (("thinking off", {"enable_thinking": False}), ("thinking on, effort low", {"reasoning_effort": "low"}),
("thinking on, default", {})):
prompt = api("/apply-template", {"messages": msgs, "tools": tool, "chat_template_kwargs": kw})["prompt"]
n = len(api("/tokenize", {"content": prompt, "add_special": False})["tokens"])
print(f" {label:24s}: {n:3d} prompt tokens, ends {prompt[-28:]!r}")
return f"KV {kv_tok // 1024} KiB/token, state {rs / 2**20:.0f} MiB; server buffers {sum(mem[k] for k in ('weights', 'KV cache', 'recurrent state', 'compute')) / 1024:.1f} GiB"
SERVER_MEM = {}
architecture_and_server()
Qwen3.8-27B is a hybrid: 48 of its 64 blocks are Gated DeltaNet linear-attention layers with a fixed recurrent state, and only every fourth block keeps full attention. The KV cache therefore costs 64 KiB per token instead of 256 KiB, plus about 150 MiB of state per sequence. We start llama-server with one slot, read its buffers from the log, check where its default sampler comes from, and look at the prompts the chat template builds with thinking off, on, and at low effort.
Copy CodeCopiedUse a different Browser
SPEED = {}
def speed_probe(tag):
"""Prefill and decode rates from the server's own timings: two prompt lengths, 128 forced output tokens."""
rng = random.Random(SEED)
words = "def class return import yield lambda tensor gradient matrix kernel cache token buffer layer".split()
rows = []
for target in (256, 1024):
text = " ".join(rng.choice(words) for _ in range(target))
ids = api("/tokenize", {"content": text, "add_special": False})["tokens"][:target]
r = api("/completion", {"prompt": ids, "n_predict": 128, "ignore_eos": True, "cache_prompt": False, **GREEDY})
t = r["timings"]
rows.append((t["prompt_n"], t["prompt_per_second"], t["predicted_n"], t["predicted_per_second"]))
print(f" {tag:7s} prompt {t['prompt_n']:5d} tok at {t['prompt_per_second']:7.1f} tok/s | "
f"decode {t['predicted_n']} tok at {t['predicted_per_second']:5.2f} tok/s")
SPEED[tag] = rows
return rows
@section("6. Prefill and decode speed, measured by the server")
def speed_step():
print(f"hardware: {HW}")
rows = speed_probe("Saluki")
return f"Saluki on {HW}: prefill {rows[1][1]:.0f} tok/s at {rows[1][0]} tokens, decode {rows[1][3]:.1f} tok/s"
speed_step()
……（长文已截断）