65 lines
1.7 KiB
Python
65 lines
1.7 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# Copyright (C) 2026 David (Corey) Herrera
|
|
"""
|
|
gguf_token_text.py — print the vocab text of token ids from a GGUF's tokenizer.ggml.tokens
|
|
(no dependencies), to read what a token-id diff in a serve log actually says.
|
|
|
|
Usage: gguf_token_text.py <file.gguf> <id>...
|
|
"""
|
|
import struct
|
|
import sys
|
|
|
|
SIZES = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8}
|
|
|
|
|
|
def rd(f, fmt):
|
|
return struct.unpack("<" + fmt, f.read(struct.calcsize("<" + fmt)))
|
|
|
|
|
|
def rd_str(f):
|
|
(n,) = rd(f, "Q")
|
|
return f.read(n).decode("utf-8", "replace")
|
|
|
|
|
|
def skip_value(f, t):
|
|
if t in SIZES:
|
|
f.read(SIZES[t])
|
|
elif t == 8:
|
|
rd_str(f)
|
|
elif t == 9:
|
|
(et,) = rd(f, "I")
|
|
(n,) = rd(f, "Q")
|
|
if et in SIZES:
|
|
f.read(SIZES[et] * n)
|
|
else:
|
|
for _ in range(n):
|
|
skip_value(f, et)
|
|
else:
|
|
raise ValueError(f"gguf value type {t}")
|
|
|
|
|
|
def main():
|
|
path, ids = sys.argv[1], [int(x) for x in sys.argv[2:]]
|
|
with open(path, "rb") as f:
|
|
magic, _version = rd(f, "4sI")
|
|
assert magic == b"GGUF", magic
|
|
_n_tensors, n_kv = rd(f, "QQ")
|
|
for _ in range(n_kv):
|
|
key = rd_str(f)
|
|
(t,) = rd(f, "I")
|
|
if key != "tokenizer.ggml.tokens":
|
|
skip_value(f, t)
|
|
continue
|
|
(_et,) = rd(f, "I")
|
|
(n,) = rd(f, "Q")
|
|
vocab = [rd_str(f) for _ in range(n)]
|
|
for i in ids:
|
|
print(i, repr(vocab[i]) if 0 <= i < n else "(out of range)")
|
|
return
|
|
sys.exit("no tokenizer.ggml.tokens")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|