596 lines
23 KiB
Plaintext
596 lines
23 KiB
Plaintext
===SYSTEM===
|
|
You are a coding agent. Apply the requested edit and re-emit the complete file in a single fenced code block. Do not explain.
|
|
===PROMPT===
|
|
I ran `read_file toolkit/lists.py` and got:
|
|
|
|
```python
|
|
"""List utilities. Implement the stubs; fix any bugs. Contracts in spec.md.
|
|
|
|
Rules: pure Python stdlib only, no third-party imports. Do NOT edit the tests.
|
|
"""
|
|
|
|
|
|
def dedup(lst):
|
|
"""Return `lst` with duplicate values removed, preserving first-seen order.
|
|
|
|
dedup([1, 2, 2, 3, 1, 4]) -> [1, 2, 3, 4].
|
|
"""
|
|
raise NotImplementedError("implement dedup (see spec.md)")
|
|
|
|
|
|
def chunk(lst, n):
|
|
"""Split `lst` into consecutive chunks of length `n` (the last may be shorter).
|
|
|
|
chunk([1, 2, 3, 4, 5], 2) -> [[1, 2], [3, 4], [5]]. Assume n >= 1.
|
|
"""
|
|
raise NotImplementedError("implement chunk (see spec.md)")
|
|
|
|
|
|
def flatten(nested):
|
|
"""Flatten exactly ONE level of nesting.
|
|
|
|
flatten([[1, 2], [3], [4, 5]]) -> [1, 2, 3, 4, 5].
|
|
Inner nesting is preserved: flatten([[1, [2]], [3]]) -> [1, [2], 3].
|
|
"""
|
|
# BUG: this recurses fully (and crashes on non-list items) instead of
|
|
# flattening a single level. Fix it to flatten ONE level only.
|
|
out = []
|
|
for item in nested:
|
|
out.extend(flatten(item))
|
|
return out
|
|
```
|
|
|
|
`flatten` recurses all the way down and crashes on non-list items. Fix it to flatten exactly ONE level, so flatten([[1, [2]], [3]]) returns [1, [2], 3]. Leave every other function exactly as it is.
|
|
|
|
Re-emit the complete updated contents of `toolkit/lists.py`.
|
|
===PROMPT===
|
|
I ran `read_file toolkit/numbers.py` and got:
|
|
|
|
```python
|
|
"""Number utilities. Implement the stubs; fix any bugs. Contracts in spec.md.
|
|
|
|
Rules: pure Python stdlib only, no third-party imports. Do NOT edit the tests.
|
|
"""
|
|
|
|
|
|
def gcd(a, b):
|
|
"""Greatest common divisor of `a` and `b` (Euclid's algorithm).
|
|
|
|
Reference implementation — already correct. Read it as an example of the
|
|
house style before you implement the stubs below.
|
|
"""
|
|
while b:
|
|
a, b = b, a % b
|
|
return abs(a)
|
|
|
|
|
|
def is_prime(n):
|
|
"""True if `n` is a prime number. n < 2 -> False. 2 -> True. 15 -> False."""
|
|
raise NotImplementedError("implement is_prime (see spec.md)")
|
|
|
|
|
|
def fib(n):
|
|
"""The nth Fibonacci number, 0-indexed: fib(0)=0, fib(1)=1, fib(2)=1,
|
|
fib(3)=2, ... Assume n >= 0.
|
|
"""
|
|
raise NotImplementedError("implement fib (see spec.md)")
|
|
```
|
|
|
|
Replace the `is_prime` stub with a working implementation (trial division up to the integer square root is fine; it must return real bools). Leave `gcd` and `fib` exactly as they are.
|
|
|
|
Re-emit the complete updated contents of `toolkit/numbers.py`.
|
|
===PROMPT===
|
|
I ran `read_file toolkit/strings.py` and got:
|
|
|
|
```python
|
|
"""String utilities. Implement the stubs; fix any bugs. Contracts in spec.md.
|
|
|
|
Rules: pure Python stdlib only, no third-party imports. Do NOT edit the tests.
|
|
Run `make test` to check your work.
|
|
"""
|
|
|
|
|
|
def reverse_words(s):
|
|
"""Return `s` with its whitespace-separated words in reverse order.
|
|
|
|
'hello world foo' -> 'foo world hello'. '' -> ''. Collapse runs of spaces:
|
|
the result is single-spaced.
|
|
"""
|
|
raise NotImplementedError("implement reverse_words (see spec.md)")
|
|
|
|
|
|
def count_vowels(s):
|
|
"""Count the vowels (a, e, i, o, u — case-insensitive) in `s`. Return an int."""
|
|
raise NotImplementedError("implement count_vowels (see spec.md)")
|
|
|
|
|
|
def is_palindrome(s):
|
|
"""True if `s` reads the same forwards and backwards, IGNORING case and any
|
|
non-alphanumeric characters.
|
|
|
|
'racecar' -> True. 'A man, a plan, a canal: Panama' -> True. 'hello' -> False.
|
|
"""
|
|
cleaned = ''.join(ch for ch in s.lower() if ch.isalnum())
|
|
return cleaned == cleaned[::-1]
|
|
```
|
|
|
|
Replace the `reverse_words` stub with a working implementation: split on whitespace, reverse the word order, join with single spaces, and map '' to ''. Leave the other functions exactly as they are.
|
|
|
|
Re-emit the complete updated contents of `toolkit/strings.py`.
|
|
===PROMPT===
|
|
I ran `read_file tests/test_toolkit.py` and got:
|
|
|
|
```python
|
|
#!/usr/bin/env python3
|
|
"""Assert-based test runner for the toolkit — no pytest needed.
|
|
|
|
Run from the repo root: `make test` (or `python3 tests/test_toolkit.py`).
|
|
Prints PASS/FAIL per check and exits non-zero if anything fails, so the agent
|
|
verification gate (and CI) can trust the exit code. DO NOT edit this file — make
|
|
the library match it.
|
|
"""
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
|
|
from toolkit import strings, numbers, lists # noqa: E402
|
|
|
|
_failures = []
|
|
|
|
|
|
def check(name, fn):
|
|
try:
|
|
ok = bool(fn())
|
|
except Exception as exc: # a crashing implementation is a failure, not a stop
|
|
print("FAIL", name, "->", repr(exc))
|
|
_failures.append(name)
|
|
return
|
|
print("PASS" if ok else "FAIL", name)
|
|
if not ok:
|
|
_failures.append(name)
|
|
|
|
|
|
# --- strings ---------------------------------------------------------------
|
|
check("reverse_words basic", lambda: strings.reverse_words("hello world foo") == "foo world hello")
|
|
check("reverse_words empty", lambda: strings.reverse_words("") == "")
|
|
check("count_vowels mixed", lambda: strings.count_vowels("Hello World") == 3)
|
|
check("count_vowels none", lambda: strings.count_vowels("xyz") == 0)
|
|
check("is_palindrome plain", lambda: strings.is_palindrome("racecar") is True)
|
|
check("is_palindrome punct", lambda: strings.is_palindrome("A man, a plan, a canal: Panama") is True)
|
|
check("is_palindrome no", lambda: strings.is_palindrome("hello") is False)
|
|
|
|
# --- numbers ---------------------------------------------------------------
|
|
check("gcd", lambda: numbers.gcd(12, 18) == 6)
|
|
check("is_prime yes", lambda: numbers.is_prime(13) is True)
|
|
check("is_prime edges", lambda: numbers.is_prime(1) is False and numbers.is_prime(2) is True and numbers.is_prime(15) is False)
|
|
check("fib sequence", lambda: [numbers.fib(i) for i in range(8)] == [0, 1, 1, 2, 3, 5, 8, 13])
|
|
|
|
# --- lists -----------------------------------------------------------------
|
|
check("dedup order", lambda: lists.dedup([1, 2, 2, 3, 1, 4]) == [1, 2, 3, 4])
|
|
check("chunk uneven", lambda: lists.chunk([1, 2, 3, 4, 5], 2) == [[1, 2], [3, 4], [5]])
|
|
check("flatten one level", lambda: lists.flatten([[1, 2], [3], [4, 5]]) == [1, 2, 3, 4, 5])
|
|
check("flatten keeps inner", lambda: lists.flatten([[1, [2]], [3]]) == [1, [2], 3])
|
|
|
|
print()
|
|
if _failures:
|
|
print("%d FAILURE(S): %s" % (len(_failures), ", ".join(_failures)))
|
|
sys.exit(1)
|
|
print("ALL %d CHECKS PASSED" % 15)
|
|
sys.exit(0)
|
|
```
|
|
|
|
Add a `--verbose` flag: when sys.argv contains it, print the expected and actual value for every check, not just PASS/FAIL. Change nothing else.
|
|
|
|
Re-emit the complete updated contents of `tests/test_toolkit.py`.
|
|
===PROMPT===
|
|
I ran `read_file src/core/ngram_draft.c` and got:
|
|
|
|
```c
|
|
/**
|
|
* @file ngram_draft.c
|
|
* @brief Prompt-lookup n-gram drafter (see ngram_draft.h). A linear backward
|
|
* scan per n-gram length: history stays a plain array (a few thousand
|
|
* tokens at serving lengths), so a scan is ~microseconds against a
|
|
* ~50 ms K80 forward — not worth an index.
|
|
*
|
|
* @status Working; covered by tests/test_spec_qwen35.c on the K80 test machine.
|
|
* @issues None
|
|
* @todo None
|
|
*/
|
|
|
|
/* 1. This module's own header */
|
|
#include "ngram_draft.h"
|
|
|
|
/* 2. Standard library */
|
|
#include <string.h>
|
|
|
|
int pn_ngram_draft(const uint32_t *hist, int n_hist, int ngram_max,
|
|
int ngram_min, int max_k, uint32_t *out)
|
|
{
|
|
if (!hist || !out || max_k <= 0 || ngram_min < 1 ||
|
|
ngram_max < ngram_min)
|
|
return 0;
|
|
|
|
for (int g = ngram_max; g >= ngram_min; g--) {
|
|
if (n_hist < g + 1)
|
|
continue;
|
|
const uint32_t *key = hist + n_hist - g;
|
|
/* Most recent earlier occurrence; start excludes the trailing
|
|
* self-match (start + g <= n_hist - 1 leaves >= 1 continuation). */
|
|
for (int start = n_hist - g - 1; start >= 0; start--) {
|
|
if (memcmp(hist + start, key, (size_t)g * sizeof(*hist)) != 0)
|
|
continue;
|
|
int avail = n_hist - (start + g);
|
|
int k = avail < max_k ? avail : max_k;
|
|
memcpy(out, hist + start + g, (size_t)k * sizeof(*hist));
|
|
return k;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
void pn_spec_throttle_init(pn_spec_throttle *t, int min_accept,
|
|
int miss_limit, int cool_steps)
|
|
{
|
|
t->min_accept = min_accept > 0 ? min_accept : 4;
|
|
t->miss_limit = miss_limit > 0 ? miss_limit : 2;
|
|
t->cool_steps = cool_steps > 0 ? cool_steps : 48;
|
|
t->miss = 0;
|
|
t->cool = 0;
|
|
}
|
|
|
|
int pn_spec_throttle_gate(pn_spec_throttle *t)
|
|
{
|
|
if (t->cool > 0) {
|
|
t->cool--;
|
|
return 0;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
void pn_spec_throttle_report(pn_spec_throttle *t, int accepted_drafts)
|
|
{
|
|
if (accepted_drafts >= t->min_accept) {
|
|
t->miss = 0;
|
|
return;
|
|
}
|
|
if (++t->miss >= t->miss_limit) {
|
|
t->miss = 0;
|
|
t->cool = t->cool_steps;
|
|
}
|
|
}
|
|
```
|
|
|
|
Add an early-out to `pn_ngram_draft`: if the last token of the history is the same as the token before it, return 0 without scanning (a repeated token is a degenerate key). Change nothing else.
|
|
|
|
Re-emit the complete updated contents of `src/core/ngram_draft.c`.
|
|
===PROMPT===
|
|
I ran `read_file src/models/bonsai_forward.c` and got:
|
|
|
|
```c
|
|
/**
|
|
* @file bonsai_forward.c
|
|
* @brief PrismML Bonsai-27B top-level forward/loader entry (PN_ARCH_BONSAI).
|
|
*
|
|
* This is Bonsai's OWN entry file, kept physically separate from the shipped
|
|
* Nex-N2 forward (qwen35moe_forward*.c). It owns the pn_forward_fn/loader/state
|
|
* seam for PN_ARCH_BONSAI and DELEGATES the heavy lifting to the shared
|
|
* qwen35moe machinery: Bonsai is the SAME hybrid GDN + sparse-full-attention
|
|
* family (general.architecture="qwen35"), differing only in a DENSE SwiGLU FFN
|
|
* (no MoE) and native-ternary (Q2_0_g128 / type-142) weights.
|
|
*
|
|
* Why delegate instead of clone: the hybrid forward is ~7.4k lines of shared,
|
|
* die-scratch-heavy CUDA orchestration (GDN block, flash-attn, Q4 KV, RoPE,
|
|
* RMSNorm, the ternary fc_matvec_q2_0). Cloning it to strip MoE -> dense would
|
|
* be a large correctness/drift risk (the exact hazard the bring-up flagged).
|
|
* Instead, the ONE Bonsai-specific behavior — the dense SwiGLU FFN block — is a
|
|
* config-driven branch inside the shared per-layer FFN helper, selected by
|
|
* cfg->dense_ffn (set at load from the GGUF metadata). The shipped Nex-N2 (qwen35moe,
|
|
* dense_ffn=0) never takes that branch and is byte-identical. Bonsai therefore
|
|
* has its own arch identity + its own forward entry file, without a risky clone.
|
|
*/
|
|
|
|
#include "bonsai.h"
|
|
#include "qwen35moe.h"
|
|
|
|
#include <string.h>
|
|
|
|
int pn_bonsai_is_bonsai_gguf(pn_gguf_file *gguf)
|
|
{
|
|
if (!gguf)
|
|
return 0;
|
|
const char *arch = pn_gguf_get_str(gguf, "general.architecture");
|
|
if (!arch || strcmp(arch, "qwen35") != 0)
|
|
return 0;
|
|
/* Dense FFN (Bonsai) vs routed MoE (Nex-N2): a plain per-layer ffn_gate is
|
|
* present and the merged expert tensor is absent. */
|
|
if (pn_gguf_find_tensor(gguf, "blk.0.ffn_gate_exps.weight") != NULL)
|
|
return 0;
|
|
if (pn_gguf_find_tensor(gguf, "blk.0.ffn_gate.weight") == NULL)
|
|
return 0;
|
|
return 1;
|
|
}
|
|
|
|
void *pn_bonsai_load(pn_gguf_file *gguf, pn_config *config)
|
|
{
|
|
/* The shared loader detects the dense-FFN variant from the metadata
|
|
* (expert_count absent + ffn_gate present) and sets cfg->dense_ffn. */
|
|
return pn_qwen35moe_load(gguf, config);
|
|
}
|
|
|
|
tm_tensor *pn_bonsai_forward(pn_model *model, uint32_t token, pn_state *state)
|
|
{
|
|
return pn_qwen35moe_forward(model, token, state);
|
|
}
|
|
|
|
#ifdef USE_CUDA
|
|
tm_tensor *pn_bonsai_forward_cuda(pn_model *model, uint32_t token,
|
|
pn_state *state)
|
|
{
|
|
return pn_qwen35moe_forward_cuda(model, token, state);
|
|
}
|
|
|
|
int pn_bonsai_gpu_upload(void *model)
|
|
{
|
|
return pn_qwen35moe_gpu_upload((pn_qwen35moe_model *)model);
|
|
}
|
|
|
|
int pn_bonsai_model_gpu_loaded(const void *model)
|
|
{
|
|
return pn_qwen35moe_model_gpu_loaded((const pn_qwen35moe_model *)model);
|
|
}
|
|
#endif
|
|
|
|
void *pn_bonsai_state_new(void *weights)
|
|
{
|
|
return pn_qwen35moe_state_new(weights);
|
|
}
|
|
|
|
void pn_bonsai_state_free(void *state)
|
|
{
|
|
pn_qwen35moe_state_free(state);
|
|
}
|
|
|
|
void pn_bonsai_state_reset(void *state)
|
|
{
|
|
pn_qwen35moe_state_reset(state);
|
|
}
|
|
|
|
void pn_bonsai_free(void *model)
|
|
{
|
|
pn_qwen35moe_free((pn_qwen35moe_model *)model);
|
|
}
|
|
|
|
pn_gguf_file *pn_bonsai_gguf(const void *model)
|
|
{
|
|
return pn_qwen35moe_gguf((const pn_qwen35moe_model *)model);
|
|
}
|
|
|
|
void pn_bonsai_model_set_dies(void *model, const int *dies, int n)
|
|
{
|
|
pn_qwen35moe_model_set_dies((pn_qwen35moe_model *)model, dies, n);
|
|
}
|
|
```
|
|
|
|
Add a short doc comment above each function describing what it returns. Change no code.
|
|
|
|
Re-emit the complete updated contents of `src/models/bonsai_forward.c`.
|
|
===PROMPT===
|
|
I ran `read_file src/serve/serve_registry.h` and got:
|
|
|
|
```c
|
|
/**
|
|
* @file serve_registry.h
|
|
* @brief Serve-side dispatch self-registration lifecycle (Phase G0 wiring).
|
|
* Ties the register / heartbeat / deregister client (registry_client.h)
|
|
* into the live serve process: one register per served model_id at
|
|
* startup, a background heartbeat thread reporting live load from
|
|
* batch_sched, and a best-effort deregister on controlled shutdown.
|
|
*
|
|
* FULLY GATED on PANTHEON_REGISTRY_URL (via rc_enabled()): with it unset
|
|
* every function here is a no-op and the serve process is byte-for-byte
|
|
* unchanged (no registration, no heartbeat thread spawned, no network).
|
|
*
|
|
* Identity semantics (see registry_wire.h): the shared `model_id` is the
|
|
* served name clients request (mr_info.name) — two serve processes
|
|
* hosting the same weights advertise the SAME model_id and become a
|
|
* router replica pool; the `instance_id` is unique per serve process
|
|
* ("<host>:<port>:<model>").
|
|
*
|
|
* @status G0 wiring. Live-registration behaviour is verified on a K80 host separately
|
|
* (no GPU/model on the Mac); the Mac build path exercises only the
|
|
* unset-env no-op and the compile.
|
|
* @issues None
|
|
* @todo None
|
|
*/
|
|
|
|
#ifndef PN_SERVE_REGISTRY_H
|
|
#define PN_SERVE_REGISTRY_H
|
|
|
|
#include "model_registry.h"
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
/**
|
|
* Startup registration: for each resident model in `reg`, build an rc_identity
|
|
* (model_id = served name, unique instance_id = "<host>:<port>:<name>",
|
|
* endpoint = "http://<host>:<port>", capacity from the model's batch_sched) and
|
|
* rc_register() it, recording it for later heartbeat + deregister. Then spawn a
|
|
* detached heartbeat thread that every RW_HEARTBEAT_SECONDS reports each
|
|
* instance's live load (bs_get_load under mr_acquire/mr_release).
|
|
*
|
|
* No-op (does not touch `reg`, spawns no thread) when rc_enabled() is false.
|
|
* Call once, after hs_listen succeeds and all models are loaded.
|
|
*
|
|
* `reg`, `host` must outlive the serve process (both are process-lifetime in
|
|
* pn_serve_main); the heartbeat thread borrows `reg` and never frees it.
|
|
*
|
|
* Thread-safe: no (call once at startup, before the accept loop).
|
|
*/
|
|
void serve_registry_start(mr_registry *reg, const char *host, int port);
|
|
|
|
/**
|
|
* Register a model loaded AFTER serve_registry_start() (an /admin/models/load),
|
|
* using the host/port captured at start. Idempotent: a model whose instance is
|
|
* already registered (e.g. a second replica of the same served name) is skipped.
|
|
* No-op when rc_enabled() is false or serve_registry_start() has not run. The
|
|
* running heartbeat thread picks the new instance up on its next cycle.
|
|
*
|
|
* Thread-safe: yes (guards the instance table; runs from the request thread).
|
|
*/
|
|
void serve_registry_add(const char *model_id);
|
|
|
|
/**
|
|
* Deregister a model unloaded via /admin/models/unload — but only if no replica
|
|
* of the same served name remains loaded (unloading one of two same-name
|
|
* replicas keeps the shared instance registered). No-op when disabled/unstarted.
|
|
*
|
|
* Thread-safe: yes.
|
|
*/
|
|
void serve_registry_remove(const char *model_id);
|
|
|
|
/**
|
|
* Best-effort deregister of every instance registered by
|
|
* serve_registry_start(). Called from the SIGINT/SIGTERM shutdown handler so a
|
|
* clean stop drops the instances from the router immediately (the registry TTL
|
|
* also evicts them if this is skipped). No-op when rc_enabled() is false or
|
|
* nothing was registered. Each rc_deregister uses a short connect timeout.
|
|
*
|
|
* Safe to call from the controlled-shutdown signal handler (same trade as the
|
|
* CUDA teardown there: operator-initiated stop, not a crash mid-kernel).
|
|
*/
|
|
void serve_registry_deregister_all(void);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif /* PN_SERVE_REGISTRY_H */
|
|
```
|
|
|
|
Add a `int max_batch;` field to the registry entry struct, documented in the same comment style as the neighbouring fields. Change nothing else.
|
|
|
|
Re-emit the complete updated contents of `src/serve/serve_registry.h`.
|
|
===PROMPT===
|
|
I ran `read_file src/core/qweight.h` and got:
|
|
|
|
```c
|
|
#ifndef PN_QWEIGHT_H
|
|
#define PN_QWEIGHT_H
|
|
|
|
#include "../formats/gguf.h"
|
|
#include "../formats/gguf_dequant.h"
|
|
#include <stddef.h>
|
|
|
|
/*
|
|
* Quantized weight matrix.
|
|
* Points directly into mmap'd GGUF data — zero-copy.
|
|
* pn_qmv() performs matvec by dequantizing 32 elements at a time.
|
|
*
|
|
* For ternary weights (PN_QUANT_TERNARY): data is packed 2-bit
|
|
* (4 elements/byte, encoding: 00=-1, 01=0, 10=+1).
|
|
* scales[] holds per-group float32 scales, laid out as
|
|
* scales[row * n_groups + group], where n_groups = ceil(cols/group_size).
|
|
*/
|
|
|
|
typedef struct {
|
|
const void *data; /* Raw quantized blocks (mmap pointer or owned) */
|
|
pn_gguf_quant type; /* Q4_0, Q8_0, ..., F32, PN_QUANT_TERNARY */
|
|
int rows; /* Output dimension */
|
|
int cols; /* Input dimension */
|
|
size_t row_stride; /* Bytes per row */
|
|
const float *scales; /* Per-group ternary scales (NULL for non-ternary) */
|
|
int group_size; /* Elements per scale group (0 if not ternary) */
|
|
void *gpu; /* Metal buffer of the weight's Q4_K blocks, or NULL.
|
|
Unconditional (not #ifdef USE_METAL) so the struct
|
|
layout never depends on the build flag — a
|
|
conditional field here silently corrupts adjacent
|
|
structs if any TU disagrees on the flag. Unused
|
|
(always NULL) in non-Metal builds. */
|
|
} pn_qweight;
|
|
|
|
/* Create a qweight from a named GGUF tensor.
|
|
* Sets rows/cols from tensor dims (reversed: GGUF innermost-first → C outermost-first).
|
|
* Returns 0 on success, -1 on failure. */
|
|
int pn_qweight_from_gguf(pn_qweight *qw, pn_gguf_file *gguf, const char *name);
|
|
|
|
/* Quantized matrix-vector multiply: out[rows] = W[rows, cols] @ vec[cols]
|
|
* Dequantizes row by row, 32 elements at a time.
|
|
* If a thread pool has been registered via pn_qmv_set_pool() and the matrix is
|
|
* large enough, the output-row loop is distributed across threads. Each output
|
|
* row is computed whole by a single thread, so the result is bit-identical to
|
|
* the single-threaded path (no cross-thread reduction). */
|
|
void pn_qmv(const pn_qweight *w, const float *vec, float *out);
|
|
|
|
/* Coalesced matvec for a group of independent weights sharing one parallel
|
|
* region. Computes out[i] = W[i] @ vec[i] for i in [0,n). All n matvecs are
|
|
* dispatched to the thread pool as a SINGLE fw_for over the concatenated row
|
|
* space, cutting per-matvec dispatch (condvar broadcast + join) overhead from n
|
|
* to 1. Each output row is still computed whole by one thread, so the result is
|
|
* bit-identical to calling pn_qmv(w[i], vec[i], out[i]) individually. Falls back
|
|
* to per-weight pn_qmv for ternary/GPU-resident weights. n up to 8. */
|
|
void pn_qmv_group(const pn_qweight *const *w, const float *const *vec,
|
|
float *const *out, int n);
|
|
|
|
/* Fused quantized matrix times B activation vectors (M=B "matmul"):
|
|
* out[b, r] = sum_c dequant(W[r, c]) * acts[b, c] for b in [0,B), r in [0,rows)
|
|
* acts is [B, cols] row-major, out is [B, rows] row-major (out[b*rows + r]).
|
|
*
|
|
* Each weight ROW is dequantized once into a small L1-resident scratch and
|
|
* dotted against all B activation vectors, so the quantized weight is read
|
|
* once and the dequant cost is amortized across the batch — unlike
|
|
* materializing the full F32 weight to DRAM before a BLAS GEMM. This is the
|
|
* decode-batch analog of pn_qmv's fused dequant-dot. Parallelized over rows via
|
|
* the registered pool; rows are disjoint so the result is order-independent.
|
|
* B up to PN_QMM_MAX_B (falls back to a per-b pn_qmv loop above that or for
|
|
* ternary/GPU-resident weights). */
|
|
#define PN_QMM_MAX_B 64
|
|
void pn_qmm(const pn_qweight *w, const float *acts, int B, float *out);
|
|
|
|
/* Prefill-oriented batched matmul. Same [B,cols] acts -> [B,rows] out contract
|
|
* as pn_qmm (out[b*rows + r]), but dispatched in fixed row-blocks so each
|
|
* per-panel cblas_sgemm keeps an AMX-friendly N dimension even for large B
|
|
* (B = P, ~1200, at prefill). Unlike pn_qmm it never degrades to a per-b matvec
|
|
* loop for large B: the panel path handles any B, and the 64 cap in pn_qmm only
|
|
* exists to bound the flag-gated GPU decode accumulators. Dequantizes each
|
|
* L2-resident weight panel once and multiplies it against all B activation
|
|
* rows, so the quantized weight is read once and never round-trips DRAM as F32.
|
|
* The only numeric difference from a per-row dequant-dot is BLAS summation
|
|
* order, the same reassociation the decode-batch pn_qmm already relies on. */
|
|
void pn_qmm_prefill(const pn_qweight *w, const float *acts, int B, float *out);
|
|
|
|
/* Register the thread pool used by pn_qmv to parallelize the row loop.
|
|
* Forward-declared opaque pool type (foundry threadpool fw_pool). Passing NULL
|
|
* disables parallelization. No-op (other than storing NULL) when the engine is
|
|
* built without USE_THREADS. Safe to call repeatedly; not thread-safe to call
|
|
* concurrently with pn_qmv. */
|
|
struct fw_pool;
|
|
void pn_qmv_set_pool(struct fw_pool *pool);
|
|
|
|
/* Quantized dot product: dequant + dot for a single row.
|
|
* Used by pn_qmv inner loop and available for direct use. */
|
|
float pn_qdot(const void *row_data, const float *vec, int n, pn_gguf_quant type);
|
|
|
|
/* GPU (Metal) weight residency for the Q4_K batched-decode path.
|
|
* pn_qmv_gpu_upload copies a Q4_K weight's blocks into a GPU buffer (sets
|
|
* w->gpu); pn_qmm then runs the batched simdgroup GEMM on the GPU. All three
|
|
* are no-ops / return 0 when built without USE_METAL or for non-Q4_K weights. */
|
|
int pn_qmv_gpu_available(void);
|
|
int pn_qmv_gpu_upload(pn_qweight *w);
|
|
void pn_qmv_gpu_free(pn_qweight *w);
|
|
|
|
/* Enable/disable running batched decode (pn_qmm) on the GPU. Off by default:
|
|
* weights are GPU-uploaded for prefill, but GPU decode is slower than CPU AMX,
|
|
* so decode stays on the CPU unless this is explicitly enabled. */
|
|
void pn_qmm_set_gpu_decode(int on);
|
|
|
|
#endif /* PN_QWEIGHT_H */
|
|
```
|
|
|
|
Rename the constant prefix `PN_QW_` to `PN_QWEIGHT_` everywhere it appears in this header, updating every use. Change nothing else.
|
|
|
|
Re-emit the complete updated contents of `src/core/qweight.h`.
|