296 lines
11 KiB
C
296 lines
11 KiB
C
/* SPDX-License-Identifier: AGPL-3.0-or-later */
|
|
/* Copyright (C) 2026 David (Corey) Herrera */
|
|
/*
|
|
* test_qwen35moe_metal_coverage.c — the Metal kernel-coverage gate.
|
|
*
|
|
* The gate asserts that every weight the qwen35moe forward actually dispatches
|
|
* a kernel at has a Metal matvec kernel registered for its quant type. Without
|
|
* it, a missing kernel for one of the six quant types in the stock Q3_K_M file
|
|
* surfaces as garbage logits at end-to-end bring-up (Phase 5), where it is easy
|
|
* to blame the SSM or attention instead.
|
|
*
|
|
* This file is compiled TWICE, and that is what proves the gate measures
|
|
* something:
|
|
*
|
|
* tests/test_qwen35moe_metal_coverage — plain. Reports the real
|
|
* uncovered set, and (with PN_METAL_COVERAGE_DROP_TYPE) shows that
|
|
* hiding a type from the dispatch derivation is caught as UNACCOUNTED
|
|
* rather than passing silently.
|
|
*
|
|
* tests/test_qwen35moe_metal_coverage_fake — built with
|
|
* -DPN_COVERAGE_FAKE_KERNEL, which links a stub `fm_matvec_q5k`. Q5_K
|
|
* must flip to covered while Q3_K/Q6_K/Q8_0 stay uncovered. A gate that
|
|
* printed a constant could not tell those two binaries apart.
|
|
*
|
|
* The model-gated tests load the real ~15 GiB GGUF, so they need
|
|
* PN_QWEN35_MODEL=<path to the .gguf>; `make test` skips them otherwise. They
|
|
* run ONE forward token (~0.25 s on CPU) and never generate.
|
|
*/
|
|
|
|
#include "lib/test.h"
|
|
#include "../include/pantheon.h"
|
|
#include "../src/core/engine.h"
|
|
#include "../src/models/qwen35moe_internal.h"
|
|
#include "../src/models/qwen35moe_metal_coverage.h"
|
|
#include "../src/models/qwen35moe_metal_kernel_registry.h"
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <unistd.h>
|
|
|
|
#ifdef PN_COVERAGE_FAKE_KERNEL
|
|
/*
|
|
* A stub standing in for the Phase-1 Q5_K matvec that foundry does not have
|
|
* yet. Never called — the registry only asks the loader whether the symbol
|
|
* resolves. Its whole job is to move exactly one type from uncovered to
|
|
* covered so the gate's output can be shown to track linkage.
|
|
*/
|
|
void fm_matvec_q5k(void);
|
|
void fm_matvec_q5k(void) { }
|
|
#endif
|
|
|
|
/* Default search paths, tried when PN_QWEN35_MODEL is unset.
|
|
*
|
|
* A skip that has to be opted OUT of is the safe default here. The model is
|
|
* large but it is normally present on a dev box, and the failure this gate
|
|
* exists to catch is a missing kernel — exactly the thing a silent skip would
|
|
* hide behind a green suite. So look for the model before giving up. */
|
|
/* Relative to the working directory, then to $HOME. */
|
|
static const char *const k_default_models[] = {
|
|
"models/Nex-N2-mini/nex-agi_Nex-N2-mini-Q3_K_M.gguf",
|
|
};
|
|
|
|
static const char *model_path(void)
|
|
{
|
|
const char *p = getenv("PN_QWEN35_MODEL");
|
|
if (p && *p && access(p, F_OK) == 0) return p;
|
|
if (p && *p) return NULL; /* explicitly pointed somewhere missing — do not second-guess */
|
|
|
|
static char home_path[4096];
|
|
const char *home = getenv("HOME");
|
|
for (size_t i = 0; i < sizeof(k_default_models)/sizeof(*k_default_models); i++) {
|
|
if (access(k_default_models[i], F_OK) == 0) return k_default_models[i];
|
|
if (home && snprintf(home_path, sizeof(home_path), "%s/%s", home, k_default_models[i])
|
|
< (int)sizeof(home_path) &&
|
|
access(home_path, F_OK) == 0)
|
|
return home_path;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
/* Load the model and hand back its qwen35moe weights, or NULL when skipping. */
|
|
static pn_qwen35moe_model *load_qwen35moe(pn_model **model_out)
|
|
{
|
|
const char *path = model_path();
|
|
if (!path) {
|
|
printf(" [SKIP] set PN_QWEN35_MODEL to the GGUF to run\n");
|
|
return NULL;
|
|
}
|
|
pn_model *m = pn_load(path);
|
|
if (!m) return NULL;
|
|
*model_out = m;
|
|
return (pn_qwen35moe_model *)m->weights;
|
|
}
|
|
|
|
static const pn_qwen35moe_metal_coverage_row *
|
|
row_for(const pn_qwen35moe_metal_coverage *cov, pn_gguf_quant type)
|
|
{
|
|
for (int i = 0; i < cov->n_rows; i++)
|
|
if (cov->rows[i].type == type)
|
|
return &cov->rows[i];
|
|
return NULL;
|
|
}
|
|
|
|
/* ============================================================
|
|
* Registry — no model needed
|
|
* ============================================================ */
|
|
|
|
/*
|
|
* The registry answers by symbol lookup, so its answer must track what is
|
|
* actually linked. Q4_K's matvec ships in fm_metal today, so it resolves in a
|
|
* USE_METAL build and cannot resolve without one. Q2_K has no planned kernel
|
|
* at all and must never resolve.
|
|
*/
|
|
static void test_registry_tracks_linkage(void)
|
|
{
|
|
const char *entry = NULL;
|
|
|
|
int q4k = pn_qwen35moe_metal_kernel_for_type(GGUF_QUANT_Q4_K, &entry);
|
|
#ifdef USE_METAL
|
|
ASSERT_INT_EQ(1, q4k);
|
|
ASSERT_NOT_NULL((void *)entry);
|
|
#else
|
|
ASSERT_INT_EQ(0, q4k);
|
|
ASSERT_NULL((void *)entry);
|
|
#endif
|
|
|
|
ASSERT_INT_EQ(0, pn_qwen35moe_metal_kernel_for_type(GGUF_QUANT_Q2_K, &entry));
|
|
ASSERT_NULL((void *)entry);
|
|
}
|
|
|
|
/*
|
|
* The Q5_K answer is the difference between the two binaries this file builds.
|
|
* Same source, same model, same gate — only a linked stub differs.
|
|
*/
|
|
static void test_registry_q5k_matches_build(void)
|
|
{
|
|
const char *entry = NULL;
|
|
int q5k = pn_qwen35moe_metal_kernel_for_type(GGUF_QUANT_Q5_K, &entry);
|
|
/* Off Apple the registry reports nothing linked by design (no Metal), so the fake
|
|
* kernel can only move the answer on macOS. */
|
|
#if defined(PN_COVERAGE_FAKE_KERNEL) && defined(__APPLE__)
|
|
ASSERT_INT_EQ(1, q5k);
|
|
ASSERT_STR_EQ("fm_matvec_q5k", entry);
|
|
#else
|
|
ASSERT_INT_EQ(0, q5k);
|
|
ASSERT_NULL((void *)entry);
|
|
#endif
|
|
}
|
|
|
|
/* ============================================================
|
|
* The gate itself — model-gated
|
|
* ============================================================ */
|
|
|
|
/*
|
|
* The headline run: trace the dispatch set off a real forward, print the
|
|
* per-type table, and hold the classification to a conservation law — every
|
|
* tensor in the file is dispatched, excluded, or unaccounted for, and nothing
|
|
* falls off the edge. That law is what makes "no false alarm" and "nothing
|
|
* silently skipped" checkable rather than asserted.
|
|
*/
|
|
static void test_coverage_gate_reports_real_state(void)
|
|
{
|
|
pn_model *model = NULL;
|
|
pn_qwen35moe_model *qw = load_qwen35moe(&model);
|
|
if (!qw) return;
|
|
|
|
pn_qwen35moe_metal_coverage cov = {0};
|
|
int rc = pn_qwen35moe_metal_coverage_run(qw, &cov);
|
|
pn_qwen35moe_metal_coverage_report(&cov, stdout);
|
|
|
|
ASSERT_INT_EQ(0, rc);
|
|
ASSERT_INT_EQ(1, cov.traced);
|
|
|
|
/* Conservation: nothing is dropped on the floor. */
|
|
ASSERT_INT_EQ(cov.file_tensors,
|
|
cov.dispatched_tensors + cov.excluded_tensors +
|
|
cov.unaccounted_tensors);
|
|
|
|
/* The derivation is complete: every bound weight was seen dispatched. */
|
|
ASSERT_INT_EQ(0, cov.unaccounted_tensors);
|
|
ASSERT_INT_EQ(0, cov.unresolved_weights);
|
|
|
|
/* Real work happened, and far more weights than tensors — the merged
|
|
* expert tensors are reached through thousands of slices. */
|
|
ASSERT_TRUE(cov.dispatched_tensors > 0);
|
|
ASSERT_TRUE(cov.dispatched_weights > cov.dispatched_tensors);
|
|
|
|
/* No false alarm: the file's norms and embeddings are never dispatched and
|
|
* must not be demanding a kernel. */
|
|
ASSERT_TRUE(cov.excluded_tensors > 0);
|
|
|
|
/* Verdict follows the findings, not a constant. */
|
|
ASSERT_INT_EQ(cov.uncovered_tensors == 0,
|
|
pn_qwen35moe_metal_coverage_passed(&cov));
|
|
|
|
pn_model_free(model);
|
|
}
|
|
|
|
/*
|
|
* Per-type expectations. Q3_K, Q6_K and Q8_0 have no kernel in either binary
|
|
* (note Q8_0 in particular: fm_matvec_q8 exists but is a single-scale int8 dot,
|
|
* the wrong layout for block-scaled Q8_0, and the registry deliberately refuses
|
|
* to count it). Q5_K is the one that moves between builds.
|
|
*/
|
|
static void test_coverage_rows_track_registration(void)
|
|
{
|
|
pn_model *model = NULL;
|
|
pn_qwen35moe_model *qw = load_qwen35moe(&model);
|
|
if (!qw) return;
|
|
|
|
pn_qwen35moe_metal_coverage cov = {0};
|
|
ASSERT_INT_EQ(0, pn_qwen35moe_metal_coverage_run(qw, &cov));
|
|
|
|
const pn_gguf_quant never[] = { GGUF_QUANT_Q3_K, GGUF_QUANT_Q6_K,
|
|
GGUF_QUANT_Q8_0 };
|
|
for (size_t i = 0; i < sizeof(never) / sizeof(never[0]); i++) {
|
|
const pn_qwen35moe_metal_coverage_row *r = row_for(&cov, never[i]);
|
|
ASSERT_NOT_NULL((void *)r);
|
|
ASSERT_INT_EQ(0, r->covered);
|
|
ASSERT_TRUE(r->tensors > 0);
|
|
ASSERT_TRUE(r->example[0] != '\0');
|
|
}
|
|
|
|
const pn_qwen35moe_metal_coverage_row *q5k = row_for(&cov, GGUF_QUANT_Q5_K);
|
|
ASSERT_NOT_NULL((void *)q5k);
|
|
#ifdef PN_COVERAGE_FAKE_KERNEL
|
|
ASSERT_INT_EQ(1, q5k->covered);
|
|
#else
|
|
ASSERT_INT_EQ(0, q5k->covered);
|
|
#endif
|
|
|
|
pn_model_free(model);
|
|
}
|
|
|
|
/*
|
|
* Prove the gate can fail for the OTHER reason: hide one dispatched quant type
|
|
* from the derivation and its tensors must resurface as UNACCOUNTED, named,
|
|
* with the verdict flipped — not quietly vanish from the table. This is the
|
|
* check that a hardcoded dispatch list would fail.
|
|
*/
|
|
static void test_dropping_a_type_is_caught(void)
|
|
{
|
|
pn_model *model = NULL;
|
|
pn_qwen35moe_model *qw = load_qwen35moe(&model);
|
|
if (!qw) return;
|
|
|
|
char drop[16];
|
|
snprintf(drop, sizeof(drop), "%d", (int)GGUF_QUANT_Q5_K);
|
|
setenv("PN_METAL_COVERAGE_DROP_TYPE", drop, 1);
|
|
|
|
pn_qwen35moe_metal_coverage cov = {0};
|
|
ASSERT_INT_EQ(0, pn_qwen35moe_metal_coverage_run(qw, &cov));
|
|
pn_qwen35moe_metal_coverage_report(&cov, stdout);
|
|
|
|
unsetenv("PN_METAL_COVERAGE_DROP_TYPE");
|
|
|
|
ASSERT_TRUE(cov.unaccounted_tensors > 0);
|
|
ASSERT_TRUE(cov.n_unaccounted_named > 0);
|
|
ASSERT_INT_EQ(0, pn_qwen35moe_metal_coverage_passed(&cov));
|
|
ASSERT_NULL((void *)row_for(&cov, GGUF_QUANT_Q5_K));
|
|
|
|
/* Still conserved — the tensors moved class, they did not disappear. */
|
|
ASSERT_INT_EQ(cov.file_tensors,
|
|
cov.dispatched_tensors + cov.excluded_tensors +
|
|
cov.unaccounted_tensors);
|
|
|
|
pn_model_free(model);
|
|
}
|
|
|
|
int main(void)
|
|
{
|
|
#ifdef PN_COVERAGE_FAKE_KERNEL
|
|
printf("Pantheon qwen35moe Metal coverage gate (FAKE Q5_K kernel linked)\n");
|
|
#else
|
|
printf("Pantheon qwen35moe Metal coverage gate\n");
|
|
#endif
|
|
printf("======================================\n\n");
|
|
|
|
RUN_TEST(registry_tracks_linkage);
|
|
RUN_TEST(registry_q5k_matches_build);
|
|
RUN_TEST(coverage_gate_reports_real_state);
|
|
RUN_TEST(coverage_rows_track_registration);
|
|
RUN_TEST(dropping_a_type_is_caught);
|
|
|
|
TEST_REPORT();
|
|
|
|
/* Three of the five tests do nothing without a model, and the shared harness
|
|
* prints "ALL TESTS PASSED" either way — a green that means nothing. Say so
|
|
* loudly, so a skipped run is never mistaken for a covered one. */
|
|
if (!model_path())
|
|
printf("\n *** WARNING: no model found — the 3 tests that actually exercise the\n"
|
|
" gate DID NOT RUN. This result proves nothing about coverage.\n"
|
|
" Set PN_QWEN35_MODEL=<path to the qwen35moe .gguf> to run them. ***\n");
|
|
|
|
return __ft_failed > 0 ? 1 : 0;
|
|
}
|