/* SPDX-License-Identifier: AGPL-3.0-or-later */ /* Copyright (C) 2026 David (Corey) Herrera */ /* * test_qwen35moe_metal_coverage.c — the Metal kernel-coverage gate. * * The gate asserts that every weight the qwen35moe forward actually dispatches * a kernel at has a Metal matvec kernel registered for its quant type. Without * it, a missing kernel for one of the six quant types in the stock Q3_K_M file * surfaces as garbage logits at end-to-end bring-up (Phase 5), where it is easy * to blame the SSM or attention instead. * * This file is compiled TWICE, and that is what proves the gate measures * something: * * tests/test_qwen35moe_metal_coverage — plain. Reports the real * uncovered set, and (with PN_METAL_COVERAGE_DROP_TYPE) shows that * hiding a type from the dispatch derivation is caught as UNACCOUNTED * rather than passing silently. * * tests/test_qwen35moe_metal_coverage_fake — built with * -DPN_COVERAGE_FAKE_KERNEL, which links a stub `fm_matvec_q5k`. Q5_K * must flip to covered while Q3_K/Q6_K/Q8_0 stay uncovered. A gate that * printed a constant could not tell those two binaries apart. * * The model-gated tests load the real ~15 GiB GGUF, so they need * PN_QWEN35_MODEL=; `make test` skips them otherwise. They * run ONE forward token (~0.25 s on CPU) and never generate. */ #include "lib/test.h" #include "../include/pantheon.h" #include "../src/core/engine.h" #include "../src/models/qwen35moe_internal.h" #include "../src/models/qwen35moe_metal_coverage.h" #include "../src/models/qwen35moe_metal_kernel_registry.h" #include #include #include #ifdef PN_COVERAGE_FAKE_KERNEL /* * A stub standing in for the Phase-1 Q5_K matvec that foundry does not have * yet. Never called — the registry only asks the loader whether the symbol * resolves. Its whole job is to move exactly one type from uncovered to * covered so the gate's output can be shown to track linkage. */ void fm_matvec_q5k(void); void fm_matvec_q5k(void) { } #endif /* Default search paths, tried when PN_QWEN35_MODEL is unset. * * A skip that has to be opted OUT of is the safe default here. The model is * large but it is normally present on a dev box, and the failure this gate * exists to catch is a missing kernel — exactly the thing a silent skip would * hide behind a green suite. So look for the model before giving up. */ /* Relative to the working directory, then to $HOME. */ static const char *const k_default_models[] = { "models/Nex-N2-mini/nex-agi_Nex-N2-mini-Q3_K_M.gguf", }; static const char *model_path(void) { const char *p = getenv("PN_QWEN35_MODEL"); if (p && *p && access(p, F_OK) == 0) return p; if (p && *p) return NULL; /* explicitly pointed somewhere missing — do not second-guess */ static char home_path[4096]; const char *home = getenv("HOME"); for (size_t i = 0; i < sizeof(k_default_models)/sizeof(*k_default_models); i++) { if (access(k_default_models[i], F_OK) == 0) return k_default_models[i]; if (home && snprintf(home_path, sizeof(home_path), "%s/%s", home, k_default_models[i]) < (int)sizeof(home_path) && access(home_path, F_OK) == 0) return home_path; } return NULL; } /* Load the model and hand back its qwen35moe weights, or NULL when skipping. */ static pn_qwen35moe_model *load_qwen35moe(pn_model **model_out) { const char *path = model_path(); if (!path) { printf(" [SKIP] set PN_QWEN35_MODEL to the GGUF to run\n"); return NULL; } pn_model *m = pn_load(path); if (!m) return NULL; *model_out = m; return (pn_qwen35moe_model *)m->weights; } static const pn_qwen35moe_metal_coverage_row * row_for(const pn_qwen35moe_metal_coverage *cov, pn_gguf_quant type) { for (int i = 0; i < cov->n_rows; i++) if (cov->rows[i].type == type) return &cov->rows[i]; return NULL; } /* ============================================================ * Registry — no model needed * ============================================================ */ /* * The registry answers by symbol lookup, so its answer must track what is * actually linked. Q4_K's matvec ships in fm_metal today, so it resolves in a * USE_METAL build and cannot resolve without one. Q2_K has no planned kernel * at all and must never resolve. */ static void test_registry_tracks_linkage(void) { const char *entry = NULL; int q4k = pn_qwen35moe_metal_kernel_for_type(GGUF_QUANT_Q4_K, &entry); #ifdef USE_METAL ASSERT_INT_EQ(1, q4k); ASSERT_NOT_NULL((void *)entry); #else ASSERT_INT_EQ(0, q4k); ASSERT_NULL((void *)entry); #endif ASSERT_INT_EQ(0, pn_qwen35moe_metal_kernel_for_type(GGUF_QUANT_Q2_K, &entry)); ASSERT_NULL((void *)entry); } /* * The Q5_K answer is the difference between the two binaries this file builds. * Same source, same model, same gate — only a linked stub differs. */ static void test_registry_q5k_matches_build(void) { const char *entry = NULL; int q5k = pn_qwen35moe_metal_kernel_for_type(GGUF_QUANT_Q5_K, &entry); /* Off Apple the registry reports nothing linked by design (no Metal), so the fake * kernel can only move the answer on macOS. */ #if defined(PN_COVERAGE_FAKE_KERNEL) && defined(__APPLE__) ASSERT_INT_EQ(1, q5k); ASSERT_STR_EQ("fm_matvec_q5k", entry); #else ASSERT_INT_EQ(0, q5k); ASSERT_NULL((void *)entry); #endif } /* ============================================================ * The gate itself — model-gated * ============================================================ */ /* * The headline run: trace the dispatch set off a real forward, print the * per-type table, and hold the classification to a conservation law — every * tensor in the file is dispatched, excluded, or unaccounted for, and nothing * falls off the edge. That law is what makes "no false alarm" and "nothing * silently skipped" checkable rather than asserted. */ static void test_coverage_gate_reports_real_state(void) { pn_model *model = NULL; pn_qwen35moe_model *qw = load_qwen35moe(&model); if (!qw) return; pn_qwen35moe_metal_coverage cov = {0}; int rc = pn_qwen35moe_metal_coverage_run(qw, &cov); pn_qwen35moe_metal_coverage_report(&cov, stdout); ASSERT_INT_EQ(0, rc); ASSERT_INT_EQ(1, cov.traced); /* Conservation: nothing is dropped on the floor. */ ASSERT_INT_EQ(cov.file_tensors, cov.dispatched_tensors + cov.excluded_tensors + cov.unaccounted_tensors); /* The derivation is complete: every bound weight was seen dispatched. */ ASSERT_INT_EQ(0, cov.unaccounted_tensors); ASSERT_INT_EQ(0, cov.unresolved_weights); /* Real work happened, and far more weights than tensors — the merged * expert tensors are reached through thousands of slices. */ ASSERT_TRUE(cov.dispatched_tensors > 0); ASSERT_TRUE(cov.dispatched_weights > cov.dispatched_tensors); /* No false alarm: the file's norms and embeddings are never dispatched and * must not be demanding a kernel. */ ASSERT_TRUE(cov.excluded_tensors > 0); /* Verdict follows the findings, not a constant. */ ASSERT_INT_EQ(cov.uncovered_tensors == 0, pn_qwen35moe_metal_coverage_passed(&cov)); pn_model_free(model); } /* * Per-type expectations. Q3_K, Q6_K and Q8_0 have no kernel in either binary * (note Q8_0 in particular: fm_matvec_q8 exists but is a single-scale int8 dot, * the wrong layout for block-scaled Q8_0, and the registry deliberately refuses * to count it). Q5_K is the one that moves between builds. */ static void test_coverage_rows_track_registration(void) { pn_model *model = NULL; pn_qwen35moe_model *qw = load_qwen35moe(&model); if (!qw) return; pn_qwen35moe_metal_coverage cov = {0}; ASSERT_INT_EQ(0, pn_qwen35moe_metal_coverage_run(qw, &cov)); const pn_gguf_quant never[] = { GGUF_QUANT_Q3_K, GGUF_QUANT_Q6_K, GGUF_QUANT_Q8_0 }; for (size_t i = 0; i < sizeof(never) / sizeof(never[0]); i++) { const pn_qwen35moe_metal_coverage_row *r = row_for(&cov, never[i]); ASSERT_NOT_NULL((void *)r); ASSERT_INT_EQ(0, r->covered); ASSERT_TRUE(r->tensors > 0); ASSERT_TRUE(r->example[0] != '\0'); } const pn_qwen35moe_metal_coverage_row *q5k = row_for(&cov, GGUF_QUANT_Q5_K); ASSERT_NOT_NULL((void *)q5k); #ifdef PN_COVERAGE_FAKE_KERNEL ASSERT_INT_EQ(1, q5k->covered); #else ASSERT_INT_EQ(0, q5k->covered); #endif pn_model_free(model); } /* * Prove the gate can fail for the OTHER reason: hide one dispatched quant type * from the derivation and its tensors must resurface as UNACCOUNTED, named, * with the verdict flipped — not quietly vanish from the table. This is the * check that a hardcoded dispatch list would fail. */ static void test_dropping_a_type_is_caught(void) { pn_model *model = NULL; pn_qwen35moe_model *qw = load_qwen35moe(&model); if (!qw) return; char drop[16]; snprintf(drop, sizeof(drop), "%d", (int)GGUF_QUANT_Q5_K); setenv("PN_METAL_COVERAGE_DROP_TYPE", drop, 1); pn_qwen35moe_metal_coverage cov = {0}; ASSERT_INT_EQ(0, pn_qwen35moe_metal_coverage_run(qw, &cov)); pn_qwen35moe_metal_coverage_report(&cov, stdout); unsetenv("PN_METAL_COVERAGE_DROP_TYPE"); ASSERT_TRUE(cov.unaccounted_tensors > 0); ASSERT_TRUE(cov.n_unaccounted_named > 0); ASSERT_INT_EQ(0, pn_qwen35moe_metal_coverage_passed(&cov)); ASSERT_NULL((void *)row_for(&cov, GGUF_QUANT_Q5_K)); /* Still conserved — the tensors moved class, they did not disappear. */ ASSERT_INT_EQ(cov.file_tensors, cov.dispatched_tensors + cov.excluded_tensors + cov.unaccounted_tensors); pn_model_free(model); } int main(void) { #ifdef PN_COVERAGE_FAKE_KERNEL printf("Pantheon qwen35moe Metal coverage gate (FAKE Q5_K kernel linked)\n"); #else printf("Pantheon qwen35moe Metal coverage gate\n"); #endif printf("======================================\n\n"); RUN_TEST(registry_tracks_linkage); RUN_TEST(registry_q5k_matches_build); RUN_TEST(coverage_gate_reports_real_state); RUN_TEST(coverage_rows_track_registration); RUN_TEST(dropping_a_type_is_caught); TEST_REPORT(); /* Three of the five tests do nothing without a model, and the shared harness * prints "ALL TESTS PASSED" either way — a green that means nothing. Say so * loudly, so a skipped run is never mistaken for a covered one. */ if (!model_path()) printf("\n *** WARNING: no model found — the 3 tests that actually exercise the\n" " gate DID NOT RUN. This result proves nothing about coverage.\n" " Set PN_QWEN35_MODEL= to run them. ***\n"); return __ft_failed > 0 ? 1 : 0; }