David
/
Foundry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Foundry/tt/tests/test_vector_ops.c

2970 lines
147 KiB
C
Raw Blame History

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
/**
* @file test_vector_ops.c
* @brief OFF-HARDWARE gate for the vector op family — the elementwise binary and the
* REDUCE binary (softmax, RMS norm, layer norm, router normalise). Runs
* without a card, and is what has to be green before the hardware pass is worth a core.
*
* ---------------------------------------------------------------------------
* WHAT THIS CAN PROVE, AND WHAT IT CANNOT
* ---------------------------------------------------------------------------
* It CAN prove:
* - the host and the KERNEL agree on every wire constant, by parsing the device headers
* rather than by a comment claiming they match (dev notes §4: "the confident sentence
* in a doc is the one to check, because nobody re-derives it");
* - the parameter block puts every field in the word the kernel reads;
* - the circular buffers fit the arena, do not overlap, and the documented Wt ceiling is
* the real one rather than a rounded-down guess;
* - the ARRANGEMENT OF PASSES computes the op, and clears a PCC ≥ 0.99 gate with bf16 at
* every circular-buffer boundary — which is a prediction of the card's score;
* - the max-subtraction pass is load-bearing, by running a row whose range makes exp
* overflow without it.
*
* It CANNOT prove the kernel implements the model. Nothing off-hardware can. dev notes §3:
* "off-hardware suites cannot cover the copy path" — and here they cannot cover the SFPU,
* the packer, the face order or the CB credit protocol either. A green run here means the
* plan is sound, never that the op works.
*
* ---------------------------------------------------------------------------
* THE DATA
* ---------------------------------------------------------------------------
* Full-mantissa random, never small integers. A bf16-exact array is bit-identical to its
* own bf16 rendering, so it cannot tell two CB descriptor sets apart and made two
* deliberately-different builds byte-equal once (dev notes §3).
*
* Environment:
* FT_TT_KERNEL_DIR where the device headers live (default ../kernels)
*
* @status Green: 198 checks, 0 failures, on the Mac (clang). Seven mutations were run
* against the original 87 and six went red — 1, 6, 1, 1, 1 and 2 failures. The seventh
* (the row max reduced over ONE tile instead of Wt) came back GREEN and that is what
* added test_softmax_max_is_over_the_whole_row(); with it, the same mutation fails.
* The 2026-08-15 ragged-W stages carry five more red proofs, each on a `make red` clean
* rebuild: the pad value forced to 0 (4 failures), the scaler built at the PADDED width
* (2 — and PCC stayed 0.999995, so max-abs is what caught it), ONE pad lane out of 31
* written wrong (5), the layer-norm refusal removed (2), and the layer-norm stage's own
* subject removed by making W exact so there are no pad lanes at all (7).
* @issues None
* @todo None
*/
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../convert/ft_float_convert.h"
#include "../ft_tt_op_dispatch.h"
#include "../ft_tt_op_l1_layout.h"
#include "../ft_tt_op_matmul.h"
#include "../ft_tt_op_vector.h"
#include "../ft_tt_queue.h"
#include "../ft_tt_tilize.h"
#include "ft_tt_test_report.h"
#include "ft_tt_vector_reference.h"
#define PCC_ACCEPT 0.99
/** Where the device headers live, so the wire-constant pin reads the real file. */
#define KERNEL_DIRECTORY_ENVIRONMENT "FT_TT_KERNEL_DIR"
#define KERNEL_DIRECTORY_DEFAULT "../kernels"
/** Longest path this assembles. */
#define PATH_MAX_BYTES 512u
static unsigned char g_cb_buffer[FT_TT_NUM_CIRCULAR_BUFFERS * FT_TT_WORDS_PER_LOCAL_CB_CONFIG * 4];
/* ================================================================
* Reading a constant out of a device header
* ================================================================ */
/**
* The value of `enum { <name> = <value> };` in `path`.
*
* The device headers are C and C++ at once (plan Phase 6.6), so the wire constants are
* declared one per line as an anonymous `enum` rather than as `constexpr uint32_t` --
* a C translation unit cannot say `constexpr`. The FORM is what changed; the intent is
* unchanged and is the whole point of this function: the value is read out of the real
* device header rather than retyped here, because a duplicated contract with nothing
* enforcing agreement is a contract that will disagree. A missing constant is a FAILURE
* and never a default -- a default would make this check pass against a header that no
* longer declares the field.
*
* Parsed out of the real header rather than retyped here, which is the same discipline
* build_tensix_program.sh applies to the kernel-text relocation map: a duplicated contract
* with nothing enforcing agreement is a contract that will disagree. A missing constant is
* a FAILURE and never a default — a default would make this check pass against a header
* that no longer declares the field.
*
* @return 0 on success, -1 if the file or the constant is missing
*/
static int read_device_constant(const char *path, const char *name, uint32_t *out_value)
{
char line[512];
FILE *file = fopen(path, "r");
int found = -1;
if (!file)
return -1;
while (fgets(line, sizeof(line), file)) {
char parsed_name[128];
char parsed_value[64];
if (sscanf(line, " enum { %127[A-Za-z0-9_] = %63[^}] };", parsed_name,
parsed_value) != 2)
continue;
if (strcmp(parsed_name, name) != 0)
continue;
*out_value = (uint32_t)strtoul(parsed_value, NULL, 0);
found = 0;
break;
}
fclose(file);
return found;
}
/** `directory`/`file`, for the two device headers this pins against. */
static void device_header_path(char *out, size_t out_bytes, const char *file)
{
const char *directory = getenv(KERNEL_DIRECTORY_ENVIRONMENT);
if (!directory || directory[0] == '\0')
directory = KERNEL_DIRECTORY_DEFAULT;
snprintf(out, out_bytes, "%s/%s", directory, file);
}
/** One host constant against its device twin, by name. */
static void check_device_constant(const char *path, const char *name, uint32_t host_value)
{
uint32_t device_value = 0;
if (read_device_constant(path, name, &device_value) != 0) {
ft_tt_test_check_fmt(name, false, "not declared in %s", path);
return;
}
ft_tt_test_check_fmt(name, device_value == host_value, "host %u, kernel %u", host_value,
device_value);
}
/**
* Every word index, every function value and both opcodes, host against kernel.
*
* This is the check that would have caught a field moving under an op author, and it is
* cheap enough that there is no reason for any future op family not to have one.
*/
static void test_wire_constants_agree(void)
{
char vector_path[PATH_MAX_BYTES];
char queue_path[PATH_MAX_BYTES];
char reduce_path[PATH_MAX_BYTES];
ft_tt_test_step("wire constants: the host enum against the kernel's own header");
device_header_path(vector_path, sizeof(vector_path), "resident_vector_protocol.h");
device_header_path(queue_path, sizeof(queue_path), "resident_queue_protocol.h");
device_header_path(reduce_path, sizeof(reduce_path), "resident_reduce_protocol.h");
check_device_constant(vector_path, "kVecFunction", FT_TT_OP_VECTOR_FUNCTION);
check_device_constant(vector_path, "kVecTileCount", FT_TT_OP_VECTOR_TILE_COUNT);
check_device_constant(vector_path, "kVecTileCols", FT_TT_OP_VECTOR_TILE_COLS);
check_device_constant(vector_path, "kVecDepth", FT_TT_OP_VECTOR_DEPTH);
check_device_constant(vector_path, "kVecAX", FT_TT_OP_VECTOR_A_X);
check_device_constant(vector_path, "kVecAY", FT_TT_OP_VECTOR_A_Y);
check_device_constant(vector_path, "kVecABase", FT_TT_OP_VECTOR_A_BASE);
check_device_constant(vector_path, "kVecBX", FT_TT_OP_VECTOR_B_X);
check_device_constant(vector_path, "kVecBY", FT_TT_OP_VECTOR_B_Y);
check_device_constant(vector_path, "kVecBBase", FT_TT_OP_VECTOR_B_BASE);
check_device_constant(vector_path, "kVecCX", FT_TT_OP_VECTOR_C_X);
check_device_constant(vector_path, "kVecCY", FT_TT_OP_VECTOR_C_Y);
check_device_constant(vector_path, "kVecCBase", FT_TT_OP_VECTOR_C_BASE);
check_device_constant(vector_path, "kVecWords", FT_TT_OP_VECTOR_WORDS);
check_device_constant(vector_path, "kVecFnSilu", FT_TT_OP_VECTOR_FN_SILU);
check_device_constant(vector_path, "kVecFnSigmoid", FT_TT_OP_VECTOR_FN_SIGMOID);
check_device_constant(vector_path, "kVecFnAdd", FT_TT_OP_VECTOR_FN_ADD);
check_device_constant(vector_path, "kVecFnMul", FT_TT_OP_VECTOR_FN_MUL);
check_device_constant(vector_path, "kVecFnSub", FT_TT_OP_VECTOR_FN_SUB);
check_device_constant(vector_path, "kVecFnSiluMul", FT_TT_OP_VECTOR_FN_SILU_MUL);
check_device_constant(queue_path, "kOpEltwise", FT_TT_QUEUE_OP_ELTWISE);
check_device_constant(queue_path, "kOpSoftmax", FT_TT_QUEUE_OP_SOFTMAX);
check_device_constant(queue_path, "kOpReduce", FT_TT_QUEUE_OP_REDUCE);
check_device_constant(reduce_path, "kRedPass0", FT_TT_OP_REDUCE_PASS0);
check_device_constant(reduce_path, "kRedPass1", FT_TT_OP_REDUCE_PASS1);
check_device_constant(reduce_path, "kRedFinal", FT_TT_OP_REDUCE_FINAL);
check_device_constant(reduce_path, "kRedBytePool", FT_TT_OP_REDUCE_BYTE_POOL);
check_device_constant(reduce_path, "kRedByteMap", FT_TT_OP_REDUCE_BYTE_MAP);
check_device_constant(reduce_path, "kRedByteVerb", FT_TT_OP_REDUCE_BYTE_VERB);
check_device_constant(reduce_path, "kRedByteAddEps", FT_TT_OP_REDUCE_BYTE_ADD_EPS);
check_device_constant(reduce_path, "kRedBytePassCount", FT_TT_OP_REDUCE_BYTE_PASS_COUNT);
check_device_constant(reduce_path, "kRedByteSource", FT_TT_OP_REDUCE_BYTE_SOURCE);
check_device_constant(reduce_path, "kRedByteChain", FT_TT_OP_REDUCE_BYTE_CHAIN);
check_device_constant(reduce_path, "kRedPoolSum", FT_TT_OP_REDUCE_POOL_SUM);
check_device_constant(reduce_path, "kRedPoolMax", FT_TT_OP_REDUCE_POOL_MAX);
check_device_constant(reduce_path, "kRedMapNone", FT_TT_OP_REDUCE_MAP_NONE);
check_device_constant(reduce_path, "kRedMapSquareSelf", FT_TT_OP_REDUCE_MAP_SQUARE_SELF);
check_device_constant(reduce_path, "kRedMapSubExp", FT_TT_OP_REDUCE_MAP_SUB_EXP);
check_device_constant(reduce_path, "kRedMapSubSquare", FT_TT_OP_REDUCE_MAP_SUB_SQUARE);
check_device_constant(reduce_path, "kRedVerbNone", FT_TT_OP_REDUCE_VERB_NONE);
check_device_constant(reduce_path, "kRedVerbRecip", FT_TT_OP_REDUCE_VERB_RECIP);
check_device_constant(reduce_path, "kRedVerbRsqrt", FT_TT_OP_REDUCE_VERB_RSQRT);
check_device_constant(reduce_path, "kRedSourceIn", FT_TT_OP_REDUCE_SOURCE_IN);
check_device_constant(reduce_path, "kRedSourceMid", FT_TT_OP_REDUCE_SOURCE_MID);
check_device_constant(reduce_path, "kRedChainCenter", FT_TT_OP_REDUCE_CHAIN_CENTER);
check_device_constant(reduce_path, "kRedChainGamma", FT_TT_OP_REDUCE_CHAIN_GAMMA);
check_device_constant(reduce_path, "kRedChainScale", FT_TT_OP_REDUCE_CHAIN_SCALE);
check_device_constant(reduce_path, "kRedChainBeta", FT_TT_OP_REDUCE_CHAIN_BETA);
check_device_constant(reduce_path, "kRedConstScalerTile", FT_TT_OP_REDUCE_CONST_SCALER_TILE);
check_device_constant(reduce_path, "kRedConstEpsTile", FT_TT_OP_REDUCE_CONST_EPS_TILE);
check_device_constant(reduce_path, "kRedConstStreamTile", FT_TT_OP_REDUCE_CONST_STREAM_TILE);
/* The parameter block and the queue's slot machinery are the same sixteen words; if
* they ever diverge, a block written by the host is read past its end by the kernel. */
ft_tt_test_check_fmt("the vector block fits a queue parameter slot",
(uint32_t)FT_TT_OP_VECTOR_WORDS == FT_TT_QUEUE_PARAM_WORDS,
"vector %u, queue slot %u", (uint32_t)FT_TT_OP_VECTOR_WORDS,
FT_TT_QUEUE_PARAM_WORDS);
}
/* ================================================================
* Arity and block depth
* ================================================================ */
/**
* The KERNEL's block-depth rule, transliterated from ftv_block_depth() in
* resident_vector_protocol.h.
*
* The two rules are deliberately different: the host SHRINKS a request to the largest
* divisor, the kernel ACCEPTS a divisor or falls back to 1. That is not a disagreement, it
* is a host that computes and a kernel that refuses to guess — but it is only safe if the
* host's answer always survives the kernel's check, which is what the sweep below asserts.
*/
static uint32_t kernel_block_depth(uint32_t run, uint32_t requested)
{
return (requested > 0u && run > 0u && run % requested == 0u) ? requested : 1u;
}
static void test_block_depth(void)
{
static const uint32_t runs[] = { 1u, 2u, 3u, 8u, 12u, 48u, 96u, 144u, 1000u };
unsigned run_index;
unsigned mismatches = 0;
unsigned non_divisors = 0;
ft_tt_test_step("block depth: the host's answer must survive the kernel's check");
for (run_index = 0; run_index < sizeof(runs) / sizeof(runs[0]); run_index++) {
const uint32_t run = runs[run_index];
uint32_t requested;
for (requested = 0; requested <= run + 4u; requested++) {
const uint32_t host = ft_tt_op_block_depth(run, requested);
if (host == 0u || run % host != 0u)
non_divisors++;
if (kernel_block_depth(run, host) != host)
mismatches++;
}
}
ft_tt_test_check_fmt("every host depth divides its run", non_divisors == 0, "%u violations",
non_divisors);
ft_tt_test_check_fmt("the kernel accepts every host depth", mismatches == 0, "%u rejected",
mismatches);
ft_tt_test_check_fmt("depth 0 becomes 1", ft_tt_op_block_depth(48u, 0u) == 1u, "%u",
ft_tt_op_block_depth(48u, 0u));
ft_tt_test_check_fmt("depth past the run is clamped to the run",
ft_tt_op_block_depth(48u, 4096u) == 48u, "%u",
ft_tt_op_block_depth(48u, 4096u));
ft_tt_test_check_fmt("a non-divisor shrinks rather than clamps",
ft_tt_op_block_depth(48u, 32u) == 24u, "%u",
ft_tt_op_block_depth(48u, 32u));
ft_tt_test_check_fmt("a prime run collapses to 1", ft_tt_op_block_depth(97u, 48u) == 1u, "%u",
ft_tt_op_block_depth(97u, 48u));
/* The matmul's named entry point is now this function; keeping the two answers equal is
* what makes collapsing them a refactor rather than a behaviour change. */
ft_tt_test_check_fmt("ft_tt_op_matmul_prefetch_depth still agrees",
ft_tt_op_matmul_prefetch_depth(144u, 64u) ==
ft_tt_op_block_depth(144u, 64u),
"%u vs %u", ft_tt_op_matmul_prefetch_depth(144u, 64u),
ft_tt_op_block_depth(144u, 64u));
}
static void test_arity(void)
{
ft_tt_test_step("arity: derived from the function, sent nowhere");
ft_tt_test_check("SILU is unary",
!ft_tt_op_vector_function_is_binary(FT_TT_OP_VECTOR_FN_SILU), "");
ft_tt_test_check("SIGMOID is unary",
!ft_tt_op_vector_function_is_binary(FT_TT_OP_VECTOR_FN_SIGMOID), "");
ft_tt_test_check("ADD is binary",
ft_tt_op_vector_function_is_binary(FT_TT_OP_VECTOR_FN_ADD), "");
ft_tt_test_check("MUL is binary",
ft_tt_op_vector_function_is_binary(FT_TT_OP_VECTOR_FN_MUL), "");
ft_tt_test_check("SUB is binary",
ft_tt_op_vector_function_is_binary(FT_TT_OP_VECTOR_FN_SUB), "");
ft_tt_test_check("SILU_MUL is binary",
ft_tt_op_vector_function_is_binary(FT_TT_OP_VECTOR_FN_SILU_MUL), "");
}
/* ================================================================
* Parameter blocks
* ================================================================ */
static ft_tt_op_operand make_operand(uint16_t x, uint16_t y, uint32_t address)
{
ft_tt_op_operand operand;
operand.x = x;
operand.y = y;
operand.address = address;
operand.tile_bytes = FT_TT_OP_TILE_BYTES_BF16;
return operand;
}
static void test_eltwise_parameters(void)
{
ft_tt_op_eltwise_request request;
uint32_t words[FT_TT_OP_VECTOR_WORDS];
int rc;
ft_tt_test_step("elementwise parameter block");
memset(&request, 0, sizeof(request));
request.function = FT_TT_OP_VECTOR_FN_MUL;
request.tile_count = 48u;
request.prefetch_tiles = 32u; /* not a divisor of 48 — must shrink to 24, not clamp */
request.input_a = make_operand(3u, 4u, 0x100000u);
request.input_b = make_operand(5u, 6u, 0x200000u);
request.output = make_operand(7u, 8u, 0x300000u);
rc = ft_tt_op_eltwise_parameters(&request, words);
ft_tt_test_check_fmt("a well-formed request encodes", rc == FT_TT_OK, "rc %d", rc);
ft_tt_test_check_fmt("function lands in word 0",
words[FT_TT_OP_VECTOR_FUNCTION] == FT_TT_OP_VECTOR_FN_MUL, "%u",
words[FT_TT_OP_VECTOR_FUNCTION]);
ft_tt_test_check_fmt("tile count lands in word 1", words[FT_TT_OP_VECTOR_TILE_COUNT] == 48u,
"%u", words[FT_TT_OP_VECTOR_TILE_COUNT]);
ft_tt_test_check_fmt("tile_cols is 0 for elementwise", words[FT_TT_OP_VECTOR_TILE_COLS] == 0u,
"%u", words[FT_TT_OP_VECTOR_TILE_COLS]);
ft_tt_test_check_fmt("depth is shrunk to a divisor", words[FT_TT_OP_VECTOR_DEPTH] == 24u, "%u",
words[FT_TT_OP_VECTOR_DEPTH]);
request.row0 = 7u; /* any nonzero */
rc = ft_tt_op_eltwise_parameters(&request, words);
ft_tt_test_check_fmt("row0 sets exactly the word-2 row-0 flag",
rc == FT_TT_OK && words[FT_TT_OP_VECTOR_TILE_COLS] ==
FT_TT_OP_ELTWISE_FLAG_ROW0,
"rc %d word2 %u", rc, words[FT_TT_OP_VECTOR_TILE_COLS]);
request.row0 = 0u;
rc = ft_tt_op_eltwise_parameters(&request, words);
ft_tt_test_check_fmt("A operand round-trips",
words[FT_TT_OP_VECTOR_A_X] == 3u && words[FT_TT_OP_VECTOR_A_Y] == 4u &&
words[FT_TT_OP_VECTOR_A_BASE] == 0x100000u,
"%u,%u,0x%X", words[FT_TT_OP_VECTOR_A_X], words[FT_TT_OP_VECTOR_A_Y],
words[FT_TT_OP_VECTOR_A_BASE]);
ft_tt_test_check_fmt("B operand round-trips",
words[FT_TT_OP_VECTOR_B_X] == 5u && words[FT_TT_OP_VECTOR_B_Y] == 6u &&
words[FT_TT_OP_VECTOR_B_BASE] == 0x200000u,
"%u,%u,0x%X", words[FT_TT_OP_VECTOR_B_X], words[FT_TT_OP_VECTOR_B_Y],
words[FT_TT_OP_VECTOR_B_BASE]);
ft_tt_test_check_fmt("C operand round-trips",
words[FT_TT_OP_VECTOR_C_X] == 7u && words[FT_TT_OP_VECTOR_C_Y] == 8u &&
words[FT_TT_OP_VECTOR_C_BASE] == 0x300000u,
"%u,%u,0x%X", words[FT_TT_OP_VECTOR_C_X], words[FT_TT_OP_VECTOR_C_Y],
words[FT_TT_OP_VECTOR_C_BASE]);
ft_tt_test_check_fmt("the reserved tail is zeroed",
words[13] == 0u && words[14] == 0u && words[15] == 0u, "%u,%u,%u",
words[13], words[14], words[15]);
/* A unary request must encode without a B operand at all — that is the whole point of
* the kernel deriving arity. */
request.function = FT_TT_OP_VECTOR_FN_SILU;
memset(&request.input_b, 0, sizeof(request.input_b));
rc = ft_tt_op_eltwise_parameters(&request, words);
ft_tt_test_check_fmt("a unary request needs no B operand", rc == FT_TT_OK, "rc %d", rc);
/* …and a BINARY one must be refused without it, because the reader would otherwise
* fetch tiles from address 0 and produce a plausible wrong answer. */
request.function = FT_TT_OP_VECTOR_FN_ADD;
rc = ft_tt_op_eltwise_parameters(&request, words);
ft_tt_test_check_fmt("a binary request without B is refused", rc == FT_TT_EINVAL, "rc %d", rc);
request.function = (ft_tt_op_vector_function)99;
request.input_b = make_operand(5u, 6u, 0x200000u);
rc = ft_tt_op_eltwise_parameters(&request, words);
ft_tt_test_check_fmt("an unknown function is refused", rc == FT_TT_EINVAL, "rc %d", rc);
request.function = FT_TT_OP_VECTOR_FN_ADD;
request.tile_count = 0u;
rc = ft_tt_op_eltwise_parameters(&request, words);
ft_tt_test_check_fmt("a zero tile count is refused", rc == FT_TT_EINVAL, "rc %d", rc);
}
/**
* The four members' PROGRAM WORDS, pinned as LITERALS.
*
* Deliberately not derived. A pin computed from the same table the encoder reads re-derives
* any mistake in that table and passes — which is exactly how eleven self-consistent asserts
* on ft_tt_mailbox.h missed a 16-byte insertion (dev notes §3). These are the hex words that
* have driven silicon; changing one is a rebuild of the reduce kernel and a re-run of the
* on-card acceptance, not a header edit.
*
* Read a word as four bytes, low to high:
* a pass = pool, map, verb, add_eps
* final = pass count, chain source, chain flags, 0
*/
static void test_reduce_programs(void)
{
static const struct {
ft_tt_op_reduce_function function;
uint32_t pass0;
uint32_t pass1;
uint32_t final_word;
} expected[] = {
/* softmax: MAX/none/none/no then SUM/sub-exp/recip/no ; scale the MID stream */
{ FT_TT_OP_REDUCE_FN_SOFTMAX, 0x00000002u, 0x00010201u, 0x00040102u },
/* rms_norm: SUM/x*x/rsqrt/eps ; gamma then scale on IN */
{ FT_TT_OP_REDUCE_FN_RMS_NORM, 0x01020101u, 0x00000000u, 0x00060001u },
/* layer_norm: SUM/none/none/no then SUM/sub-square/rsqrt/eps ; centre+gamma+scale+beta */
{ FT_TT_OP_REDUCE_FN_LAYER_NORM, 0x00000001u, 0x01020301u, 0x000F0002u },
/* router: SUM/none/recip/no ; scale on IN */
{ FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE, 0x00010001u, 0x00000000u, 0x00040001u },
/* l2_norm: SUM/x*x/rsqrt/eps ; scale on IN, no gamma.
* RMS norm's PASS word beside the router's FINAL word — which is the whole of the
* claim that a fifth member is a table row: not one new bit reaches the wire. */
{ FT_TT_OP_REDUCE_FN_L2_NORM, 0x01020101u, 0x00000000u, 0x00040001u }
};
unsigned index;
ft_tt_test_step("the reduce family's five programs, pinned as literals");
for (index = 0; index < sizeof(expected) / sizeof(expected[0]); index++) {
uint32_t pass0 = 0u;
uint32_t pass1 = 0u;
uint32_t final_word = 0u;
const int rc = ft_tt_op_reduce_program(expected[index].function, &pass0, &pass1,
&final_word);
char label[96];
snprintf(label, sizeof(label), "%s compiles to its pinned program",
ft_tt_op_reduce_function_name(expected[index].function));
ft_tt_test_check_fmt(label,
rc == FT_TT_OK && pass0 == expected[index].pass0 &&
pass1 == expected[index].pass1 &&
final_word == expected[index].final_word,
"rc %d, got 0x%08X 0x%08X 0x%08X, want 0x%08X 0x%08X 0x%08X", rc,
pass0, pass1, final_word, expected[index].pass0,
expected[index].pass1, expected[index].final_word);
}
{
uint32_t pass0 = 0u;
uint32_t pass1 = 0u;
uint32_t final_word = 0u;
const int rc = ft_tt_op_reduce_program((ft_tt_op_reduce_function)99, &pass0, &pass1,
&final_word);
ft_tt_test_check_fmt("an unknown member is refused rather than encoded",
rc == FT_TT_EINVAL, "rc %d", rc);
}
/* The derived facts, checked against the definitions rather than restated: gamma and
* beta are what the CHAIN says, eps is what a PASS says, and the pass count is what the
* final word's low byte says. Two spellings of one fact is the shape of bug this file
* exists to catch. */
ft_tt_test_check("only the two norms use gamma",
!ft_tt_op_reduce_uses_gamma(FT_TT_OP_REDUCE_FN_SOFTMAX) &&
ft_tt_op_reduce_uses_gamma(FT_TT_OP_REDUCE_FN_RMS_NORM) &&
ft_tt_op_reduce_uses_gamma(FT_TT_OP_REDUCE_FN_LAYER_NORM) &&
!ft_tt_op_reduce_uses_gamma(FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE), "");
ft_tt_test_check("only layer norm uses beta",
ft_tt_op_reduce_uses_beta(FT_TT_OP_REDUCE_FN_LAYER_NORM) &&
!ft_tt_op_reduce_uses_beta(FT_TT_OP_REDUCE_FN_RMS_NORM), "");
ft_tt_test_check("only the two norms use eps",
ft_tt_op_reduce_uses_eps(FT_TT_OP_REDUCE_FN_RMS_NORM) &&
ft_tt_op_reduce_uses_eps(FT_TT_OP_REDUCE_FN_LAYER_NORM) &&
!ft_tt_op_reduce_uses_eps(FT_TT_OP_REDUCE_FN_SOFTMAX) &&
!ft_tt_op_reduce_uses_eps(FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE), "");
/* The scaler is 1/W exactly when a pass wants a MEAN. Softmax's and the router's reduces
* are a max and a plain sum, so theirs is 1.0 — one tile, two meanings, one slot. */
ft_tt_test_check("only the two norms take a 1/W scaler",
ft_tt_op_reduce_scaler_is_mean(FT_TT_OP_REDUCE_FN_RMS_NORM) &&
ft_tt_op_reduce_scaler_is_mean(FT_TT_OP_REDUCE_FN_LAYER_NORM) &&
!ft_tt_op_reduce_scaler_is_mean(FT_TT_OP_REDUCE_FN_SOFTMAX) &&
!ft_tt_op_reduce_scaler_is_mean(FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE),
"");
ft_tt_test_check("softmax and layer norm are two passes, the others one",
ft_tt_op_reduce_pass_count(FT_TT_OP_REDUCE_FN_SOFTMAX) == 2u &&
ft_tt_op_reduce_pass_count(FT_TT_OP_REDUCE_FN_LAYER_NORM) == 2u &&
ft_tt_op_reduce_pass_count(FT_TT_OP_REDUCE_FN_RMS_NORM) == 1u &&
ft_tt_op_reduce_pass_count(FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE) == 1u,
"");
/* L2 norm, stated where its neighbours are, because the two cells it differs from RMS
* norm in are exactly the two a copy-paste of that row would keep. */
ft_tt_test_check("L2 norm takes no gamma, no beta, one pass, eps, and a 1.0 scaler",
!ft_tt_op_reduce_uses_gamma(FT_TT_OP_REDUCE_FN_L2_NORM) &&
!ft_tt_op_reduce_uses_beta(FT_TT_OP_REDUCE_FN_L2_NORM) &&
ft_tt_op_reduce_uses_eps(FT_TT_OP_REDUCE_FN_L2_NORM) &&
!ft_tt_op_reduce_scaler_is_mean(FT_TT_OP_REDUCE_FN_L2_NORM) &&
ft_tt_op_reduce_pass_count(FT_TT_OP_REDUCE_FN_L2_NORM) == 1u,
"");
}
/** `pn_qwen35moe_l2_norm()` in Pantheon, spelled here so the two cannot drift: a double
* sum of squares, and a CLAMP at eps rather than an epsilon under the root. */
static void oracle_l2_norm_rows(const float *x, float eps, float *out, uint32_t rows,
uint32_t cols)
{
uint32_t row;
for (row = 0; row < rows; row++) {
const float *xr = x + (size_t)row * cols;
float *outr = out + (size_t)row * cols;
double sumsq = 0.0;
float inv;
uint32_t col;
for (col = 0; col < cols; col++)
sumsq += (double)xr[col] * (double)xr[col];
inv = 1.0f / fmaxf((float)sqrt(sumsq), eps);
for (col = 0; col < cols; col++)
outr[col] = xr[col] * inv;
}
}
/** Max-norm relative error of one row, folded into a running worst. The score
* `PN_TT_GDN_NORMCHECK` prints on the card, spelled here so the two are comparable. */
static double row_max_norm_relative(const float *want, const float *got, uint32_t cols,
double worst)
{
double max_abs = 0.0;
double max_want = 0.0;
uint32_t col;
for (col = 0; col < cols; col++) {
const double diff = fabs((double)got[col] - (double)want[col]);
const double magnitude = fabs((double)want[col]);
if (diff > max_abs)
max_abs = diff;
if (magnitude > max_want)
max_want = magnitude;
}
if (max_want > 0.0 && max_abs / max_want > worst)
worst = max_abs / max_want;
return worst;
}
/**
* THE L2 MEMBER AGAINST THE ORACLE IT EXISTS TO REPRODUCE — and against the RMS norm it is
* one table row away from.
*
* The shape is the GDN's, not the model's: 16 rows of 128, which is `[k_heads, S]` at
* Ornith's shape and the ONE reduce descriptor the q/k head norm compiles to. The red arm
* runs the same data through RMS_NORM's program and requires the score to COLLAPSE: an L2
* norm and an RMS norm over the same row differ by exactly sqrt(W), so a test that cannot
* separate them is measuring nothing.
*/
static void test_l2_norm_against_the_oracle(void)
{
const uint32_t rows = 16u;
const uint32_t cols = 128u;
const size_t count = (size_t)rows * cols;
const float eps = 1e-6f;
float *x = malloc(count * sizeof(float));
float *want = malloc(count * sizeof(float));
float *got = malloc(count * sizeof(float));
uint32_t pass0 = 0u;
uint32_t pass1 = 0u;
uint32_t final_word = 0u;
size_t element;
ft_tt_test_step("the L2 member reproduces pn_qwen35moe_l2_norm at the GDN's head shape");
if (!x || !want || !got) {
ft_tt_test_check("allocation", false, "out of memory");
goto done;
}
ft_tt_vector_random_seed(0x1220E7ULL);
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(ft_tt_vector_random_value()));
oracle_l2_norm_rows(x, eps, want, rows, cols);
/* eps*eps, because the oracle clamps the MAGNITUDE at eps and this folds into the
* SQUARE. eps_tile is that divided by 32 — the value build_constants writes. */
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_L2_NORM, &pass0, &pass1, &final_word);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x, NULL, NULL, 1.0f,
ft_float_from_bf16(ft_bf16_from_float(
eps * eps / (float)FT_TT_OP_TILE_DIM)),
got, rows, cols);
{
const double pcc = ft_tt_vector_pearson(want, got, count);
const double max_abs = ft_tt_vector_max_abs_difference(want, got, count);
ft_tt_test_check_fmt("PCC against the fp64 oracle is at least 0.9999",
pcc >= 0.9999, "PCC %.9f, max abs %.6g", pcc, max_abs);
/* bf16 through the circular buffers is ~2^-8 a value, and the modelled norm scales
* a whole row by one number, so the bar is the FORMAT and not the arithmetic. */
ft_tt_test_check_fmt("max abs is inside the bf16 budget", max_abs < 0.01,
"max abs %.6g", max_abs);
}
/* RED ARM: RMS norm's program on the same data. Same pass, same verb — only the scaler
* and the gamma step differ, and that is a factor of sqrt(128) = 11.3 on every element. */
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_RMS_NORM, &pass0, &pass1, &final_word);
{
float *gamma = malloc(cols * sizeof(float));
double max_abs;
uint32_t index;
if (!gamma) {
ft_tt_test_check("red-arm allocation", false, "out of memory");
goto done;
}
for (index = 0; index < cols; index++)
gamma[index] = 1.0f;
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x, gamma, NULL,
1.0f / (float)cols,
ft_float_from_bf16(ft_bf16_from_float(
eps * (float)cols / (float)FT_TT_OP_TILE_DIM)),
got, rows, cols);
max_abs = ft_tt_vector_max_abs_difference(want, got, count);
ft_tt_test_check_fmt("RED: the RMS member fails the same bar by sqrt(W)",
max_abs > 0.1, "max abs %.6g (want > 0.1)", max_abs);
free(gamma);
}
/* THE OPERAND'S FORMAT, PRICED SEPARATELY FROM THE REDUCE'S — because on silicon the two
* are quoted as one number and the lever is not the same.
*
* Everything above is bf16-EXACT input: `x` was rounded before the oracle saw it, so the
* only error measured is the reduce's own. A real operand is not bf16-exact — the GDN's
* `conv_out` and `y` are fp32 — and a bf16 activation strip rounds it on the way in. This
* arm gives the ORACLE the fp32 values and the CARD MODEL the bf16 rendering of them,
* which is exactly the comparison `PN_TT_GDN_NORMCHECK` makes on the card. Scored the
* same way the probe scores: max-norm relative, PER ROW. */
{
float *fp32 = malloc(count * sizeof(float));
double worst_reduce = 0.0;
double worst_storage = 0.0;
uint32_t row;
if (!fp32) {
ft_tt_test_check("format-floor allocation", false, "out of memory");
goto done;
}
ft_tt_vector_random_seed(0x1220E8ULL);
for (element = 0; element < count; element++)
fp32[element] = ft_tt_vector_random_value();
oracle_l2_norm_rows(fp32, eps, want, rows, cols);
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(fp32[element]));
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_L2_NORM, &pass0, &pass1, &final_word);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x, NULL, NULL, 1.0f,
ft_float_from_bf16(ft_bf16_from_float(
eps * eps / (float)FT_TT_OP_TILE_DIM)),
got, rows, cols);
for (row = 0; row < rows; row++)
worst_storage = row_max_norm_relative(want + (size_t)row * cols,
got + (size_t)row * cols, cols,
worst_storage);
/* And the same reduce over the bf16-exact operand, which is the control: whatever is
* left there is the reduce, and the difference between the two is the FORMAT. */
oracle_l2_norm_rows(x, eps, want, rows, cols);
for (row = 0; row < rows; row++)
worst_reduce = row_max_norm_relative(want + (size_t)row * cols,
got + (size_t)row * cols, cols, worst_reduce);
/* AND THE SPLIT IS NOT WHERE IT LOOKED. The operand's own bf16 rendering is the
* SMALLER term: the REDUCE's format is what puts a narrow norm past the gate, so an
* fp32 activation strip in front of a bf16 reduce would buy almost nothing.
* Measured on silicon 2026-08-19 by PN_TT_GDN_NORMCHECK on Ornith's real conv_out:
* q/k L2 worst 8.599e-03, gated worst 1.283e-02 — this model lands within 20% of
* the first, which is what licenses using it to price the alternative. */
ft_tt_test_check_fmt("the bf16 REDUCE alone is past the 1e-3 gate", worst_reduce > 1e-3,
"%.3e per-row max-norm relative", worst_reduce);
ft_tt_test_check_fmt("and the operand's own bf16 storage adds LESS than the reduce",
worst_storage - worst_reduce < worst_reduce,
"storage %.3e, reduce %.3e — the CB's format is the lever, not "
"the strip's", worst_storage, worst_reduce);
free(fp32);
}
done:
free(x);
free(want);
free(got);
}
/**
* THE Float32 `reduce` PROGRAM AGAINST THE bf16 ONE, ON THE GDN HEAD SHAPE, OFF-HARDWARE.
*
* The question this answers before any card time: a Float32 CB table costs a doubled page
* and a second program in the blob, so what does it BUY, and is what it buys enough?
*
* Both models run the SAME program words over the SAME data against the SAME fp64 oracle.
* The only difference is the CB table, which is two roundings (ft_tt_vector_reference.c):
* bf16 rounds on the way into L1 and is exact on the way out; Float32 is exact on the way
* in and TRUNCATES TO Tf32 — 11 mantissa bits, toward zero — on the way out. That second
* one is a floor no table can move, so the honest question is not "does Float32 help" but
* "does Tf32 clear 1e-3 at this shape".
*
* Scored per row, max-norm relative, which is exactly what `PN_TT_GDN_NORMCHECK` prints per
* head on the card. 16 rows of 128 is `[k_heads, S]` at Ornith's shape.
*/
static void test_float32_reduce_at_the_head_shape(void)
{
const uint32_t rows = 16u;
const uint32_t cols = 128u;
const size_t count = (size_t)rows * cols;
const float eps = 1e-6f;
float *fp32 = malloc(count * sizeof(float));
float *stored_bf16 = malloc(count * sizeof(float));
float *gamma = malloc(cols * sizeof(float));
float *want = malloc(count * sizeof(float));
float *got = malloc(count * sizeof(float));
double l2_bf16 = 0.0;
double l2_float32 = 0.0;
double rms_bf16 = 0.0;
double rms_float32 = 0.0;
uint32_t pass0 = 0u;
uint32_t pass1 = 0u;
uint32_t final_word = 0u;
uint32_t row;
size_t element;
ft_tt_test_step("the Float32 reduce program against the bf16 one, at the GDN head shape");
if (!fp32 || !stored_bf16 || !gamma || !want || !got) {
ft_tt_test_check("allocation", false, "out of memory");
goto done;
}
ft_tt_vector_random_seed(0x1220E8ULL);
for (element = 0; element < count; element++) {
fp32[element] = ft_tt_vector_random_value();
stored_bf16[element] = ft_float_from_bf16(ft_bf16_from_float(fp32[element]));
}
for (element = 0; element < cols; element++)
gamma[element] = 0.5f + ft_tt_vector_random_value() * 0.25f;
/* ---- the q/k arm: L2 norm, no gamma, scaler 1.0, eps as eps*eps/32 ---- */
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_L2_NORM, &pass0, &pass1, &final_word);
oracle_l2_norm_rows(fp32, eps, want, rows, cols);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, stored_bf16, NULL, NULL, 1.0f,
ft_float_from_bf16(ft_bf16_from_float(
eps * eps / (float)FT_TT_OP_TILE_DIM)),
got, rows, cols);
for (row = 0; row < rows; row++)
l2_bf16 = row_max_norm_relative(want + (size_t)row * cols, got + (size_t)row * cols,
cols, l2_bf16);
/* The Float32 strip stores the fp32 values as given — that is the whole point of it — so
* the operand this model reduces is `fp32` and not the bf16 rendering of it. */
ft_tt_vector_model_reduce_program_float32(pass0, pass1, final_word, fp32, NULL, NULL, 1.0f,
eps * eps / (float)FT_TT_OP_TILE_DIM, got, rows,
cols);
for (row = 0; row < rows; row++)
l2_float32 = row_max_norm_relative(want + (size_t)row * cols, got + (size_t)row * cols,
cols, l2_float32);
/* ---- the gated arm's reduce half: RMS norm with a real gamma, scaler 1/W ---- */
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_RMS_NORM, &pass0, &pass1, &final_word);
ft_tt_vector_reference_rms_norm(fp32, gamma, eps, want, rows, cols);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, stored_bf16, gamma, NULL,
1.0f / (float)cols,
ft_float_from_bf16(ft_bf16_from_float(
eps * (float)cols / (float)FT_TT_OP_TILE_DIM)),
got, rows, cols);
for (row = 0; row < rows; row++)
rms_bf16 = row_max_norm_relative(want + (size_t)row * cols, got + (size_t)row * cols,
cols, rms_bf16);
ft_tt_vector_model_reduce_program_float32(pass0, pass1, final_word, fp32, gamma, NULL,
1.0f / (float)cols,
eps * (float)cols / (float)FT_TT_OP_TILE_DIM,
got, rows, cols);
for (row = 0; row < rows; row++)
rms_float32 = row_max_norm_relative(want + (size_t)row * cols, got + (size_t)row * cols,
cols, rms_float32);
ft_tt_test_check_fmt("the bf16 program misses the 1e-3 gate on BOTH members",
l2_bf16 > 1e-3 && rms_bf16 > 1e-3,
"L2 %.3e, RMS %.3e per-row max-norm relative", l2_bf16, rms_bf16);
ft_tt_test_check_fmt("the Float32 program is at least 4x better on both",
l2_float32 * 4.0 < l2_bf16 && rms_float32 * 4.0 < rms_bf16,
"L2 %.3e -> %.3e (%.1fx), RMS %.3e -> %.3e (%.1fx)", l2_bf16,
l2_float32, l2_bf16 / l2_float32, rms_bf16, rms_float32,
rms_bf16 / rms_float32);
/* AND THE TWO MEMBERS LAND ON OPPOSITE SIDES OF THE GATE, which is the result. The L2
* member's chain is ONE step; the RMS member's is TWO (gamma, then scale) and every step
* is a CB round trip, so it pays two more Tf32 unpacks on the value path. */
ft_tt_test_check_fmt("the L2 member CLEARS 1e-3 at Float32", l2_float32 < 1e-3,
"%.3e against 1e-3", l2_float32);
ft_tt_test_check_fmt("the RMS member does NOT, and it is inside the Tf32 budget of 2e-3",
rms_float32 > 1e-3 && rms_float32 < 2e-3,
"%.3e — over the gate and under twice it", rms_float32);
/* THE ATTRIBUTION, and it is the whole finding: run the SAME program with an fp32 unpack
* the hardware does not have. If the residue collapses, every bit of what is left at
* Float32 is Tf32 TRUNCATION and no CB table can touch it — the next lever is the chain's
* step count, i.e. kernel source, which this round does not open. */
{
double l2_exact = 0.0;
double rms_exact = 0.0;
ft_tt_vector_model_reduce_program_exact(pass0, pass1, final_word, fp32, gamma, NULL,
1.0f / (float)cols,
eps * (float)cols / (float)FT_TT_OP_TILE_DIM,
got, rows, cols);
for (row = 0; row < rows; row++)
rms_exact = row_max_norm_relative(want + (size_t)row * cols,
got + (size_t)row * cols, cols, rms_exact);
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_L2_NORM, &pass0, &pass1, &final_word);
oracle_l2_norm_rows(fp32, eps, want, rows, cols);
ft_tt_vector_model_reduce_program_exact(pass0, pass1, final_word, fp32, NULL, NULL, 1.0f,
eps * eps / (float)FT_TT_OP_TILE_DIM, got, rows,
cols);
for (row = 0; row < rows; row++)
l2_exact = row_max_norm_relative(want + (size_t)row * cols,
got + (size_t)row * cols, cols, l2_exact);
ft_tt_test_check_fmt("an fp32 unpack would clear the gate by 100x on BOTH — so the "
"whole Float32 residue is Tf32 TRUNCATION, not the pass structure",
l2_exact * 100.0 < l2_float32 && rms_exact * 100.0 < rms_float32,
"counterfactual L2 %.3e RMS %.3e against Tf32's %.3e and %.3e",
l2_exact, rms_exact, l2_float32, rms_float32);
}
done:
free(fp32);
free(stored_bf16);
free(gamma);
free(want);
free(got);
}
/* ---------------------------------------------------------------------------
* THE THREE WALLS UNDER THE Float32 HEAD NORMS, PRICED APART
*
* On silicon the two GDN members score 1.737e-03 (L2) and 3.165e-03 (RMS) against a 1e-3
* gate, and the model above says 8.406e-04 and 1.566e-03 — the card is 2.07x and 2.02x
* ABOVE it, the SAME factor on both members. Three things were named for that gap and none
* had been measured apart:
*
* 1. the SFPU's rsqrt polynomial, which the model does not have (it uses libm);
* 2. the RMS member's chain STEP COUNT, two steps against the L2 member's one;
* 3. the sample size — the card takes the worst of 48 layers x 32 heads, the model of 16
* rows in one draw.
*
* Each is an arm here. The rsqrt one is the important one to run FIRST, because if it is
* real then the chain-step work cannot reach the gate on its own and is not worth grinding.
* --------------------------------------------------------------------------- */
static void test_the_three_walls_priced_apart(void)
{
const uint32_t rows = 1536u; /* 48 SSM layers x 32 heads — what the card's worst is over */
const uint32_t cols = 128u;
const size_t count = (size_t)rows * cols;
const float eps = 1e-6f;
float *x = malloc(count * sizeof(float));
float *gamma = malloc(cols * sizeof(float));
/* The "gamma folded into a downstream weight" arm's oracle is the same RMS norm with
* gamma = 1, which is a gamma vector rather than a second reference function. */
float *unit_gamma = malloc(cols * sizeof(float));
float *want = malloc(cols * sizeof(float));
float *got = malloc(cols * sizeof(float));
float *control = malloc((size_t)16u * cols * sizeof(float));
uint32_t l2_pass0 = 0u;
uint32_t l2_pass1 = 0u;
uint32_t l2_final = 0u;
uint32_t rms_pass0 = 0u;
uint32_t rms_pass1 = 0u;
uint32_t rms_final = 0u;
size_t element;
/* Every arm is scored the same way: worst per-row max-norm relative over `rows`, which is
* exactly `normcheck_score()` per head on the card. */
struct arm_result {
double l2;
double rms;
double rms_no_gamma;
};
ft_tt_test_step("the three walls under the Float32 head norms, priced apart");
if (!x || !gamma || !unit_gamma || !want || !got || !control) {
ft_tt_test_check("allocation", false, "out of memory");
goto done;
}
ft_tt_vector_random_seed(0x1220E8ULL);
for (element = 0; element < count; element++)
x[element] = ft_tt_vector_random_value();
for (element = 0; element < cols; element++) {
gamma[element] = 0.5f + ft_tt_vector_random_value() * 0.25f;
unit_gamma[element] = 1.0f;
}
/* THE STREAM, PINNED AS LITERALS. The card-side probe that settles the model-vs-silicon
* gap (`PN_TT_RED=head_norm_synthetic`, Pantheon `qwen35moe_gdn_normcheck_tt.c`) cannot
* link this file, so it spells the same xorshift64* itself and checks it against exactly
* these eight words. Two spellings of one stream is a thing that must agree; this is what
* makes a disagreement a refusal instead of an incomparable number. */
{
static const uint32_t pinned[8] = { 0xBF508D2Au, 0x3EF4CC04u, 0x3EE46400u, 0xBDF7F450u,
0xBE494EE8u, 0xBF4FF826u, 0x3E88AB9Cu,
0xBF4CD48Cu };
bool matches = true;
size_t index;
for (index = 0; index < 8u; index++) {
uint32_t bits;
memcpy(&bits, &x[index], sizeof(bits));
if (bits != pinned[index])
matches = false;
}
ft_tt_test_check("the synthetic stream at seed 0x1220E8 is pinned as literals", matches,
"the card-side probe checks the same eight words");
}
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_L2_NORM, &l2_pass0, &l2_pass1, &l2_final);
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_RMS_NORM, &rms_pass0, &rms_pass1,
&rms_final);
/* One arm, both members plus the gamma-free RMS the "fold gamma downstream" item would run. */
#define ARM(name, arms, row_count) \
do { \
uint32_t row; \
(name).l2 = 0.0; \
(name).rms = 0.0; \
(name).rms_no_gamma = 0.0; \
for (row = 0; row < (row_count); row++) { \
const float *x_row = x + (size_t)row * cols; \
oracle_l2_norm_rows(x_row, eps, want, 1u, cols); \
ft_tt_vector_model_head_norm_float32( \
l2_pass0, l2_pass1, l2_final, x_row, NULL, 1.0f, \
eps *eps / (float)FT_TT_OP_TILE_DIM, got, 1u, cols, (arms)); \
(name).l2 = row_max_norm_relative(want, got, cols, (name).l2); \
ft_tt_vector_reference_rms_norm(x_row, gamma, eps, want, 1u, cols); \
ft_tt_vector_model_head_norm_float32( \
rms_pass0, rms_pass1, rms_final, x_row, gamma, 1.0f / (float)cols, \
eps * (float)cols / (float)FT_TT_OP_TILE_DIM, got, 1u, cols, (arms)); \
(name).rms = row_max_norm_relative(want, got, cols, (name).rms); \
ft_tt_vector_model_head_norm_float32( \
rms_pass0, rms_pass1, rms_final, x_row, NULL, 1.0f / (float)cols, \
eps * (float)cols / (float)FT_TT_OP_TILE_DIM, got, 1u, cols, \
(arms) | FT_TT_REDUCE_ARM_NO_GAMMA); \
ft_tt_vector_reference_rms_norm(x_row, unit_gamma, eps, want, 1u, cols); \
(name).rms_no_gamma = \
row_max_norm_relative(want, got, cols, (name).rms_no_gamma); \
} \
} while (0)
/* THE CONTROL THAT LICENSES EVERY ARM: at arms = 0 the switchable model must be the
* shipped Float32 model BIT FOR BIT. Without this the arms below measure a second
* program that merely resembles the first. */
{
bool identical = true;
bool armed_differs = false;
uint32_t row;
ft_tt_vector_model_reduce_program_float32(rms_pass0, rms_pass1, rms_final, x, gamma,
NULL, 1.0f / (float)cols,
eps * (float)cols / (float)FT_TT_OP_TILE_DIM,
control, 16u, cols);
for (row = 0; row < 16u; row++) {
const size_t offset = (size_t)row * cols;
ft_tt_vector_model_head_norm_float32(
rms_pass0, rms_pass1, rms_final, x + offset, gamma, 1.0f / (float)cols,
eps * (float)cols / (float)FT_TT_OP_TILE_DIM, got, 1u, cols, 0u);
if (memcmp(got, control + offset, (size_t)cols * sizeof(float)) != 0)
identical = false;
/* The same comparison with ONE arm on must DIFFER, or the check above is a
* memcmp of two buffers that were never going to disagree. */
ft_tt_vector_model_head_norm_float32(
rms_pass0, rms_pass1, rms_final, x + offset, gamma, 1.0f / (float)cols,
eps * (float)cols / (float)FT_TT_OP_TILE_DIM, got, 1u, cols,
FT_TT_REDUCE_ARM_ROUND_NEAREST);
if (memcmp(got, control + offset, (size_t)cols * sizeof(float)) != 0)
armed_differs = true;
}
ft_tt_test_check("the switchable model IS the shipped Float32 model at arms = 0",
identical, "bit-for-bit over 16 rows of the RMS member");
ft_tt_test_check("RED: one arm on and the same comparison DIFFERS", armed_differs,
"FT_TT_REDUCE_ARM_ROUND_NEAREST against the same 16 rows");
}
/* A program this arm does not model must be REFUSED, not approximated — softmax through
* it is all-NaN, which is the red proof that the shape guard is not decoration. */
{
uint32_t soft0 = 0u;
uint32_t soft1 = 0u;
uint32_t soft_final = 0u;
bool all_nan = true;
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_SOFTMAX, &soft0, &soft1, &soft_final);
ft_tt_vector_model_head_norm_float32(soft0, soft1, soft_final, x, NULL, 1.0f, 0.0f, got,
1u, cols, 0u);
for (element = 0; element < cols; element++) {
if (!isnan(got[element]))
all_nan = false;
}
ft_tt_test_check_fmt(
"RED: softmax through the head-norm arm is refused, not approximated", all_nan,
"all %u outputs NaN", cols);
}
{
struct arm_result shipped;
struct arm_result sfpu;
struct arm_result perturbed;
struct arm_result premultiply;
struct arm_result nearest;
struct arm_result nearest_host;
struct arm_result small;
ARM(shipped, 0u, rows);
ARM(small, 0u, 16u);
ARM(sfpu, FT_TT_REDUCE_ARM_SFPU_RSQRT, rows);
ARM(perturbed, FT_TT_REDUCE_ARM_RSQRT_PERTURB, rows);
ARM(premultiply, FT_TT_REDUCE_ARM_PREMULTIPLY, rows);
ARM(nearest, FT_TT_REDUCE_ARM_ROUND_NEAREST, rows);
ARM(nearest_host,
FT_TT_REDUCE_ARM_ROUND_NEAREST | FT_TT_REDUCE_ARM_HOST_ROUND, rows);
/* WALL 1: the SFPU rsqrt. Its own accuracy first, so the null below is a number and
* not only an absence — swept against fp64 across the binades a sum of 128 squares
* plus eps can land in. */
{
double worst = 0.0;
int exponent;
for (exponent = -20; exponent <= 20; exponent++) {
uint32_t step;
for (step = 0; step < 4096u; step++) {
const float value =
ldexpf(1.0f + (float)step / 4096.0f, exponent);
const double reference = 1.0 / sqrt((double)value);
const double relative =
fabs((double)ft_tt_vector_sfpu_rsqrt(value) - reference) / reference;
if (relative > worst)
worst = relative;
}
}
ft_tt_test_check_fmt("the SFPU rsqrt polynomial is 4 orders BELOW the residue it "
"was nominated for",
worst < 1e-6, "%.3e worst relative over 2^-20..2^21, against "
"a residue of 8.4e-04",
worst);
}
/* And in the program, where it is a NULL — the perturbed arm below is what says the
* score could have seen it. */
ft_tt_test_check_fmt(
"the SFPU rsqrt polynomial moves NEITHER member — the 2x candidate is refuted",
fabs(sfpu.l2 - shipped.l2) < shipped.l2 * 0.01 &&
fabs(sfpu.rms - shipped.rms) < shipped.rms * 0.01,
"libm L2 %.3e RMS %.3e -> SFPU %.3e / %.3e", shipped.l2, shipped.rms, sfpu.l2,
sfpu.rms);
/* THE RED ARM, and its DIRECTION is a second result. A +1e-3 rsqrt moves the L2
* member UP by 1.7x and the RMS member DOWN by 1.5x — because the RMS residue is a
* downward-biased scale error that an upward perturbation partly cancels. That is the
* same signature the GDN value path carries (`card/want` 0.9955-0.9977, below 1 on
* 768 of 768 head-layers), measured here on the reduce instead of asserted. So the
* check is that a 1e-3 rsqrt MOVES the score, not that it raises it. */
ft_tt_test_check_fmt(
"RED: an rsqrt off by 1e-3 relative moves BOTH members by >30% — so the null above "
"means something",
fabs(perturbed.l2 - shipped.l2) > shipped.l2 * 0.3 &&
fabs(perturbed.rms - shipped.rms) > shipped.rms * 0.3,
"L2 %.3e -> %.3e (up), RMS %.3e -> %.3e (DOWN — the residue is biased low)",
shipped.l2, perturbed.l2, shipped.rms, perturbed.rms);
/* WALL 2: the chain step count. Reordering it is a null — two FPU multiplies through
* circular buffers cost four Tf32 unpacks whichever operand pair goes first — and the
* whole of the RMS member's excess is gamma plus the scratch round trip. */
ft_tt_test_check_fmt(
"reordering the chain is a NULL: the unpack count, not the order, is what costs",
fabs(premultiply.rms - shipped.rms) < shipped.rms * 0.15,
"as shipped %.3e, premultiplied %.3e", shipped.rms, premultiply.rms);
ft_tt_test_check_fmt(
"and DROPPING gamma is the whole of the RMS member's excess over L2",
shipped.rms_no_gamma < shipped.rms * 0.5 &&
shipped.rms_no_gamma < shipped.l2 * 1.2,
"RMS %.3e -> %.3e with a one-step chain, against the L2 member's %.3e",
shipped.rms, shipped.rms_no_gamma, shipped.l2);
/* AND THAT IS THE RESULT THAT PRICES THE WHOLE ITEM: the L2 member's chain is ALREADY
* one step and it is 1.737e-03 on the card. A perfect chain-step fix lands the RMS
* member on the L2 member's number, which is itself 1.7x past the gate. */
ft_tt_test_check_fmt(
"so a PERFECT chain-step fix lands RMS on L2's figure — which still misses 1e-3 "
"on silicon",
shipped.rms_no_gamma > 1e-3 * 0.5,
"modelled one-step RMS %.3e; the card's one-step member measures 1.737e-03",
shipped.rms_no_gamma);
/* WALL 3: the sample size. It is real and it is small — and it is on the RMS member
* only, because the L2 member's residue is a common-mode scale error that every row
* carries equally. */
ft_tt_test_check_fmt("the card's 48x32 sample costs the RMS member a little and the "
"L2 member nothing",
small.l2 == shipped.l2 && shipped.rms > small.rms,
"16 rows: L2 %.3e RMS %.3e; 1536 rows: L2 %.3e RMS %.3e", small.l2,
small.rms, shipped.l2, shipped.rms);
/* The remaining kernel-source lever, priced: an SFPU pre-round to the NEAREST Tf32
* before every pack. Worth ~1.3x on RMS, and it makes the L2 member WORSE, because
* the truncation biases of the map and the chain partly CANCEL. */
ft_tt_test_check_fmt(
"rounding to nearest before the pack is worth ~1.3x on RMS and is NOT free on L2",
nearest.rms < shipped.rms && nearest.l2 > shipped.l2 * 0.9,
"RMS %.3e -> %.3e, L2 %.3e -> %.3e (host pre-round too: %.3e / %.3e)", shipped.rms,
nearest.rms, shipped.l2, nearest.l2, nearest_host.l2, nearest_host.rms);
/* WHAT IS LEFT OF THE 2x, AND IT IS ONE MANTISSA BIT. "E8M10" is written down two
* ways — ten stored plus a hidden one, or ten of significand of which nine are
* stored — and the difference is exactly a factor of two on a truncating unpack. The
* nine-bit reading reproduces the card's q/k L2 figure to four significant figures on
* uniform data, where the ten-bit reading is 2.07x under it. It is a HYPOTHESIS with
* one strong prediction behind it; `PN_TT_RED=head_norm_synthetic` settles it on
* silicon by removing the other difference, the data. */
{
struct arm_result nine;
struct arm_result nine_best;
ARM(nine, FT_TT_REDUCE_ARM_TF32_NINE_BITS, rows);
ft_tt_test_check_fmt("a NINE-bit Tf32 unpack reproduces the card's L2 member, and "
"the ten-bit one is 2.07x under it",
nine.l2 > 1.6e-3 && nine.l2 < 1.9e-3,
"ten bits %.3e, nine bits %.3e, silicon 1.737e-03 (RMS %.3e "
"/ %.3e against silicon's 3.165e-03)",
shipped.l2, nine.l2, shipped.rms, nine.rms);
/* AND THE CONSEQUENCE, which is why the bit matters at all: at nine bits every
* lever this family has, composed, still misses the gate. */
{
struct arm_result nine_kernel_rne;
ARM(nine_best,
FT_TT_REDUCE_ARM_TF32_NINE_BITS | FT_TT_REDUCE_ARM_ROUND_NEAREST |
FT_TT_REDUCE_ARM_HOST_ROUND,
rows);
ARM(nine_kernel_rne,
FT_TT_REDUCE_ARM_TF32_NINE_BITS | FT_TT_REDUCE_ARM_ROUND_NEAREST, rows);
ft_tt_test_check_fmt(
"at nine bits EVERY lever composed still misses 1e-3 — the gate is not "
"reachable in this family at Wt = 4",
nine_best.l2 > 1e-3 && nine_best.rms_no_gamma > 1e-3 &&
nine_kernel_rne.l2 > 1e-3 && nine_kernel_rne.rms_no_gamma > 1e-3,
"one-step chain, round-to-nearest on the kernel packs: L2 %.3e RMS %.3e; "
"and with the host operands too: L2 %.3e RMS %.3e",
nine_kernel_rne.l2, nine_kernel_rne.rms_no_gamma, nine_best.l2,
nine_best.rms_no_gamma);
}
}
}
/* The per-stage attribution, REPORTED rather than gated: which unpack spends the budget.
* Removing a stage can make the score WORSE, which is the evidence that the truncation
* biases partly cancel and that "fewer stages" is not automatically "less error". */
{
static const struct {
const char *name;
uint32_t arm;
} stages[] = {
{ "the MAP's x unpack", FT_TT_REDUCE_ARM_EXACT_MAP },
{ "the reduce's mid unpack", FT_TT_REDUCE_ARM_EXACT_MID },
{ "the chain's x unpack", FT_TT_REDUCE_ARM_EXACT_CHAIN_X },
{ "the rsqrt column's unpack", FT_TT_REDUCE_ARM_EXACT_COLUMN },
{ "gamma's unpack", FT_TT_REDUCE_ARM_EXACT_GAMMA },
{ "the chain's scratch round trip", FT_TT_REDUCE_ARM_EXACT_STAGE },
};
size_t index;
for (index = 0; index < sizeof(stages) / sizeof(stages[0]); index++) {
struct arm_result without;
char label[160];
ARM(without, stages[index].arm, rows);
snprintf(label, sizeof(label), "REPORTED: with %s made exact", stages[index].name);
ft_tt_test_check_fmt(label, true, "L2 %.3e, RMS %.3e", without.l2, without.rms);
}
}
#undef ARM
done:
free(x);
free(gamma);
free(unit_gamma);
free(want);
free(got);
free(control);
}
/* ---------------------------------------------------------------------------
* `gated` IS BOUND BY THE bf16 BRIDGE, NOT BY THE REDUCE
*
* The GDN's output gated norm is TWO descriptors on TWO residencies at TWO formats:
* `FT_TT_OP_REDUCE_FN_RMS_NORM` on the Float32 `reduce_fp32` core, and then
* `pn_tt_silu_mul(z, normed)` on the bf16 ELEMENTWISE residency. So the fp32 answer is
* RENDERED INTO A bf16 STRIP before the gate ever sees it.
*
* On silicon the Float32 program took the reduce alone from 8.599e-03 to 3.165e-03 (2.7x)
* and moved `gated` only 1.283e-02 -> 1.023e-02 (1.25x). That is the shape of a term the
* reduce does not own, and this prices it: the same two steps with the bridge and without.
* --------------------------------------------------------------------------- */
static void test_gated_is_bound_by_the_bf16_bridge(void)
{
/* FT_TT_OP_VECTOR_FN_SILU_MUL models a whole tile at a time, so the block is 1024
* elements = 8 heads of 128. */
const uint32_t cols = 128u;
const uint32_t rows = 128u;
const uint32_t block_rows = 8u;
const size_t count = (size_t)rows * cols;
const float eps = 1e-6f;
float *y = malloc(count * sizeof(float));
float *z = malloc(count * sizeof(float));
float *gamma = malloc(cols * sizeof(float));
float *normed = malloc((size_t)block_rows * cols * sizeof(float));
float *want = malloc((size_t)block_rows * cols * sizeof(float));
float *bridged = malloc((size_t)block_rows * cols * sizeof(float));
double reduce_alone = 0.0;
double gated_bridged = 0.0;
double gated_no_bridge = 0.0;
uint32_t pass0 = 0u;
uint32_t pass1 = 0u;
uint32_t final_word = 0u;
uint32_t block;
size_t element;
ft_tt_test_step("`gated`: the fp32 reduce against the bf16 bridge it crosses");
if (!y || !z || !gamma || !normed || !want || !bridged) {
ft_tt_test_check("allocation", false, "out of memory");
goto done;
}
ft_tt_vector_random_seed(0x1220E8ULL);
for (element = 0; element < count; element++)
y[element] = ft_tt_vector_random_value();
for (element = 0; element < count; element++)
z[element] = ft_tt_vector_random_value();
for (element = 0; element < cols; element++)
gamma[element] = 0.5f + ft_tt_vector_random_value() * 0.25f;
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_RMS_NORM, &pass0, &pass1, &final_word);
for (block = 0; block < rows / block_rows; block++) {
const size_t base = (size_t)block * block_rows * cols;
uint32_t row;
/* Step one: the Float32 reduce, exactly as it ships. */
ft_tt_vector_model_head_norm_float32(pass0, pass1, final_word, y + base, gamma,
1.0f / (float)cols,
eps * (float)cols / (float)FT_TT_OP_TILE_DIM,
normed, block_rows, cols, 0u);
ft_tt_vector_reference_rms_norm(y + base, gamma, eps, want, block_rows, cols);
for (row = 0; row < block_rows; row++)
reduce_alone = row_max_norm_relative(want + (size_t)row * cols,
normed + (size_t)row * cols, cols,
reduce_alone);
/* Step two AS SHIPPED: that answer rendered into a bf16 strip and gated on the bf16
* ELEMENTWISE residency. `pn_tt_silu_mul(z, normed)` is silu(a) * b with a = z. */
ft_tt_vector_model_eltwise(FT_TT_OP_VECTOR_FN_SILU_MUL, z + base, normed, bridged,
(size_t)block_rows * cols);
for (element = 0; element < (size_t)block_rows * cols; element++)
want[element] = (float)((double)want[element] *
((double)z[base + element] /
(1.0 + exp(-(double)z[base + element]))));
for (row = 0; row < block_rows; row++)
gated_bridged = row_max_norm_relative(want + (size_t)row * cols,
bridged + (size_t)row * cols, cols,
gated_bridged);
/* THE COUNTERFACTUAL: the same gate with no render in front of it. Not a thing the
* card can do today — that is the qkv/z strip bridge — and it exists to attribute. */
for (element = 0; element < (size_t)block_rows * cols; element++) {
const float gate = z[base + element] / (1.0f + expf(-z[base + element]));
bridged[element] = normed[element] * gate;
}
for (row = 0; row < block_rows; row++)
gated_no_bridge = row_max_norm_relative(want + (size_t)row * cols,
bridged + (size_t)row * cols, cols,
gated_no_bridge);
}
ft_tt_test_check_fmt("the bf16 gate costs MORE than the whole Float32 reduce does",
gated_bridged > reduce_alone * 2.0,
"reduce alone %.3e, then gated %.3e", reduce_alone, gated_bridged);
ft_tt_test_check_fmt("and removing the render puts `gated` back at the reduce's own figure",
gated_no_bridge < gated_bridged * 0.5,
"with the bf16 bridge %.3e, without it %.3e — a %.1fx term the "
"reduce does not own",
gated_bridged, gated_no_bridge, gated_bridged / gated_no_bridge);
ft_tt_test_check_fmt("so `gated` cannot reach 1e-3 while it crosses into bf16, whatever "
"the reduce does",
gated_bridged > 1e-3, "%.3e against a 1e-3 gate", gated_bridged);
done:
free(y);
free(z);
free(gamma);
free(normed);
free(want);
free(bridged);
}
static void test_reduce_parameters(void)
{
ft_tt_op_reduce_request request;
ft_tt_op_reduce_residency residency;
ft_tt_op_reduce_function softmax = FT_TT_OP_REDUCE_FN_SOFTMAX;
uint32_t words[FT_TT_OP_VECTOR_WORDS];
uint32_t pass0 = 0u;
uint32_t pass1 = 0u;
uint32_t final_word = 0u;
uint32_t ceiling;
int rc;
ft_tt_test_step("reduce parameter block");
memset(&request, 0, sizeof(request));
request.function = FT_TT_OP_REDUCE_FN_SOFTMAX;
request.row_blocks = 2u;
request.tile_cols = 48u;
request.prefetch_tiles = 32u;
request.input = make_operand(3u, 4u, 0x100000u);
request.constants = make_operand(5u, 6u, 0x200000u);
request.output = make_operand(7u, 8u, 0x300000u);
rc = ft_tt_op_reduce_parameters(&request, words);
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_SOFTMAX, &pass0, &pass1, &final_word);
ft_tt_test_check_fmt("a well-formed request encodes", rc == FT_TT_OK, "rc %d", rc);
ft_tt_test_check_fmt("tile count is row_blocks * tile_cols",
words[FT_TT_OP_VECTOR_TILE_COUNT] == 96u, "%u",
words[FT_TT_OP_VECTOR_TILE_COUNT]);
ft_tt_test_check_fmt("tile_cols lands in word 2", words[FT_TT_OP_VECTOR_TILE_COLS] == 48u,
"%u", words[FT_TT_OP_VECTOR_TILE_COLS]);
/* The run a reduce block may not span is Wt, not the total: a block that crossed a row
* boundary would hand the compute thread half of two different rows. */
ft_tt_test_check_fmt("depth divides Wt, not the total",
words[FT_TT_OP_VECTOR_DEPTH] == 24u, "%u", words[FT_TT_OP_VECTOR_DEPTH]);
ft_tt_test_check_fmt("the program lands in words 13, 14 and 15",
words[FT_TT_OP_REDUCE_PASS0] == pass0 &&
words[FT_TT_OP_REDUCE_PASS1] == pass1 &&
words[FT_TT_OP_REDUCE_FINAL] == final_word,
"0x%08X 0x%08X 0x%08X", words[FT_TT_OP_REDUCE_PASS0],
words[FT_TT_OP_REDUCE_PASS1], words[FT_TT_OP_REDUCE_FINAL]);
/* The one word the family must NOT use, because it means something else to the
* elementwise binary that shares this block layout. */
ft_tt_test_check_fmt("word 0 stays 0, so it cannot be read as an eltwise function",
words[FT_TT_OP_VECTOR_FUNCTION] == 0u, "%u",
words[FT_TT_OP_VECTOR_FUNCTION]);
memset(&residency, 0, sizeof(residency));
residency.max_tile_cols = 1u;
residency.tile_bytes = FT_TT_OP_TILE_BYTES_BF16;
ceiling = ft_tt_op_reduce_max_tile_cols(&residency);
ft_tt_test_check_fmt("a softmax residency's Wt ceiling is the arena's", ceiling >= 300u,
"Wt <= %u, i.e. W <= %u", ceiling, ceiling * FT_TT_OP_TILE_DIM);
request.tile_cols = ceiling;
rc = ft_tt_op_reduce_parameters(&request, words);
ft_tt_test_check_fmt("the ceiling itself is accepted", rc == FT_TT_OK, "rc %d at Wt %u", rc,
ceiling);
request.tile_cols = ceiling + 1u;
rc = ft_tt_op_reduce_parameters(&request, words);
ft_tt_test_check_fmt("one past the ceiling is refused", rc == FT_TT_EINVAL, "rc %d at Wt %u",
rc, ceiling + 1u);
/* 32k of attention context is Wt = 1024, and it does not fit. This assertion is the
* capacity argument for flash-decode attention, written down where it can go red. */
ft_tt_test_check_fmt("a 32k attention row does NOT fit a standalone softmax",
ceiling < 1024u, "Wt ceiling %u vs 1024 needed", ceiling);
/* A residency that also carries gamma and beta holds FOUR rows, not two, so its ceiling
* is half. Naming both is what stops "the softmax number" being quoted for a norm. */
{
static const ft_tt_op_reduce_function all[] = {
FT_TT_OP_REDUCE_FN_SOFTMAX, FT_TT_OP_REDUCE_FN_RMS_NORM,
FT_TT_OP_REDUCE_FN_LAYER_NORM, FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE
};
ft_tt_op_reduce_residency full;
uint32_t full_ceiling;
rc = ft_tt_op_reduce_residency_for(all, sizeof(all) / sizeof(all[0]), 48u,
FT_TT_OP_TILE_BYTES_BF16, &full);
full_ceiling = ft_tt_op_reduce_max_tile_cols(&full);
ft_tt_test_check_fmt("a residency for all four carries gamma and beta",
rc == FT_TT_OK && full.carries_gamma && full.carries_beta, "rc %d",
rc);
ft_tt_test_check_fmt("and its ceiling is about half the softmax-only one",
full_ceiling * 2u <= ceiling + 4u && full_ceiling * 2u + 8u >= ceiling,
"four rows Wt <= %u against two rows Wt <= %u", full_ceiling,
ceiling);
rc = ft_tt_op_reduce_residency_for(all, sizeof(all) / sizeof(all[0]), full_ceiling + 1u,
FT_TT_OP_TILE_BYTES_BF16, &full);
ft_tt_test_check_fmt("one past THAT ceiling is refused at residency time",
rc == FT_TT_ENOMEM, "rc %d", rc);
}
request.tile_cols = 48u;
request.row_blocks = 0u;
rc = ft_tt_op_reduce_parameters(&request, words);
ft_tt_test_check_fmt("a zero row-block count is refused", rc == FT_TT_EINVAL, "rc %d", rc);
request.row_blocks = 2u;
request.function = (ft_tt_op_reduce_function)0;
rc = ft_tt_op_reduce_parameters(&request, words);
ft_tt_test_check_fmt("an unknown member is refused", rc == FT_TT_EINVAL, "rc %d", rc);
(void)softmax;
}
/* ================================================================
* Circular-buffer placement
* ================================================================ */
/** Every placed CB must lie in the arena, be 64-byte aligned and overlap nothing. */
static void check_placement(const char *label, const ft_tt_launch_cb_entry *entries,
unsigned count)
{
unsigned index;
unsigned misaligned = 0;
unsigned outside = 0;
unsigned overlapping = 0;
for (index = 0; index < count; index++) {
const uint32_t base = entries[index].address;
const uint32_t bytes = entries[index].size_bytes;
unsigned other;
if ((base & 63u) != 0u)
misaligned++;
if (base < FT_TT_OP_CB_ARENA_BASE || base + bytes > FT_TT_OP_CB_ARENA_END)
outside++;
for (other = 0; other < index; other++) {
const uint32_t other_base = entries[other].address;
const uint32_t other_bytes =
entries[other].size_bytes;
if (base < other_base + other_bytes && other_base < base + bytes)
overlapping++;
}
}
ft_tt_test_check_fmt(label, misaligned == 0 && outside == 0 && overlapping == 0,
"%u misaligned, %u outside the arena, %u overlapping", misaligned,
outside, overlapping);
}
static void test_eltwise_placement(void)
{
ft_tt_op_eltwise_request request;
ft_tt_launch_cb_entry entries[FT_TT_OP_ELTWISE_CB_COUNT];
uint64_t mask = 0;
int rc;
ft_tt_test_step("elementwise circular buffers");
memset(&request, 0, sizeof(request));
request.function = FT_TT_OP_VECTOR_FN_SILU_MUL;
request.tile_count = 96u;
request.prefetch_tiles = 48u;
request.input_a = make_operand(3u, 4u, 0x100000u);
request.input_b = make_operand(5u, 6u, 0x200000u);
request.output = make_operand(7u, 8u, 0x300000u);
rc = ft_tt_op_eltwise_place_cbs(&request, entries, g_cb_buffer, sizeof(g_cb_buffer), &mask);
ft_tt_test_check_fmt("four CBs place", rc == FT_TT_OK, "rc %d", rc);
check_placement("they are aligned, inside the arena and disjoint", entries,
FT_TT_OP_ELTWISE_CB_COUNT);
ft_tt_test_check_fmt("both inputs hold two blocks",
entries[0].page_count == 96u && entries[1].page_count == 96u, "%u, %u",
entries[0].page_count, entries[1].page_count);
ft_tt_test_check_fmt("the mask names exactly the four CBs used",
mask == ((1ull << FT_TT_OP_ELTWISE_CB_IN0) |
(1ull << FT_TT_OP_ELTWISE_CB_IN1) |
(1ull << FT_TT_OP_ELTWISE_CB_SCRATCH) |
(1ull << FT_TT_OP_ELTWISE_CB_OUT)),
"0x%llX", (unsigned long long)mask);
/* A unary function still places cb_in1, so the KERNEL alone decides arity and the host
* never has a table that can disagree with it. */
request.function = FT_TT_OP_VECTOR_FN_SILU;
memset(&request.input_b, 0, sizeof(request.input_b));
rc = ft_tt_op_eltwise_place_cbs(&request, entries, g_cb_buffer, sizeof(g_cb_buffer), &mask);
ft_tt_test_check_fmt("a unary request still places cb_in1",
rc == FT_TT_OK && entries[1].page_size_bytes == FT_TT_OP_TILE_BYTES_BF16,
"rc %d, page %u", rc, entries[1].page_size_bytes);
/* Past the arena it must REFUSE, not truncate: a truncated CB is a kernel writing over
* the queue region at the top of L1. */
request.function = FT_TT_OP_VECTOR_FN_ADD;
request.input_b = make_operand(5u, 6u, 0x200000u);
request.tile_count = 4096u;
request.prefetch_tiles = 4096u;
rc = ft_tt_op_eltwise_place_cbs(&request, entries, g_cb_buffer, sizeof(g_cb_buffer), &mask);
ft_tt_test_check_fmt("a depth past the arena is refused", rc == FT_TT_ENOMEM, "rc %d", rc);
}
static void test_reduce_placement(void)
{
static const ft_tt_op_reduce_function all[] = {
FT_TT_OP_REDUCE_FN_SOFTMAX, FT_TT_OP_REDUCE_FN_RMS_NORM,
FT_TT_OP_REDUCE_FN_LAYER_NORM, FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE
};
ft_tt_op_reduce_residency residency;
ft_tt_launch_cb_entry entries[FT_TT_OP_REDUCE_CB_COUNT];
uint64_t mask = 0;
uint32_t ceiling;
int rc;
ft_tt_test_step("reduce circular buffers");
rc = ft_tt_op_reduce_residency_for(all, sizeof(all) / sizeof(all[0]), 48u,
FT_TT_OP_TILE_BYTES_BF16, &residency);
if (rc == FT_TT_OK)
rc = ft_tt_op_reduce_place_cbs(&residency, entries, g_cb_buffer, sizeof(g_cb_buffer),
&mask);
ft_tt_test_check_fmt("eleven CBs place", rc == FT_TT_OK, "rc %d", rc);
check_placement("they are aligned, inside the arena and disjoint", entries,
FT_TT_OP_REDUCE_CB_COUNT);
ft_tt_test_check_fmt("cb_in, cb_mid, gamma and beta are each a whole row deep",
entries[0].page_count == 48u && entries[4].page_count == 48u &&
entries[6].page_count == 48u && entries[7].page_count == 48u,
"%u, %u, %u, %u", entries[0].page_count, entries[4].page_count,
entries[6].page_count, entries[7].page_count);
/* A softmax-only residency must still PLACE gamma and beta, at one page. An unplaced CB
* a descriptor then asks for is a NoC write to an address nobody chose (spec §7.3); a
* one-page CB is a cb_reserve_back that blocks and is retired by its bound. */
{
ft_tt_op_reduce_function softmax = FT_TT_OP_REDUCE_FN_SOFTMAX;
rc = ft_tt_op_reduce_residency_for(&softmax, 1u, 48u, FT_TT_OP_TILE_BYTES_BF16,
&residency);
if (rc == FT_TT_OK)
rc = ft_tt_op_reduce_place_cbs(&residency, entries, g_cb_buffer, sizeof(g_cb_buffer),
&mask);
ft_tt_test_check_fmt("a softmax-only residency still places gamma and beta, at one page",
rc == FT_TT_OK && entries[6].page_count == 1u &&
entries[7].page_count == 1u,
"rc %d, gamma %u pages, beta %u pages", rc, entries[6].page_count,
entries[7].page_count);
ft_tt_test_check_fmt("the mask names all eleven",
mask == ((1ull << FT_TT_OP_REDUCE_CB_IN) |
(1ull << FT_TT_OP_REDUCE_CB_SCALER) |
(1ull << FT_TT_OP_REDUCE_CB_COL0) |
(1ull << FT_TT_OP_REDUCE_CB_COL1) |
(1ull << FT_TT_OP_REDUCE_CB_MID) |
(1ull << FT_TT_OP_REDUCE_CB_EPS) |
(1ull << FT_TT_OP_REDUCE_CB_GAMMA) |
(1ull << FT_TT_OP_REDUCE_CB_BETA) |
(1ull << FT_TT_OP_REDUCE_CB_STAGE0) |
(1ull << FT_TT_OP_REDUCE_CB_STAGE1) |
(1ull << FT_TT_OP_REDUCE_CB_OUT)),
"0x%llX", (unsigned long long)mask);
}
/* The documented ceiling has to be the REAL one. A ceiling that is merely conservative is
* a number nobody can plan against, and one that is optimistic is an ENOMEM on the card. */
ceiling = ft_tt_op_reduce_max_tile_cols(&residency);
residency.max_tile_cols = ceiling;
rc = ft_tt_op_reduce_place_cbs(&residency, entries, g_cb_buffer, sizeof(g_cb_buffer), &mask);
ft_tt_test_check_fmt("the ceiling places", rc == FT_TT_OK, "rc %d at Wt %u", rc, ceiling);
residency.max_tile_cols = ceiling + 1u;
rc = ft_tt_op_reduce_place_cbs(&residency, entries, g_cb_buffer, sizeof(g_cb_buffer), &mask);
ft_tt_test_check_fmt("one past the ceiling does not place", rc == FT_TT_ENOMEM,
"rc %d at Wt %u", rc, ceiling + 1u);
}
/* ================================================================
* The constants buffer
* ================================================================ */
/** Untilize one constant tile and return its first element, plus a count of the elements
* that disagree with it — a constant tile that is not constant is the interesting failure. */
static float constant_tile_value(const unsigned char *tile, unsigned *out_wrong)
{
uint16_t plain[FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM];
float first;
size_t index;
*out_wrong = 0u;
if (ft_tt_untilize_strip(tile, sizeof(uint16_t), FT_TT_OP_TILE_DIM, FT_TT_OP_TILE_DIM,
FT_TT_OP_TILE_DIM * sizeof(uint16_t), plain) != 0) {
*out_wrong = 1u;
return 0.0f;
}
first = ft_float_from_bf16(plain[0]);
for (index = 0; index < FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM; index++) {
if (ft_float_from_bf16(plain[index]) != first)
(*out_wrong)++;
}
return first;
}
/** `constant_tile_value()` for a Float32 constants buffer: 4-byte elements, and the value is
* compared EXACTLY because an fp32 tile has nothing to round it. */
static float constant_tile_value_float32(const unsigned char *tile, unsigned *out_wrong)
{
float plain[FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM];
float first;
size_t index;
*out_wrong = 0u;
if (ft_tt_untilize_strip(tile, sizeof(float), FT_TT_OP_TILE_DIM, FT_TT_OP_TILE_DIM,
FT_TT_OP_TILE_DIM * sizeof(float), plain) != 0) {
*out_wrong = 1u;
return 0.0f;
}
first = plain[0];
for (index = 0; index < FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM; index++) {
if (plain[index] != first)
(*out_wrong)++;
}
return first;
}
/**
* THE SAME CONSTANTS BUFFER AT Float32, WHICH IS THE ONE HOST PIECE `reduce_float32.cbtable`
* NEEDS AND THE ONE THAT CANNOT FAIL LOUDLY.
*
* A constants buffer built at 2048 under a program placed at 4096 is not refused by anything
* on the card: the reader issues 4096-byte reads, there is no descriptor header on a
* data-movement core, and the result is a scaler tile made of half a tile and whatever
* followed it — a smoothly wrong norm. So the page size is a parameter of the builder and
* this is what pins it.
*
* The GDN head width is 128, which is what the two per-head norms run at.
*/
static void test_reduce_constants_float32(void)
{
const uint32_t width = 128u;
const uint32_t tile_cols = width / FT_TT_OP_TILE_DIM;
const float eps = 1e-6f;
const size_t bf16_bytes =
ft_tt_op_reduce_constants_bytes(FT_TT_OP_REDUCE_FN_RMS_NORM, tile_cols,
FT_TT_OP_TILE_BYTES_BF16);
const size_t bytes = ft_tt_op_reduce_constants_bytes(FT_TT_OP_REDUCE_FN_RMS_NORM, tile_cols,
FT_TT_OP_TILE_BYTES_FP32);
float *gamma = malloc(width * sizeof(float));
unsigned char *buffer = malloc(bytes);
unsigned wrong = 0u;
uint32_t index;
int rc;
ft_tt_test_step("the constants buffer at Float32 — the host half of reduce_float32");
if (!gamma || !buffer) {
ft_tt_test_check("allocation", false, "out of memory");
goto done;
}
ft_tt_vector_random_seed(0x1220E9ULL);
for (index = 0; index < width; index++)
gamma[index] = 0.5f + ft_tt_vector_random_value() * 0.25f;
ft_tt_test_check_fmt("an fp32 constants buffer is exactly twice a bf16 one",
bytes == 2u * bf16_bytes &&
bytes == (size_t)(2u + tile_cols) * FT_TT_OP_TILE_BYTES_FP32,
"%zu vs %zu bytes at Wt=%u", bytes, bf16_bytes, tile_cols);
rc = ft_tt_op_reduce_build_constants(FT_TT_OP_REDUCE_FN_RMS_NORM, width, eps, gamma, NULL,
FT_TT_OP_TILE_BYTES_FP32, buffer, bytes);
ft_tt_test_check_fmt("it builds at Float32", rc == FT_TT_OK, "rc %d", rc);
if (rc != FT_TT_OK)
goto done;
/* EXACT equality, not a bf16 round trip — that IS the difference this table buys, and a
* builder that still went through bf16 would fail here and pass every bf16 check. */
ft_tt_test_check_fmt("slot 0 is 1/W to the bit, with no bf16 rounding in it",
constant_tile_value_float32(buffer, &wrong) == 1.0f / (float)width &&
wrong == 0u,
"%.9g against %.9g, %u elements disagreeing",
(double)constant_tile_value_float32(buffer, &wrong),
(double)(1.0f / (float)width), wrong);
ft_tt_test_check_fmt("slot 1 is eps*W/32 to the bit",
constant_tile_value_float32(buffer + FT_TT_OP_TILE_BYTES_FP32,
&wrong) ==
eps * (float)width / (float)FT_TT_OP_TILE_DIM &&
wrong == 0u,
"%.9g",
(double)constant_tile_value_float32(buffer + FT_TT_OP_TILE_BYTES_FP32,
&wrong));
/* gamma, ROW-REPLICATED at the fp32 stride. A builder that kept the bf16 tile stride
* would put tile 1 half a tile early and this is what sees it. */
{
float plain[FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM];
unsigned bad = 0u;
uint32_t tile;
for (tile = 0; tile < tile_cols; tile++) {
const unsigned char *source =
buffer + (size_t)(2u + tile) * FT_TT_OP_TILE_BYTES_FP32;
uint32_t row;
uint32_t column;
(void)ft_tt_untilize_strip(source, sizeof(float), FT_TT_OP_TILE_DIM,
FT_TT_OP_TILE_DIM, FT_TT_OP_TILE_DIM * sizeof(float),
plain);
for (row = 0; row < FT_TT_OP_TILE_DIM; row++) {
for (column = 0; column < FT_TT_OP_TILE_DIM; column++) {
if (plain[row * FT_TT_OP_TILE_DIM + column] !=
gamma[tile * FT_TT_OP_TILE_DIM + column])
bad++;
}
}
}
ft_tt_test_check_fmt("gamma is row-replicated at the fp32 tile stride, bit-exact",
bad == 0u, "%u of %u elements wrong", bad,
tile_cols * FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM);
}
/* The L2 member, which is the q/k arm and the one whose eps FOLLOWS THE SCALER: it sums
* rather than means, so the eps tile is eps/32 and not eps*W/32. */
rc = ft_tt_op_reduce_build_constants(FT_TT_OP_REDUCE_FN_L2_NORM, width, eps * eps, NULL,
NULL, FT_TT_OP_TILE_BYTES_FP32, buffer, bytes);
ft_tt_test_check_fmt("the L2 member builds at Float32 with scaler 1.0 and eps/32",
rc == FT_TT_OK &&
constant_tile_value_float32(buffer, &wrong) == 1.0f &&
constant_tile_value_float32(buffer + FT_TT_OP_TILE_BYTES_FP32,
&wrong) ==
eps * eps / (float)FT_TT_OP_TILE_DIM,
"rc %d, scaler %.9g, eps tile %.9g", rc,
(double)constant_tile_value_float32(buffer, &wrong),
(double)constant_tile_value_float32(buffer + FT_TT_OP_TILE_BYTES_FP32,
&wrong));
/* THE TWO REFUSALS THAT MATTER, because both are the silent-wrong-answer shape. */
rc = ft_tt_op_reduce_build_constants(FT_TT_OP_REDUCE_FN_RMS_NORM, width, eps, gamma, NULL,
FT_TT_OP_TILE_BYTES_FP32, buffer, bf16_bytes);
ft_tt_test_check_fmt("a buffer SIZED at bf16 and BUILT at fp32 is refused, not overrun",
rc == FT_TT_EINVAL, "rc %d", rc);
rc = ft_tt_op_reduce_build_constants(FT_TT_OP_REDUCE_FN_RMS_NORM, width, eps, gamma, NULL,
1024u, buffer, bytes);
ft_tt_test_check_fmt("a page size that is neither bf16 nor fp32 is refused",
rc == FT_TT_EINVAL, "rc %d", rc);
ft_tt_test_check_fmt("and so is sizing one",
ft_tt_op_reduce_constants_bytes(FT_TT_OP_REDUCE_FN_RMS_NORM, tile_cols,
0u) == 0u,
"0 bytes at page size 0, Wt %u", tile_cols);
done:
free(gamma);
free(buffer);
}
static void test_reduce_constants(void)
{
const uint32_t width = 1536u;
const uint32_t tile_cols = width / FT_TT_OP_TILE_DIM;
const float eps = 1e-5f;
float *gamma = malloc(width * sizeof(float));
float *beta = malloc(width * sizeof(float));
unsigned char *buffer;
size_t bytes;
unsigned wrong = 0u;
uint32_t index;
int rc;
ft_tt_test_step("the reduce constants buffer");
bytes = ft_tt_op_reduce_constants_bytes(FT_TT_OP_REDUCE_FN_LAYER_NORM, tile_cols,
FT_TT_OP_TILE_BYTES_BF16);
buffer = malloc(bytes);
if (!gamma || !beta || !buffer) {
ft_tt_test_check("host buffers allocate", false, "out of memory");
goto done;
}
for (index = 0; index < width; index++) {
gamma[index] = 0.5f + (float)index * 0.001f;
beta[index] = -0.25f + (float)index * 0.0005f;
}
ft_tt_test_check_fmt("layer norm asks for 2 + 2*Wt tiles",
bytes == (size_t)(2u + 2u * tile_cols) * FT_TT_OP_TILE_BYTES_BF16,
"%zu bytes for Wt %u", bytes, tile_cols);
ft_tt_test_check_fmt("softmax asks for the two constant tiles and no stream",
ft_tt_op_reduce_constants_bytes(FT_TT_OP_REDUCE_FN_SOFTMAX, tile_cols,
FT_TT_OP_TILE_BYTES_BF16) ==
2u * FT_TT_OP_TILE_BYTES_BF16,
"%zu",
ft_tt_op_reduce_constants_bytes(FT_TT_OP_REDUCE_FN_SOFTMAX, tile_cols,
FT_TT_OP_TILE_BYTES_BF16));
rc = ft_tt_op_reduce_build_constants(FT_TT_OP_REDUCE_FN_LAYER_NORM, width, eps, gamma, beta,
FT_TT_OP_TILE_BYTES_BF16, buffer, bytes);
ft_tt_test_check_fmt("it builds", rc == FT_TT_OK, "rc %d", rc);
/* Slot 0: the scaler. 1/W for a member that wants a MEAN. */
ft_tt_test_check_fmt("slot 0 is the 1/W scaler",
constant_tile_value(buffer, &wrong) ==
ft_float_from_bf16(ft_bf16_from_float(1.0f / (float)width)) &&
wrong == 0u,
"%g with %u elements disagreeing",
(double)constant_tile_value(buffer, &wrong), wrong);
/* Slot 1: eps*W/32, which is the whole trick — each row of that tile has 32 columns, so
* 32 * (eps*W/32) * (1/W) = eps, and eps costs one extra reduce_tile instead of a
* broadcast add and a CB round trip. */
ft_tt_test_check_fmt("slot 1 is eps*W/32, so one reduce_tile adds exactly eps",
constant_tile_value(buffer + FT_TT_OP_TILE_BYTES_BF16, &wrong) ==
ft_float_from_bf16(ft_bf16_from_float(
eps * (float)width / (float)FT_TT_OP_TILE_DIM)) &&
wrong == 0u,
"%g",
(double)constant_tile_value(buffer + FT_TT_OP_TILE_BYTES_BF16, &wrong));
/* The gamma stream: each tile's every ROW must repeat that tile's 32 gamma values, which
* is what turns the multiply into a plain mul_tiles rather than a broadcast. */
{
uint16_t plain[FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM];
unsigned bad_columns = 0u;
unsigned bad_rows = 0u;
uint32_t tile;
for (tile = 0; tile < tile_cols; tile++) {
const unsigned char *source =
buffer + (size_t)(2u + tile) * FT_TT_OP_TILE_BYTES_BF16;
uint32_t row;
uint32_t column;
(void)ft_tt_untilize_strip(source, sizeof(uint16_t), FT_TT_OP_TILE_DIM,
FT_TT_OP_TILE_DIM, FT_TT_OP_TILE_DIM * sizeof(uint16_t),
plain);
for (column = 0; column < FT_TT_OP_TILE_DIM; column++) {
const float want =
ft_float_from_bf16(ft_bf16_from_float(gamma[tile * FT_TT_OP_TILE_DIM +
column]));
if (ft_float_from_bf16(plain[column]) != want)
bad_columns++;
for (row = 1; row < FT_TT_OP_TILE_DIM; row++) {
if (ft_float_from_bf16(plain[row * FT_TT_OP_TILE_DIM + column]) != want)
bad_rows++;
}
}
}
ft_tt_test_check_fmt("gamma expands to full tiles, row-replicated",
bad_columns == 0u && bad_rows == 0u,
"%u wrong columns, %u wrong replications", bad_columns, bad_rows);
}
rc = ft_tt_op_reduce_build_constants(FT_TT_OP_REDUCE_FN_LAYER_NORM, width, eps, gamma, NULL,
FT_TT_OP_TILE_BYTES_BF16, buffer, bytes);
ft_tt_test_check_fmt("layer norm without a beta is refused, not silently zeroed",
rc == FT_TT_EINVAL, "rc %d", rc);
rc = ft_tt_op_reduce_build_constants(FT_TT_OP_REDUCE_FN_LAYER_NORM, width, eps, gamma, beta,
FT_TT_OP_TILE_BYTES_BF16, buffer, 8u);
ft_tt_test_check_fmt("a short buffer is refused", rc == FT_TT_EINVAL, "rc %d", rc);
done:
free(gamma);
free(beta);
free(buffer);
}
/* ================================================================
* The model against the reference — the PCC prediction
* ================================================================ */
/** Score one array pair the way the hardware pass will, and print both metrics. */
static void score(const char *label, const float *expected, const float *actual, size_t count)
{
char census[192];
const size_t non_finite =
ft_tt_vector_count_non_finite(expected, actual, count, census, sizeof(census));
double pcc;
double worst;
if (non_finite != 0u) {
ft_tt_test_check_fmt(label, false, "NON-FINITE: %s", census);
return;
}
pcc = ft_tt_vector_pearson(expected, actual, count);
worst = ft_tt_vector_max_abs_difference(expected, actual, count);
ft_tt_test_check_fmt(label, pcc >= PCC_ACCEPT, "PCC %.9f, max abs %.6g", pcc, worst);
}
/**
* THE EPS TILE FOLLOWS THE SCALER, and a member that SUMS needs a different value from one
* that MEANS.
*
* The eps trick is one more accumulated row reduce, so the pass contributes
* `32 * tile_value * scaler`. At scaler 1/W that wants `eps*W/32`; at scaler 1.0 it wants
* `eps/32`. The unconditional `eps*W/32` that shipped before L2_NORM existed is wrong by a
* factor of W for any summing member — 128x at the GDN's head width — and nothing on the
* card can see it, because an eps that is too large is a smoothly wrong norm.
*/
static void test_eps_tile_follows_the_scaler(void)
{
const uint32_t width = 128u;
const uint32_t tile_cols = width / FT_TT_OP_TILE_DIM;
const float eps = 1e-5f;
unsigned char *buffer = NULL;
size_t bytes;
unsigned wrong = 0u;
int rc;
ft_tt_test_step("the eps tile is scaled by the member's own reduce scaler");
bytes = ft_tt_op_reduce_constants_bytes(FT_TT_OP_REDUCE_FN_L2_NORM, tile_cols,
FT_TT_OP_TILE_BYTES_BF16);
buffer = malloc(bytes);
if (!buffer) {
ft_tt_test_check("host buffer allocates", false, "out of memory");
return;
}
ft_tt_test_check_fmt("L2 norm asks for the two constant tiles and no stream",
bytes == 2u * FT_TT_OP_TILE_BYTES_BF16, "%zu bytes", bytes);
rc = ft_tt_op_reduce_build_constants(FT_TT_OP_REDUCE_FN_L2_NORM, width, eps, NULL, NULL,
FT_TT_OP_TILE_BYTES_BF16, buffer, bytes);
ft_tt_test_check_fmt("it builds with no gamma and no beta", rc == FT_TT_OK, "rc %d", rc);
ft_tt_test_check_fmt("slot 0 is 1.0, not 1/W — the reduce is a SUM",
constant_tile_value(buffer, &wrong) == 1.0f && wrong == 0u,
"%g with %u elements disagreeing",
(double)constant_tile_value(buffer, &wrong), wrong);
ft_tt_test_check_fmt("slot 1 is eps/32, not eps*W/32",
constant_tile_value(buffer + FT_TT_OP_TILE_BYTES_BF16, &wrong) ==
ft_float_from_bf16(ft_bf16_from_float(
eps / (float)FT_TT_OP_TILE_DIM)) &&
wrong == 0u,
"%g with %u elements disagreeing",
(double)constant_tile_value(buffer + FT_TT_OP_TILE_BYTES_BF16, &wrong),
wrong);
/* The two members that already shipped are UNMOVED: their scaler is a mean, so the
* value is the one they have always carried. */
{
float gamma[128];
unsigned char *rms;
uint32_t index;
for (index = 0; index < width; index++)
gamma[index] = 0.5f + (float)index * 0.001f;
bytes = ft_tt_op_reduce_constants_bytes(FT_TT_OP_REDUCE_FN_RMS_NORM, tile_cols,
FT_TT_OP_TILE_BYTES_BF16);
rms = malloc(bytes);
if (!rms) {
ft_tt_test_check("host buffer allocates", false, "out of memory");
free(buffer);
return;
}
rc = ft_tt_op_reduce_build_constants(FT_TT_OP_REDUCE_FN_RMS_NORM, width, eps, gamma,
NULL, FT_TT_OP_TILE_BYTES_BF16, rms, bytes);
ft_tt_test_check_fmt("RMS norm's eps tile is still eps*W/32",
rc == FT_TT_OK &&
constant_tile_value(rms + FT_TT_OP_TILE_BYTES_BF16, &wrong) ==
ft_float_from_bf16(ft_bf16_from_float(
eps * (float)width / (float)FT_TT_OP_TILE_DIM)),
"rc %d, %g", rc,
(double)constant_tile_value(rms + FT_TT_OP_TILE_BYTES_BF16,
&wrong));
free(rms);
}
free(buffer);
}
static void test_eltwise_model(void)
{
static const ft_tt_op_vector_function functions[] = {
FT_TT_OP_VECTOR_FN_SILU, FT_TT_OP_VECTOR_FN_SIGMOID, FT_TT_OP_VECTOR_FN_ADD,
FT_TT_OP_VECTOR_FN_MUL, FT_TT_OP_VECTOR_FN_SUB, FT_TT_OP_VECTOR_FN_SILU_MUL
};
const size_t count = 48u * FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM;
float *a = malloc(count * sizeof(float));
float *b = malloc(count * sizeof(float));
float *expected = malloc(count * sizeof(float));
float *actual = malloc(count * sizeof(float));
unsigned index;
ft_tt_test_step("elementwise: the tile-space model against the fp32 reference");
if (!a || !b || !expected || !actual) {
ft_tt_test_check("allocation", false, "out of memory");
free(a);
free(b);
free(expected);
free(actual);
return;
}
for (index = 0; index < sizeof(functions) / sizeof(functions[0]); index++) {
const ft_tt_op_vector_function function = functions[index];
const bool binary = ft_tt_op_vector_function_is_binary(function);
size_t element;
char label[96];
ft_tt_vector_random_seed(0x51EDULL + index);
for (element = 0; element < count; element++) {
a[element] = ft_tt_vector_random_value();
b[element] = ft_tt_vector_random_value();
}
/* The reference consumes the values the CARD will see — bf16-rounded — so the score
* is about the OP and not about a rounding the kernel is not responsible for. */
for (element = 0; element < count; element++) {
a[element] = ft_float_from_bf16(ft_bf16_from_float(a[element]));
b[element] = ft_float_from_bf16(ft_bf16_from_float(b[element]));
}
ft_tt_vector_reference_eltwise(function, a, binary ? b : NULL, expected, count);
ft_tt_vector_model_eltwise(function, a, binary ? b : NULL, actual, count);
snprintf(label, sizeof(label), "%s over %u tiles",
ft_tt_op_vector_function_name(function), 48u);
score(label, expected, actual, count);
}
free(a);
free(b);
free(expected);
free(actual);
}
static void test_softmax_model(void)
{
static const uint32_t widths[] = { 32u, 64u, 1536u, 4096u };
const uint32_t rows = 64u;
unsigned index;
ft_tt_test_step("softmax: the tile-space model against the fp32 reference");
for (index = 0; index < sizeof(widths) / sizeof(widths[0]); index++) {
const uint32_t cols = widths[index];
const size_t count = (size_t)rows * cols;
float *x = malloc(count * sizeof(float));
float *expected = malloc(count * sizeof(float));
float *actual = malloc(count * sizeof(float));
size_t element;
char label[96];
if (!x || !expected || !actual) {
ft_tt_test_check("allocation", false, "out of memory");
free(x);
free(expected);
free(actual);
return;
}
ft_tt_vector_random_seed(0x50F7ULL + index);
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(ft_tt_vector_random_value()));
ft_tt_vector_reference_softmax(x, expected, rows, cols);
ft_tt_vector_model_softmax(x, actual, rows, cols);
snprintf(label, sizeof(label), "%u x %u (Wt = %u)", rows, cols,
cols / FT_TT_OP_TILE_DIM);
score(label, expected, actual, count);
free(x);
free(expected);
free(actual);
}
}
/**
* The max-subtraction pass, proved load-bearing by the only data that can prove it.
*
* At the [-1, 1) range every other test uses, a softmax with NO max subtraction scores
* identically — exp(1) is 2.7 and nothing overflows. So a PCC gate on random small
* operands cannot see the difference, and a kernel that dropped the pass would ship green.
* At a range where exp overflows, the same arrangement is the difference between a result
* and a page of infinities.
*/
static void test_softmax_dynamic_range(void)
{
const uint32_t rows = 32u;
const uint32_t cols = 128u;
const size_t count = (size_t)rows * cols;
float *x = malloc(count * sizeof(float));
float *expected = malloc(count * sizeof(float));
float *actual = malloc(count * sizeof(float));
size_t element;
unsigned naive_non_finite = 0;
ft_tt_test_step("softmax: the range where the max pass stops being cosmetic");
if (!x || !expected || !actual) {
ft_tt_test_check("allocation", false, "out of memory");
free(x);
free(expected);
free(actual);
return;
}
ft_tt_vector_random_seed(0xBADFULL);
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(
ft_tt_vector_random_value_scaled(120.0f)));
/* What a max-free implementation would produce, computed here rather than assumed. */
for (element = 0; element < count; element++) {
if (!isfinite(expf(x[element])))
naive_non_finite++;
}
ft_tt_test_check_fmt("exp(x) alone overflows on this data", naive_non_finite > 0u,
"%u of %zu elements", naive_non_finite, count);
ft_tt_vector_reference_softmax(x, expected, rows, cols);
ft_tt_vector_model_softmax(x, actual, rows, cols);
score("with the max pass, 32 x 128 at range +/-120", expected, actual, count);
free(x);
free(expected);
free(actual);
}
/**
* The max must be taken over the WHOLE ROW, not over the first tile — and this is the only
* test in the file that can tell.
*
* FOUND BY A MUTATION THAT CAME BACK GREEN. Reducing the model's max over one tile instead
* of all Wt fails nothing above, and the reason is mathematics rather than a weak
* assertion: softmax(x - c) = softmax(x) for ANY constant c, so subtracting the wrong
* maximum changes nothing at all as long as no exponential overflows. Every PCC check in
* this file is therefore structurally blind to a partial-row max, and so is the on-card
* acceptance, and so would a reviewer be.
*
* What is not blind to it is a row whose largest element is NOT in the first tile and is
* more than ~88 above it — then a partial max leaves exp() overflowing and the answer is a
* page of infinities instead of a slightly different constant. That is this test: a plain
* [-1, 1) row with ONE outlier of +200 planted in the LAST tile.
*
* The general lesson is dev notes §3's, one op further on: a check with a mathematical
* invariance in it is a check that cannot see whatever the invariance absorbs.
*/
static void test_softmax_max_is_over_the_whole_row(void)
{
const uint32_t rows = 32u;
const uint32_t cols = 256u; /* Wt = 8, so "the first tile" is an eighth of the row */
const size_t count = (size_t)rows * cols;
float *x = malloc(count * sizeof(float));
float *expected = malloc(count * sizeof(float));
float *actual = malloc(count * sizeof(float));
size_t element;
uint32_t row;
ft_tt_test_step("softmax: the max is over the WHOLE row, not the first tile");
if (!x || !expected || !actual) {
ft_tt_test_check("allocation", false, "out of memory");
free(x);
free(expected);
free(actual);
return;
}
ft_tt_vector_random_seed(0x0177ULL);
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(ft_tt_vector_random_value()));
/* One outlier per row, in the LAST tile. 200 is well past the ~88 where exp overflows
* fp32, so a max taken over any earlier tile cannot rescue it. */
for (row = 0; row < rows; row++)
x[(size_t)row * cols + (cols - 3u)] = 200.0f;
ft_tt_vector_reference_softmax(x, expected, rows, cols);
ft_tt_vector_model_softmax(x, actual, rows, cols);
score("an outlier of +200 in the last tile, 32 x 256", expected, actual, count);
free(x);
free(expected);
free(actual);
}
/**
* The three new members' tile-space models against their fp32 references.
*
* Same purpose as the softmax model test: predict the on-card PCC before spending a core on
* it. §22 recorded that the model called silicon to ~6 decimals for the two ops it had, so
* a prediction here is worth making and worth checking against the hardware pass afterwards.
*/
static void test_reduce_models(void)
{
static const uint32_t widths[] = { 32u, 1536u };
const uint32_t rows = 64u;
const float eps = 1e-5f;
unsigned index;
ft_tt_test_step("RMS norm, layer norm and the router: model against reference");
for (index = 0; index < sizeof(widths) / sizeof(widths[0]); index++) {
const uint32_t cols = widths[index];
const size_t count = (size_t)rows * cols;
float *x = malloc(count * sizeof(float));
float *gamma = malloc(cols * sizeof(float));
float *beta = malloc(cols * sizeof(float));
float *expected = malloc(count * sizeof(float));
float *actual = malloc(count * sizeof(float));
size_t element;
char label[96];
if (!x || !gamma || !beta || !expected || !actual) {
ft_tt_test_check("allocation", false, "out of memory");
goto next;
}
ft_tt_vector_random_seed(0x4EDCE0ULL + index);
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(ft_tt_vector_random_value()));
for (element = 0; element < cols; element++) {
gamma[element] =
ft_float_from_bf16(ft_bf16_from_float(0.5f + ft_tt_vector_random_value() * 0.25f));
beta[element] =
ft_float_from_bf16(ft_bf16_from_float(ft_tt_vector_random_value() * 0.1f));
}
ft_tt_vector_reference_rms_norm(x, gamma, eps, expected, rows, cols);
ft_tt_vector_model_rms_norm(x, gamma, eps, actual, rows, cols);
snprintf(label, sizeof(label), "RMS norm %u x %u (Wt = %u)", rows, cols,
cols / FT_TT_OP_TILE_DIM);
score(label, expected, actual, count);
ft_tt_vector_reference_layer_norm(x, gamma, beta, eps, expected, rows, cols);
ft_tt_vector_model_layer_norm(x, gamma, beta, eps, actual, rows, cols);
snprintf(label, sizeof(label), "layer norm %u x %u (Wt = %u)", rows, cols,
cols / FT_TT_OP_TILE_DIM);
score(label, expected, actual, count);
/* Positive gates: x/sum(x) on signed noise divides by something near zero, which is
* a property of the TEST DATA and not of the op. */
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(0.05f + fabsf(x[element])));
ft_tt_vector_reference_router_normalise(x, expected, rows, cols);
ft_tt_vector_model_router_normalise(x, actual, rows, cols);
snprintf(label, sizeof(label), "router normalise %u x %u (Wt = %u)", rows, cols,
cols / FT_TT_OP_TILE_DIM);
score(label, expected, actual, count);
next:
free(x);
free(gamma);
free(beta);
free(expected);
free(actual);
}
}
/**
* PCC IS BLIND TO A GLOBAL SCALE, AND THREE OF THE FOUR MEMBERS ARE SCALE-SHAPED.
*
* This is §22.2's lesson in its second form. Softmax hides a wrong row maximum behind
* shift-invariance; RMS norm, layer norm and the router hide a wrong SCALE behind the fact
* that a correlation is invariant under multiplying one side by a constant. A reduce that
* summed the row twice, a scaler tile off by two, an rsqrt where a reciprocal belongs — all
* of them are exactly that, and every PCC gate in this file and in the on-card acceptance
* would pass them.
*
* So this test does the mutation itself, in the model, and asserts BOTH things: PCC does not
* move, and max-abs does. That is the evidence for scoring the family on a relative max-abs
* budget as well as on PCC, and it is what the on-card kernel mutation then confirms.
*/
static void test_pcc_is_blind_to_scale(void)
{
const uint32_t rows = 32u;
const uint32_t cols = 1536u;
const size_t count = (size_t)rows * cols;
float *x = malloc(count * sizeof(float));
float *gamma = malloc(cols * sizeof(float));
float *reference = malloc(count * sizeof(float));
float *scaled = malloc(count * sizeof(float));
size_t element;
double pcc;
double worst;
double peak = 0.0;
ft_tt_test_step("PCC cannot see a global scale — which is why max-abs is a CHECK here");
if (!x || !gamma || !reference || !scaled) {
ft_tt_test_check("allocation", false, "out of memory");
goto done;
}
ft_tt_vector_random_seed(0x5CA1EULL);
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(ft_tt_vector_random_value()));
for (element = 0; element < cols; element++)
gamma[element] = 1.0f;
ft_tt_vector_reference_rms_norm(x, gamma, 1e-5f, reference, rows, cols);
/* A reduce that accumulated each tile TWICE gives a mean square 2x too large, hence an
* output 1/sqrt(2) too small. That is a real mutation of the kernel's inner loop, and
* this is what it does to the answer. */
for (element = 0; element < count; element++)
scaled[element] = reference[element] * (float)(1.0 / sqrt(2.0));
for (element = 0; element < count; element++) {
if (fabs((double)reference[element]) > peak)
peak = fabs((double)reference[element]);
}
pcc = ft_tt_vector_pearson(reference, scaled, count);
worst = ft_tt_vector_max_abs_difference(reference, scaled, count);
ft_tt_test_check_fmt("a 29% scale error keeps PCC at 1.0 — the gate does not see it",
pcc >= 0.999999, "PCC %.9f", pcc);
ft_tt_test_check_fmt("and max-abs sees it at 100x the 5% budget",
worst > 0.05 * peak, "max abs %.6g against a %.6g budget", worst,
0.05 * peak);
done:
free(x);
free(gamma);
free(reference);
free(scaled);
}
/* ================================================================
* A RAGGED W, SERVED BY PADDING — and the one member it cannot serve
* ================================================================
*
* The reduce triad has no mask tile: it reduces over `tile_cols * 32` columns
* unconditionally, and a ragged W has no wire representation at all. What closes that for
* three of the four members is a caller that PADS — pad value from the member's own passes,
* DIVISOR from the real W — and what these checks are for is that the padded pipeline lands
* on the true-W reference, not merely near it.
*
* THE MODEL IS NOT TOLD HOW THE PADDING WAS DONE. It is handed the padded activation, the
* scaler and eps values UNTILIZED OUT OF the constants buffer the host built, the gamma row
* read back the same way, and the program words — i.e. only what the card would get. A model
* that recomputed 1/W itself would agree with a wrong builder and prove nothing (dev notes
* §3).
*
* AND LAYER NORM IS MEASURED, NOT ASSERTED. Its second pass squares (x - row mean), so a
* padded lane contributes (pad - mean)^2 and vanishes only at pad == the row's own mean. The
* last stage sweeps constant pads across rows with deliberately DIFFERENT means and shows
* every one of them wrong — which is why the builder refuses it instead of shipping it with
* a caveat.
*/
/** Widest padded row any stage below builds, for the fixed-size scratch the sweep uses. */
#define RAGGED_MAX_PADDED_COLS 1536u
/** Read a gamma/beta stream back out of a constants buffer, the way the card will index it:
* row 0 of each 32x32 tile, `tile_cols` tiles. Reading the BYTES rather than the caller's
* array is what makes the padded tail an observation instead of an assumption. */
static void read_broadcast_row(const unsigned char *tiles, uint32_t tile_cols, float *out)
{
uint32_t tile;
for (tile = 0; tile < tile_cols; tile++) {
uint16_t plain[FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM];
uint32_t column;
(void)ft_tt_untilize_strip(tiles + (size_t)tile * FT_TT_OP_TILE_BYTES_BF16,
sizeof(uint16_t), FT_TT_OP_TILE_DIM, FT_TT_OP_TILE_DIM,
FT_TT_OP_TILE_DIM * sizeof(uint16_t), plain);
for (column = 0; column < FT_TT_OP_TILE_DIM; column++)
out[tile * FT_TT_OP_TILE_DIM + column] = ft_float_from_bf16(plain[column]);
}
}
/** Test data shaped for the member: the router divides by a row sum, so it needs positive
* gates; softmax gets an ALL-NEGATIVE row, which is the arm a zero pad gets wrong twice
* over (a max clamped at 0, and a denominator with exp(0 - max) in it). */
static void fill_member_input(ft_tt_op_reduce_function function, float *out, size_t count)
{
size_t index;
for (index = 0; index < count; index++) {
const float value = ft_tt_vector_random_value();
if (function == FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE)
out[index] = ft_float_from_bf16(ft_bf16_from_float(0.05f + fabsf(value)));
else if (function == FT_TT_OP_REDUCE_FN_SOFTMAX)
out[index] = ft_float_from_bf16(ft_bf16_from_float(value - 4.0f));
else
out[index] = ft_float_from_bf16(ft_bf16_from_float(value));
}
}
static void reference_for(ft_tt_op_reduce_function function, const float *x, const float *gamma,
float eps, float *out, uint32_t rows, uint32_t cols)
{
if (function == FT_TT_OP_REDUCE_FN_SOFTMAX)
ft_tt_vector_reference_softmax(x, out, rows, cols);
else if (function == FT_TT_OP_REDUCE_FN_RMS_NORM)
ft_tt_vector_reference_rms_norm(x, gamma, eps, out, rows, cols);
else
ft_tt_vector_reference_router_normalise(x, out, rows, cols);
}
/**
* One member at one W through the padding caller, scored against the true-W reference, with
* an exact-W control at the PADDED width as the rounding floor.
*
* The control is what turns the verdict into a comparison instead of a magic constant: a
* padded lane contributes the pool's identity, so padding must cost nothing beyond the bf16
* rounding the same shape already pays.
*
* @return false only if the run could not be made (allocation); the checks report themselves
*/
static bool score_padded_member(ft_tt_op_reduce_function function, uint32_t width, float eps,
unsigned seed)
{
const uint32_t tile_cols = ft_tt_op_reduce_padded_tile_cols(width);
const uint32_t padded = tile_cols * FT_TT_OP_TILE_DIM;
const uint32_t rows = 32u;
const size_t padded_count = (size_t)rows * padded;
float *x = malloc((size_t)rows * width * sizeof(float));
float *x_padded = malloc(padded_count * sizeof(float));
float *gamma = malloc(padded * sizeof(float));
float *gamma_wire = malloc(padded * sizeof(float));
float *expected = malloc(padded_count * sizeof(float));
float *modelled = malloc(padded_count * sizeof(float));
float *sliced = malloc(padded_count * sizeof(float));
unsigned char *constants = NULL;
size_t constants_bytes =
ft_tt_op_reduce_constants_bytes(function, tile_cols, FT_TT_OP_TILE_BYTES_BF16);
double ragged_worst = 0.0;
double control_worst = 0.0;
double ragged_pcc = 0.0;
uint32_t pass0 = 0u;
uint32_t pass1 = 0u;
uint32_t final_word = 0u;
float scaler = 0.0f;
float eps_tile = 0.0f;
unsigned wrong = 0u;
uint32_t row;
size_t element;
char label[128];
bool made = false;
int rc;
constants = malloc(constants_bytes);
if (!x || !x_padded || !gamma || !gamma_wire || !expected || !modelled || !sliced ||
!constants)
goto done;
ft_tt_vector_random_seed(0x5AD9E0ULL + seed);
fill_member_input(function, x, (size_t)rows * width);
for (element = 0; element < padded; element++)
gamma[element] =
ft_float_from_bf16(ft_bf16_from_float(0.5f + ft_tt_vector_random_value() * 0.25f));
/* ---- the ragged arm: everything the card sees comes from the caller ---- */
rc = ft_tt_op_reduce_pad_rows(function, x, rows, width, x_padded, padded_count);
snprintf(label, sizeof(label), "%s W=%u: the row pads to Wt=%u",
ft_tt_op_reduce_function_name(function), width, tile_cols);
ft_tt_test_check_fmt(label, rc == FT_TT_OK, "rc %d", rc);
rc = ft_tt_op_reduce_build_constants_ragged(function, width, eps, gamma, NULL,
FT_TT_OP_TILE_BYTES_BF16, constants,
constants_bytes);
snprintf(label, sizeof(label), "%s W=%u: the constants build at the real W",
ft_tt_op_reduce_function_name(function), width);
ft_tt_test_check_fmt(label, rc == FT_TT_OK, "rc %d", rc);
if (rc != FT_TT_OK)
goto done;
scaler = constant_tile_value(constants, &wrong);
eps_tile = constant_tile_value(constants + FT_TT_OP_TILE_BYTES_BF16, &wrong);
if (ft_tt_op_reduce_uses_gamma(function))
read_broadcast_row(constants + 2u * FT_TT_OP_TILE_BYTES_BF16, tile_cols, gamma_wire);
else
memset(gamma_wire, 0, padded * sizeof(float));
(void)ft_tt_op_reduce_program(function, &pass0, &pass1, &final_word);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x_padded, gamma_wire, NULL,
scaler, eps_tile, modelled, rows, padded);
/* Only the real columns are the answer; the padded tail is scratch by construction. */
for (row = 0; row < rows; row++)
memcpy(sliced + (size_t)row * width, modelled + (size_t)row * padded,
width * sizeof(float));
reference_for(function, x, gamma, eps, expected, rows, width);
ragged_worst = ft_tt_vector_max_abs_difference(expected, sliced, (size_t)rows * width);
ragged_pcc = ft_tt_vector_pearson(expected, sliced, (size_t)rows * width);
/* ---- the control: the same shape at an exact W, which needs no padding at all ---- */
fill_member_input(function, x_padded, padded_count);
rc = ft_tt_op_reduce_build_constants(function, padded, eps, gamma, NULL,
FT_TT_OP_TILE_BYTES_BF16, constants, constants_bytes);
if (rc != FT_TT_OK) {
ft_tt_test_check_fmt("the exact-width control builds", false, "rc %d", rc);
goto done;
}
scaler = constant_tile_value(constants, &wrong);
eps_tile = constant_tile_value(constants + FT_TT_OP_TILE_BYTES_BF16, &wrong);
if (ft_tt_op_reduce_uses_gamma(function))
read_broadcast_row(constants + 2u * FT_TT_OP_TILE_BYTES_BF16, tile_cols, gamma_wire);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x_padded, gamma_wire, NULL,
scaler, eps_tile, modelled, rows, padded);
reference_for(function, x_padded, gamma, eps, expected, rows, padded);
control_worst = ft_tt_vector_max_abs_difference(expected, modelled, padded_count);
snprintf(label, sizeof(label), "%s W=%u: padding costs no more than the bf16 floor",
ft_tt_op_reduce_function_name(function), width);
ft_tt_test_check_fmt(label,
ragged_pcc >= PCC_ACCEPT &&
ragged_worst <= 4.0 * control_worst + 1e-9,
"max abs %.6g against an exact-W floor of %.6g, PCC %.9f",
ragged_worst, control_worst, ragged_pcc);
made = true;
done:
if (!made && !constants)
ft_tt_test_check("allocation", false, "out of memory");
free(x);
free(x_padded);
free(gamma);
free(gamma_wire);
free(expected);
free(modelled);
free(sliced);
free(constants);
return made;
}
/** The emulator against the four models it has to agree with, at an exact W. A model driven
* by the program words is only a fair instrument if it reproduces the hand-written ones
* where both apply — and it has to be BIT-exact, because both apply the same rounding at
* the same places and anything else would be an unexplained difference. */
static void test_program_model_matches_the_member_models(void)
{
const uint32_t rows = 32u;
const uint32_t cols = 1536u;
const uint32_t tile_cols = cols / FT_TT_OP_TILE_DIM;
const size_t count = (size_t)rows * cols;
const float eps = 1e-5f;
float *x = malloc(count * sizeof(float));
float *gamma = malloc(cols * sizeof(float));
float *beta = malloc(cols * sizeof(float));
float *from_member = malloc(count * sizeof(float));
float *from_program = malloc(count * sizeof(float));
const float scaler_mean = ft_float_from_bf16(ft_bf16_from_float(1.0f / (float)cols));
const float eps_tile = ft_float_from_bf16(
ft_bf16_from_float(eps * (float)cols / (float)FT_TT_OP_TILE_DIM));
uint32_t pass0 = 0u;
uint32_t pass1 = 0u;
uint32_t final_word = 0u;
size_t element;
(void)tile_cols;
ft_tt_test_step("the program-driven model reproduces the four member models exactly");
if (!x || !gamma || !beta || !from_member || !from_program) {
ft_tt_test_check("allocation", false, "out of memory");
goto done;
}
ft_tt_vector_random_seed(0x9C0DEFULL);
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(ft_tt_vector_random_value()));
for (element = 0; element < cols; element++) {
gamma[element] =
ft_float_from_bf16(ft_bf16_from_float(0.5f + ft_tt_vector_random_value() * 0.25f));
beta[element] = ft_float_from_bf16(ft_bf16_from_float(ft_tt_vector_random_value() * 0.1f));
}
ft_tt_vector_model_rms_norm(x, gamma, eps, from_member, rows, cols);
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_RMS_NORM, &pass0, &pass1, &final_word);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x, gamma, beta, scaler_mean,
eps_tile, from_program, rows, cols);
ft_tt_test_check_fmt("RMS norm", ft_tt_vector_max_abs_difference(from_member, from_program,
count) == 0.0,
"max abs %.6g",
ft_tt_vector_max_abs_difference(from_member, from_program, count));
ft_tt_vector_model_layer_norm(x, gamma, beta, eps, from_member, rows, cols);
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_LAYER_NORM, &pass0, &pass1, &final_word);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x, gamma, beta, scaler_mean,
eps_tile, from_program, rows, cols);
ft_tt_test_check_fmt("layer norm", ft_tt_vector_max_abs_difference(from_member, from_program,
count) == 0.0,
"max abs %.6g",
ft_tt_vector_max_abs_difference(from_member, from_program, count));
ft_tt_vector_model_softmax(x, from_member, rows, cols);
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_SOFTMAX, &pass0, &pass1, &final_word);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x, gamma, beta, 1.0f, 0.0f,
from_program, rows, cols);
ft_tt_test_check_fmt("softmax",
ft_tt_vector_max_abs_difference(from_member, from_program, count) == 0.0,
"max abs %.6g",
ft_tt_vector_max_abs_difference(from_member, from_program, count));
for (element = 0; element < count; element++)
x[element] = ft_float_from_bf16(ft_bf16_from_float(0.05f + fabsf(x[element])));
ft_tt_vector_model_router_normalise(x, from_member, rows, cols);
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE, &pass0, &pass1,
&final_word);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x, gamma, beta, 1.0f, 0.0f,
from_program, rows, cols);
ft_tt_test_check_fmt("router normalise",
ft_tt_vector_max_abs_difference(from_member, from_program, count) == 0.0,
"max abs %.6g",
ft_tt_vector_max_abs_difference(from_member, from_program, count));
done:
free(x);
free(gamma);
free(beta);
free(from_member);
free(from_program);
}
/** The derivation itself: which member takes which pad, and what ceil(W/32) is. */
static void test_ragged_padding_contract(void)
{
float value = 0.0f;
int rc;
ft_tt_test_step("a ragged W: the pad value each member's own passes ask for");
ft_tt_test_check_fmt("Wt is ceil(W/32)",
ft_tt_op_reduce_padded_tile_cols(1500u) == 47u &&
ft_tt_op_reduce_padded_tile_cols(1536u) == 48u &&
ft_tt_op_reduce_padded_tile_cols(1u) == 1u &&
ft_tt_op_reduce_padded_tile_cols(0u) == 0u,
"1500 -> %u, 1536 -> %u", ft_tt_op_reduce_padded_tile_cols(1500u),
ft_tt_op_reduce_padded_tile_cols(1536u));
/* Asserted as the PROPERTY the pad has to have — far below any activation, and finite so
* the subtraction cannot overflow — rather than against the macro, which would make the
* check agree with whatever the macro says (dev notes §3). */
rc = ft_tt_op_reduce_pad_value(FT_TT_OP_REDUCE_FN_SOFTMAX, &value);
ft_tt_test_check_fmt("softmax pads far below any activation, not with 0 — its sum would "
"pick up exp(0 - max) per pad lane",
rc == FT_TT_OK && value <= -1.0e20f && value > -HUGE_VALF, "rc %d, %g",
rc, (double)value);
rc = ft_tt_op_reduce_pad_value(FT_TT_OP_REDUCE_FN_RMS_NORM, &value);
ft_tt_test_check_fmt("RMS norm pads with 0 — a zero lane squares to zero",
rc == FT_TT_OK && value == 0.0f, "rc %d, %g", rc, (double)value);
rc = ft_tt_op_reduce_pad_value(FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE, &value);
ft_tt_test_check_fmt("the router pads with 0 — a zero lane adds nothing to the row sum",
rc == FT_TT_OK && value == 0.0f, "rc %d, %g", rc, (double)value);
rc = ft_tt_op_reduce_pad_value(FT_TT_OP_REDUCE_FN_LAYER_NORM, &value);
ft_tt_test_check_fmt("layer norm has NO constant pad value, and says so",
rc == FT_TT_EINVAL &&
!ft_tt_op_reduce_padding_is_exact(FT_TT_OP_REDUCE_FN_LAYER_NORM),
"rc %d", rc);
ft_tt_test_check("the other three are exact under padding",
ft_tt_op_reduce_padding_is_exact(FT_TT_OP_REDUCE_FN_SOFTMAX) &&
ft_tt_op_reduce_padding_is_exact(FT_TT_OP_REDUCE_FN_RMS_NORM) &&
ft_tt_op_reduce_padding_is_exact(FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE),
NULL);
{
/* Six tiles: scaler, eps, and TWO tiles each of gamma and beta at Wt = 2. Sizing it
* to what layer norm actually asks for is what keeps the refusals below about the
* ragged W rather than about a short buffer — two grounds for one failure is a check
* that tests whichever is cheaper (dev notes §3). */
unsigned char scratch[6u * FT_TT_OP_TILE_BYTES_BF16];
static float gamma[64];
unsigned index;
for (index = 0; index < 64u; index++)
gamma[index] = 1.0f;
rc = ft_tt_op_reduce_build_constants_ragged(FT_TT_OP_REDUCE_FN_LAYER_NORM, 33u, 1e-5f,
gamma, gamma, FT_TT_OP_TILE_BYTES_BF16,
scratch, sizeof(scratch));
ft_tt_test_check_fmt("a RAGGED layer norm is refused rather than served wrong",
rc == FT_TT_EINVAL, "rc %d", rc);
rc = ft_tt_op_reduce_build_constants_ragged(FT_TT_OP_REDUCE_FN_LAYER_NORM, 64u, 1e-5f,
gamma, gamma, FT_TT_OP_TILE_BYTES_BF16,
scratch, sizeof(scratch));
ft_tt_test_check_fmt("an EXACT layer norm still builds — the refusal is about "
"raggedness, not the member",
rc == FT_TT_OK, "rc %d", rc);
}
}
/** The divisor, checked as bytes: the scaler and eps tiles must carry the REAL W. This is
* the mutation PCC cannot see, so it is asserted directly and not only through a score. */
static void test_ragged_divisor_is_the_real_width(void)
{
const uint32_t width = 1057u;
const uint32_t tile_cols = ft_tt_op_reduce_padded_tile_cols(width);
const uint32_t padded = tile_cols * FT_TT_OP_TILE_DIM;
const float eps = 1e-5f;
const float want_scaler = ft_float_from_bf16(ft_bf16_from_float(1.0f / (float)width));
const float padded_scaler = ft_float_from_bf16(ft_bf16_from_float(1.0f / (float)padded));
const float want_eps =
ft_float_from_bf16(ft_bf16_from_float(eps * (float)width / (float)FT_TT_OP_TILE_DIM));
const float padded_eps = ft_float_from_bf16(
ft_bf16_from_float(eps * (float)padded / (float)FT_TT_OP_TILE_DIM));
size_t bytes = ft_tt_op_reduce_constants_bytes(FT_TT_OP_REDUCE_FN_RMS_NORM, tile_cols,
FT_TT_OP_TILE_BYTES_BF16);
unsigned char *constants = malloc(bytes);
float *gamma = malloc(width * sizeof(float));
unsigned wrong = 0u;
uint32_t index;
int rc;
ft_tt_test_step("a ragged W: the reduce divisor stays the REAL W");
if (!constants || !gamma) {
ft_tt_test_check("allocation", false, "out of memory");
goto done;
}
for (index = 0; index < width; index++)
gamma[index] = 1.0f;
/* Non-vacuity FIRST: at this W the two candidate divisors are different bf16 values, so
* the checks below can tell them apart. At a W where they round together they could
* not, and a green would mean nothing. */
ft_tt_test_check_fmt("1/W and 1/padded-W are distinct bf16 values at this W",
want_scaler != padded_scaler && want_eps != padded_eps,
"%g vs %g", (double)want_scaler, (double)padded_scaler);
rc = ft_tt_op_reduce_build_constants_ragged(FT_TT_OP_REDUCE_FN_RMS_NORM, width, eps, gamma,
NULL, FT_TT_OP_TILE_BYTES_BF16, constants,
bytes);
ft_tt_test_check_fmt("it builds at W=1057 into Wt=34", rc == FT_TT_OK && tile_cols == 34u,
"rc %d, Wt %u", rc, tile_cols);
if (rc != FT_TT_OK)
goto done;
ft_tt_test_check_fmt("the scaler tile is 1/1057, not 1/1088",
constant_tile_value(constants, &wrong) == want_scaler && wrong == 0u,
"%g", (double)constant_tile_value(constants, &wrong));
ft_tt_test_check_fmt("the eps tile is eps*1057/32, not eps*1088/32",
constant_tile_value(constants + FT_TT_OP_TILE_BYTES_BF16, &wrong) ==
want_eps &&
wrong == 0u,
"%g",
(double)constant_tile_value(constants + FT_TT_OP_TILE_BYTES_BF16,
&wrong));
/* The gamma tail is the other thing a padded width invites: reading 31 floats past the
* caller's array. It has to be a written zero, not whatever was next in memory. */
{
float *wire = malloc(padded * sizeof(float));
unsigned bad = 0u;
if (wire) {
read_broadcast_row(constants + 2u * FT_TT_OP_TILE_BYTES_BF16, tile_cols, wire);
for (index = 0; index < padded; index++) {
const float want = (index < width) ? 1.0f : 0.0f;
if (wire[index] != want)
bad++;
}
ft_tt_test_check_fmt("gamma's padded tail is a written zero, not read past the "
"caller's array",
bad == 0u, "%u of %u columns wrong", bad, padded);
free(wire);
} else {
ft_tt_test_check("allocation", false, "out of memory");
}
}
done:
free(constants);
free(gamma);
}
/** The three paddable members, end to end, at three ragged widths. */
static void test_ragged_padded_pipeline(void)
{
static const uint32_t widths[] = { 33u, 1057u, 1500u };
static const ft_tt_op_reduce_function members[] = { FT_TT_OP_REDUCE_FN_SOFTMAX,
FT_TT_OP_REDUCE_FN_RMS_NORM,
FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE };
unsigned member;
unsigned index;
ft_tt_test_step("a ragged W: the padded pipeline against the true-W reference");
for (member = 0; member < sizeof(members) / sizeof(members[0]); member++) {
for (index = 0; index < sizeof(widths) / sizeof(widths[0]); index++)
(void)score_padded_member(members[member], widths[index], 1e-5f,
member * 8u + index);
}
}
/**
* WHY LAYER NORM IS REFUSED, MEASURED.
*
* Its variance pass reduces (x - row mean)^2, so a padded lane contributes (pad - mean)^2 —
* zero only where the pad equals THAT ROW's mean. The rows here are given deliberately
* different means (row r is offset by r), which is the input that makes the claim
* discriminating: on zero-centred data the same error is ~1e-5 relative and would let a
* broken padding through, and that is exactly the "vacuous because the input never
* exercises it" trap in dev notes §3. Both regimes are reported.
*/
static void test_layer_norm_cannot_be_padded(void)
{
static const float pads[] = { 0.0f, 8.0f, 16.0f, 31.0f, -1.0e30f };
const uint32_t width = 1057u;
const uint32_t tile_cols = ft_tt_op_reduce_padded_tile_cols(width);
const uint32_t padded = tile_cols * FT_TT_OP_TILE_DIM;
const uint32_t rows = 32u;
const float eps = 1e-5f;
const float scaler = ft_float_from_bf16(ft_bf16_from_float(1.0f / (float)width));
const float eps_tile =
ft_float_from_bf16(ft_bf16_from_float(eps * (float)width / (float)FT_TT_OP_TILE_DIM));
float *x = malloc((size_t)rows * width * sizeof(float));
float *x_padded = malloc((size_t)rows * padded * sizeof(float));
float *gamma = malloc(padded * sizeof(float));
float *beta = malloc(padded * sizeof(float));
float *expected = malloc((size_t)rows * width * sizeof(float));
float *modelled = malloc((size_t)rows * padded * sizeof(float));
/* padded, not width: the exact-W control at the end scores a whole padded row through
* this same buffer. */
float *sliced = malloc((size_t)rows * padded * sizeof(float));
uint32_t pass0 = 0u;
uint32_t pass1 = 0u;
uint32_t final_word = 0u;
double best = 0.0;
double floor_worst = 0.0;
double broken = 0.0;
unsigned failures_seen = 0u;
unsigned arm;
uint32_t row;
uint32_t column;
ft_tt_test_step("layer norm under padding: measured wrong for EVERY constant pad");
if (!x || !x_padded || !gamma || !beta || !expected || !modelled || !sliced) {
ft_tt_test_check("allocation", false, "out of memory");
goto done;
}
ft_tt_vector_random_seed(0x1A4E70ULL);
for (column = 0; column < padded; column++) {
gamma[column] = 1.0f;
beta[column] = 0.0f;
}
(void)ft_tt_op_reduce_program(FT_TT_OP_REDUCE_FN_LAYER_NORM, &pass0, &pass1, &final_word);
/* THE FLOOR FIRST, AND IT IS NOT A GUESS. The same member, the same generator, the same
* model, at an EXACT W where no padding happens at all — so the bar every arm below has
* to clear is four times what this arrangement's bf16 rounding already costs, rather
* than a constant somebody picked. On this deliberately offset data the floor is large
* (the CENTER step subtracts two numbers near 31 in bf16), which is exactly why a fixed
* threshold would have been wrong. */
for (row = 0; row < rows; row++) {
for (column = 0; column < padded; column++)
x_padded[(size_t)row * padded + column] = ft_float_from_bf16(
ft_bf16_from_float((float)row + ft_tt_vector_random_value()));
}
ft_tt_vector_reference_layer_norm(x_padded, gamma, beta, eps, modelled, rows, padded);
ft_tt_vector_model_reduce_program(
pass0, pass1, final_word, x_padded, gamma, beta,
ft_float_from_bf16(ft_bf16_from_float(1.0f / (float)padded)),
ft_float_from_bf16(ft_bf16_from_float(eps * (float)padded / (float)FT_TT_OP_TILE_DIM)),
sliced, rows, padded);
floor_worst = ft_tt_vector_max_abs_difference(modelled, sliced, (size_t)rows * padded);
broken = 4.0 * floor_worst;
ft_tt_test_check_fmt("CONTROL: at an EXACT W the same layer norm needs no padding and "
"lands on the reference — this is the floor the arms below clear",
floor_worst > 0.0 && floor_worst < 0.25, "max abs %.6g, so the bar "
"is %.6g", floor_worst, broken);
for (column = 0; column < padded; column++)
gamma[column] = (column < width) ? 1.0f : 0.0f;
/* Row r is centred on r, so no single constant can sit near every row's mean. */
for (row = 0; row < rows; row++) {
for (column = 0; column < width; column++)
x[(size_t)row * width + column] = ft_float_from_bf16(
ft_bf16_from_float((float)row + ft_tt_vector_random_value()));
}
ft_tt_vector_reference_layer_norm(x, gamma, beta, eps, expected, rows, width);
for (arm = 0; arm < sizeof(pads) / sizeof(pads[0]); arm++) {
double worst;
for (row = 0; row < rows; row++) {
for (column = 0; column < padded; column++)
x_padded[(size_t)row * padded + column] =
(column < width) ? x[(size_t)row * width + column] : pads[arm];
}
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x_padded, gamma, beta,
scaler, eps_tile, modelled, rows, padded);
for (row = 0; row < rows; row++)
memcpy(sliced + (size_t)row * width, modelled + (size_t)row * padded,
width * sizeof(float));
worst = ft_tt_vector_max_abs_difference(expected, sliced, (size_t)rows * width);
if (arm == 0u || worst < best)
best = worst;
if (worst > broken)
failures_seen++;
ft_tt_test_check_fmt("a constant pad does not save layer norm", worst > broken,
"pad %g: max abs %.6g against a floor of %.6g", (double)pads[arm],
worst, floor_worst);
}
ft_tt_test_check_fmt("the BEST constant pad is still wrong",
failures_seen == sizeof(pads) / sizeof(pads[0]) && best > broken,
"best of %u pads: max abs %.6g, %.1fx the floor", failures_seen, best,
best / floor_worst);
/* AND A PER-ROW PAD DOES NOT RESCUE IT EITHER, which is the stronger form of the finding
* and was not obvious until it was run. The row's own mean makes pass 1's (pad - mean)^2
* vanish — and CORRUPTS pass 0, which is a plain SUM over the padded row with a 1/W_true
* scaler, so the "mean" it produces is m*(padded/W). The two passes want different
* padding of the SAME buffer. Nothing a caller writes into the input satisfies both. */
{
double worst;
for (row = 0; row < rows; row++) {
double mean = 0.0;
for (column = 0; column < width; column++)
mean += (double)x[(size_t)row * width + column];
mean /= (double)width;
for (column = 0; column < padded; column++)
x_padded[(size_t)row * padded + column] =
(column < width) ? x[(size_t)row * width + column] : (float)mean;
}
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x_padded, gamma, beta,
scaler, eps_tile, modelled, rows, padded);
for (row = 0; row < rows; row++)
memcpy(sliced + (size_t)row * width, modelled + (size_t)row * padded,
width * sizeof(float));
worst = ft_tt_vector_max_abs_difference(expected, sliced, (size_t)rows * width);
ft_tt_test_check_fmt("nor does a PER-ROW pad — pass 0 wants 0 and pass 1 wants the "
"row mean, and there is one buffer",
worst > broken, "row-mean pad: max abs %.6g, %.1fx the floor",
worst, worst / floor_worst);
}
/* The honest nuance, reported rather than asserted: on a zero-centred row the same hole
* is invisible, which is why "it scored fine on our data" is not evidence here. */
for (row = 0; row < rows; row++) {
for (column = 0; column < width; column++)
x[(size_t)row * width + column] =
ft_float_from_bf16(ft_bf16_from_float(ft_tt_vector_random_value()));
for (column = 0; column < padded; column++)
x_padded[(size_t)row * padded + column] =
(column < width) ? x[(size_t)row * width + column] : 0.0f;
}
ft_tt_vector_reference_layer_norm(x, gamma, beta, eps, expected, rows, width);
ft_tt_vector_model_reduce_program(pass0, pass1, final_word, x_padded, gamma, beta, scaler,
eps_tile, modelled, rows, padded);
for (row = 0; row < rows; row++)
memcpy(sliced + (size_t)row * width, modelled + (size_t)row * padded,
width * sizeof(float));
ft_tt_test_check_fmt("REPORTED, not a gate: on zero-centred rows the same hole hides",
true, "zero pad, zero-mean data: max abs %.6g",
ft_tt_vector_max_abs_difference(expected, sliced, (size_t)rows * width));
done:
free(x);
free(x_padded);
free(gamma);
free(beta);
free(expected);
free(modelled);
free(sliced);
}
/* ================================================================ */
int main(void)
{
ft_tt_test_step("Phase 6 vector ops — OFF-HARDWARE gate (no card touched)");
test_wire_constants_agree();
test_arity();
test_block_depth();
test_eltwise_parameters();
test_reduce_programs();
test_reduce_parameters();
test_eltwise_placement();
test_reduce_placement();
test_reduce_constants();
test_reduce_constants_float32();
test_eps_tile_follows_the_scaler();
test_l2_norm_against_the_oracle();
test_float32_reduce_at_the_head_shape();
test_the_three_walls_priced_apart();
test_gated_is_bound_by_the_bf16_bridge();
test_eltwise_model();
test_softmax_model();
test_reduce_models();
test_pcc_is_blind_to_scale();
test_softmax_dynamic_range();
test_softmax_max_is_over_the_whole_row();
test_program_model_matches_the_member_models();
test_ragged_padding_contract();
test_ragged_divisor_is_the_real_width();
test_ragged_padded_pipeline();
test_layer_norm_cannot_be_padded();
return ft_tt_test_finish();
}
Reference in New Issue View Git Blame Copy Permalink