David
/
Foundry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Foundry/tt/tests/test_memory_ops.c

1314 lines
60 KiB
C
Raw Permalink Blame History

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
/**
* @file test_memory_ops.c
* @brief OFF-HARDWARE gate for the memory op family — EMBEDDING and KV_APPEND — and for
* the KV cache layout contract (ft_tt_op_memory.h PART 1).
*
* ---------------------------------------------------------------------------
* A COPY OP IS WHERE A TEST LIES MOST EASILY
* ---------------------------------------------------------------------------
* A gather that returns the right BYTES from the WRONG ROW looks right whenever every row
* holds similar data, and a face-order mistake is invisible to any metric computed through
* the same wrong arithmetic (docs/tt_kernel_model.md §9). So this suite does four things
* deliberately, and every one of them was needed to make some mutation go red:
*
* 1. **Every source row holds different content**, from a per-row generator whose values
* are full-mantissa bit patterns — never small integers and never a per-row constant.
* 2. **Every destination is poisoned first**, and the poison is checked to SURVIVE
* everywhere the op was not supposed to write. That half is what proves an APPEND
* preserves the positions already in the cache, and no PCC-shaped check can see it.
* 3. **The comparison is BYTE-FOR-BYTE, never a correlation.** PCC is blind to a
* consistent permutation, which is exactly the failure mode of a layout op (dev notes
* §3). The expected image is built element by element through
* `ft_tt_tile_face_index()` — ft_tt_tilize.c, written for the weight converter and
* verified against tt-metal's own tilize_utils.cpp — so it shares no code with the
* thing under test.
* 4. **The functions under test are THE KERNEL'S**, not a C copy of them. The arithmetic
* half of kernels/resident_memory_protocol.h compiles on the host and is reached
* through tests/ft_tt_memory_device_shim.c. A transcription would have agreed with
* itself.
*
* ---------------------------------------------------------------------------
* WHAT THIS CANNOT PROVE
* ---------------------------------------------------------------------------
* That a 32-byte DRAM write is legal, which is the fact the whole family rests on. Nothing
* off-hardware can — "off-hardware suites cannot cover the copy path" (dev notes §3), and
* here they cannot cover the NoC's alignment rules either. tests/test_phase6_memory.c
* measures it on silicon, against neighbours as well as against the row.
*
* Environment:
* FT_TT_KERNEL_DIR where the device headers live (default ../kernels)
*
* @status GREEN: 95 checks, 0 failures, on the Mac (clang) and on the card host (gcc). TWELVE
* mutations were run against it and ALL TWELVE went red — 14, 11, 10, 2, 6, 9, 15, 3, 6,
* 2, 1 and 1 failures. SIX were in the KERNEL's own header (the face index, a run that
* emits one column half, a run one face row too long, the face pairing of a row-major
* source, a place that reads the wrong source row, and a run split that ignores the
* 16-row face group boundary); TWO in the host layout arithmetic (the cache axis order,
* and a split rule that ignores the source tile row); FOUR in the model's control flow
* (the reader's fast path, the gather's index array, the whole-tiles flag, the
* zero-fill).
* @issues None
* @todo None
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../ft_tt_op_dispatch.h"
#include "../ft_tt_op_l1_layout.h"
#include "../ft_tt_op_memory.h"
#include "../ft_tt_queue.h"
#include "../ft_tt_tilize.h"
#include "ft_tt_test_report.h"
/** Where the device headers live, so the wire-constant pin reads the real file. */
#define KERNEL_DIRECTORY_ENVIRONMENT "FT_TT_KERNEL_DIR"
#define KERNEL_DIRECTORY_DEFAULT "../kernels"
#define PATH_MAX_BYTES 512u
#define TILE_BYTES FT_TT_OP_TILE_BYTES_BF16
#define TILE_DIM FT_TT_OP_TILE_DIM
/** Two patterns, and they answer different questions. POISON is what a destination holds
* before the op, so "the op did not write here" and "the op wrote a zero" are different
* bit patterns. STALE is what a circular-buffer page holds before the reader fills it, so
* a missing zero-fill publishes something recognisable instead of a plausible zero. */
#define POISON_BYTE 0xA5u
#define STALE_BYTE 0xCCu
/* Declared by tests/ft_tt_memory_device_shim.c — the KERNEL's own arithmetic. */
uint32_t ft_tt_device_chunk_bytes(uint32_t tile_bytes);
uint32_t ft_tt_device_face_bytes(uint32_t tile_bytes);
uint32_t ft_tt_device_chunk_offset(uint32_t row, uint32_t half, uint32_t tile_bytes);
uint32_t ft_tt_device_group_row_bytes(uint32_t group_tiles, uint32_t tile_bytes);
void ft_tt_device_place_row_from_row_major(uint32_t *destination_block,
const uint32_t *source_row, uint32_t destination_row,
uint32_t group_tiles, uint32_t tile_bytes);
void ft_tt_device_place_row_from_tiles(uint32_t *destination_block, const uint32_t *source_block,
uint32_t source_row, uint32_t destination_row,
uint32_t group_tiles, uint32_t tile_bytes);
void ft_tt_device_zero_words(uint32_t *destination, uint32_t words);
uint32_t ft_tt_device_destination_runs(uint32_t row0, uint32_t row_count, uint32_t tile_bytes,
uint32_t *offsets, uint32_t *bytes);
uint32_t ft_tt_device_max_runs(void);
int ft_tt_device_length_is_issuable(uint32_t bytes);
static unsigned char g_cb_buffer[FT_TT_NUM_CIRCULAR_BUFFERS * FT_TT_WORDS_PER_LOCAL_CB_CONFIG * 4];
/* ================================================================
* Test data that cannot be confused between rows
* ================================================================ */
/**
* The bf16 bit pattern for logical element (row, column) of source `stream`.
*
* Every (stream, row, column) gets a DIFFERENT value, and the row contributes a large
* multiplier, so "the right bytes from the wrong row" is a mismatch in every element rather
* than in none. The pattern also avoids POISON_BYTE and STALE_BYTE in both halves, so a
* value can never be mistaken for un-written memory.
*/
static uint16_t source_pattern(uint32_t stream, uint32_t row, uint32_t column)
{
uint32_t mixed = (stream * 2654435761u) ^ (row * 40503u) ^ (column * 2246822519u);
mixed ^= mixed >> 13;
mixed = (mixed * 1103515245u) + 12345u;
mixed >>= 9;
mixed &= 0xFFFFu;
if ((mixed & 0xFFu) == POISON_BYTE || (mixed & 0xFFu) == STALE_BYTE)
mixed ^= 0x0001u;
if (((mixed >> 8) & 0xFFu) == POISON_BYTE || ((mixed >> 8) & 0xFFu) == STALE_BYTE)
mixed ^= 0x0100u;
return (uint16_t)mixed;
}
/** Write the bf16 element (row, column) of a face-ordered tile grid `tiles` wide. */
static void put_tiled_element(unsigned char *base, uint32_t tiles_wide, uint32_t row,
uint32_t column, uint16_t value)
{
const uint32_t tile = column / TILE_DIM;
const uint32_t tile_row = row / TILE_DIM;
const size_t offset = ((size_t)tile_row * tiles_wide + tile) * TILE_BYTES +
(size_t)ft_tt_tile_face_index(row % TILE_DIM, column % TILE_DIM) * 2u;
(void)tiles_wide;
memcpy(base + offset, &value, sizeof(value));
}
/* ================================================================
* A model of the two kernels, driven by the KERNEL's own arithmetic
* ================================================================ */
/**
* Run one descriptor the way resident_gather_reader.c and resident_place_writer.c do,
* over simulated device memory.
*
* The control flow is a transcription — that much is unavoidable off-hardware — but every
* address it computes comes from the kernel's own inline functions through the shim, and
* the ANSWER is scored against an expectation built out of ft_tt_tilize.c. So a bug in the
* face arithmetic goes red here; a bug in the control flow goes red on the card.
*/
static void model_run(const uint32_t words[FT_TT_OP_MEMORY_WORDS], const unsigned char *source,
unsigned char *destination, const uint32_t *indices)
{
const uint32_t flags = words[FT_TT_OP_MEMORY_FLAGS];
const uint32_t row_count = words[FT_TT_OP_MEMORY_ROW_COUNT];
const uint32_t group_count = words[FT_TT_OP_MEMORY_GROUP_COUNT];
const uint32_t group_tiles = words[FT_TT_OP_MEMORY_GROUP_TILES];
const uint32_t source_base = words[FT_TT_OP_MEMORY_SRC_BASE];
const uint32_t source_row_stride = words[FT_TT_OP_MEMORY_SRC_ROW_STRIDE];
const uint32_t source_row0 = words[FT_TT_OP_MEMORY_SRC_ROW0];
const uint32_t source_group_stride = words[FT_TT_OP_MEMORY_SRC_GROUP_STRIDE];
const uint32_t destination_base = words[FT_TT_OP_MEMORY_DST_BASE];
const uint32_t destination_group_stride = words[FT_TT_OP_MEMORY_DST_GROUP_STRIDE];
const uint32_t destination_row0 = words[FT_TT_OP_MEMORY_DST_ROW0];
const uint32_t row_in_tile = destination_row0 % TILE_DIM;
const uint32_t tile_row = destination_row0 / TILE_DIM;
const uint32_t group_bytes = group_tiles * TILE_BYTES;
const uint32_t row_bytes = ft_tt_device_group_row_bytes(group_tiles, TILE_BYTES);
const bool whole_tiles = (flags & (uint32_t)FT_TT_OP_MEMORY_FLAG_DST_WHOLE_TILES) != 0u;
const bool tiled_source = (flags & (uint32_t)FT_TT_OP_MEMORY_FLAG_SRC_TILED) != 0u;
const bool gather = (flags & (uint32_t)FT_TT_OP_MEMORY_FLAG_GATHER) != 0u;
unsigned char *block = malloc(group_bytes);
unsigned char *stage = malloc(group_bytes);
uint32_t group;
if (!block || !stage) {
free(block);
free(stage);
return;
}
for (group = 0; group < group_count; group++) {
const uint32_t source_group = source_base + group * source_group_stride;
const uint32_t destination_group = destination_base + group * destination_group_stride +
tile_row * group_bytes;
uint32_t row;
/* A circular-buffer page is REUSED, so it starts as whatever the previous descriptor
* left. Modelling that is what makes a missing zero-fill visible. */
memset(block, (int)STALE_BYTE, group_bytes);
if (whole_tiles && row_count < TILE_DIM)
ft_tt_device_zero_words((uint32_t *)(void *)block, group_bytes / 4u);
if (tiled_source) {
const bool direct =
source_row0 == row_in_tile && !(whole_tiles && row_count < TILE_DIM);
memcpy(direct ? block : stage, source + source_group, group_bytes);
if (!direct) {
for (row = 0; row < row_count; row++) {
ft_tt_device_place_row_from_tiles((uint32_t *)(void *)block,
(const uint32_t *)(const void *)stage,
source_row0 + row, row_in_tile + row,
group_tiles, TILE_BYTES);
}
}
} else {
for (row = 0; row < row_count; row++) {
const uint32_t index = gather ? indices[row] : source_row0 + row;
memcpy(stage + row * row_bytes,
source + source_group + index * source_row_stride, row_bytes);
}
for (row = 0; row < row_count; row++) {
ft_tt_device_place_row_from_row_major(
(uint32_t *)(void *)block,
(const uint32_t *)(const void *)(stage + row * row_bytes),
row_in_tile + row, group_tiles, TILE_BYTES);
}
}
if (whole_tiles || (row_in_tile == 0u && row_count == TILE_DIM)) {
memcpy(destination + destination_group, block, group_bytes);
} else {
uint32_t offsets[8];
uint32_t lengths[8];
const uint32_t run_count =
ft_tt_device_destination_runs(row_in_tile, row_count, TILE_BYTES, offsets,
lengths);
uint32_t tile;
for (tile = 0; tile < group_tiles; tile++) {
uint32_t index;
for (index = 0; index < run_count; index++) {
const uint32_t offset = tile * TILE_BYTES + offsets[index];
memcpy(destination + destination_group + offset, block + offset,
lengths[index]);
}
}
}
}
free(block);
free(stage);
}
/* ================================================================
* Reading a constant out of a device header
* ================================================================ */
static int read_device_constant(const char *path, const char *name, uint32_t *out_value)
{
char line[512];
FILE *file = fopen(path, "r");
int found = -1;
if (!file)
return -1;
while (fgets(line, sizeof(line), file)) {
char parsed_name[128];
char parsed_value[64];
if (sscanf(line, " enum { %127[A-Za-z0-9_] = %63[^}] };", parsed_name,
parsed_value) != 2)
continue;
if (strcmp(parsed_name, name) != 0)
continue;
*out_value = (uint32_t)strtoul(parsed_value, NULL, 0);
found = 0;
break;
}
fclose(file);
return found;
}
static void device_header_path(char *out, size_t out_bytes, const char *file)
{
const char *directory = getenv(KERNEL_DIRECTORY_ENVIRONMENT);
if (!directory || directory[0] == '\0')
directory = KERNEL_DIRECTORY_DEFAULT;
snprintf(out, out_bytes, "%s/%s", directory, file);
}
static void check_device_constant(const char *path, const char *name, uint32_t host_value)
{
uint32_t device_value = 0;
if (read_device_constant(path, name, &device_value) != 0) {
ft_tt_test_check_fmt(name, false, "not declared in %s", path);
return;
}
ft_tt_test_check_fmt(name, device_value == host_value, "host %u, kernel %u", host_value,
device_value);
}
static void test_wire_constants_agree(void)
{
char memory_path[PATH_MAX_BYTES];
char queue_path[PATH_MAX_BYTES];
ft_tt_test_step("wire constants: the host enum against the kernel's own header");
device_header_path(memory_path, sizeof(memory_path), "resident_memory_protocol.h");
device_header_path(queue_path, sizeof(queue_path), "resident_queue_protocol.h");
check_device_constant(memory_path, "kMemFlags", FT_TT_OP_MEMORY_FLAGS);
check_device_constant(memory_path, "kMemRowCount", FT_TT_OP_MEMORY_ROW_COUNT);
check_device_constant(memory_path, "kMemGroupCount", FT_TT_OP_MEMORY_GROUP_COUNT);
check_device_constant(memory_path, "kMemGroupTiles", FT_TT_OP_MEMORY_GROUP_TILES);
check_device_constant(memory_path, "kMemSrcX", FT_TT_OP_MEMORY_SRC_X);
check_device_constant(memory_path, "kMemSrcY", FT_TT_OP_MEMORY_SRC_Y);
check_device_constant(memory_path, "kMemSrcBase", FT_TT_OP_MEMORY_SRC_BASE);
check_device_constant(memory_path, "kMemSrcRowStride", FT_TT_OP_MEMORY_SRC_ROW_STRIDE);
check_device_constant(memory_path, "kMemSrcRow0", FT_TT_OP_MEMORY_SRC_ROW0);
check_device_constant(memory_path, "kMemSrcGroupStride", FT_TT_OP_MEMORY_SRC_GROUP_STRIDE);
check_device_constant(memory_path, "kMemDstX", FT_TT_OP_MEMORY_DST_X);
check_device_constant(memory_path, "kMemDstY", FT_TT_OP_MEMORY_DST_Y);
check_device_constant(memory_path, "kMemDstBase", FT_TT_OP_MEMORY_DST_BASE);
check_device_constant(memory_path, "kMemDstGroupStride", FT_TT_OP_MEMORY_DST_GROUP_STRIDE);
check_device_constant(memory_path, "kMemDstRow0", FT_TT_OP_MEMORY_DST_ROW0);
check_device_constant(memory_path, "kMemWords", FT_TT_OP_MEMORY_WORDS);
check_device_constant(memory_path, "kMemMaxRows", FT_TT_OP_MEMORY_MAX_ROWS);
check_device_constant(memory_path, "kMemFlagSrcTiled", FT_TT_OP_MEMORY_FLAG_SRC_TILED);
check_device_constant(memory_path, "kMemFlagDstWholeTiles",
FT_TT_OP_MEMORY_FLAG_DST_WHOLE_TILES);
check_device_constant(memory_path, "kMemFlagGather", FT_TT_OP_MEMORY_FLAG_GATHER);
check_device_constant(memory_path, "kMemFaceRows", FT_TT_OP_MEMORY_FACE_ROWS);
check_device_constant(queue_path, "kOpEmbedding", FT_TT_QUEUE_OP_EMBEDDING);
check_device_constant(queue_path, "kOpKvAppend", FT_TT_QUEUE_OP_KV_APPEND);
}
/* ================================================================
* The face arithmetic, against an independent implementation
* ================================================================ */
static void test_chunk_offset_is_the_tilize_layout(void)
{
uint32_t row;
unsigned mismatches = 0;
unsigned host_mismatches = 0;
ft_tt_test_step("face arithmetic: the KERNEL's chunk offsets against ft_tt_tilize.c");
for (row = 0; row < TILE_DIM; row++) {
uint32_t half;
for (half = 0; half < 2u; half++) {
const uint32_t expected =
ft_tt_tile_face_index(row, half * 16u) * 2u; /* bf16 */
const uint32_t device = ft_tt_device_chunk_offset(row, half, TILE_BYTES);
const uint32_t host = ft_tt_op_memory_chunk_offset(row, half, TILE_BYTES);
if (device != expected)
mismatches++;
if (host != device)
host_mismatches++;
}
}
ft_tt_test_check_fmt("all 64 face rows land where ft_tt_tilize puts them",
mismatches == 0u, "%u of 64 wrong", mismatches);
ft_tt_test_check_fmt("and the host copy agrees with the kernel's", host_mismatches == 0u,
"%u of 64 differ", host_mismatches);
ft_tt_test_check_fmt("a face row is 32 bytes at bf16",
ft_tt_device_chunk_bytes(TILE_BYTES) == 32u, "%u",
ft_tt_device_chunk_bytes(TILE_BYTES));
ft_tt_test_check_fmt("and 64 at fp32",
ft_tt_device_chunk_bytes(FT_TT_OP_TILE_BYTES_FP32) == 64u, "%u",
ft_tt_device_chunk_bytes(FT_TT_OP_TILE_BYTES_FP32));
ft_tt_test_check_fmt("a face is a quarter of a tile",
ft_tt_device_face_bytes(TILE_BYTES) == 512u, "%u",
ft_tt_device_face_bytes(TILE_BYTES));
/* The property the whole family rests on: every partial-write address and length is a
* multiple of the 16-byte DRAM write alignment. */
ft_tt_test_check("every face row is a multiple of the 16-byte DRAM write alignment",
ft_tt_device_chunk_bytes(TILE_BYTES) % 16u == 0u &&
ft_tt_device_chunk_bytes(FT_TT_OP_TILE_BYTES_FP32) % 16u == 0u,
"32 and 64");
}
/**
* The runs of a row range cover EXACTLY that range's bytes — no byte of another row, and
* no byte of the range missed.
*
* Checked by painting: mark every byte the runs claim, then walk the tile element by
* element through ft_tt_tile_face_index() and demand that a byte is marked if and only if
* its row is in the range. A run that spilled into row 16 would be invisible to any check
* that only looked at the rows it meant to write.
*/
static void test_destination_runs_cover_exactly_the_rows(void)
{
uint32_t row0;
unsigned bad_ranges = 0;
unsigned worst_runs = 0;
ft_tt_test_step("destination runs: exactly the rows asked for, and nothing beside them");
for (row0 = 0; row0 < TILE_DIM; row0++) {
uint32_t count;
for (count = 1u; row0 + count <= TILE_DIM; count++) {
unsigned char marked[FT_TT_OP_TILE_BYTES_BF16];
uint32_t offsets[8];
uint32_t lengths[8];
uint32_t run_count =
ft_tt_device_destination_runs(row0, count, TILE_BYTES, offsets, lengths);
uint32_t index;
uint32_t row;
bool ok = true;
memset(marked, 0, sizeof(marked));
if (run_count == 0u || run_count > ft_tt_device_max_runs()) {
bad_ranges++;
continue;
}
if (run_count > worst_runs)
worst_runs = run_count;
for (index = 0; index < run_count; index++) {
uint32_t byte;
for (byte = 0; byte < lengths[index]; byte++)
marked[offsets[index] + byte] = 1u;
}
for (row = 0; row < TILE_DIM && ok; row++) {
const bool wanted = row >= row0 && row < row0 + count;
uint32_t column;
for (column = 0; column < TILE_DIM; column++) {
const uint32_t at = ft_tt_tile_face_index(row, column) * 2u;
if ((marked[at] != 0u) != wanted || (marked[at + 1u] != 0u) != wanted) {
ok = false;
break;
}
}
}
if (!ok)
bad_ranges++;
}
}
ft_tt_test_check_fmt("every one of the 528 row ranges is covered exactly",
bad_ranges == 0u, "%u wrong", bad_ranges);
ft_tt_test_check_fmt("and never needs more than four runs", worst_runs <= 4u, "worst %u",
worst_runs);
ft_tt_test_check("a zero-length or out-of-range request yields no runs",
ft_tt_device_destination_runs(0u, 0u, TILE_BYTES, NULL, NULL) == 0u &&
ft_tt_device_destination_runs(30u, 4u, TILE_BYTES, NULL, NULL) == 0u,
"");
}
/* ================================================================
* The KV cache layout contract
* ================================================================ */
static void test_kv_cache_layout(void)
{
ft_tt_kv_cache_layout layout;
ft_tt_kv_cache_layout model;
uint64_t offset;
uint32_t bytes;
int rc;
ft_tt_test_step("KV cache layout: the contract as arithmetic");
/* MiniCPM5-1B, read from the GGUF header (docs/tt_phase0_decisions.md §3) and never
* assumed: 2 KV heads, head_dim 128, context 131072. */
rc = ft_tt_kv_cache_layout_init(2u, 128u, 131072u, TILE_BYTES, &model);
ft_tt_test_check_fmt("MiniCPM5-1B's shape derives", rc == FT_TT_OK, "rc=%d", rc);
ft_tt_test_check_fmt("dim_tiles is 4 and capacity_tiles 4096",
model.dim_tiles == 4u && model.capacity_tiles == 4096u, "%u, %u",
model.dim_tiles, model.capacity_tiles);
ft_tt_test_check_fmt("one region is 64 MiB, so K+V over 24 layers is 3 GiB",
model.region_bytes == 64ull * 1024ull * 1024ull, "%llu bytes",
(unsigned long long)model.region_bytes);
rc = ft_tt_kv_cache_layout_init(2u, 128u, 200u, TILE_BYTES, &layout);
ft_tt_test_check_fmt("a capacity is rounded UP to a whole position tile",
rc == FT_TT_OK && layout.capacity_positions == 224u &&
layout.capacity_tiles == 7u,
"rc=%d capacity=%u tiles=%u", rc, layout.capacity_positions,
layout.capacity_tiles);
/* F2: the axis order. Consecutive dim tiles are adjacent; the next position tile
* follows the last dim tile; a head is head_stride away. */
ft_tt_test_check(
"dim tiles are adjacent",
ft_tt_kv_cache_tile_offset(&layout, 0u, 0u, 1u) -
ft_tt_kv_cache_tile_offset(&layout, 0u, 0u, 0u) == TILE_BYTES,
"");
ft_tt_test_check("the next position tile follows the last dim tile",
ft_tt_kv_cache_tile_offset(&layout, 0u, 32u, 0u) -
ft_tt_kv_cache_tile_offset(&layout, 0u, 31u, 3u) == TILE_BYTES,
"");
ft_tt_test_check("a head is head_stride away",
ft_tt_kv_cache_tile_offset(&layout, 1u, 0u, 0u) == layout.head_stride,
"");
ft_tt_test_check("an out-of-range index refuses rather than wrapping",
ft_tt_kv_cache_tile_offset(&layout, 2u, 0u, 0u) == UINT64_MAX &&
ft_tt_kv_cache_tile_offset(&layout, 0u, 224u, 0u) == UINT64_MAX &&
ft_tt_kv_cache_tile_offset(&layout, 0u, 0u, 4u) == UINT64_MAX,
"");
/* §1.2: the property attention is written against. */
rc = ft_tt_kv_cache_run(&layout, 1u, 64u, 96u, &offset, &bytes);
ft_tt_test_check_fmt("a position range is ONE run", rc == FT_TT_OK, "rc=%d", rc);
ft_tt_test_check_fmt("it starts at the first position's tile",
offset == ft_tt_kv_cache_tile_offset(&layout, 1u, 64u, 0u),
"%llu", (unsigned long long)offset);
ft_tt_test_check_fmt("and it is exactly the positions asked for",
bytes == 3u * layout.position_tile_bytes, "%u bytes", bytes);
ft_tt_test_check("the run ends where the next one begins — no gap, no stride",
offset + bytes == ft_tt_kv_cache_tile_offset(&layout, 1u, 160u, 0u), "");
ft_tt_test_check(
"a run that is not a whole number of position tiles is refused, not rounded",
ft_tt_kv_cache_run(&layout, 0u, 16u, 32u, &offset, &bytes) == FT_TT_EINVAL &&
ft_tt_kv_cache_run(&layout, 0u, 0u, 33u, &offset, &bytes) == FT_TT_EINVAL,
"");
ft_tt_test_check("a run past the capacity is refused",
ft_tt_kv_cache_run(&layout, 0u, 192u, 64u, &offset, &bytes) ==
FT_TT_EINVAL,
"");
/* §1.6: block float is refused BY NAME. */
ft_tt_test_check("a Bfp8_b cache is refused, and so is Bfp4_b",
ft_tt_kv_cache_layout_init(2u, 128u, 64u, FT_TT_OP_TILE_BYTES_BFP8_B,
&layout) == FT_TT_EINVAL &&
ft_tt_kv_cache_layout_init(2u, 128u, 64u, FT_TT_OP_TILE_BYTES_BFP4_B,
&layout) == FT_TT_EINVAL,
"");
ft_tt_test_check("a head dimension that is not a whole number of tiles is refused",
ft_tt_kv_cache_layout_init(2u, 100u, 64u, TILE_BYTES, &layout) ==
FT_TT_EINVAL,
"");
}
/* ================================================================
* KV_APPEND, byte for byte
* ================================================================ */
#define APPEND_HEADS 2u
#define APPEND_HEAD_DIM 128u
#define APPEND_DIM_TILES (APPEND_HEAD_DIM / TILE_DIM)
#define APPEND_SOURCE_TILES (APPEND_HEADS * APPEND_DIM_TILES)
#define APPEND_CAPACITY 128u
/**
* One append, scored against an expectation built element by element.
*
* `stream` seeds the source content so no two calls in this file can pass on each other's
* data.
*/
static bool run_append_case(const char *label, uint32_t position, uint32_t source_row0,
uint32_t token_count, uint32_t stream)
{
ft_tt_kv_cache_layout layout;
ft_tt_op_kv_append_request request;
uint32_t words[FT_TT_OP_MEMORY_WORDS];
unsigned char *source = NULL;
unsigned char *cache = NULL;
unsigned char *expected = NULL;
size_t source_bytes;
size_t cache_bytes;
uint32_t row;
uint32_t head;
size_t wrong = 0;
size_t disturbed = 0;
bool ok = false;
int rc;
rc = ft_tt_kv_cache_layout_init(APPEND_HEADS, APPEND_HEAD_DIM, APPEND_CAPACITY, TILE_BYTES,
&layout);
if (rc != FT_TT_OK) {
ft_tt_test_check_fmt(label, false, "layout rc=%d", rc);
return false;
}
source_bytes = (size_t)APPEND_SOURCE_TILES * TILE_BYTES * 2u; /* two source tile rows */
cache_bytes = (size_t)layout.region_bytes;
source = malloc(source_bytes);
cache = malloc(cache_bytes);
expected = malloc(cache_bytes);
if (!source || !cache || !expected) {
ft_tt_test_check(label, false, "out of memory");
goto done;
}
/* Per-ROW distinguishable content: a gather that returned the right bytes from the
* wrong row cannot pass this. */
memset(source, 0, source_bytes);
for (row = 0; row < TILE_DIM * 2u; row++) {
uint32_t column;
for (column = 0; column < APPEND_HEADS * APPEND_HEAD_DIM; column++)
put_tiled_element(source, APPEND_SOURCE_TILES, row, column,
source_pattern(stream, row, column));
}
memset(cache, (int)POISON_BYTE, cache_bytes);
memcpy(expected, cache, cache_bytes);
for (head = 0; head < APPEND_HEADS; head++) {
uint32_t token;
for (token = 0; token < token_count; token++) {
uint32_t dimension;
for (dimension = 0; dimension < APPEND_HEAD_DIM; dimension++) {
const uint64_t tile = ft_tt_kv_cache_tile_offset(&layout, head, position + token,
dimension / TILE_DIM);
const size_t at = (size_t)tile + (size_t)ft_tt_tile_face_index(
(position + token) % TILE_DIM,
dimension % TILE_DIM) *
2u;
const uint16_t value =
source_pattern(stream, source_row0 + token, head * APPEND_HEAD_DIM + dimension);
memcpy(expected + at, &value, sizeof(value));
}
}
}
memset(&request, 0, sizeof(request));
request.cache = &layout;
request.position = position;
request.token_count = token_count;
request.source.x = 1u;
request.source.y = 2u;
request.source.address = 0u;
request.source.tile_bytes = TILE_BYTES;
request.source_row0 = source_row0;
request.source_tiles = APPEND_SOURCE_TILES;
request.region.x = 3u;
request.region.y = 4u;
request.region.address = 0u;
request.region.tile_bytes = TILE_BYTES;
rc = ft_tt_op_kv_append_parameters(&request, words);
if (rc != FT_TT_OK) {
ft_tt_test_check_fmt(label, false, "encode rc=%d", rc);
goto done;
}
model_run(words, source, cache, NULL);
{
size_t index;
for (index = 0; index < cache_bytes; index++) {
if (cache[index] == expected[index])
continue;
if (expected[index] == POISON_BYTE)
disturbed++;
else
wrong++;
}
}
ft_tt_test_check_fmt(label, wrong == 0u && disturbed == 0u,
"%zu bytes wrong, %zu bytes of an untouched position disturbed",
wrong, disturbed);
ok = wrong == 0u && disturbed == 0u;
done:
free(source);
free(cache);
free(expected);
return ok;
}
static void test_kv_append(void)
{
ft_tt_test_step("KV_APPEND: byte for byte, against a poisoned cache");
/* Decode: one token, at four positions that exercise both face row groups and both
* position tiles. */
(void)run_append_case("one token at position 0", 0u, 0u, 1u, 1u);
(void)run_append_case("one token at position 7 (first face group)", 7u, 0u, 1u, 2u);
(void)run_append_case("one token at position 20 (second face group)", 20u, 0u, 1u, 3u);
(void)run_append_case("one token at position 63 (last row of the second tile)", 63u, 0u,
1u, 4u);
/* The source row need not be the destination row — decode reads row 0 of a projection
* whose position is anything. */
(void)run_append_case("source row 5 into position 19", 19u, 5u, 1u, 5u);
/* A range that CROSSES the face row group boundary: the case with four runs. */
(void)run_append_case("six tokens across rows 14..19", 14u, 14u, 6u, 6u);
(void)run_append_case("six tokens from source row 0 into rows 14..19", 14u, 0u, 6u, 7u);
/* Prefill: a whole aligned tile row, which collapses to one transaction per head. */
(void)run_append_case("a whole 32-token block at position 32", 32u, 32u, 32u, 8u);
(void)run_append_case("a whole 32-token block from source row 0", 96u, 0u, 32u, 9u);
/* The last position of the cache. */
(void)run_append_case("the last position of the cache", APPEND_CAPACITY - 1u, 3u, 1u, 10u);
}
/**
* Two appends in a row: the second must not disturb the first.
*
* This is the check that makes "an append PRESERVES" more than a claim about poison — the
* bytes it must preserve are real data at a neighbouring position, in the SAME tile, in the
* same face rows.
*/
static void test_kv_append_preserves_earlier_positions(void)
{
ft_tt_kv_cache_layout layout;
ft_tt_op_kv_append_request request;
uint32_t words[FT_TT_OP_MEMORY_WORDS];
unsigned char *source = NULL;
unsigned char *cache = NULL;
size_t source_bytes;
size_t cache_bytes;
uint32_t position;
uint32_t row;
size_t wrong = 0;
int rc;
ft_tt_test_step("KV_APPEND: appending position p+1 leaves position p alone");
rc = ft_tt_kv_cache_layout_init(APPEND_HEADS, APPEND_HEAD_DIM, APPEND_CAPACITY, TILE_BYTES,
&layout);
source_bytes = (size_t)APPEND_SOURCE_TILES * TILE_BYTES;
cache_bytes = (size_t)layout.region_bytes;
source = malloc(source_bytes);
cache = malloc(cache_bytes);
if (rc != FT_TT_OK || !source || !cache) {
ft_tt_test_check("the case sets up", false, "");
goto done;
}
memset(source, 0, source_bytes);
for (row = 0; row < TILE_DIM; row++) {
uint32_t column;
for (column = 0; column < APPEND_HEADS * APPEND_HEAD_DIM; column++)
put_tiled_element(source, APPEND_SOURCE_TILES, row, column,
source_pattern(99u, row, column));
}
memset(cache, (int)POISON_BYTE, cache_bytes);
/* Append 24 positions, ONE AT A TIME, exactly as decode does. Source row p is what
* position p receives, so every position holds different content. */
for (position = 0; position < 24u; position++) {
memset(&request, 0, sizeof(request));
request.cache = &layout;
request.position = position;
request.token_count = 1u;
request.source.address = 0u;
request.source.tile_bytes = TILE_BYTES;
request.source_row0 = position;
request.source_tiles = APPEND_SOURCE_TILES;
request.region.address = 0u;
request.region.tile_bytes = TILE_BYTES;
if (ft_tt_op_kv_append_parameters(&request, words) != FT_TT_OK) {
ft_tt_test_check_fmt("every step encodes", false, "position %u", position);
goto done;
}
model_run(words, source, cache, NULL);
}
for (position = 0; position < 24u; position++) {
uint32_t head;
for (head = 0; head < APPEND_HEADS; head++) {
uint32_t dimension;
for (dimension = 0; dimension < APPEND_HEAD_DIM; dimension++) {
const uint64_t tile =
ft_tt_kv_cache_tile_offset(&layout, head, position, dimension / TILE_DIM);
const size_t at =
(size_t)tile +
(size_t)ft_tt_tile_face_index(position % TILE_DIM, dimension % TILE_DIM) * 2u;
const uint16_t want =
source_pattern(99u, position, head * APPEND_HEAD_DIM + dimension);
uint16_t got;
memcpy(&got, cache + at, sizeof(got));
if (got != want)
wrong++;
}
}
}
ft_tt_test_check_fmt("all 24 positions survive 24 single-position appends", wrong == 0u,
"%zu of %u elements wrong", wrong,
24u * APPEND_HEADS * APPEND_HEAD_DIM);
done:
free(source);
free(cache);
}
/* ================================================================
* EMBEDDING, byte for byte
* ================================================================ */
#define EMBEDDING_VOCABULARY 64u
#define EMBEDDING_DIM 64u
#define EMBEDDING_TILES (EMBEDDING_DIM / TILE_DIM)
/**
* A gather, with a PERMUTED, repeating id list.
*
* The permutation is the point: a kernel that ignored the index array and read rows in
* order would return the right bytes from the wrong rows, which is the single most likely
* bug in a gather and the one a similar-looking table hides.
*/
static void test_embedding(bool zero_fill)
{
static const uint32_t ids[8] = { 40u, 3u, 17u, 3u, 63u, 0u, 55u, 8u };
ft_tt_op_embedding_request request;
uint32_t words[FT_TT_OP_MEMORY_WORDS];
unsigned char *table = NULL;
unsigned char *output = NULL;
unsigned char *expected = NULL;
const uint32_t row_bytes = EMBEDDING_DIM * 2u;
const uint32_t output_row0 = zero_fill ? 3u : 11u;
size_t table_bytes = (size_t)EMBEDDING_VOCABULARY * row_bytes;
size_t output_bytes = (size_t)EMBEDDING_TILES * TILE_BYTES;
uint32_t token;
uint32_t row;
size_t wrong = 0;
int rc;
ft_tt_test_step("EMBEDDING: a permuted gather with a repeat, %s",
zero_fill ? "zero-filling the rest of the tiles"
: "preserving the rest of the tiles");
table = malloc(table_bytes);
output = malloc(output_bytes);
expected = malloc(output_bytes);
if (!table || !output || !expected) {
ft_tt_test_check("the case sets up", false, "out of memory");
goto done;
}
/* A ROW-MAJOR table: row `id` is one contiguous run, which is the whole reason the
* table is not tilized (a face row is 32 B and a 32-byte DRAM READ is not legal). */
for (row = 0; row < EMBEDDING_VOCABULARY; row++) {
uint32_t column;
for (column = 0; column < EMBEDDING_DIM; column++) {
const uint16_t value = source_pattern(0x1234u, row, column);
memcpy(table + (size_t)row * row_bytes + (size_t)column * 2u, &value, sizeof(value));
}
}
memset(output, (int)POISON_BYTE, output_bytes);
memcpy(expected, output, output_bytes);
if (zero_fill)
memset(expected, 0, output_bytes);
for (token = 0; token < 8u; token++) {
uint32_t column;
for (column = 0; column < EMBEDDING_DIM; column++) {
const size_t at = (size_t)(column / TILE_DIM) * TILE_BYTES +
(size_t)ft_tt_tile_face_index(output_row0 + token,
column % TILE_DIM) *
2u;
const uint16_t value = source_pattern(0x1234u, ids[token], column);
memcpy(expected + at, &value, sizeof(value));
}
}
memset(&request, 0, sizeof(request));
request.token_count = 8u;
request.embedding_dim = EMBEDDING_DIM;
request.table.address = 0u;
request.table.tile_bytes = TILE_BYTES;
request.table_row_stride = row_bytes;
request.table_row_count = EMBEDDING_VOCABULARY;
request.output.address = 0u;
request.output.tile_bytes = TILE_BYTES;
request.output_bytes = (uint32_t)output_bytes;
request.output_row0 = output_row0;
request.zero_fill = zero_fill;
rc = ft_tt_op_embedding_parameters(&request, words);
if (rc != FT_TT_OK) {
ft_tt_test_check_fmt("the descriptor encodes", false, "rc=%d", rc);
goto done;
}
model_run(words, table, output, ids);
{
size_t index;
for (index = 0; index < output_bytes; index++) {
if (output[index] != expected[index])
wrong++;
}
}
ft_tt_test_check_fmt("the gathered rows land byte for byte, and nothing else moves",
wrong == 0u, "%zu of %zu bytes differ", wrong, output_bytes);
done:
free(table);
free(output);
free(expected);
}
/* ================================================================
* The split rule, the encoders' refusals, and the placement
* ================================================================ */
static void test_split_rule(void)
{
uint32_t positions[64];
uint32_t source_rows[64];
uint32_t counts[64];
uint32_t count = 0;
uint32_t index;
uint32_t total = 0;
bool contiguous = true;
int rc;
ft_tt_test_step("the split rule: a long append becomes descriptors that each fit one "
"tile row");
rc = ft_tt_op_kv_append_split(0u, 32u, 0u, positions, source_rows, counts, 64u, &count);
ft_tt_test_check_fmt("an aligned 32-token block is ONE descriptor",
rc == FT_TT_OK && count == 1u && counts[0] == 32u, "rc=%d count=%u",
rc, count);
rc = ft_tt_op_kv_append_split(5u, 100u, 0u, positions, source_rows, counts, 64u, &count);
ft_tt_test_check_fmt("100 tokens from position 5 split", rc == FT_TT_OK, "rc=%d", rc);
for (index = 0; index < count; index++) {
if ((positions[index] % TILE_DIM) + counts[index] > TILE_DIM ||
(source_rows[index] % TILE_DIM) + counts[index] > TILE_DIM)
contiguous = false;
if (index > 0 && (positions[index] != positions[index - 1] + counts[index - 1] ||
source_rows[index] != source_rows[index - 1] + counts[index - 1]))
contiguous = false;
total += counts[index];
}
ft_tt_test_check_fmt("every piece fits one source AND one destination tile row",
contiguous, "%u pieces", count);
ft_tt_test_check_fmt("and they account for every token", total == 100u, "%u of 100",
total);
/* Both ends bind independently: a source row and a position that are out of phase split
* more often than either would alone. */
rc = ft_tt_op_kv_append_split(0u, 64u, 16u, positions, source_rows, counts, 64u, &count);
ft_tt_test_check_fmt("an out-of-phase source splits on the SOURCE boundary too",
rc == FT_TT_OK && count == 4u && counts[0] == 16u, "count=%u first=%u",
count, counts[0]);
/* And keeping the two in phase is worth doing: 64 aligned tokens are 2 descriptors, the
* same 64 out of phase are 4. A projection output whose row 0 is position p0 is in
* phase by construction, so this is a property to preserve rather than a cost to pay. */
rc = ft_tt_op_kv_append_split(0u, 64u, 0u, positions, source_rows, counts, 64u, &count);
ft_tt_test_check_fmt("in phase, the same 64 tokens are half as many descriptors",
rc == FT_TT_OK && count == 2u, "count=%u", count);
rc = ft_tt_op_kv_append_split(5u, 100u, 0u, positions, source_rows, counts, 2u, &count);
ft_tt_test_check_fmt("too small an output array is refused, and still says how many",
rc == FT_TT_ENOMEM && count == 7u, "rc=%d count=%u", rc, count);
}
static void test_encoders_refuse_what_the_kernel_cannot_do(void)
{
ft_tt_kv_cache_layout layout;
ft_tt_op_kv_append_request append;
ft_tt_op_embedding_request embedding;
uint32_t words[FT_TT_OP_MEMORY_WORDS];
ft_tt_test_step("the encoders refuse what the kernel cannot do");
(void)ft_tt_kv_cache_layout_init(APPEND_HEADS, APPEND_HEAD_DIM, APPEND_CAPACITY, TILE_BYTES,
&layout);
memset(&append, 0, sizeof(append));
append.cache = &layout;
append.position = 0u;
append.token_count = 1u;
append.source.tile_bytes = TILE_BYTES;
append.source_tiles = APPEND_SOURCE_TILES;
append.region.tile_bytes = TILE_BYTES;
ft_tt_test_check("the baseline request encodes",
ft_tt_op_kv_append_parameters(&append, words) == FT_TT_OK, "");
append.position = 30u;
append.token_count = 4u;
ft_tt_test_check("a range crossing a destination tile row is refused",
ft_tt_op_kv_append_parameters(&append, words) == FT_TT_EINVAL, "");
append.position = 0u;
append.source_row0 = 30u;
ft_tt_test_check("a range crossing a SOURCE tile row is refused",
ft_tt_op_kv_append_parameters(&append, words) == FT_TT_EINVAL, "");
append.source_row0 = 0u;
append.token_count = 33u;
ft_tt_test_check("more than 32 rows is refused",
ft_tt_op_kv_append_parameters(&append, words) == FT_TT_EINVAL, "");
append.token_count = 1u;
append.position = APPEND_CAPACITY;
ft_tt_test_check("a position past the capacity is refused",
ft_tt_op_kv_append_parameters(&append, words) == FT_TT_EINVAL, "");
append.position = 0u;
append.source_tiles = APPEND_SOURCE_TILES - 1u;
ft_tt_test_check("a source too narrow for every head is refused",
ft_tt_op_kv_append_parameters(&append, words) == FT_TT_EINVAL, "");
append.source_tiles = APPEND_SOURCE_TILES;
append.source.tile_bytes = FT_TT_OP_TILE_BYTES_BFP8_B;
ft_tt_test_check("a source whose tile size disagrees with the cache is refused",
ft_tt_op_kv_append_parameters(&append, words) == FT_TT_EINVAL, "");
memset(&embedding, 0, sizeof(embedding));
embedding.token_count = 4u;
embedding.embedding_dim = 1536u;
embedding.table.tile_bytes = TILE_BYTES;
embedding.table_row_stride = 1536u * 2u;
embedding.table_row_count = 73448u;
embedding.output.tile_bytes = TILE_BYTES;
embedding.output_bytes = 48u * TILE_BYTES;
ft_tt_test_check("MiniCPM5-1B's 1536-wide embedding encodes",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_OK, "");
ft_tt_test_check_fmt("and asks for 48 tiles in one group",
words[FT_TT_OP_MEMORY_GROUP_TILES] == 48u &&
words[FT_TT_OP_MEMORY_GROUP_COUNT] == 1u,
"%u tiles, %u groups", words[FT_TT_OP_MEMORY_GROUP_TILES],
words[FT_TT_OP_MEMORY_GROUP_COUNT]);
ft_tt_test_check("every embedding descriptor is a GATHER",
(words[FT_TT_OP_MEMORY_FLAGS] &
(uint32_t)FT_TT_OP_MEMORY_FLAG_GATHER) != 0u,
"");
embedding.table_row_stride = 1536u * 2u + 32u;
ft_tt_test_check("a table row stride that breaks the 64-byte DRAM read alignment is "
"refused",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_EINVAL, "");
embedding.table_row_stride = 1536u * 2u;
embedding.embedding_dim = 1500u;
ft_tt_test_check("a width that is not a whole number of tiles is refused",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_EINVAL, "");
embedding.embedding_dim = 1536u;
embedding.output_row0 = 30u;
embedding.token_count = 4u;
ft_tt_test_check("a token block crossing a destination tile row is refused",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_EINVAL, "");
/* THE TWO BOUNDS THE KERNEL REFUSES WITHOUT (gaps 4 and 5). Each is asserted against
* FT_TT_EINVAL specifically and re-armed afterwards, so the case that follows it is not
* being carried by the previous one still being broken — the failure mode a check with
* several sufficient grounds always has (DEV_NOTES.md §3). */
embedding.output_row0 = 0u;
embedding.token_count = 4u;
ft_tt_test_check("the request is green again before the bound cases",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_OK, "");
embedding.table_row_count = 0u;
ft_tt_test_check("a table with no declared row count is refused — an unbounded token id "
"is an arbitrary DRAM read",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_EINVAL, "");
embedding.table_row_count = 73448u;
embedding.output_bytes = 0u;
ft_tt_test_check("an output with no declared extent is refused — an absolute row0 is an "
"unbounded byte offset",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_EINVAL, "");
embedding.output_bytes = 48u * TILE_BYTES;
embedding.output_row0 = 32u;
ft_tt_test_check("and a destination tile row past that extent is refused, not clamped",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_EINVAL, "");
embedding.output_bytes = 2u * 48u * TILE_BYTES;
ft_tt_test_check("the same row0 against an extent that HOLDS it encodes",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_OK, "");
ft_tt_test_check_fmt("and both bounds ride the descriptor",
words[FT_TT_OP_MEMORY_SRC_ROW_LIMIT] == 73448u &&
words[FT_TT_OP_MEMORY_DST_GROUP_STRIDE] == 2u * 48u * TILE_BYTES,
"row limit %u, group bytes %u", words[FT_TT_OP_MEMORY_SRC_ROW_LIMIT],
words[FT_TT_OP_MEMORY_DST_GROUP_STRIDE]);
}
static void test_circular_buffers_fit_the_arena(void)
{
ft_tt_launch_cb_entry entries[FT_TT_OP_MEMORY_CB_COUNT];
uint64_t mask = 0;
uint32_t index_address;
int rc;
ft_tt_test_step("circular buffers: ONE OUTPUT CB PER GROUP SIZE, and each is a whole "
"multiple of its own group");
/* MiniCPM5-1B: 48 tiles of hidden width for the embedding, 4 dim-tiles a KV head. */
rc = ft_tt_op_memory_place_cbs(48u, 4u, TILE_BYTES, entries, g_cb_buffer,
sizeof(g_cb_buffer), &mask);
ft_tt_test_check_fmt("a 1536-wide embedding + 4-tile KV residency places", rc == FT_TT_OK,
"rc=%d", rc);
index_address = ft_tt_op_memory_index_address(entries, FT_TT_OP_MEMORY_CB_COUNT);
ft_tt_test_check_fmt("and the index array has an address in the arena",
index_address >= FT_TT_OP_CB_ARENA_BASE &&
index_address < FT_TT_OP_CB_ARENA_END,
"0x%X", index_address);
ft_tt_test_check_fmt("the mask names exactly the four circular buffers this op uses",
mask == ((1ull << FT_TT_OP_MEMORY_CB_STAGE) |
(1ull << FT_TT_OP_MEMORY_CB_INDEX) |
(1ull << FT_TT_OP_MEMORY_CB_OUT_EMBEDDING) |
(1ull << FT_TT_OP_MEMORY_CB_OUT_KV)),
"0x%llX", (unsigned long long)mask);
/* THE INVARIANT THE SPLIT EXISTS FOR, stated as the arithmetic the kernel's guard runs:
* each output CB holds a whole number of ITS OWN blocks, so its pointer is on that grid
* at every descriptor boundary. The two are DIFFERENT buffers — the check that would
* have failed before the split is that they do not overlap. */
{
const ft_tt_launch_cb_entry *embedding_out =
ft_tt_op_cb_find(entries, FT_TT_OP_MEMORY_CB_COUNT, FT_TT_OP_MEMORY_CB_OUT_EMBEDDING);
const ft_tt_launch_cb_entry *kv_out =
ft_tt_op_cb_find(entries, FT_TT_OP_MEMORY_CB_COUNT, FT_TT_OP_MEMORY_CB_OUT_KV);
ft_tt_test_check("both output circular buffers are placed",
embedding_out != NULL && kv_out != NULL, "");
if (embedding_out && kv_out) {
ft_tt_test_check_fmt("the embedding output holds a whole number of 48-tile blocks",
embedding_out->page_count % 48u == 0u, "%u pages",
embedding_out->page_count);
ft_tt_test_check_fmt("the KV output holds a whole number of 4-tile blocks",
kv_out->page_count % 4u == 0u, "%u pages", kv_out->page_count);
ft_tt_test_check_fmt("and they are DISJOINT byte ranges — the whole point of the "
"split, since one buffer cannot carry two group sizes",
embedding_out->address + embedding_out->size_bytes <=
kv_out->address ||
kv_out->address + kv_out->size_bytes <=
embedding_out->address,
"embedding 0x%X+%u kv 0x%X+%u", embedding_out->address,
embedding_out->size_bytes, kv_out->address,
kv_out->size_bytes);
}
}
rc = ft_tt_op_memory_place_cbs(0u, 4u, TILE_BYTES, entries, g_cb_buffer,
sizeof(g_cb_buffer), &mask);
ft_tt_test_check("a zero-width embedding group is refused", rc == FT_TT_EINVAL, "");
rc = ft_tt_op_memory_place_cbs(48u, 0u, TILE_BYTES, entries, g_cb_buffer,
sizeof(g_cb_buffer), &mask);
ft_tt_test_check("a zero-width KV group is refused", rc == FT_TT_EINVAL, "");
rc = ft_tt_op_memory_place_cbs(48u, 4u, FT_TT_OP_TILE_BYTES_BFP8_B, entries, g_cb_buffer,
sizeof(g_cb_buffer), &mask);
ft_tt_test_check("a block-float page size is refused", rc == FT_TT_EINVAL, "");
}
/* ================================================================
* The zero-length guard, and the cross-op row phase
* ================================================================ */
/**
* Nothing this family issues can be zero bytes long.
*
* A zero-length NoC read is suspected of taking the WHOLE CARD down — the RoPE round's
* card-down on 2026-08-12 happened at exactly the first descriptor that computed one
* (INFERRED, and deliberately not re-confirmed, because confirming it costs the card for
* every worker on it). Both ops here are gathers built out of computed lengths, so a zero
* arrives by arithmetic rather than by a typo: a token count of zero, a collapsed range, an
* underflowed subtraction.
*
* So the guard is asserted three ways — the kernel HAS one, every length it can compute is
* non-zero, and the host REFUSES the requests that would produce one.
*/
static void test_no_transfer_can_be_zero_length(void)
{
ft_tt_kv_cache_layout layout;
ft_tt_op_kv_append_request append;
ft_tt_op_embedding_request embedding;
uint32_t words[FT_TT_OP_MEMORY_WORDS];
uint32_t row0;
unsigned zero_runs = 0;
unsigned zero_rows = 0;
uint32_t group_tiles;
ft_tt_test_step("no transfer this family issues can be zero bytes long");
ft_tt_test_check("the kernel has the guard, and it answers",
ft_tt_device_length_is_issuable(0u) == 0 &&
ft_tt_device_length_is_issuable(32u) == 1,
"");
for (row0 = 0; row0 < TILE_DIM; row0++) {
uint32_t count;
for (count = 1u; row0 + count <= TILE_DIM; count++) {
uint32_t offsets[8];
uint32_t lengths[8];
const uint32_t run_count =
ft_tt_device_destination_runs(row0, count, TILE_BYTES, offsets, lengths);
uint32_t index;
for (index = 0; index < run_count; index++) {
if (!ft_tt_device_length_is_issuable(lengths[index]))
zero_runs++;
}
}
}
ft_tt_test_check_fmt("every run of every one of the 528 row ranges is non-empty",
zero_runs == 0u, "%u empty", zero_runs);
for (group_tiles = 1u; group_tiles <= 64u; group_tiles++) {
if (!ft_tt_device_length_is_issuable(
ft_tt_device_group_row_bytes(group_tiles, TILE_BYTES)))
zero_rows++;
}
ft_tt_test_check_fmt("a source row is non-empty at every group width 1..64",
zero_rows == 0u, "%u empty", zero_rows);
/* And the host refuses the requests that would compute one, rather than sending a
* descriptor the kernel has to refuse on the card. */
(void)ft_tt_kv_cache_layout_init(APPEND_HEADS, APPEND_HEAD_DIM, APPEND_CAPACITY, TILE_BYTES,
&layout);
memset(&append, 0, sizeof(append));
append.cache = &layout;
append.token_count = 0u;
append.source.tile_bytes = TILE_BYTES;
append.source_tiles = APPEND_SOURCE_TILES;
append.region.tile_bytes = TILE_BYTES;
ft_tt_test_check("a zero-token append is REFUSED, not sent as a no-op",
ft_tt_op_kv_append_parameters(&append, words) == FT_TT_EINVAL, "");
memset(&embedding, 0, sizeof(embedding));
embedding.token_count = 0u;
embedding.embedding_dim = 1536u;
embedding.table.tile_bytes = TILE_BYTES;
embedding.table_row_stride = 1536u * 2u;
embedding.output.tile_bytes = TILE_BYTES;
ft_tt_test_check("a zero-token gather is REFUSED",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_EINVAL, "");
embedding.token_count = 4u;
embedding.embedding_dim = 0u;
ft_tt_test_check("a zero-width gather is REFUSED",
ft_tt_op_embedding_parameters(&embedding, words) == FT_TT_EINVAL, "");
}
/**
* §1.5b: a token at position p occupies tile row p % 32 — in BOTH ops, and RoPE depends on
* it.
*
* This is the check that keeps a cross-op constraint from being lost between two authors:
* RoPE multiplies activation tile row `r` by sin/cos table row `r`, so a hidden state that
* the embedding op put at the wrong row is rotated by the wrong angle — a plausible wrong
* answer with no error anywhere.
*/
static void test_row_phase_invariant(void)
{
ft_tt_kv_cache_layout layout;
ft_tt_op_kv_append_request append;
ft_tt_op_embedding_request embedding;
uint32_t append_words[FT_TT_OP_MEMORY_WORDS];
uint32_t embedding_words[FT_TT_OP_MEMORY_WORDS];
uint32_t position;
unsigned disagreements = 0;
ft_tt_test_step("the row-phase invariant: position p lands at tile row p %% 32, in BOTH "
"ops");
for (position = 0; position < 100u; position++) {
if (ft_tt_op_memory_position_row(position) != position % TILE_DIM)
disagreements++;
}
ft_tt_test_check_fmt("the helper is position modulo 32", disagreements == 0u, "%u wrong",
disagreements);
(void)ft_tt_kv_cache_layout_init(APPEND_HEADS, APPEND_HEAD_DIM, APPEND_CAPACITY, TILE_BYTES,
&layout);
for (position = 0; position < APPEND_CAPACITY; position++) {
memset(&append, 0, sizeof(append));
append.cache = &layout;
append.position = position;
append.token_count = 1u;
append.source.tile_bytes = TILE_BYTES;
append.source_tiles = APPEND_SOURCE_TILES;
append.region.tile_bytes = TILE_BYTES;
if (ft_tt_op_kv_append_parameters(&append, append_words) != FT_TT_OK) {
disagreements++;
continue;
}
memset(&embedding, 0, sizeof(embedding));
embedding.token_count = 1u;
embedding.embedding_dim = 1536u;
embedding.table.tile_bytes = TILE_BYTES;
embedding.table_row_stride = 1536u * 2u;
embedding.table_row_count = 73448u;
embedding.output.tile_bytes = TILE_BYTES;
embedding.output_bytes = 48u * TILE_BYTES;
embedding.output_row0 = ft_tt_op_memory_position_row(position);
if (ft_tt_op_embedding_parameters(&embedding, embedding_words) != FT_TT_OK) {
disagreements++;
continue;
}
/* The two ops must put the same position on the same tile row — that is the whole
* statement, and it is what RoPE reads. */
if (append_words[FT_TT_OP_MEMORY_DST_ROW0] % TILE_DIM !=
embedding_words[FT_TT_OP_MEMORY_DST_ROW0] % TILE_DIM)
disagreements++;
if (embedding_words[FT_TT_OP_MEMORY_DST_ROW0] % TILE_DIM != position % TILE_DIM)
disagreements++;
}
ft_tt_test_check_fmt("EMBEDDING and KV_APPEND agree on the row for all 128 positions",
disagreements == 0u, "%u disagreements", disagreements);
}
/* ================================================================ */
int main(void)
{
ft_tt_test_step("Phase 6 memory ops — OFF-HARDWARE gate");
test_wire_constants_agree();
test_chunk_offset_is_the_tilize_layout();
test_destination_runs_cover_exactly_the_rows();
test_kv_cache_layout();
test_kv_append();
test_kv_append_preserves_earlier_positions();
test_embedding(true);
test_embedding(false);
test_split_rule();
test_encoders_refuse_what_the_kernel_cannot_do();
test_circular_buffers_fit_the_arena();
test_no_transfer_can_be_zero_length();
test_row_phase_invariant();
return ft_tt_test_finish();
}
Reference in New Issue View Git Blame Copy Permalink