1127 lines
54 KiB
C
1127 lines
54 KiB
C
/* SPDX-License-Identifier: AGPL-3.0-or-later */
|
|
/**
|
|
* @file test_matmul_block.c
|
|
* @brief The BLOCKED resident matmul, checked off hardware: the reader's DRAM reads and L1
|
|
* layout, the compute thread's indexed unpacks, the pairing of A and B tiles that reaches
|
|
* each DST tile, and the destination the writer places that tile at.
|
|
*
|
|
* WHAT THIS SUITE IS FOR. `kMatmulColumnBlock` carries TWO blocks now. `NB` moves the reader
|
|
* from one output column per pass to NB, which is what stops the A column being re-read once
|
|
* per column. `MB` (the high half of the same word) holds MB A tile-rows and streams B past
|
|
* all of them, which is what stops the WEIGHT column being re-read once per row. Nothing
|
|
* about either is visible on the card: a wrong L1 stride pairs the right A tile with the
|
|
* wrong weight tile, a wrong destination index writes a correct tile to the wrong address,
|
|
* every kernel completes, every status word is clean, and the answer is silently wrong. So
|
|
* the arithmetic is modelled here and checked against the DRAM tile identities, before a
|
|
* card is involved.
|
|
*
|
|
* THE MODEL IS THE THREE KERNELS' OWN ARITHMETIC, transliterated:
|
|
* - resident_matmul_reader.c: for each pass it reserves `depth * MB` pages of CB_IN0 and
|
|
* `depth * NB` of CB_IN1, reads `rows` A blocks ROW-MAJOR and `columns` B blocks
|
|
* COLUMN-MAJOR, and pushes the FULL chunk on both. The pointers wrap on an EXACT hit of
|
|
* the limit and nowhere else.
|
|
* - resident_matmul_compute.c: for each k-step and each row it names A tile
|
|
* `row*depth + step` and B tile `step`, and the unpacker's MOP then walks `columns` SrcA
|
|
* tiles at a stride of `depth` tiles — which is what
|
|
* `ft_tensix_unpack_matmul_set_srca_tile_stride()` programs. DST base is `row*columns`.
|
|
* - resident_matmul_writer.c: it re-walks those loops and places output tile `(m, n)` at
|
|
* `m * pitch + n`, where the pitch is descriptor word a2 (0 meaning `tile_cols`).
|
|
*
|
|
* THE THREE CHECKS THAT MATTER:
|
|
* 1. the (A tile, B tile) PAIRS reaching output tile `(m, n)` are exactly
|
|
* `{ (m*Kt + k, tile(n, k)) : k < Kt }` — that IS the matmul, stated as a multiset;
|
|
* 2. the DESTINATION the writer computes for that tile is `m * pitch + n`;
|
|
* 3. the DRAM tiles read obey `A = Mt*ceil(Nt/NB)*Kt` and `B = ceil(Mt/MB)*Nt*Kt`, whose
|
|
* sum is `Mt*Nt*Kt*(1/NB + 1/MB)` — the law the whole MB lever rests on. **In TILES
|
|
* that is symmetric and its optimum is MB = NB = 2. In BYTES it is not**, because NB
|
|
* divides the 2048 B activation tile and MB divides the 1088 B Bfp8_b weight tile: the
|
|
* byte optimum is MB=2/NB=2 by **2.0%** at Bfp8_b, MB=1/NB=4 at Bfp4_b, and MB=2/NB=2
|
|
* by 25% only at bf16 weights. Both laws are checked, because checking only the first
|
|
* is how "the row block is worth 1.25x" survives contact with a Q8_0 model.
|
|
*
|
|
* NO CARD. It opens no device and reads no artifact, so it runs on the Mac where the code is
|
|
* written. It reads `kernels/resident_queue_protocol.h` through FT_TT_KERNEL_DIR to pin the
|
|
* parameter-word contract; running it without that is a FAILURE, not a skip.
|
|
*
|
|
* @status Complete for the shapes Ornith dispatches (Mt = 1, N-major and K-major panels, NB
|
|
* in {1, 2, 4}, and the ragged Nt = 1 weight that 48 of Ornith's 249 projections are) and
|
|
* for every (MB, NB) the DST section admits.
|
|
* @issues None
|
|
* @todo None
|
|
*/
|
|
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "ft_tt_queue.h"
|
|
#include "ft_tt_test_report.h"
|
|
#include "resident_matmul_block_protocol.h"
|
|
|
|
/** Ornith's activation tile (bf16 32x32) and weight tile (Bfp8_b). */
|
|
#define A_TILE_BYTES 2048u
|
|
#define B_TILE_BYTES 1088u
|
|
|
|
/** The DST section at `dst_accum_mode 1`, SyncHalf: `(FT_TENSIX_DEST_REGISTER_HALF_SIZE >> 1)` rows
|
|
* over `1 << 6` rows a tile = 4. */
|
|
#define DST_SECTION_TILES FT_TT_MATMUL_DST_SECTION_TILES
|
|
|
|
#define MAX_TILES 4096u
|
|
|
|
/** One circular buffer as the firmware keeps it: a base, a limit and a page size, with the
|
|
* exact-hit wrap that `ft_tensix_cb_push_back()` and `..._pop_front()` perform. */
|
|
typedef struct model_cb {
|
|
uint32_t page_bytes;
|
|
uint32_t page_count;
|
|
uint32_t base;
|
|
uint32_t limit; /* exclusive here; the kernel's `fifo_limit` is the same address */
|
|
uint32_t write_pointer;
|
|
uint32_t read_pointer;
|
|
/** Which DRAM tile each page currently holds, indexed by page. -1 is never written. */
|
|
int32_t page_tile[MAX_TILES];
|
|
} model_cb;
|
|
|
|
/** Everything one descriptor says, plus the residency constants the CBs were sized for. */
|
|
typedef struct model_shape {
|
|
const char *name;
|
|
uint32_t tile_rows; /* Mt */
|
|
uint32_t tile_inner; /* Kt */
|
|
uint32_t tile_cols; /* Nt */
|
|
uint32_t depth;
|
|
uint32_t column_block; /* NB */
|
|
uint32_t row_block; /* MB */
|
|
int b_n_major;
|
|
/** The DST section this shape's KERNEL was built for, in tiles. 0 means this build's
|
|
* own `FT_TT_MATMUL_DST_SECTION_TILES`, which is the SyncHalf four. */
|
|
uint32_t section_tiles;
|
|
} model_shape;
|
|
|
|
/** The section a shape is checked against: its own, or this build's if it names none. */
|
|
static uint32_t shape_section(const model_shape *shape)
|
|
{
|
|
return shape->section_tiles ? shape->section_tiles : DST_SECTION_TILES;
|
|
}
|
|
|
|
/** What the compute thread issued: one (A tile, B tile) pair into one DST tile. */
|
|
typedef struct model_pair {
|
|
uint32_t row; /* the OUTPUT row this DST tile becomes */
|
|
uint32_t column; /* the OUTPUT column this DST tile becomes */
|
|
uint32_t dst;
|
|
int32_t a_tile;
|
|
int32_t b_tile;
|
|
} model_pair;
|
|
|
|
/** One tile leaving CB_OUT: which output element it is, and where the writer put it. */
|
|
typedef struct model_out {
|
|
uint32_t row;
|
|
uint32_t column;
|
|
uint32_t destination; /* the tile index the writer computed */
|
|
} model_out;
|
|
|
|
typedef struct model_result {
|
|
uint64_t a_tiles_read;
|
|
uint64_t b_tiles_read;
|
|
uint32_t max_dst;
|
|
int off_grid; /* a pointer left the chunk grid */
|
|
int out_of_bounds; /* a pointer or an indexed read left the buffer */
|
|
int unwritten; /* the compute named a page the reader never filled */
|
|
uint64_t pair_count;
|
|
model_pair *pairs;
|
|
uint64_t pair_capacity;
|
|
/* CB_OUT: what the writer did with each tile, in the order the packer pushed them. */
|
|
model_out *out;
|
|
uint64_t out_count;
|
|
} model_result;
|
|
|
|
/** The mutations this suite proves red with. The shipped values are all zero. */
|
|
typedef struct model_mutation {
|
|
uint32_t stride_tiles; /* the MOP's SrcA step; shipped is `depth` */
|
|
int b_k_major; /* lay B out K-major in L1 instead of column-major */
|
|
int a_row_shared; /* every output row unpacks A tile `step` — i.e. row 0's */
|
|
int writer_counter; /* the writer places tiles with a running counter, not (m, n) */
|
|
} model_mutation;
|
|
|
|
/* ------------------------------------------------------------------------------------
|
|
* The kernels' pointer arithmetic
|
|
* ---------------------------------------------------------------------------------- */
|
|
|
|
static void model_cb_init(model_cb *cb, uint32_t page_bytes, uint32_t page_count, uint32_t base)
|
|
{
|
|
uint32_t page;
|
|
|
|
cb->page_bytes = page_bytes;
|
|
cb->page_count = page_count;
|
|
cb->base = base;
|
|
cb->limit = base + page_bytes * page_count;
|
|
cb->write_pointer = base;
|
|
cb->read_pointer = base;
|
|
for (page = 0; page < MAX_TILES; page++)
|
|
cb->page_tile[page] = -1;
|
|
}
|
|
|
|
/** `ft_tensix_cb_push_back()`: advance, then wrap on an EXACT hit of the limit. */
|
|
static void model_cb_push(model_cb *cb, uint32_t pages)
|
|
{
|
|
cb->write_pointer += pages * cb->page_bytes;
|
|
if (cb->write_pointer == cb->limit)
|
|
cb->write_pointer = cb->base;
|
|
}
|
|
|
|
/** `ft_tensix_compute_cb_pop_front()`: the same rule on the read side. */
|
|
static void model_cb_pop(model_cb *cb, uint32_t pages)
|
|
{
|
|
cb->read_pointer += pages * cb->page_bytes;
|
|
if (cb->read_pointer >= cb->limit)
|
|
cb->read_pointer -= cb->page_bytes * cb->page_count;
|
|
}
|
|
|
|
/** `ft_tensix_cb_block_grid_is_safe()`, the producer's form. */
|
|
static int model_cb_on_grid(const model_cb *cb, uint32_t chunk_pages, uint32_t pointer)
|
|
{
|
|
if (chunk_pages == 0u || chunk_pages > cb->page_count || cb->page_count % chunk_pages != 0u)
|
|
return 0;
|
|
if (pointer < cb->base || pointer >= cb->limit)
|
|
return 0;
|
|
return ((pointer - cb->base) % (chunk_pages * cb->page_bytes)) == 0;
|
|
}
|
|
|
|
/** Which page of the buffer an L1 address names, or -1 if it is outside. */
|
|
static int32_t model_cb_page_of(const model_cb *cb, uint32_t address)
|
|
{
|
|
if (address < cb->base || address >= cb->limit)
|
|
return -1;
|
|
if ((address - cb->base) % cb->page_bytes != 0u)
|
|
return -1;
|
|
return (int32_t)((address - cb->base) / cb->page_bytes);
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------------------
|
|
* The three kernels, run in lockstep one pass at a time
|
|
* ---------------------------------------------------------------------------------- */
|
|
|
|
/** The DRAM tile identity of weight tile (column, k), as the reader computes its offset. */
|
|
static int32_t model_b_tile(const model_shape *shape, uint32_t column, uint32_t k)
|
|
{
|
|
return shape->b_n_major ? (int32_t)(column * shape->tile_inner + k)
|
|
: (int32_t)(k * shape->tile_cols + column);
|
|
}
|
|
|
|
/**
|
|
* Run one descriptor through all three thirds of the triad.
|
|
*
|
|
* LOCKSTEP IS A SIMPLIFICATION AND IT IS THE CONSERVATIVE ONE. On the card the reader may be
|
|
* a whole chunk ahead; here it fills a pass and the compute thread drains it immediately.
|
|
* That cannot hide a layout error — the addresses are the same either way — and it lets the
|
|
* model check that the page the compute NAMES is the page the reader WROTE, which is the
|
|
* property a stride error breaks.
|
|
*/
|
|
static void model_run(const model_shape *shape, const model_mutation *mutate, model_result *out)
|
|
{
|
|
model_cb in0;
|
|
model_cb in1;
|
|
const uint32_t depth = shape->depth;
|
|
const uint32_t block = shape->column_block;
|
|
const uint32_t rblock = shape->row_block;
|
|
const uint32_t chunk = depth * block;
|
|
const uint32_t a_chunk = depth * rblock;
|
|
const uint32_t stride_tiles = mutate->stride_tiles ? mutate->stride_tiles : depth;
|
|
/* The host sends the weight's FULL Nt; a single member's descriptor carries its own
|
|
* slice in `tile_cols`. Modelled as the whole tensor, which is the un-sharded case. */
|
|
const uint32_t pitch = shape->tile_cols;
|
|
uint32_t mt;
|
|
uint32_t nt;
|
|
uint32_t kt;
|
|
uint32_t step;
|
|
uint32_t column;
|
|
uint32_t row;
|
|
uint32_t counter = 0u;
|
|
|
|
model_cb_init(&in0, A_TILE_BYTES, a_chunk * 2u, 0x10000u);
|
|
model_cb_init(&in1, B_TILE_BYTES, chunk * 2u, 0x40000u);
|
|
out->a_tiles_read = 0;
|
|
out->b_tiles_read = 0;
|
|
out->max_dst = 0;
|
|
out->off_grid = 0;
|
|
out->out_of_bounds = 0;
|
|
out->unwritten = 0;
|
|
out->pair_count = 0;
|
|
out->out_count = 0;
|
|
|
|
for (mt = 0; mt < shape->tile_rows; mt += rblock) {
|
|
const uint32_t rows =
|
|
(shape->tile_rows - mt) < rblock ? (shape->tile_rows - mt) : rblock;
|
|
|
|
for (nt = 0; nt < shape->tile_cols; nt += block) {
|
|
const uint32_t columns =
|
|
(shape->tile_cols - nt) < block ? (shape->tile_cols - nt) : block;
|
|
|
|
for (kt = 0; kt < shape->tile_inner; kt += depth) {
|
|
uint32_t a_write;
|
|
uint32_t b_write;
|
|
|
|
/* --- the reader --- */
|
|
if (!model_cb_on_grid(&in0, a_chunk, in0.write_pointer) ||
|
|
!model_cb_on_grid(&in1, chunk, in1.write_pointer))
|
|
out->off_grid = 1;
|
|
a_write = in0.write_pointer;
|
|
b_write = in1.write_pointer;
|
|
if (a_write + a_chunk * A_TILE_BYTES > in0.limit ||
|
|
b_write + chunk * B_TILE_BYTES > in1.limit)
|
|
out->out_of_bounds = 1;
|
|
|
|
for (row = 0; row < rows; row++) {
|
|
for (step = 0; step < depth; step++) {
|
|
/* ROW-MAJOR IN L1: row i's `depth` tiles at `i * depth`. */
|
|
const uint32_t slot = row * depth + step;
|
|
int32_t page = model_cb_page_of(&in0, a_write + slot * A_TILE_BYTES);
|
|
|
|
if (page < 0) {
|
|
out->out_of_bounds = 1;
|
|
continue;
|
|
}
|
|
in0.page_tile[page] =
|
|
(int32_t)((mt + row) * shape->tile_inner + kt + step);
|
|
out->a_tiles_read++;
|
|
}
|
|
}
|
|
for (column = 0; column < columns; column++) {
|
|
for (step = 0; step < depth; step++) {
|
|
/* COLUMN-MAJOR IN L1: column j's `depth` tiles at `j * depth`, which
|
|
* is what keeps the N-major panel's per-column read whole. The
|
|
* mutation lays them K-major instead. */
|
|
const uint32_t slot = mutate->b_k_major ? step * columns + column
|
|
: column * depth + step;
|
|
int32_t page = model_cb_page_of(&in1, b_write + slot * B_TILE_BYTES);
|
|
|
|
if (page < 0) {
|
|
out->out_of_bounds = 1;
|
|
continue;
|
|
}
|
|
in1.page_tile[page] = model_b_tile(shape, nt + column, kt + step);
|
|
out->b_tiles_read++;
|
|
}
|
|
}
|
|
model_cb_push(&in0, a_chunk);
|
|
model_cb_push(&in1, chunk);
|
|
|
|
/* --- the compute thread --- */
|
|
for (step = 0; step < depth; step++) {
|
|
for (row = 0; row < rows; row++) {
|
|
/* `tile_index_a` is `row * depth + step`; the mutation shares row
|
|
* 0's activation across every output row. */
|
|
const uint32_t a_index = mutate->a_row_shared ? step : row * depth + step;
|
|
const uint32_t a_address = in0.read_pointer + a_index * A_TILE_BYTES;
|
|
const uint32_t b_base = in1.read_pointer + step * B_TILE_BYTES;
|
|
|
|
for (column = 0; column < columns; column++) {
|
|
/* The MOP bumps SrcA by the tile-size GPR once per iteration. */
|
|
const uint32_t b_address =
|
|
b_base + column * stride_tiles * B_TILE_BYTES;
|
|
const uint32_t dst = row * columns + column;
|
|
int32_t a_page = model_cb_page_of(&in0, a_address);
|
|
int32_t b_page = model_cb_page_of(&in1, b_address);
|
|
model_pair *pair;
|
|
|
|
if (dst >= shape_section(shape))
|
|
out->max_dst = shape_section(shape) + 1u;
|
|
else if (dst + 1u > out->max_dst)
|
|
out->max_dst = dst + 1u;
|
|
if (a_page < 0 || b_page < 0) {
|
|
out->out_of_bounds = 1;
|
|
continue;
|
|
}
|
|
if (in0.page_tile[a_page] < 0 || in1.page_tile[b_page] < 0) {
|
|
out->unwritten = 1;
|
|
continue;
|
|
}
|
|
if (out->pair_count >= out->pair_capacity)
|
|
continue;
|
|
pair = &out->pairs[out->pair_count++];
|
|
pair->row = mt + row;
|
|
pair->column = nt + column;
|
|
pair->dst = dst;
|
|
pair->a_tile = in0.page_tile[a_page];
|
|
pair->b_tile = in1.page_tile[b_page];
|
|
}
|
|
}
|
|
}
|
|
model_cb_pop(&in0, a_chunk);
|
|
model_cb_pop(&in1, chunk);
|
|
}
|
|
|
|
/* --- the packer, then the writer: one tile at a time, ROW-MAJOR within the
|
|
* block because that is the order the DST indices were assigned in. --- */
|
|
for (row = 0; row < rows; row++) {
|
|
for (column = 0; column < columns; column++) {
|
|
model_out *entry = &out->out[out->out_count++];
|
|
|
|
entry->row = mt + row;
|
|
entry->column = nt + column;
|
|
entry->destination = mutate->writer_counter
|
|
? counter
|
|
: (mt + row) * pitch + nt + column;
|
|
counter++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------------------
|
|
* The checks
|
|
* ---------------------------------------------------------------------------------- */
|
|
|
|
/**
|
|
* Every pair that reached output tile `(m, c)` must be `(m*Kt + k, tile(c, k))` for a
|
|
* distinct k, and there must be exactly Kt of them. THIS IS THE MATMUL, and it is what a
|
|
* stride error, an L1 layout error, a DST-index error or a shared activation row all break.
|
|
*/
|
|
static int pairs_are_the_matmul(const model_shape *shape, const model_result *result)
|
|
{
|
|
uint32_t column;
|
|
uint32_t m;
|
|
unsigned char *seen = calloc(shape->tile_inner, 1u);
|
|
int ok = 1;
|
|
|
|
if (!seen)
|
|
return 0;
|
|
for (m = 0; m < shape->tile_rows && ok; m++) {
|
|
for (column = 0; column < shape->tile_cols && ok; column++) {
|
|
uint64_t index;
|
|
uint32_t found = 0;
|
|
|
|
memset(seen, 0, shape->tile_inner);
|
|
for (index = 0; index < result->pair_count; index++) {
|
|
const model_pair *pair = &result->pairs[index];
|
|
uint32_t k;
|
|
|
|
if (pair->column != column || pair->row != m)
|
|
continue;
|
|
if (pair->a_tile < 0 ||
|
|
(uint32_t)pair->a_tile < m * shape->tile_inner ||
|
|
(uint32_t)pair->a_tile >= (m + 1u) * shape->tile_inner) {
|
|
ok = 0;
|
|
break;
|
|
}
|
|
k = (uint32_t)pair->a_tile - m * shape->tile_inner;
|
|
if (seen[k] || pair->b_tile != model_b_tile(shape, column, k)) {
|
|
ok = 0;
|
|
break;
|
|
}
|
|
seen[k] = 1u;
|
|
found++;
|
|
}
|
|
if (found != shape->tile_inner)
|
|
ok = 0;
|
|
}
|
|
}
|
|
free(seen);
|
|
return ok;
|
|
}
|
|
|
|
/** Output tile `(m, n)` must leave for destination `m * Nt + n`, exactly once. */
|
|
static int writer_places_every_tile(const model_shape *shape, const model_result *result)
|
|
{
|
|
const uint64_t total = (uint64_t)shape->tile_rows * shape->tile_cols;
|
|
unsigned char *seen;
|
|
uint64_t index;
|
|
int ok = 1;
|
|
|
|
if (result->out_count != total)
|
|
return 0;
|
|
seen = calloc((size_t)total, 1u);
|
|
if (!seen)
|
|
return 0;
|
|
for (index = 0; index < result->out_count; index++) {
|
|
const model_out *entry = &result->out[index];
|
|
const uint64_t want = (uint64_t)entry->row * shape->tile_cols + entry->column;
|
|
|
|
if (entry->destination != want || want >= total || seen[want]) {
|
|
ok = 0;
|
|
break;
|
|
}
|
|
seen[want] = 1u;
|
|
}
|
|
free(seen);
|
|
return ok;
|
|
}
|
|
|
|
/** The A tiles a descriptor costs: one column per ROW per column BLOCK. Unmoved by MB. */
|
|
static uint64_t expected_a_tiles(const model_shape *shape)
|
|
{
|
|
const uint64_t passes = (shape->tile_cols + shape->column_block - 1u) / shape->column_block;
|
|
|
|
return (uint64_t)shape->tile_rows * passes * shape->tile_inner;
|
|
}
|
|
|
|
/** The B tiles a descriptor costs: one column per COLUMN per row BLOCK. Divided by MB. */
|
|
static uint64_t expected_b_tiles(const model_shape *shape)
|
|
{
|
|
const uint64_t passes = (shape->tile_rows + shape->row_block - 1u) / shape->row_block;
|
|
|
|
return passes * shape->tile_cols * shape->tile_inner;
|
|
}
|
|
|
|
static const model_mutation kShipped = { 0u, 0, 0, 0 };
|
|
|
|
static void check_shape(const model_shape *shape, model_result *result)
|
|
{
|
|
uint64_t index;
|
|
int order_ok = 1;
|
|
|
|
model_run(shape, &kShipped, result);
|
|
|
|
ft_tt_test_step("shape %s — Mt %u Kt %u Nt %u depth %u MBxNB %ux%u %s, DST %u",
|
|
shape->name, shape->tile_rows, shape->tile_inner, shape->tile_cols,
|
|
shape->depth, shape->row_block, shape->column_block,
|
|
shape->b_n_major ? "N-major" : "K-major", shape_section(shape));
|
|
ft_tt_test_check_fmt(" the A x B pairs reaching every output TILE ARE the matmul",
|
|
pairs_are_the_matmul(shape, result),
|
|
"%s: %llu pairs over %u x %u tiles", shape->name,
|
|
(unsigned long long)result->pair_count, shape->tile_rows,
|
|
shape->tile_cols);
|
|
ft_tt_test_check_fmt(" A tiles read = Mt * ceil(Nt/NB) * Kt, UNMOVED by MB",
|
|
result->a_tiles_read == expected_a_tiles(shape),
|
|
"%s: %llu, want %llu", shape->name,
|
|
(unsigned long long)result->a_tiles_read,
|
|
(unsigned long long)expected_a_tiles(shape));
|
|
ft_tt_test_check_fmt(" B tiles read = ceil(Mt/MB) * Nt * Kt, DIVIDED by MB",
|
|
result->b_tiles_read == expected_b_tiles(shape),
|
|
"%s: %llu, want %llu (MB=%u divides the %llu an unblocked reader "
|
|
"costs)", shape->name, (unsigned long long)result->b_tiles_read,
|
|
(unsigned long long)expected_b_tiles(shape), shape->row_block,
|
|
(unsigned long long)((uint64_t)shape->tile_rows * shape->tile_cols *
|
|
shape->tile_inner));
|
|
ft_tt_test_check_fmt(" every CB pointer stays on its chunk grid", !result->off_grid, "%s",
|
|
shape->name);
|
|
ft_tt_test_check_fmt(" no read or write leaves a circular buffer", !result->out_of_bounds,
|
|
"%s", shape->name);
|
|
ft_tt_test_check_fmt(" the compute thread names no page the reader did not fill",
|
|
!result->unwritten, "%s", shape->name);
|
|
ft_tt_test_check_fmt(" DST tiles held at once are MB*NB and fit the section",
|
|
result->max_dst <= shape_section(shape) &&
|
|
result->max_dst <= shape->row_block * shape->column_block,
|
|
"%s: %u of %u", shape->name, result->max_dst, shape_section(shape));
|
|
/* THE REASON NO DST WRITE-ADDRESS SITE CHANGES AT SyncFull. Every one of them computes
|
|
* `(tile_index << DstTileSizeLog2) + (dest_offset_id ? HALF : 0)`, and SyncFull pins the
|
|
* id at 0, so the highest address a section of `section_tiles` reaches is
|
|
* `(section_tiles - 1) << 6`. At eight tiles that is 448, one tile inside
|
|
* `DEST_REGISTER_FULL_SIZE >> 1 = 512`. The formula fits the bigger section exactly. */
|
|
ft_tt_test_check_fmt(" the DST write address of the last tile is inside the section",
|
|
result->max_dst == 0u ||
|
|
((result->max_dst - 1u) << 6) + 64u <= (shape_section(shape) << 6),
|
|
"%s: tile %u lands at row %u of %u", shape->name,
|
|
result->max_dst ? result->max_dst - 1u : 0u,
|
|
result->max_dst ? (result->max_dst - 1u) << 6 : 0u,
|
|
shape_section(shape) << 6);
|
|
ft_tt_test_check_fmt(" the writer places every output tile at m*Nt + n, exactly once",
|
|
writer_places_every_tile(shape, result), "%s: %llu tiles",
|
|
shape->name, (unsigned long long)result->out_count);
|
|
|
|
/* At MB = 1 the destination sequence is the RUNNING COUNTER the pre-MB writer used, and
|
|
* that identity is what makes a decode `cmp` evidence about this change. */
|
|
if (shape->row_block == 1u) {
|
|
for (index = 0; index < result->out_count; index++)
|
|
if (result->out[index].destination != (uint32_t)index)
|
|
order_ok = 0;
|
|
ft_tt_test_check_fmt(" at MB=1 the destinations ARE the old running counter",
|
|
order_ok, "%s: %llu tiles", shape->name,
|
|
(unsigned long long)result->out_count);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The mutations that must be caught, run on a shape where both blocks are real.
|
|
*
|
|
* A suite that has never been seen to fail is not evidence (best-practices §9), and none of
|
|
* these are synthetic: an SrcA stride of one tile is what the hardware does if
|
|
* `ft_tensix_unpack_matmul_set_srca_tile_stride()` is not called; a K-major L1 layout is the
|
|
* other design the reader could have used; a shared activation index is what the compute
|
|
* thread does if the row loop is added without moving `tile_index_a`; and a running-counter
|
|
* writer is the loop this file's writer USED TO BE, which is correct at MB=1 and silently
|
|
* transposes the output block at MB=2.
|
|
*/
|
|
static void check_red_arms(model_result *result)
|
|
{
|
|
const model_shape wide = { "red-NB", 1u, 32u, 8u, 8u, 4u, 1u, 1, 0u };
|
|
const model_shape square = { "red-MBxNB", 4u, 32u, 8u, 8u, 2u, 2u, 1, 0u };
|
|
model_mutation mutate;
|
|
|
|
ft_tt_test_step("RED — the mutations that must NOT pass");
|
|
|
|
mutate = kShipped;
|
|
mutate.stride_tiles = 1u;
|
|
model_run(&wide, &mutate, result);
|
|
ft_tt_test_check_fmt("an SrcA stride of ONE tile mis-pairs the column block",
|
|
!pairs_are_the_matmul(&wide, result),
|
|
"the un-strided MOP reads tiles 0..NB-1 of column 0's block");
|
|
|
|
mutate = kShipped;
|
|
mutate.b_k_major = 1;
|
|
model_run(&wide, &mutate, result);
|
|
ft_tt_test_check_fmt("a K-major L1 layout under the shipped stride mis-pairs the block",
|
|
!pairs_are_the_matmul(&wide, result),
|
|
"the reader's slot and the MOP's stride must agree");
|
|
|
|
mutate = kShipped;
|
|
mutate.a_row_shared = 1;
|
|
model_run(&square, &mutate, result);
|
|
ft_tt_test_check_fmt("a compute thread that does NOT advance tile_index_a by row*depth "
|
|
"feeds every output row row 0's activation",
|
|
!pairs_are_the_matmul(&square, result),
|
|
"MB is a loop AND an index; the loop alone is a wrong answer");
|
|
|
|
mutate = kShipped;
|
|
mutate.writer_counter = 1;
|
|
model_run(&square, &mutate, result);
|
|
ft_tt_test_check_fmt("a running-counter writer places the MB>1 block at the wrong "
|
|
"destination",
|
|
!writer_places_every_tile(&square, result),
|
|
"correct at MB=1, silently transposed at MB=2");
|
|
|
|
/* THE FAILURE THE `PN_TT_MATMUL_DST_TILES` INTERLOCK EXISTS FOR: a host that believes
|
|
* the section is eight tiles driving a kernel built at four. Nothing refuses on the
|
|
* card -- DST has no bounds check and tiles 4..7 are the OTHER half, which the packer is
|
|
* reading -- so the model is the only place it can be seen. It is the same shape twice,
|
|
* differing ONLY in the section it is checked against. */
|
|
{
|
|
model_shape too_wide = { "red-section", 1u, 32u, 32u, 8u, 8u, 1u, 1, 4u };
|
|
|
|
model_run(&too_wide, &kShipped, result);
|
|
ft_tt_test_check_fmt("NB=8 against a FOUR-tile section writes outside it",
|
|
result->max_dst > 4u, "highest DST tile reached is %u of 4",
|
|
result->max_dst);
|
|
too_wide.section_tiles = 8u;
|
|
model_run(&too_wide, &kShipped, result);
|
|
ft_tt_test_check_fmt(" CONTROL: the same NB=8 against EIGHT tiles is inside it",
|
|
result->max_dst == 8u, "highest DST tile reached is %u of 8",
|
|
result->max_dst);
|
|
}
|
|
|
|
/* THE CONTROL FOR THE TWO ABOVE. Both mutations are run on a shape the SHIPPED model
|
|
* passes, so a red that fired because the shape itself was broken is excluded. */
|
|
model_run(&square, &kShipped, result);
|
|
ft_tt_test_check_fmt("CONTROL: the same MB=2 NB=2 shape PASSES unmutated",
|
|
pairs_are_the_matmul(&square, result) &&
|
|
writer_places_every_tile(&square, result),
|
|
"the reds above are about the mutation, not the shape");
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------------------
|
|
* The byte law, and where its optimum is
|
|
* ---------------------------------------------------------------------------------- */
|
|
|
|
/**
|
|
* THE LAW IN TILES: `total = Mt*Nt*Kt*(1/NB + 1/MB)`, symmetric in the two blocks, and the
|
|
* DST section caps `MB*NB` at four. In TILES the optimum is therefore MB = NB = 2.
|
|
*
|
|
* THE LAW IN BYTES IS NOT SYMMETRIC AND THAT IS THE RESULT:
|
|
* `bytes = Mt*T*(a/NB + b/MB)`, because NB divides the operand whose tile is `a` (the bf16
|
|
* ACTIVATION, 2048 B) and MB divides the one whose tile is `b` (the WEIGHT — 1088 B at
|
|
* Bfp8_b, 576 at Bfp4_b). So the blocking budget belongs to whichever operand has the
|
|
* bigger tile, and on a Q8_0 model that is the ACTIVATION. `check_the_byte_law` asks the
|
|
* model both questions, because answering only the first is how "MB = NB = 2 is worth 1.25x"
|
|
* survives — it is true in tiles and false in bytes at every weight format Ornith uses.
|
|
*/
|
|
static void check_the_tile_law(model_result *result)
|
|
{
|
|
static const struct { uint32_t rows; uint32_t columns; } pairs[] = {
|
|
{ 1u, 1u }, { 1u, 2u }, { 1u, 4u }, { 2u, 1u }, { 2u, 2u }, { 4u, 1u },
|
|
};
|
|
/* Divisible on both axes so the closed form is exact rather than a ceiling. */
|
|
model_shape shape = { "law", 4u, 32u, 8u, 8u, 1u, 1u, 1, 0u };
|
|
uint64_t best = 0;
|
|
uint32_t best_rows = 0;
|
|
uint32_t best_columns = 0;
|
|
unsigned index;
|
|
|
|
ft_tt_test_step("the TILE law, and the (MB, NB) the DST section makes optimal IN TILES");
|
|
for (index = 0; index < sizeof(pairs) / sizeof(pairs[0]); index++) {
|
|
const uint64_t unblocked =
|
|
(uint64_t)shape.tile_rows * shape.tile_cols * shape.tile_inner;
|
|
uint64_t total;
|
|
|
|
shape.row_block = pairs[index].rows;
|
|
shape.column_block = pairs[index].columns;
|
|
ft_tt_test_check_fmt("the pair fits the DST section",
|
|
ftmm_block_fits_dst(shape.row_block, shape.column_block) != 0,
|
|
"MB=%u NB=%u: %u tiles of %u", shape.row_block, shape.column_block,
|
|
shape.row_block * shape.column_block, DST_SECTION_TILES);
|
|
model_run(&shape, &kShipped, result);
|
|
total = result->a_tiles_read + result->b_tiles_read;
|
|
ft_tt_test_check_fmt(" it reads Mt*Nt*Kt*(1/NB + 1/MB) tiles",
|
|
total * shape.row_block * shape.column_block ==
|
|
unblocked * (shape.row_block + shape.column_block),
|
|
"MB=%u NB=%u: %llu tiles against the law's %llu", shape.row_block,
|
|
shape.column_block, (unsigned long long)total,
|
|
(unsigned long long)(unblocked *
|
|
(shape.row_block + shape.column_block) /
|
|
(shape.row_block * shape.column_block)));
|
|
if (best == 0 || total < best) {
|
|
best = total;
|
|
best_rows = shape.row_block;
|
|
best_columns = shape.column_block;
|
|
}
|
|
}
|
|
ft_tt_test_check_fmt("the DST-4 optimum IN TILES is MB = NB = 2, not either extreme",
|
|
best_rows == 2u && best_columns == 2u, "best is MB=%u NB=%u",
|
|
best_rows, best_columns);
|
|
|
|
shape.row_block = 4u;
|
|
shape.column_block = 1u;
|
|
model_run(&shape, &kShipped, result);
|
|
{
|
|
const uint64_t rows4 = result->a_tiles_read + result->b_tiles_read;
|
|
|
|
shape.row_block = 1u;
|
|
shape.column_block = 4u;
|
|
model_run(&shape, &kShipped, result);
|
|
ft_tt_test_check_fmt("MB=4/NB=1 reads EXACTLY as many tiles as the shipped MB=1/NB=4",
|
|
rows4 == result->a_tiles_read + result->b_tiles_read,
|
|
"%llu against %llu — 128 positions a weight sweep buys nothing "
|
|
"when A pays it all back",
|
|
(unsigned long long)rows4,
|
|
(unsigned long long)(result->a_tiles_read + result->b_tiles_read));
|
|
}
|
|
|
|
ft_tt_test_check_fmt("MB*NB = 5 is REFUSED, which is what the kernel guard asks",
|
|
ftmm_block_fits_dst(5u, 1u) == 0 && ftmm_block_fits_dst(2u, 3u) == 0 &&
|
|
ftmm_block_fits_dst(0u, 4u) == 0,
|
|
"the section is %u tiles", DST_SECTION_TILES);
|
|
}
|
|
|
|
/** `Mt * T * (a/NB + b/MB)` in BYTES a weight tile, from the model's own tile counts. */
|
|
static uint64_t law_bytes(const model_result *result, uint32_t weight_tile)
|
|
{
|
|
return result->a_tiles_read * A_TILE_BYTES + result->b_tiles_read * weight_tile;
|
|
}
|
|
|
|
/**
|
|
* THE BYTE LAW, WHICH IS NOT THE TILE LAW, AND THE REASON THE ROW BLOCK IS WORTH 2%.
|
|
*
|
|
* NB divides the operand whose tile is the bf16 ACTIVATION (2048 B) and MB divides the one
|
|
* whose tile is the WEIGHT (1088 B at Bfp8_b, 576 at Bfp4_b, 2048 at bf16). So the two
|
|
* blocks are symmetric in tiles and NOT in bytes, and the budget belongs to whichever
|
|
* operand has the bigger tile.
|
|
*
|
|
* THE LAW IS VALIDATED AGAINST SILICON WITH NO FREE PARAMETER and that check is here:
|
|
* at Mt=1/MB=1/NB=4 with Bfp8_b it says `A/B = a/(4b) = 2048/4352 = 0.4706`, and Ornith's
|
|
* measured decode split is A 3.978 GB / B 8.432 GB = 0.4718. Nothing was fitted.
|
|
*/
|
|
static void check_the_byte_law(model_result *result)
|
|
{
|
|
static const struct { const char *name; uint32_t tile; uint32_t rows; uint32_t columns; }
|
|
formats[] = {
|
|
/* the shipped Ornith weight (Q8_0 -> Bfp8_b) and the two others in the tree */
|
|
{ "Bfp8_b", 1088u, 2u, 2u },
|
|
{ "Bfp4_b", 576u, 1u, 4u },
|
|
{ "bf16", 2048u, 2u, 2u },
|
|
};
|
|
static const struct { uint32_t rows; uint32_t columns; } pairs[] = {
|
|
{ 1u, 1u }, { 1u, 2u }, { 1u, 4u }, { 2u, 1u }, { 2u, 2u }, { 4u, 1u },
|
|
};
|
|
model_shape shape = { "law", 4u, 32u, 8u, 8u, 1u, 1u, 1, 0u };
|
|
unsigned format;
|
|
unsigned index;
|
|
|
|
ft_tt_test_step("the BYTE law, and why the optimum moves with the WEIGHT FORMAT");
|
|
for (format = 0; format < sizeof(formats) / sizeof(formats[0]); format++) {
|
|
uint64_t best = 0;
|
|
uint32_t best_rows = 0;
|
|
uint32_t best_columns = 0;
|
|
|
|
for (index = 0; index < sizeof(pairs) / sizeof(pairs[0]); index++) {
|
|
uint64_t bytes;
|
|
|
|
shape.row_block = pairs[index].rows;
|
|
shape.column_block = pairs[index].columns;
|
|
model_run(&shape, &kShipped, result);
|
|
bytes = law_bytes(result, formats[format].tile);
|
|
if (best == 0 || bytes < best) {
|
|
best = bytes;
|
|
best_rows = shape.row_block;
|
|
best_columns = shape.column_block;
|
|
}
|
|
}
|
|
ft_tt_test_check_fmt(" the byte optimum at this weight format",
|
|
best_rows == formats[format].rows &&
|
|
best_columns == formats[format].columns,
|
|
"%s (%u B/tile): best MB=%u NB=%u, want MB=%u NB=%u",
|
|
formats[format].name, formats[format].tile, best_rows,
|
|
best_columns, formats[format].rows, formats[format].columns);
|
|
}
|
|
|
|
/* THE NUMBER THE WHOLE RUNG TURNS ON. At Ornith's Q8_0 weights the row block moves the
|
|
* matmul's DRAM bytes by 2%, not by 1.25x -- the 1.25x is the TILE answer and it holds
|
|
* only when the two tiles are the same size, which is the bf16 row above. */
|
|
{
|
|
uint64_t shipped, square;
|
|
|
|
shape.row_block = 1u;
|
|
shape.column_block = 4u;
|
|
model_run(&shape, &kShipped, result);
|
|
shipped = law_bytes(result, 1088u);
|
|
shape.row_block = 2u;
|
|
shape.column_block = 2u;
|
|
model_run(&shape, &kShipped, result);
|
|
square = law_bytes(result, 1088u);
|
|
ft_tt_test_check_fmt("at Bfp8_b, MB=2/NB=2 beats the shipped MB=1/NB=4 by 1.5-2.5%%",
|
|
square < shipped && square * 1000u > shipped * 970u,
|
|
"%llu B against %llu B — %.2f%%",
|
|
(unsigned long long)square, (unsigned long long)shipped,
|
|
100.0 * (1.0 - (double)square / (double)shipped));
|
|
|
|
/* `a/(4b)` with no fitted constant, against Ornith's own measured 0.4718. */
|
|
shape.row_block = 1u;
|
|
shape.column_block = 4u;
|
|
model_run(&shape, &kShipped, result);
|
|
{
|
|
const double ratio = (double)(result->a_tiles_read * A_TILE_BYTES) /
|
|
(double)(result->b_tiles_read * 1088u);
|
|
|
|
ft_tt_test_check_fmt("the law's A/B at the shipped shape reproduces the measured "
|
|
"0.4718 with NO fitted constant",
|
|
ratio > 0.469 && ratio < 0.473, "law %.4f, silicon 0.4718",
|
|
ratio);
|
|
}
|
|
}
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------------------
|
|
* The DST section: its two sizes, which of them this build is, and what the bigger one
|
|
* actually reaches at the Mt every descriptor carries
|
|
* ---------------------------------------------------------------------------------- */
|
|
|
|
/**
|
|
* THE SECTION IS FIXED BY THE HARDWARE AND IT HAS EXACTLY TWO VALUES AT `dst_accum_mode 1`.
|
|
* The section is
|
|
* `(FT_TENSIX_DEST_REGISTER_{HALF,FULL}_SIZE >> 1) >> 6` for 32x32 tiles, i.e.
|
|
* `(512 >> 1) >> 6 = 4` at SyncHalf and `(1024 >> 1) >> 6 = 8` at SyncFull. Sixteen would
|
|
* need `dst_accum_mode 0`, which is worth 13-35x accuracy and is not on the table.
|
|
*
|
|
* AND THE WRITE-ADDRESS FORMULA FITS THE BIGGER SECTION EXACTLY, which is the reason twelve
|
|
* DST address sites across seven headers need no edit: they compute
|
|
* `(tile_index << 6) + (dest_offset_id ? HALF : 0)`, SyncFull pins the id at 0, and tile 7
|
|
* lands at row 448 -- the last 64-row tile inside `DEST_REGISTER_FULL_SIZE >> 1 = 512`.
|
|
*/
|
|
static void check_the_dst_section(void)
|
|
{
|
|
static const struct { uint32_t rows; uint32_t columns; int half; int full; } lattice[] = {
|
|
{ 1u, 1u, 1, 1 }, { 1u, 4u, 1, 1 }, { 2u, 2u, 1, 1 }, { 4u, 1u, 1, 1 },
|
|
{ 1u, 8u, 0, 1 }, { 2u, 4u, 0, 1 }, { 4u, 2u, 0, 1 }, { 8u, 1u, 0, 1 },
|
|
{ 3u, 3u, 0, 0 }, { 2u, 8u, 0, 0 }, { 5u, 1u, 0, 1 }, { 1u, 9u, 0, 0 },
|
|
{ 0u, 4u, 0, 0 }, { 4u, 0u, 0, 0 },
|
|
};
|
|
const uint32_t half = FT_TT_MATMUL_DST_SECTION_TILES_HALF;
|
|
const uint32_t full = FT_TT_MATMUL_DST_SECTION_TILES_FULL;
|
|
unsigned index;
|
|
int agrees = 1;
|
|
|
|
ft_tt_test_step("the DST section: SyncHalf 4, SyncFull 8, and what each admits");
|
|
ft_tt_test_check_fmt("the two sections ARE upstream's get_dest_max_tiles at accum mode",
|
|
half == ((512u >> 1) >> 6) && full == ((1024u >> 1) >> 6),
|
|
"half %u, full %u", half, full);
|
|
ft_tt_test_check_fmt("ftmm_dst_section_tiles() answers for a kernel that is not this one",
|
|
ftmm_dst_section_tiles(0) == half && ftmm_dst_section_tiles(1) == full,
|
|
"SyncHalf %u, SyncFull %u", ftmm_dst_section_tiles(0),
|
|
ftmm_dst_section_tiles(1));
|
|
|
|
for (index = 0; index < sizeof(lattice) / sizeof(lattice[0]); index++) {
|
|
const uint32_t rows = lattice[index].rows;
|
|
const uint32_t columns = lattice[index].columns;
|
|
|
|
ft_tt_test_check_fmt(" the pair is admitted by the right sections",
|
|
ftmm_block_fits_section(rows, columns, half) ==
|
|
lattice[index].half &&
|
|
ftmm_block_fits_section(rows, columns, full) ==
|
|
lattice[index].full,
|
|
"MB=%u NB=%u: four %d eight %d, want %d and %d", rows, columns,
|
|
ftmm_block_fits_section(rows, columns, half),
|
|
ftmm_block_fits_section(rows, columns, full),
|
|
lattice[index].half, lattice[index].full);
|
|
if (ftmm_block_fits_dst(rows, columns) !=
|
|
ftmm_block_fits_section(rows, columns, FT_TT_MATMUL_DST_SECTION_TILES))
|
|
agrees = 0;
|
|
}
|
|
/* THE ONE THAT MATTERS FOR THE INTERLOCK: four pairs fit at eight and NOT at four, so a
|
|
* host that believed the wrong section would write past it with nothing refusing. */
|
|
ft_tt_test_check_fmt("four pairs SEPARATE the two sections, so the size is not decorative",
|
|
ftmm_block_fits_section(1u, 8u, full) &&
|
|
!ftmm_block_fits_section(1u, 8u, half) &&
|
|
ftmm_block_fits_section(2u, 4u, full) &&
|
|
!ftmm_block_fits_section(2u, 4u, half),
|
|
"(1,8) and (2,4) fit 8 and not 4");
|
|
ft_tt_test_check_fmt("ftmm_block_fits_dst() asks about THIS BUILD's own section",
|
|
agrees, "FT_TT_MATMUL_DST_SECTION_TILES = %u",
|
|
FT_TT_MATMUL_DST_SECTION_TILES);
|
|
ft_tt_test_check_fmt("this build is SyncHalf, which is what the shipped blob carries",
|
|
FT_TT_MATMUL_DST_SYNC == FT_TT_MATMUL_DST_SYNC_HALF &&
|
|
FT_TT_MATMUL_DST_SECTION_TILES == half,
|
|
"mode %d, section %u", (int)FT_TT_MATMUL_DST_SYNC,
|
|
FT_TT_MATMUL_DST_SECTION_TILES);
|
|
|
|
/* The write-address formula, which is why no address site is edited -- and why NINE
|
|
* tiles would not merely be refused, it would not fit. */
|
|
ft_tt_test_check_fmt("the LAST SyncFull tile's 64 DST rows end exactly at FULL_SIZE >> 1",
|
|
((full - 1u) << 6) + 64u == (1024u >> 1) &&
|
|
(full << 6) + 64u > (1024u >> 1),
|
|
"tile %u ends at row %u of %u; tile %u would end at %u", full - 1u,
|
|
((full - 1u) << 6) + 64u, 1024u >> 1, full, (full << 6) + 64u);
|
|
ft_tt_test_check_fmt("and the last SyncHalf tile ends exactly at HALF_SIZE >> 1",
|
|
((half - 1u) << 6) + 64u == (512u >> 1),
|
|
"tile %u ends at row %u of %u", half - 1u, ((half - 1u) << 6) + 64u,
|
|
512u >> 1);
|
|
}
|
|
|
|
/**
|
|
* WHAT THE BIGGER SECTION IS WORTH, AND THE CLAUSE THAT DECIDES IT: `Mt`.
|
|
*
|
|
* The DST-8 lattice's Bfp8_b optimum is MB=2/NB=4 at `512 + 544 = 1056` bytes a unit,
|
|
* 1.52x the shipped 1600. **That point needs `Mt >= 2` and this tree has none** --
|
|
* `pn_tt_matmul_group.c` writes `FT_TT_QUEUE_MATMUL_TILE_ROWS` from a literal 1 on the
|
|
* decode path and on the batched-prefill path alike, because `rows` is capped at 32
|
|
* positions, which is one tile-row. At `Mt = 1` the row loop runs once whatever MB says, so
|
|
* MB is INERT and the reachable DST-8 point is MB=1/NB=8 at `256 + 1088 = 1344` -- 1.19x.
|
|
*
|
|
* The inertness is not asserted, it is MEASURED off the same model that produces the tile
|
|
* counts, with an `Mt = 2` control beside it so the check is able to fail.
|
|
*/
|
|
static void check_what_the_bigger_section_reaches(model_result *result)
|
|
{
|
|
static const struct { uint32_t rows; uint32_t columns; } lattice[] = {
|
|
{ 1u, 1u }, { 1u, 2u }, { 1u, 4u }, { 1u, 8u },
|
|
{ 2u, 1u }, { 2u, 2u }, { 2u, 4u },
|
|
{ 4u, 1u }, { 4u, 2u }, { 8u, 1u },
|
|
};
|
|
model_shape shape = { "dst8", 8u, 32u, 8u, 8u, 1u, 1u, 1, 8u };
|
|
uint64_t best = 0;
|
|
uint32_t best_rows = 0;
|
|
uint32_t best_columns = 0;
|
|
uint64_t best_at_mt1 = 0;
|
|
uint32_t best_mt1_columns = 0;
|
|
uint64_t shipped;
|
|
uint64_t rows1_columns8;
|
|
uint64_t rows2_columns4;
|
|
unsigned index;
|
|
|
|
ft_tt_test_step("the DST-8 lattice in BYTES, and which of it `Mt = 1` can reach");
|
|
for (index = 0; index < sizeof(lattice) / sizeof(lattice[0]); index++) {
|
|
uint64_t bytes;
|
|
|
|
shape.row_block = lattice[index].rows;
|
|
shape.column_block = lattice[index].columns;
|
|
ft_tt_test_check_fmt(" the pair fits the eight-tile section",
|
|
ftmm_block_fits_section(shape.row_block, shape.column_block,
|
|
FT_TT_MATMUL_DST_SECTION_TILES_FULL) != 0,
|
|
"MB=%u NB=%u: %u tiles", shape.row_block, shape.column_block,
|
|
shape.row_block * shape.column_block);
|
|
model_run(&shape, &kShipped, result);
|
|
bytes = law_bytes(result, 1088u);
|
|
if (best == 0 || bytes < best) {
|
|
best = bytes;
|
|
best_rows = shape.row_block;
|
|
best_columns = shape.column_block;
|
|
}
|
|
if (shape.row_block == 1u && (best_at_mt1 == 0 || bytes < best_at_mt1)) {
|
|
best_at_mt1 = bytes;
|
|
best_mt1_columns = shape.column_block;
|
|
}
|
|
}
|
|
ft_tt_test_check_fmt("the DST-8 Bfp8_b optimum is MB=2/NB=4 — and it needs Mt >= 2",
|
|
best_rows == 2u && best_columns == 4u, "best is MB=%u NB=%u",
|
|
best_rows, best_columns);
|
|
ft_tt_test_check_fmt("restricted to MB=1, which Mt=1 forces, the optimum is NB=8",
|
|
best_mt1_columns == 8u, "best MB=1 column block is NB=%u",
|
|
best_mt1_columns);
|
|
|
|
/* The two multiples, per unit of `T`, from the model's own counts rather than by hand. */
|
|
shape.row_block = 1u;
|
|
shape.column_block = 4u;
|
|
model_run(&shape, &kShipped, result);
|
|
shipped = law_bytes(result, 1088u);
|
|
shape.column_block = 8u;
|
|
model_run(&shape, &kShipped, result);
|
|
rows1_columns8 = law_bytes(result, 1088u);
|
|
shape.row_block = 2u;
|
|
shape.column_block = 4u;
|
|
model_run(&shape, &kShipped, result);
|
|
rows2_columns4 = law_bytes(result, 1088u);
|
|
|
|
ft_tt_test_check_fmt("MB=1/NB=8 is 1.18-1.20x the shipped MB=1/NB=4 — the REACHABLE win",
|
|
rows1_columns8 * 120u > shipped * 100u &&
|
|
rows1_columns8 * 118u < shipped * 100u,
|
|
"%llu B against %llu B — %.4fx", (unsigned long long)rows1_columns8,
|
|
(unsigned long long)shipped,
|
|
(double)shipped / (double)rows1_columns8);
|
|
ft_tt_test_check_fmt("MB=2/NB=4 is 1.50-1.53x it — the figure the spec carried",
|
|
rows2_columns4 * 153u > shipped * 100u &&
|
|
rows2_columns4 * 150u < shipped * 100u,
|
|
"%llu B against %llu B — %.4fx", (unsigned long long)rows2_columns4,
|
|
(unsigned long long)shipped,
|
|
(double)shipped / (double)rows2_columns4);
|
|
|
|
/* THE CLAUSE. At Mt = 1 the row block moves NOTHING -- not a tile, not a byte -- so the
|
|
* 1.52x is gated on a host shape change and not on the DST section. The Mt = 2 arm
|
|
* beside it is the control that makes this check able to fail. */
|
|
{
|
|
model_shape mt1 = { "Mt=1 MB=1", 1u, 128u, 64u, 8u, 4u, 1u, 1, 8u };
|
|
uint64_t a_at_mb1, b_at_mb1, a_at_mb2, b_at_mb2;
|
|
|
|
model_run(&mt1, &kShipped, result);
|
|
a_at_mb1 = result->a_tiles_read;
|
|
b_at_mb1 = result->b_tiles_read;
|
|
mt1.row_block = 2u;
|
|
model_run(&mt1, &kShipped, result);
|
|
a_at_mb2 = result->a_tiles_read;
|
|
b_at_mb2 = result->b_tiles_read;
|
|
ft_tt_test_check_fmt("at Mt=1 the ROW BLOCK IS INERT — same A tiles, same B tiles",
|
|
a_at_mb1 == a_at_mb2 && b_at_mb1 == b_at_mb2,
|
|
"A %llu -> %llu, B %llu -> %llu",
|
|
(unsigned long long)a_at_mb1, (unsigned long long)a_at_mb2,
|
|
(unsigned long long)b_at_mb1, (unsigned long long)b_at_mb2);
|
|
|
|
mt1.tile_rows = 2u;
|
|
mt1.row_block = 1u;
|
|
model_run(&mt1, &kShipped, result);
|
|
b_at_mb1 = result->b_tiles_read;
|
|
mt1.row_block = 2u;
|
|
model_run(&mt1, &kShipped, result);
|
|
b_at_mb2 = result->b_tiles_read;
|
|
ft_tt_test_check_fmt(" and at Mt=2 it HALVES B — the control that makes that a result",
|
|
b_at_mb2 * 2u == b_at_mb1, "B %llu -> %llu",
|
|
(unsigned long long)b_at_mb1, (unsigned long long)b_at_mb2);
|
|
}
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------------------
|
|
* The wire contract: the parameter word, parsed out of the kernel's own header
|
|
* ---------------------------------------------------------------------------------- */
|
|
|
|
/** Read `enum { name = N };` out of resident_queue_protocol.h, or -1. */
|
|
static long kernel_enum_value(const char *name)
|
|
{
|
|
const char *directory = getenv("FT_TT_KERNEL_DIR");
|
|
char path[512];
|
|
char line[512];
|
|
FILE *file;
|
|
long value = -1;
|
|
|
|
snprintf(path, sizeof(path), "%s/resident_queue_protocol.h",
|
|
directory ? directory : "../kernels");
|
|
file = fopen(path, "r");
|
|
if (!file)
|
|
return -1;
|
|
while (fgets(line, sizeof(line), file)) {
|
|
char found[128];
|
|
long parsed;
|
|
|
|
if (sscanf(line, " enum { %127[A-Za-z0-9_] = %ld }", found, &parsed) == 2 &&
|
|
strcmp(found, name) == 0) {
|
|
value = parsed;
|
|
break;
|
|
}
|
|
}
|
|
fclose(file);
|
|
return value;
|
|
}
|
|
|
|
static void check_wire_contract(void)
|
|
{
|
|
const long block_word = kernel_enum_value("kMatmulColumnBlock");
|
|
const long words = kernel_enum_value("kMatmulWords");
|
|
const long a2 = kernel_enum_value("kDescA2");
|
|
uint32_t block;
|
|
|
|
ft_tt_test_step("the parameter word, PARSED out of the kernel's own header");
|
|
ft_tt_test_check_fmt("kMatmulColumnBlock == FT_TT_QUEUE_MATMUL_COLUMN_BLOCK",
|
|
block_word == (long)FT_TT_QUEUE_MATMUL_COLUMN_BLOCK,
|
|
"kernel %ld, host %d", block_word,
|
|
(int)FT_TT_QUEUE_MATMUL_COLUMN_BLOCK);
|
|
ft_tt_test_check_fmt("the block still fits sixteen words — the row block bought NO word",
|
|
words == (long)FT_TT_QUEUE_MATMUL_WORDS && words == 16,
|
|
"kernel %ld, host %d", words, (int)FT_TT_QUEUE_MATMUL_WORDS);
|
|
ft_tt_test_check_fmt("FT_TT_QUEUE_PARAM_WORDS is still 16 for every family",
|
|
FT_TT_QUEUE_PARAM_WORDS == 16u, "%u", FT_TT_QUEUE_PARAM_WORDS);
|
|
ft_tt_test_check_fmt("kDescA2 exists and is the pitch's home", a2 == 4, "kernel %ld", a2);
|
|
|
|
ft_tt_test_step("the packed encoding, and its zero");
|
|
for (block = 1u; block <= 4u; block++)
|
|
ft_tt_test_check_fmt("a pre-encoding word still decodes to (MB=1, NB=itself)",
|
|
ftmm_column_block(block) == block && ftmm_row_block(block) == 1u,
|
|
"word %u -> NB %u, MB %u", block, ftmm_column_block(block),
|
|
ftmm_row_block(block));
|
|
ft_tt_test_check_fmt("a zero word is (MB=1, NB=1)",
|
|
ftmm_column_block(0u) == 1u && ftmm_row_block(0u) == 1u, "NB %u, MB %u",
|
|
ftmm_column_block(0u), ftmm_row_block(0u));
|
|
ft_tt_test_check_fmt("ftmm_block_word(1, NB) is BIT-IDENTICAL to the pre-encoding word",
|
|
ftmm_block_word(1u, 4u) == 4u && ftmm_block_word(1u, 1u) == 1u,
|
|
"%u and %u", ftmm_block_word(1u, 4u), ftmm_block_word(1u, 1u));
|
|
ft_tt_test_check_fmt("ftmm_block_word(2, 2) round-trips",
|
|
ftmm_row_block(ftmm_block_word(2u, 2u)) == 2u &&
|
|
ftmm_column_block(ftmm_block_word(2u, 2u)) == 2u,
|
|
"word 0x%x", ftmm_block_word(2u, 2u));
|
|
}
|
|
|
|
int main(void)
|
|
{
|
|
static const model_shape shapes[] = {
|
|
/* name Mt Kt Nt depth NB MB n-major DST */
|
|
{ "unblocked control", 1u, 32u, 16u, 8u, 1u, 1u, 1, 0u },
|
|
{ "qkv-shaped, NB=4", 1u, 128u, 64u, 8u, 4u, 1u, 1, 0u },
|
|
{ "ffn-shaped, NB=4", 1u, 384u, 96u, 8u, 4u, 1u, 1, 0u },
|
|
{ "NB=2", 1u, 64u, 12u, 8u, 2u, 1u, 1, 0u },
|
|
{ "ragged: Nt=6 under NB=4", 1u, 32u, 6u, 8u, 4u, 1u, 1, 0u },
|
|
{ "Nt=1, the ssm_alpha case", 1u, 32u, 1u, 8u, 4u, 1u, 1, 0u },
|
|
{ "K-major panel, NB=4", 1u, 64u, 16u, 8u, 4u, 1u, 0, 0u },
|
|
{ "depth 32, NB=4", 1u, 64u, 16u, 32u, 4u, 1u, 1, 0u },
|
|
{ "depth 1, NB=4", 1u, 16u, 8u, 1u, 4u, 1u, 1, 0u },
|
|
/* The row block. Mt = 2 is 64 prompt positions, Mt = 4 is 128. */
|
|
{ "MB=2 NB=2, 64 positions", 2u, 128u, 64u, 8u, 2u, 2u, 1, 0u },
|
|
{ "MB=4 NB=1, 128 positions", 4u, 128u, 64u, 8u, 1u, 4u, 1, 0u },
|
|
{ "MB=2 NB=1", 2u, 64u, 16u, 8u, 1u, 2u, 1, 0u },
|
|
{ "MB=2 NB=2, Nt=1 weight", 2u, 32u, 1u, 8u, 2u, 2u, 1, 0u },
|
|
{ "MB=2 NB=2, ragged Mt=3", 3u, 32u, 8u, 8u, 2u, 2u, 1, 0u },
|
|
{ "MB=4 NB=1, ragged Mt=6", 6u, 32u, 8u, 8u, 1u, 4u, 1, 0u },
|
|
{ "MB=2 NB=2, ragged Nt=7", 4u, 32u, 7u, 8u, 2u, 2u, 1, 0u },
|
|
{ "MB=2 NB=2, K-major panel", 2u, 64u, 16u, 8u, 2u, 2u, 0, 0u },
|
|
{ "MB=2 NB=2, depth 1", 2u, 16u, 8u, 1u, 2u, 2u, 1, 0u },
|
|
{ "MB=1 NB=4 at Mt=4 (today)", 4u, 128u, 64u, 8u, 4u, 1u, 1, 0u },
|
|
/* DstSync::SyncFull: eight tiles. At Mt = 1 -- which is every descriptor the tree
|
|
* writes -- the whole section belongs to NB, because MB has no second tile-row to
|
|
* divide. The MB=2/NB=4 rows below are what a host that batched more than 32
|
|
* positions would reach; they are checked so the arithmetic is on record, not
|
|
* because anything dispatches them. */
|
|
{ "SyncFull: Mt=1 NB=8", 1u, 128u, 64u, 8u, 8u, 1u, 1, 8u },
|
|
{ "SyncFull: Mt=1 NB=8, Nt=1", 1u, 32u, 1u, 8u, 8u, 1u, 1, 8u },
|
|
{ "SyncFull: Mt=1 NB=8, ragged",1u, 64u, 20u, 8u, 8u, 1u, 1, 8u },
|
|
{ "SyncFull: Mt=1 NB=8 K-major",1u, 64u, 16u, 8u, 8u, 1u, 0, 8u },
|
|
{ "SyncFull: MB=2 NB=4", 2u, 128u, 64u, 8u, 4u, 2u, 1, 8u },
|
|
{ "SyncFull: MB=2 NB=4 ragged", 3u, 64u, 22u, 8u, 4u, 2u, 1, 8u },
|
|
{ "SyncFull: MB=4 NB=2", 4u, 64u, 16u, 8u, 2u, 4u, 1, 8u },
|
|
};
|
|
model_result result;
|
|
unsigned index;
|
|
|
|
result.pair_capacity = 1u << 22;
|
|
result.pairs = malloc((size_t)result.pair_capacity * sizeof(*result.pairs));
|
|
result.out = malloc((size_t)(256u << 10) * sizeof(*result.out));
|
|
if (!result.pairs || !result.out) {
|
|
fprintf(stderr, "test_matmul_block: out of memory\n");
|
|
return 1;
|
|
}
|
|
|
|
check_wire_contract();
|
|
ft_tt_test_step("the blocked reader, compute thread and writer, shape by shape");
|
|
for (index = 0; index < sizeof(shapes) / sizeof(shapes[0]); index++)
|
|
check_shape(&shapes[index], &result);
|
|
check_the_tile_law(&result);
|
|
check_the_byte_law(&result);
|
|
check_the_dst_section();
|
|
check_what_the_bigger_section_reaches(&result);
|
|
check_red_arms(&result);
|
|
|
|
free(result.pairs);
|
|
free(result.out);
|
|
return ft_tt_test_finish();
|
|
}
|