David
/
Foundry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Foundry/tt/ft_tt_op_gdnc.c

939 lines
43 KiB
C
Raw Permalink Blame History

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
/**
* @file ft_tt_op_gdnc.c
* @brief Implementation of the chunkwise Gated-Delta-Net host driver. See ft_tt_op_gdnc.h.
*
* @status NEW. Parameter layout, CB placement, band construction and the per-operand DRAM
* sizes a host stager allocates from — pure host arithmetic, pinned off hardware by
* tests/test_gdnc_ops.c.
* @issues None
* @todo None
*/
#include "ft_tt_op_gdnc.h"
#include <math.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "ft_tt_device.h"
#include "ft_tt_op_l1_layout.h"
#include "ft_tt_tilize.h"
/* The device's L1 pointer attribute belongs to the SFPI GCC fork, not to the contract; the
* protocol header guards it so a HOST compiler can read the file. Including it here is what
* lets the sizing functions below compute their tile counts with the SAME `ftgc_*` accessors
* both dataflow kernels compute their offsets with, instead of respelling the layout a third
* time. */
#define FTQ_L1_PTR
#include "kernels/resident_gdnc_protocol.h"
/** Pages of the streamed per-column operands. FOUR columns of look-ahead, not the usual two:
* this reader issues `1 + 2*(taps-1) + taps` NoC reads a column against a compute thread
* that issues `4*taps + 2` tile operations, so the two are close enough in cost that a
* two-page buffer would hard-handshake them. Four pages of 4 KiB is 16 KiB against a
* 1.29 MiB arena. */
#define GDNC_CB_COLUMNS 4u
/** Pages of CB_OUT: four column groups of two. The decode family measured CB_OUT capacity
* worth 3.8% between two pages and ten and NOTHING beyond that — with only one group live
* the compute thread hard-handshakes the writer on every column. */
#define GDNC_CB_OUT_PAGES 8u
static bool operand_is_set(const ft_tt_op_operand *operand)
{
return operand->tile_bytes != 0u;
}
/* ================================================================
* The CONV block depth — one number, three sides. See ft_tt_op_gdnc.h.
* ================================================================ */
uint32_t ft_tt_op_gdnc_conv_depth_parse(const char *value)
{
char *end = NULL;
unsigned long parsed;
if (!value || !*value)
return FT_TT_OP_GDNC_CONV_DEPTH;
parsed = strtoul(value, &end, 10);
/* NO SILENT FALLBACK TO THE DEFAULT. A knob that ignores what it was told is how a sweep
* measures its control four times and reports a ladder. It must also fit a BYTE, because
* that is where it rides on the wire. */
if (end == value || *end != '\0' || parsed < 1u || parsed > 255u)
return 0u;
return (uint32_t)parsed;
}
uint32_t ft_tt_op_gdnc_conv_depth(void)
{
static uint32_t resolved;
static bool known;
const char *value;
if (known)
return resolved;
known = true;
value = getenv(FT_TT_OP_GDNC_CONV_DEPTH_ENV);
resolved = ft_tt_op_gdnc_conv_depth_parse(value);
/* ONE LINE PER PROCESS, ALWAYS — this is the line that says which rung actually placed. */
if (resolved == 0u)
fprintf(stderr,
"ft_tt: gdnc CONV block depth REFUSED — %s=\"%s\" is not a depth in 1..255, "
"so every GDNC descriptor refuses rather than quietly placing for %u\n",
FT_TT_OP_GDNC_CONV_DEPTH_ENV, value ? value : "",
(unsigned)FT_TT_OP_GDNC_CONV_DEPTH);
else
fprintf(stderr,
"ft_tt: gdnc CONV block depth %u (compile-time default %u, %s%s) — declared on "
"every CONV descriptor; a program built at another depth refuses\n",
(unsigned)resolved, (unsigned)FT_TT_OP_GDNC_CONV_DEPTH,
FT_TT_OP_GDNC_CONV_DEPTH_ENV, (value && *value) ? " set" : " unset");
return resolved;
}
static uint32_t gdnc_gcd(uint32_t a, uint32_t b)
{
while (b != 0u) {
const uint32_t r = a % b;
a = b;
b = r;
}
return a;
}
static uint32_t gdnc_lcm(uint32_t a, uint32_t b)
{
if (a == 0u || b == 0u)
return a > b ? a : b;
return a / gdnc_gcd(a, b) * b;
}
static uint32_t gdnc_round_up(uint32_t value, uint32_t grid)
{
if (grid == 0u)
return value;
return ((value + grid - 1u) / grid) * grid;
}
/**
* EVERY OPERAND IS AN fp32 TILE, and this refusal is what makes the device's one-word check
* sufficient.
*
* The family publishes ONE `kGdncTileBytes` word covering four DRAM endpoints, so the device
* cannot tell a 2048-byte activation from a 4096-byte one when the word came from a different
* operand — which is exactly the defect the decode family shipped for a round: bf16 pages
* placed under a reader issuing 4096-byte tile reads, with the tile-size refusal reading the
* other operand and passing.
*/
static bool operands_are_fp32(const ft_tt_op_gdnc_request *request)
{
const ft_tt_op_operand *operands[4] = { &request->activation_in, &request->activation_out,
&request->constant, &request->state_conv };
unsigned index;
for (index = 0; index < 4u; index++) {
if (operand_is_set(operands[index]) &&
operands[index]->tile_bytes != FT_TT_OP_GDNC_FP32_TILE_BYTES)
return false;
}
return true;
}
/** Pack a NoC coordinate pair into one word. */
static uint32_t pack_xy(const ft_tt_op_operand *operand)
{
return (uint32_t)operand->x | ((uint32_t)operand->y << 16);
}
/** Tile columns of the stream. */
static uint32_t gdnc_tile_cols(const ft_tt_op_gdnc_request *request)
{
return request->conv_dim / FT_TT_OP_TILE_DIM;
}
/** The tile columns a descriptor owns. A `col_count` of 0 means the whole stream — the
* convention the device decodes, and what an unsliced caller leaves the field at. */
static uint32_t gdnc_col_count(const ft_tt_op_gdnc_request *request)
{
return request->col_count ? request->col_count : gdnc_tile_cols(request);
}
/** The v-heads a DELTA descriptor owns. A `col_count` of 0 means all of them. */
static uint32_t gdnc_head_count(const ft_tt_op_gdnc_request *request)
{
return request->col_count ? request->col_count : request->v_heads;
}
int ft_tt_op_gdnc_shape_is_supported(const ft_tt_op_gdnc_request *request)
{
if (!request)
return 0;
if (request->chunk_rows == 0u || request->chunk_rows > FT_TT_OP_GDNC_MAX_CHUNK_ROWS)
return 0;
/* The slice is a HALF-WORD each because `conv_tile_cols` is 256 at Ornith and a byte
* would truncate to 0 — silently making every descriptor a whole-stream one. */
if (request->first_col > 0xFFFFu || request->col_count > 0xFFFFu)
return 0;
if (request->phase == FT_TT_OP_GDNC_PHASE_CONV) {
/* THE ROLL SOURCE ROW IS `chunk_rows - (taps-1) + plane` AND IT KNOWS NOTHING ABOUT AN
* OFFSET, so a CONV descriptor naming one rolls the wrong rows into the window: right
* in chunk 0 and wrong from chunk 1 on. CONV does not need the field — see
* `kGdncFirstRow` — so it is refused rather than given a meaning. */
if (request->first_row != 0u)
return 0;
if (request->conv_taps < 2u || request->conv_taps > 8u)
return 0;
/* A chunk shorter than the window cannot REPLACE it, only shift it, and a partial
* roll is a different writer. THAT FLOOR BELONGS TO THE ROLL, so PHASE DELTA — which
* rolls nothing — does not carry it. See ft_tt_op_gdnc.h. */
if (request->chunk_rows < request->conv_taps - 1u)
return 0;
if (request->conv_dim == 0u || (request->conv_dim % FT_TT_OP_TILE_DIM) != 0u)
return 0;
if (request->first_col + gdnc_col_count(request) > gdnc_tile_cols(request))
return 0;
return 1;
}
if (request->phase == FT_TT_OP_GDNC_PHASE_DELTA) {
/* `state_tiles` squares into L1 — 8 is 64 pages of 4 KiB for ONE head's state, which
* is where the arena stops. It must also fit the shape word's byte. */
if (request->state_tiles == 0u || request->state_tiles > 8u)
return 0;
if (request->v_heads == 0u || request->v_heads > 0xFFFFu)
return 0;
if (request->k_heads == 0u || request->k_heads > 0xFFFFu)
return 0;
/* The GQA divisibility, checked on BOTH sides because getting the repeat backwards is
* a KNOWN bug in this architecture rather than a hypothetical one. */
if (request->v_heads % request->k_heads != 0u)
return 0;
if (request->first_col + gdnc_head_count(request) > request->v_heads)
return 0;
/* THE IN-PLACE SUB-CHUNK'S WINDOW. Off a face boundary the y write is a gather rather
* than a contiguous suffix; past the tile it is a negative length. Both retire clean
* and corrupt a neighbouring head's y. */
if ((request->first_row % FT_TT_OP_GDNC_SUB_CHUNK_ALIGN) != 0u)
return 0;
if (request->first_row + request->chunk_rows > FT_TT_OP_GDNC_MAX_CHUNK_ROWS)
return 0;
return 1;
}
return 0;
}
int ft_tt_op_gdnc_parameters(const ft_tt_op_gdnc_request *request,
uint32_t out_words[FT_TT_OP_GDNC_WORDS])
{
if (!request || !out_words)
return FT_TT_EINVAL;
/* An unusable depth knob refuses HERE, before a descriptor exists, rather than declaring a
* zero the kernels would refuse one descriptor at a time. */
if (ft_tt_op_gdnc_conv_depth() == 0u)
return FT_TT_EINVAL;
if (!ft_tt_op_gdnc_shape_is_supported(request))
return FT_TT_EINVAL;
if (!operand_is_set(&request->activation_in) ||
!operand_is_set(&request->activation_out) || !operand_is_set(&request->constant) ||
!operand_is_set(&request->state_conv))
return FT_TT_EINVAL;
if (!operands_are_fp32(request))
return FT_TT_EINVAL;
/* THE ACTIVATION PAIR SHARES ONE NoC COORDINATE WORD, so both buffers must sit on the same
* DRAM endpoint. NOTHING ON THE DEVICE CAN CATCH A VIOLATION: the reader reads the right
* bank and the writer puts the answer at the OUTPUT's offset inside the INPUT's bank, so
* the descriptor completes normally, the output buffer is never written, and the host
* reads back whatever was already there. Measured on the decode family, silicon
* 2026-08-18 — and what named it was that the conv answer came back BIT-IDENTICAL with
* the kernel's SiLU removed. */
if (request->activation_in.x != request->activation_out.x ||
request->activation_in.y != request->activation_out.y)
return FT_TT_EINVAL;
/* PHASE DELTA's gate block shares the CONSTANT operand's packed NoC coordinate, exactly as
* the activation pair shares its own, and it fails the same way: the reader would pull
* `kGdncGateStride` floats from the right bank at an address that belongs to nothing, the
* descriptor would retire clean, and every gate column would be whatever was there. */
if (request->phase == FT_TT_OP_GDNC_PHASE_DELTA && request->gate_address == 0u)
return FT_TT_EINVAL;
memset(out_words, 0, sizeof(uint32_t) * FT_TT_OP_GDNC_WORDS);
out_words[FT_TT_OP_GDNC_SHAPE] = (request->phase & 0xFFu) |
((request->chunk_rows & 0xFFu) << 8) |
((request->conv_taps & 0xFFu) << 16) |
((request->state_tiles & 0xFFu) << 24);
/* ONE WORD, TWO READINGS, selected by the phase byte. PHASE CONV keeps it byte-identical
* to what shipped; PHASE DELTA packs the two head counts into it, because
* `FT_TT_QUEUE_PARAM_WORDS` is 16 for EVERY family and a seventeenth moves the shared slot
* stride. */
out_words[FT_TT_OP_GDNC_CONV_TILE_COLS] =
(request->phase == FT_TT_OP_GDNC_PHASE_DELTA)
? ((request->v_heads & 0xFFFFu) | ((request->k_heads & 0xFFFFu) << 16))
: gdnc_tile_cols(request);
/* The COUNT is written as the caller gave it, NOT as resolved: a 0 count means "the whole
* stream" on both sides, and writing the resolved value would make an unsliced descriptor
* encode differently from the one an encoder that predates the slice would emit. */
out_words[FT_TT_OP_GDNC_COL_SLICE] =
(request->first_col & 0xFFFFu) | ((request->col_count & 0xFFFFu) << 16);
out_words[FT_TT_OP_GDNC_ACTIVATION_XY] = pack_xy(&request->activation_in);
out_words[FT_TT_OP_GDNC_ACTIVATION_IN] = request->activation_in.address;
out_words[FT_TT_OP_GDNC_ACTIVATION_OUT] = request->activation_out.address;
out_words[FT_TT_OP_GDNC_CONSTANT_XY] = pack_xy(&request->constant);
out_words[FT_TT_OP_GDNC_CONSTANT_WEIGHT] = request->constant.address;
out_words[FT_TT_OP_GDNC_STATE_XY] = pack_xy(&request->state_conv);
out_words[FT_TT_OP_GDNC_STATE_CONV] = request->state_conv.address;
out_words[FT_TT_OP_GDNC_DELTA_GATE] =
(request->phase == FT_TT_OP_GDNC_PHASE_DELTA) ? request->gate_address : 0u;
/* PHASE CONV encodes a ZERO here whatever the request said, and the request is separately
* refused for a nonzero one: the word must be zero on a CONV descriptor so this family's
* shipped encoding is unchanged for the phase that does not use it. */
out_words[FT_TT_OP_GDNC_FIRST_ROW] =
(request->phase == FT_TT_OP_GDNC_PHASE_DELTA) ? request->first_row : 0u;
/* ONE tile size for the whole family, not whichever operand this branch reached for. */
out_words[FT_TT_OP_GDNC_TILE_BYTES] = FT_TT_OP_GDNC_FP32_TILE_BYTES;
/* BYTE 0 OF THE FLAGS WORD IS THIS HOST'S OWN CONV BLOCK DEPTH, and both kernels compare
* it for STRICT EQUALITY against the constant they were compiled with. It is declared on
* CONV descriptors ONLY, because PHASE DELTA never reads it — so a DELTA descriptor still
* leaves the whole word zero and its shipped encoding is unchanged. There is deliberately
* NO "0 means unset" escape: an encoder that forgets to declare must REFUSE, not pass.
* `kGdncByteConvDepth` in kernels/resident_gdnc_protocol.h carries the drift this
* catches — a host that placed DEEPER than the kernels were built at passes every capacity
* check, runs at the kernels' depth, and reports the control's timing as the arm's. */
out_words[FT_TT_OP_GDNC_FLAGS] = ((request->phase == FT_TT_OP_GDNC_PHASE_CONV)
? (ft_tt_op_gdnc_conv_depth() & 0xFFu)
: (request->delta_flags &
FT_TT_OP_GDNC_DELTA_FLAG_COMPACT)) |
(request->serialize ? FT_TT_OP_GDNC_FLAG_SERIALIZE : 0u);
/* PHASE CONV: two pages a column, the conv_out tile and then the RAW input tile the writer
* takes the window roll's rows out of. PHASE DELTA: `state_tiles*(state_tiles+1)` a head.
* The kernels test `tile_count` for EQUALITY against this, not for `>=`: it is fully
* determined by the slice, so an inequality would be a host that disagreed with itself. */
out_words[FT_TT_OP_GDNC_TILE_COUNT] =
(request->phase == FT_TT_OP_GDNC_PHASE_DELTA)
? gdnc_head_count(request) * request->state_tiles * (request->state_tiles + 1u)
: 2u * gdnc_col_count(request);
return FT_TT_OK;
}
/* ================================================================
* Circular buffer placement
* ================================================================ */
/**
* Pages per buffer, for a shape with `taps` taps.
*
* EVERY COUNT IS A CONSUMER GRID, not a comfort setting. `ft_tensix_unpack_ab(cb, ..., tile)`
* is `read_pointer + tile*page_size` with NO WRAP, so a buffer a consumer indexes `n` tiles
* across needs `pages % n == 0` as well as `pages >= n`:
*
* CB_BAND indexed `2*taps` wide -> exactly `2*taps` pages, which also makes its push an
* EXACT HIT on the limit and therefore self-rebasing
* CB_VECTOR indexed `taps` wide -> a multiple of `taps`
* CB_PROD indexed `2*taps` wide -> a multiple of `2*taps`, and at least TWO of them so
* the compute thread can stage the next column's products while the matmul
* reduces this one
* CB_IN, CB_STATE indexed one tile at a time -> any count; four for the look-ahead
* CB_OUT never indexed (the writer spells its own wrap) -> any count
*
* The reader re-checks all of it against the PLACED counts and refuses a descriptor that does
* not fit, because the host and the kernels are compiled separately.
*/
static void resolve_cb_pages_at(uint32_t taps, uint32_t state_tiles, uint32_t depth,
uint32_t out_pages[FT_TT_OP_GDNC_CB_COUNT])
{
const uint32_t band = 2u * taps;
const uint32_t input = 3u * state_tiles;
const uint32_t state = state_tiles * state_tiles;
/* THE CONV BLOCK'S THREE INDEX SPANS AT THIS DEPTH. A group is `depth` input tiles,
* `depth` assembled window tiles and `depth*taps` weight planes, and the compute thread
* indexes across the whole of each — so these are grids, not capacities. */
const uint32_t conv_group = depth;
const uint32_t conv_weight_group = depth * taps;
/* ONE PLACEMENT SERVES BOTH PHASES, because a residency is armed once and then answers
* whatever the queue hands it. Every count below is therefore a COMMON MULTIPLE of the two
* phases' index spans, not the larger of them:
*
* CB_IN CONV indexes 1 wide, DELTA `3*state_tiles` -> two DELTA groups
* CB_STATE CONV 1, DELTA `state_tiles^2` -> two DELTA groups
* CB_BAND CONV `2*taps`, DELTA 4 -> lcm, so both divide
* CB_VECTOR CONV `taps`, DELTA 5 -> `5*taps` divides both
* CB_PROD CONV `2*taps`, DELTA unused
* CB_CC DELTA 24 -> exactly one span, so its push is an EXACT HIT and self-rebasing
* CB_WORK DELTA `9*state_tiles` -> the same
* CB_OUT never indexed (the writer spells its own wrap) -> any count
*
* A count that satisfied only one phase would be a residency that runs one kind of
* descriptor and reads out of its buffers on the other, at some head that is not the
* first -- and the reader refuses exactly that, which is what makes this arithmetic
* checked rather than asserted. */
/* AT DEPTH D EVERY CONV SPAN MULTIPLIES BY D, so each count is rounded onto the lcm of the
* two phases' spans and then held to two blocks of the larger one. At D = 1 every
* expression below collapses to exactly what shipped — 24 / 32 / 8 / 20 / 16 at Ornith —
* which is what makes `FT_TT_OP_GDNC_CONV_DEPTH=1` a control and not a second sizing. */
out_pages[0] = gdnc_round_up(2u * (input > conv_group ? input : conv_group),
gdnc_lcm(input, conv_group)); /* CB_IN */
out_pages[1] = gdnc_round_up(2u * (state > conv_group ? state : conv_group),
gdnc_lcm(state, conv_group)); /* CB_STATE */
out_pages[2] = (band % 4u == 0u) ? band : 4u * band; /* CB_BAND */
out_pages[3] = gdnc_round_up(2u * (FT_TT_OP_GDNC_DELTA_VECTOR_SPAN > conv_weight_group
? FT_TT_OP_GDNC_DELTA_VECTOR_SPAN
: conv_weight_group),
gdnc_lcm(FT_TT_OP_GDNC_DELTA_VECTOR_SPAN,
conv_weight_group)); /* CB_VECTOR */
out_pages[4] = 2u * band; /* CB_PROD */
out_pages[5] = FT_TT_OP_GDNC_DELTA_WORK_SPAN(state_tiles); /* CB_WORK */
out_pages[6] = 1u; /* CB_SCRATCH — one gate run */
out_pages[7] = 2u; /* CB_DECAY */
out_pages[8] = FT_TT_OP_GDNC_DELTA_CC_SPAN; /* CB_CC */
/* CB_OUT holds TWO of the WRITER'S GROUPS and not one head. A head is
* `state_tiles*(state_tiles+1)` = 20 pages at Ornith and this buffer holds eight: a writer
* that waited for a whole head would block on page twenty while the compute thread blocked
* reserving page nine, and neither would ever move. The writer drains `state_tiles` at a
* time -- the y row, then one state tile-row each -- so this is the count that matters. */
out_pages[9] = (GDNC_CB_OUT_PAGES > 2u * state_tiles) ? GDNC_CB_OUT_PAGES
: 2u * state_tiles; /* CB_OUT */
(void)GDNC_CB_COLUMNS;
}
int ft_tt_op_gdnc_cb_pages_at_depth(uint32_t taps, uint32_t state_tiles, uint32_t depth,
uint32_t out_pages[FT_TT_OP_GDNC_CB_COUNT])
{
if (!out_pages || taps < 2u || taps > 8u || state_tiles == 0u || state_tiles > 8u ||
depth < 1u || depth > 255u)
return FT_TT_EINVAL;
resolve_cb_pages_at(taps, state_tiles, depth, out_pages);
return FT_TT_OK;
}
/** The sizing the SHIPPING path uses: this process's own depth, read from the environment
* once. Split from the arithmetic above so a pricing caller can ask about a rung it is not
* running. */
static void resolve_cb_pages(uint32_t taps, uint32_t state_tiles,
uint32_t out_pages[FT_TT_OP_GDNC_CB_COUNT])
{
resolve_cb_pages_at(taps, state_tiles, ft_tt_op_gdnc_conv_depth(), out_pages);
}
int ft_tt_op_gdnc_place_cbs(const ft_tt_op_gdnc_request *request,
ft_tt_launch_cb_entry out_entries[FT_TT_OP_GDNC_CB_COUNT],
void *buffer, size_t buffer_bytes, uint64_t *out_mask)
{
static const unsigned indices[FT_TT_OP_GDNC_CB_COUNT] = {
FT_TT_OP_GDNC_CB_IN, FT_TT_OP_GDNC_CB_STATE, FT_TT_OP_GDNC_CB_BAND,
FT_TT_OP_GDNC_CB_VECTOR, FT_TT_OP_GDNC_CB_PROD, FT_TT_OP_GDNC_CB_WORK,
FT_TT_OP_GDNC_CB_SCRATCH, FT_TT_OP_GDNC_CB_DECAY, FT_TT_OP_GDNC_CB_CC,
FT_TT_OP_GDNC_CB_OUT
};
ft_tt_op_cb_request requests[FT_TT_OP_GDNC_CB_COUNT];
uint32_t pages[FT_TT_OP_GDNC_CB_COUNT];
unsigned index;
if (!request || !out_entries || !buffer || !out_mask)
return FT_TT_EINVAL;
/* An unusable depth refuses HERE, because a placement sized from one is the single thing
* this file must never produce: it is the placement the kernels then find perfectly
* legal. */
if (ft_tt_op_gdnc_conv_depth() == 0u)
return FT_TT_EINVAL;
if (!ft_tt_op_gdnc_shape_is_supported(request))
return FT_TT_EINVAL;
/* A PLACEMENT IS THE WHOLE FAMILY'S, NOT ONE PHASE'S. The residency is armed once and then
* answers whatever the queue hands it, so a request that names only its own phase's shape
* would place buffers the OTHER phase reads out of. Both shapes are required here even
* though `shape_is_supported` only checks the phase's own. */
if (request->conv_taps < 2u || request->conv_taps > 8u)
return FT_TT_EINVAL;
if (request->state_tiles == 0u || request->state_tiles > 8u)
return FT_TT_EINVAL;
resolve_cb_pages(request->conv_taps, request->state_tiles, pages);
/* EVERY page is FT_TT_OP_GDNC_FP32_TILE_BYTES and none of them is taken from an operand.
* The CB table makes all six buffers Float32 and all three kernels are built with
* TILE_BYTES = 4096; a page sized from a bf16 operand would put 2048-byte pages under a
* reader that writes 4096-byte tiles, which overruns the buffer into whatever
* `ft_tt_op_cb_place` put above it. */
for (index = 0; index < FT_TT_OP_GDNC_CB_COUNT; index++) {
requests[index].index = indices[index];
requests[index].page_bytes = FT_TT_OP_GDNC_FP32_TILE_BYTES;
requests[index].page_count = pages[index];
}
return ft_tt_op_cb_place(requests, FT_TT_OP_GDNC_CB_COUNT, out_entries, buffer,
buffer_bytes, out_mask);
}
/* ================================================================
* The band matrices
* ================================================================ */
size_t ft_tt_op_gdnc_band_bytes(uint32_t taps)
{
if (taps < 2u || taps > 8u)
return 0u;
return (size_t)(2u * taps) * FT_TT_OP_GDNC_FP32_TILE_BYTES;
}
int ft_tt_op_gdnc_build_band(uint32_t taps, void *out_tiles, size_t out_bytes)
{
/* One 32x32 row-major fp32 matrix at a time, tilized into place. The stack cost is 4 KiB,
* which is the same tile it produces. */
float plane[FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM];
unsigned char *out = (unsigned char *)out_tiles;
uint32_t tap;
if (!out_tiles || taps < 2u || taps > 8u)
return FT_TT_EINVAL;
if (out_bytes < ft_tt_op_gdnc_band_bytes(taps))
return FT_TT_EINVAL;
for (tap = 0; tap < taps; tap++) {
const uint32_t shift = (taps - 1u) - tap; /* `d` in the derivation */
uint32_t row;
uint32_t which;
for (which = 0; which < 2u; which++) {
memset(plane, 0, sizeof(plane));
for (row = 0; row < FT_TT_OP_TILE_DIM; row++) {
if (which == 0u) {
/* S_tau: the chunk's OWN input, row t-d, for rows at or past the shift. */
if (row >= shift)
plane[row * FT_TT_OP_TILE_DIM + (row - shift)] = 1.0f;
} else {
/* C_tau: the WINDOW, plane p = t + tau, for rows below the shift. Empty
* at tau = taps-1, where the shift is 0 — the zero tile, emitted anyway
* so the product count stays `2*taps`. */
if (row < shift)
plane[row * FT_TT_OP_TILE_DIM + (row + tap)] = 1.0f;
}
}
if (ft_tt_tilize_strip(plane, sizeof(float), FT_TT_OP_TILE_DIM, FT_TT_OP_TILE_DIM,
(size_t)FT_TT_OP_TILE_DIM * sizeof(float),
out + (size_t)(2u * tap + which) *
FT_TT_OP_GDNC_FP32_TILE_BYTES) != FT_TT_TILIZE_OK)
return FT_TT_EINVAL;
}
}
return FT_TT_OK;
}
/* ================================================================
* What one layer costs in card DRAM
* ================================================================
*
* EVERY SIZE BELOW IS `<an ftgc_* TILE COUNT> * FT_TT_OP_GDNC_FP32_TILE_BYTES`, and the tile
* counts come from `resident_gdnc_protocol.h` itself rather than from arithmetic respelled
* here. That header is the settled layout and both dataflow kernels compute their offsets
* from those same accessors, so a stager sized from them cannot disagree with the reader
* about where a plane ends — which is the one disagreement nothing checks at run time.
*/
/** The device's own view of this request. The slice and `tile_count` are not size inputs. */
static ftgc_shape shape_of(const ft_tt_op_gdnc_request *request)
{
ftgc_shape shape;
shape.phase = request->phase;
shape.chunk_rows = request->chunk_rows;
shape.conv_taps = request->conv_taps;
shape.conv_tile_cols = gdnc_tile_cols(request);
shape.first_col = 0u;
shape.col_count = shape.conv_tile_cols;
shape.tile_bytes = FT_TT_OP_GDNC_FP32_TILE_BYTES;
shape.tile_count = 0u;
return shape;
}
static size_t gdnc_tile_bytes(uint32_t tiles)
{
return (size_t)tiles * FT_TT_OP_GDNC_FP32_TILE_BYTES;
}
size_t ft_tt_op_gdnc_conv_stream_bytes(const ft_tt_op_gdnc_request *request)
{
if (!ft_tt_op_gdnc_shape_is_supported(request))
return 0u;
return gdnc_tile_bytes(gdnc_tile_cols(request));
}
size_t ft_tt_op_gdnc_constant_bytes(const ft_tt_op_gdnc_request *request)
{
ftgc_shape shape;
if (!ft_tt_op_gdnc_shape_is_supported(request))
return 0u;
shape = shape_of(request);
return gdnc_tile_bytes(ftgc_constant_tiles(&shape));
}
size_t ft_tt_op_gdnc_conv_state_bytes(const ft_tt_op_gdnc_request *request)
{
if (!ft_tt_op_gdnc_shape_is_supported(request))
return 0u;
return gdnc_tile_bytes((request->conv_taps - 1u) * gdnc_tile_cols(request));
}
/* ================================================================
* PHASE DELTA's operands
* ================================================================ */
size_t ft_tt_op_gdnc_delta_constant_tile_bytes(void)
{
return (size_t)FT_TT_OP_GDNC_DELTA_CONST_TILES * FT_TT_OP_GDNC_FP32_TILE_BYTES;
}
int ft_tt_op_gdnc_build_delta_constants(uint32_t state_size, void *out_tiles, size_t out_bytes)
{
float plane[FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM];
unsigned char *out = (unsigned char *)out_tiles;
float scale;
uint32_t which;
if (!out_tiles || state_size == 0u)
return FT_TT_EINVAL;
if (out_bytes < ft_tt_op_gdnc_delta_constant_tile_bytes())
return FT_TT_EINVAL;
/* DERIVED ONCE, HERE. The card never computes a reciprocal square root, so the two arms of
* `Y = scale (Q_hat N_0^T) + (scale Pm) U` cannot disagree about a factor that multiplies
* every output of a recurrence. The decode family sends the same number as raw fp32 bits
* for the same reason. */
scale = 1.0f / sqrtf((float)state_size);
for (which = 0; which < FT_TT_OP_GDNC_DELTA_CONST_TILES; which++) {
uint32_t row;
uint32_t column;
memset(plane, 0, sizeof(plane));
for (row = 0; row < FT_TT_OP_TILE_DIM; row++) {
for (column = 0; column < FT_TT_OP_TILE_DIM; column++) {
float value = 0.0f;
if (which == FT_TT_OP_GDNC_DELTA_CONST_IDENTITY)
value = (row == column) ? 1.0f : 0.0f;
else if (which == FT_TT_OP_GDNC_DELTA_CONST_TWO_I)
value = (row == column) ? 2.0f : 0.0f;
else if (which == FT_TT_OP_GDNC_DELTA_CONST_STRICT)
value = (column < row) ? 1.0f : 0.0f;
else
/* A COLUMN, not a full tile: `bcast_col` reads column 0 and replicates it
* across the row. The other lanes are never read; they are written zero
* so a dump of this tile is readable rather than because it matters. */
value = (column == 0u) ? scale : 0.0f;
plane[row * FT_TT_OP_TILE_DIM + column] = value;
}
}
if (ft_tt_tilize_strip(plane, sizeof(float), FT_TT_OP_TILE_DIM, FT_TT_OP_TILE_DIM,
(size_t)FT_TT_OP_TILE_DIM * sizeof(float),
out + (size_t)which * FT_TT_OP_GDNC_FP32_TILE_BYTES) !=
FT_TT_TILIZE_OK)
return FT_TT_EINVAL;
}
return FT_TT_OK;
}
int ft_tt_op_gdnc_build_decay_at(const float *decay, uint32_t first_row, uint32_t chunk_rows,
void *out_tile, size_t out_bytes)
{
float plane[FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM];
uint32_t row;
if (!decay || !out_tile || chunk_rows == 0u)
return FT_TT_EINVAL;
if (first_row + chunk_rows > FT_TT_OP_GDNC_MAX_CHUNK_ROWS)
return FT_TT_EINVAL;
if ((first_row % FT_TT_OP_GDNC_SUB_CHUNK_ALIGN) != 0u)
return FT_TT_EINVAL;
if (out_bytes < FT_TT_OP_GDNC_FP32_TILE_BYTES)
return FT_TT_EINVAL;
/* THE UNSLICED CASE IS THE SHIPPED FUNCTION AND NOT A RE-DERIVATION OF IT. Delegating is
* what makes `first_row = 0` BIT-IDENTICAL by construction instead of by inspection —
* this file has already paid once for two spellings of one recurrence. */
if (first_row == 0u)
return ft_tt_op_gdnc_build_decay(decay, chunk_rows, out_tile, out_bytes);
/* EVERYTHING THIS DESCRIPTOR DOES NOT OWN IS ZERO, AND THAT INCLUDES THE COLUMNS.
* Zeroing the whole plane first is what confines `A_off` to the sub-chunk's own [C,C]
* block — see ft_tt_op_gdnc_build_decay_at()'s declaration. */
memset(plane, 0, sizeof(plane));
for (row = first_row; row < first_row + chunk_rows; row++) {
uint32_t column = row;
/* THE PRODUCT RECURRENCE, run over the sub-chunk's own rows and stopped at its first:
* `D[t][t] = 1`, then walk LEFT multiplying by one decay at a time, every partial in
* (0,1] and nothing divided by a cumulative product. */
plane[row * FT_TT_OP_TILE_DIM + row] = 1.0f;
while (column > first_row) {
plane[row * FT_TT_OP_TILE_DIM + (column - 1u)] =
plane[row * FT_TT_OP_TILE_DIM + column] * decay[column - first_row];
column--;
}
}
if (ft_tt_tilize_strip(plane, sizeof(float), FT_TT_OP_TILE_DIM, FT_TT_OP_TILE_DIM,
(size_t)FT_TT_OP_TILE_DIM * sizeof(float),
out_tile) != FT_TT_TILIZE_OK)
return FT_TT_EINVAL;
return FT_TT_OK;
}
int ft_tt_op_gdnc_build_decay(const float *decay, uint32_t chunk_rows, void *out_tile,
size_t out_bytes)
{
float plane[FT_TT_OP_TILE_DIM * FT_TT_OP_TILE_DIM];
uint32_t row;
if (!decay || !out_tile || chunk_rows == 0u || chunk_rows > FT_TT_OP_GDNC_MAX_CHUNK_ROWS)
return FT_TT_EINVAL;
if (out_bytes < FT_TT_OP_GDNC_FP32_TILE_BYTES)
return FT_TT_EINVAL;
memset(plane, 0, sizeof(plane));
for (row = 0; row < FT_TT_OP_TILE_DIM; row++) {
uint32_t column = row;
/* THE PRODUCT RECURRENCE. `D[t][t] = 1`, then walk LEFT multiplying by one decay at a
* time: every partial is in (0,1], nothing is divided by a cumulative product, and no
* logarithm is taken. Above the diagonal stays zero, which is what makes the same tile
* serve as the CAUSAL MASK on the y path. */
plane[row * FT_TT_OP_TILE_DIM + row] = 1.0f;
while (column > 0u) {
const float step = (column <= chunk_rows - 1u) ? decay[column] : 1.0f;
plane[row * FT_TT_OP_TILE_DIM + (column - 1u)] =
plane[row * FT_TT_OP_TILE_DIM + column] * step;
column--;
}
}
if (ft_tt_tilize_strip(plane, sizeof(float), FT_TT_OP_TILE_DIM, FT_TT_OP_TILE_DIM,
(size_t)FT_TT_OP_TILE_DIM * sizeof(float),
out_tile) != FT_TT_TILIZE_OK)
return FT_TT_EINVAL;
return FT_TT_OK;
}
int ft_tt_op_gdnc_build_gate(const float *decay, const float *beta, uint32_t chunk_rows,
uint32_t state_size, float *out_floats, size_t out_count)
{
const uint32_t rows = FT_TT_OP_TILE_DIM;
float scale;
float last;
uint32_t row;
if (!decay || !beta || !out_floats || state_size == 0u)
return FT_TT_EINVAL;
if (chunk_rows == 0u || chunk_rows > FT_TT_OP_GDNC_MAX_CHUNK_ROWS)
return FT_TT_EINVAL;
if (out_count < FT_TT_OP_GDNC_GATE_STRIDE)
return FT_TT_EINVAL;
scale = 1.0f / sqrtf((float)state_size);
/* P over the LIVE rows only, and then FROZEN. A ragged chunk's rows at and above
* `chunk_rows` carry zero q, k and v, so what they must not do is move the carried state's
* decay -- `G` is `P_{chunk_rows-1}` and not `P_31`. This is the whole of the ragged-chunk
* handling, and it is why no kernel has a `chunk_rows` branch in PHASE DELTA. */
for (row = 0; row < rows; row++) {
const float step = (row < chunk_rows) ? decay[row] : 1.0f;
out_floats[FT_TT_OP_GDNC_GATE_P * rows + row] =
(row == 0u) ? step : out_floats[FT_TT_OP_GDNC_GATE_P * rows + row - 1u] * step;
}
last = out_floats[FT_TT_OP_GDNC_GATE_P * rows + chunk_rows - 1u];
/* ---------------------------------------------------------------------------
* F IS ITS OWN BACKWARD PRODUCT RECURRENCE, AND `last / P_t` IS THE BUG IT REPLACES
* ---------------------------------------------------------------------------
* `F_t = prod_{r = t+1 .. chunk_rows-1} decay_r` — mathematically `P_{chunk_rows-1}/P_t`,
* and that spelling is the one thing in this file that is not a product recurrence.
*
* MEASURED ON SILICON 2026-08-21, at Ornith's own decays and C = 32: P UNDERFLOWS. The
* decay is `exp(ssm_a * softplus(alpha))` and `ssm_a` reaches the tens, so a head can
* carry a decay of 1e-7 — and 32 of those multiply to 1e-224, which is ZERO in fp32 after
* about five positions. `last / P_t` is then `0/0 = NaN`, the NaN reaches the carried
* state, and the whole prefill is garbage. It is INVISIBLE at small C — at C = 4 the same
* decay gives P_3 = 1.5e-28, comfortably normal — which is exactly the shape the arm
* showed: `cmp` IDENTICAL to the control at 4 rows a pass and a completely different
* completion at 32.
*
* Nothing had exercised it: `tests/test_gdnc_ops.c` and `tests/test_phase6_gdnc.c` both
* draw decays in [0.90, 0.999], where `0.90^32 = 0.034` and no denominator is ever small.
* The suites are green at C = 32 and the model is not.
*
* The backward recurrence has no denominator at all: `F_{chunk_rows-1} = 1` and
* `F_t = F_{t+1} * decay_{t+1}`, so every entry is a product of decays in (0,1], every
* partial is in (0,1], and an underflow gives the ZERO the mathematics asks for instead
* of a NaN. It is the same argument `ft_tt_op_gdnc_build_decay()` is built on, applied to
* the one column that was left out of it.
*
* P, PS and G still underflow to zero and that is CORRECT rather than tolerated: a decay
* that small really does annihilate the incoming state, and zero is the value the
* recurrence wants. Only a ratio could turn it into a NaN.
* --------------------------------------------------------------------------- */
out_floats[FT_TT_OP_GDNC_GATE_F * rows + chunk_rows - 1u] = 1.0f;
for (row = chunk_rows - 1u; row > 0u; row--)
out_floats[FT_TT_OP_GDNC_GATE_F * rows + row - 1u] =
out_floats[FT_TT_OP_GDNC_GATE_F * rows + row] * decay[row];
for (row = 0; row < rows; row++) {
const float p = out_floats[FT_TT_OP_GDNC_GATE_P * rows + row];
out_floats[FT_TT_OP_GDNC_GATE_PS * rows + row] = scale * p;
/* beta ZERO above the chunk: `U`'s rows there are then exactly zero rather than
* merely small, so the carried state takes no contribution from a position that does
* not exist. */
out_floats[FT_TT_OP_GDNC_GATE_B * rows + row] = (row < chunk_rows) ? beta[row] : 0.0f;
if (row >= chunk_rows)
out_floats[FT_TT_OP_GDNC_GATE_F * rows + row] = 1.0f;
out_floats[FT_TT_OP_GDNC_GATE_G * rows + row] = last;
}
return FT_TT_OK;
}
int ft_tt_op_gdnc_build_gate_at(const float *decay, const float *beta, uint32_t first_row,
uint32_t chunk_rows, uint32_t state_size, float *out_floats,
size_t out_count)
{
const uint32_t rows = FT_TT_OP_TILE_DIM;
const uint32_t last_row = first_row + chunk_rows - 1u;
float scale;
float last;
uint32_t row;
if (!decay || !beta || !out_floats || state_size == 0u || chunk_rows == 0u)
return FT_TT_EINVAL;
if (first_row + chunk_rows > FT_TT_OP_GDNC_MAX_CHUNK_ROWS)
return FT_TT_EINVAL;
if ((first_row % FT_TT_OP_GDNC_SUB_CHUNK_ALIGN) != 0u)
return FT_TT_EINVAL;
if (out_count < FT_TT_OP_GDNC_GATE_STRIDE)
return FT_TT_EINVAL;
/* THE UNSLICED CASE IS THE SHIPPED FUNCTION, for the reason build_decay_at() states. */
if (first_row == 0u)
return ft_tt_op_gdnc_build_gate(decay, beta, chunk_rows, state_size, out_floats,
out_count);
scale = 1.0f / sqrtf((float)state_size);
/* `P` OVER THE SUB-CHUNK'S OWN ROWS, AND EXACTLY ZERO EVERYWHERE ELSE. Zero and not the
* ragged tail's frozen `P_{last}`: `Y`'s first term is `scale * diag(P) Q N_0^T` and the
* writer lands whole tile suffixes, so a nonzero `P` on a row this descriptor does not own
* writes a whole chunk's worth of `Q N_0^T` into another sub-chunk's answer. */
for (row = 0; row < rows; row++) {
if (row < first_row || row > last_row) {
out_floats[FT_TT_OP_GDNC_GATE_P * rows + row] = 0.0f;
continue;
}
out_floats[FT_TT_OP_GDNC_GATE_P * rows + row] =
(row == first_row) ? decay[row - first_row]
: out_floats[FT_TT_OP_GDNC_GATE_P * rows + row - 1u] *
decay[row - first_row];
}
last = out_floats[FT_TT_OP_GDNC_GATE_P * rows + last_row];
/* `F` IS THE SAME BACKWARD PRODUCT RECURRENCE, run inside the sub-chunk. No denominator,
* every partial in (0,1], and an underflow gives the zero the mathematics asks for — see
* ft_tt_op_gdnc_build_gate() for what the `last / P_t` spelling cost on silicon. */
for (row = 0; row < rows; row++)
out_floats[FT_TT_OP_GDNC_GATE_F * rows + row] = 1.0f;
for (row = last_row; row > first_row; row--)
out_floats[FT_TT_OP_GDNC_GATE_F * rows + row - 1u] =
out_floats[FT_TT_OP_GDNC_GATE_F * rows + row] * decay[row - first_row];
for (row = 0; row < rows; row++) {
const bool live = (row >= first_row && row <= last_row);
out_floats[FT_TT_OP_GDNC_GATE_PS * rows + row] =
scale * out_floats[FT_TT_OP_GDNC_GATE_P * rows + row];
/* beta ZERO off the sub-chunk: row t of `A` is then exactly `e_t`, so row t of `A^-1`
* is exactly `e_t`, so `U` row t is exactly zero and the carried state takes no
* contribution from a position this descriptor does not own. */
out_floats[FT_TT_OP_GDNC_GATE_B * rows + row] = live ? beta[row - first_row] : 0.0f;
out_floats[FT_TT_OP_GDNC_GATE_G * rows + row] = last;
}
return FT_TT_OK;
}
/** The device's own view of a DELTA request. */
static ftgc_shape delta_shape_of(const ft_tt_op_gdnc_request *request)
{
ftgc_shape shape;
memset(&shape, 0, sizeof(shape));
shape.phase = request->phase;
shape.chunk_rows = request->chunk_rows;
shape.conv_taps = request->conv_taps;
shape.state_tiles = request->state_tiles;
shape.v_heads = request->v_heads;
shape.k_heads = request->k_heads;
shape.head_first = 0u;
shape.head_count = request->v_heads;
shape.first_row = request->first_row;
shape.tile_bytes = FT_TT_OP_GDNC_FP32_TILE_BYTES;
return shape;
}
size_t ft_tt_op_gdnc_gate_bytes(const ft_tt_op_gdnc_request *request)
{
if (!ft_tt_op_gdnc_shape_is_supported(request) ||
request->phase != FT_TT_OP_GDNC_PHASE_DELTA)
return 0u;
return (size_t)request->v_heads * FT_TT_OP_GDNC_GATE_STRIDE * sizeof(float);
}
size_t ft_tt_op_gdnc_delta_constant_bytes(const ft_tt_op_gdnc_request *request)
{
ftgc_shape shape;
if (!ft_tt_op_gdnc_shape_is_supported(request) ||
request->phase != FT_TT_OP_GDNC_PHASE_DELTA)
return 0u;
shape = delta_shape_of(request);
return gdnc_tile_bytes(ftgc_delta_constant_tiles(&shape));
}
size_t ft_tt_op_gdnc_delta_input_bytes(const ft_tt_op_gdnc_request *request)
{
ftgc_shape shape;
if (!ft_tt_op_gdnc_shape_is_supported(request) ||
request->phase != FT_TT_OP_GDNC_PHASE_DELTA)
return 0u;
shape = delta_shape_of(request);
return gdnc_tile_bytes(ftgc_delta_input_tiles(&shape));
}
size_t ft_tt_op_gdnc_delta_output_bytes(const ft_tt_op_gdnc_request *request)
{
ftgc_shape shape;
if (!ft_tt_op_gdnc_shape_is_supported(request) ||
request->phase != FT_TT_OP_GDNC_PHASE_DELTA)
return 0u;
shape = delta_shape_of(request);
return gdnc_tile_bytes(ftgc_delta_y_tiles(&shape));
}
size_t ft_tt_op_gdnc_delta_state_bytes(const ft_tt_op_gdnc_request *request)
{
ftgc_shape shape;
if (!ft_tt_op_gdnc_shape_is_supported(request) ||
request->phase != FT_TT_OP_GDNC_PHASE_DELTA)
return 0u;
shape = delta_shape_of(request);
return gdnc_tile_bytes(ftgc_delta_state_tiles(&shape));
}
Reference in New Issue View Git Blame Copy Permalink