384 lines
18 KiB
C
384 lines
18 KiB
C
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
//
|
|
// resident_memory_protocol.h
|
|
// The parameter block and the face arithmetic of the MEMORY op family — EMBEDDING and
|
|
// KV_APPEND — shared by resident_gather_reader.c and resident_place_writer.c.
|
|
// The host half is ft_tt_op_memory.h; the two must agree word for word and
|
|
// tests/test_memory_ops.c pins the pair off-hardware.
|
|
//
|
|
// **The KV cache layout contract lives in ft_tt_op_memory.h PART 1.** Read it there;
|
|
// it is not repeated here, because a contract with two copies has two versions.
|
|
//
|
|
// ===========================================================================
|
|
// ONE OPERATION, TWO OPCODES
|
|
// ===========================================================================
|
|
// Both ops are:
|
|
//
|
|
// place ROW_COUNT rows into a face-ordered tile destination starting at row DST_ROW0,
|
|
// for GROUP_COUNT independent destination groups of GROUP_TILES tiles each.
|
|
//
|
|
// EMBEDDING gathers its rows out of a ROW-MAJOR table by token id and owns the whole
|
|
// destination tile row; KV_APPEND takes its rows out of a TILED projection output, splits
|
|
// them across one group per KV head, and must PRESERVE every destination row it does not
|
|
// write. Two flag bits, one binary.
|
|
//
|
|
// ===========================================================================
|
|
// WHY A PARTIAL ROW IS WRITABLE AT ALL — the arithmetic the whole family rests on
|
|
// ===========================================================================
|
|
// A tile is four 16x16 faces in face order. One ROW of a tile is therefore not contiguous:
|
|
// it is TWO face rows, one per face column-half, `tile_bytes/4` apart. Each is 16 elements
|
|
// = `tile_bytes/64` bytes — 32 B for bf16.
|
|
//
|
|
// 32 bytes is a legal DRAM WRITE (the NoC's write alignment is 16 B; only the READ
|
|
// direction needs 64 B — ft_tt_noc.c, ft_tt_noc_alignment()). That single asymmetry is
|
|
// what makes a one-position KV append a copy instead of a read-modify-write, and it is
|
|
// MEASURED by tests/test_phase6_memory.c stage 1 rather than assumed: the test writes one
|
|
// row into a poisoned destination and checks BOTH that the row landed byte-exact AND that
|
|
// its neighbours are untouched. If a future part refuses a 32-byte DRAM write, this family
|
|
// needs a read-modify-write path and the KV layout does not change.
|
|
//
|
|
// Consecutive rows MERGE: rows [r, r+n) of one face half are `n * chunk` contiguous bytes,
|
|
// so a 32-row block is one write of a whole tile and a whole group of tiles is one write of
|
|
// `group_tiles * tile_bytes`. The single-row case and the prefill case are the same code.
|
|
//
|
|
// ===========================================================================
|
|
// NO COMPUTE THREAD, AND WHY THAT IS SAFE RATHER THAN JUST SMALL
|
|
// ===========================================================================
|
|
// `enables` is DM0|DM1. The hazard that would make that wrong is TRISC0's
|
|
// stock sync init, which zeroes all 64 CB page-counter pairs and would break a
|
|
// BRISC<->NCRISC handshake underneath it. It runs ONLY when BRISC writes
|
|
// RUN_SYNC_MSG_INIT_SYNC_REGISTERS into trisc0's sync slot (at boot, and
|
|
// after a kernel RETURNS). A resident BRISC never returns, so the only
|
|
// zeroing happens at firmware boot, before the queue is armed.
|
|
//
|
|
#ifndef FOUNDRY_RESIDENT_MEMORY_PROTOCOL_H
|
|
#define FOUNDRY_RESIDENT_MEMORY_PROTOCOL_H
|
|
|
|
#include <stdint.h>
|
|
|
|
/* Header-only helper linkage, spelled the same way in both languages -- `inline` in C++ and
|
|
* `static inline` in C. NOT cosmetic: internal linkage changes GCC's inlining and cloning
|
|
* decisions, and the COMPUTE images that also include this header must not move for a
|
|
* change that is about the data-movement ones. */
|
|
#ifndef FTQ_INLINE
|
|
#ifdef __cplusplus
|
|
#define FTQ_INLINE inline
|
|
#else
|
|
#define FTQ_INLINE static inline
|
|
#endif
|
|
#endif
|
|
|
|
#ifndef __cplusplus
|
|
#include <stdbool.h>
|
|
#endif
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The sixteen words. Mirrors ft_tt_op_memory.h -- change both or neither.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
enum { kMemFlags = 0 };
|
|
enum { kMemRowCount = 1 };
|
|
enum { kMemGroupCount = 2 };
|
|
enum { kMemGroupTiles = 3 };
|
|
enum { kMemSrcX = 4 };
|
|
enum { kMemSrcY = 5 };
|
|
/** ROW-MAJOR: base of table row 0. TILED: base of the ONE source TILE ROW this descriptor
|
|
* reads — the host folds (source_row0 / 32) * source_tile_row_bytes in, because the split
|
|
* rule already guarantees one tile row per descriptor. */
|
|
enum { kMemSrcBase = 6 };
|
|
/** ROW-MAJOR only: bytes between table rows. Unread when the source is tiled. */
|
|
enum { kMemSrcRowStride = 7 };
|
|
/** TILED: the first source row WITHIN its tile row, 0..31. ROW-MAJOR without GATHER: the
|
|
* first table row index. GATHER: unread — the index array names every row. */
|
|
enum { kMemSrcRow0 = 8 };
|
|
enum { kMemSrcGroupStride = 9 };
|
|
enum { kMemDstX = 10 };
|
|
enum { kMemDstY = 11 };
|
|
enum { kMemDstBase = 12 };
|
|
/** Bytes each destination group OWNS: the stride when there are several, and the EXTENT
|
|
* when there is one. Both readings are the same number whenever the groups are contiguous,
|
|
* which they always are here — a KV region is `head_count` head strides end to end and an
|
|
* embedding output is one group. It is the only field that knows how big the destination
|
|
* is, and `destination_row0` is an ABSOLUTE position whose tile row multiplies into a raw
|
|
* byte offset, so without it a position past the cache writes into whatever DRAM follows,
|
|
* at full length, with nothing faulting. 0 is a REFUSAL. */
|
|
enum { kMemDstGroupStride = 13 };
|
|
/** ABSOLUTE destination row — the KV position. Its tile row is DST_ROW0 / 32 and its row
|
|
* within that tile is DST_ROW0 % 32. */
|
|
enum { kMemDstRow0 = 14 };
|
|
/** ROW-MAJOR sources only: how many rows the source table HAS. The gather index and the
|
|
* sequential row index are both raw wire values that become a DRAM address as
|
|
* `source_base + index * source_row_stride`, and nothing else in this block bounds them —
|
|
* so without this word a wrong index is an arbitrary DRAM read at full length rather than
|
|
* an error. 0 is a REFUSAL in row-major mode, never "unchecked": a bound a zero turns off
|
|
* is a bound a zeroed descriptor turns off. Unread when the source is tiled.
|
|
*
|
|
* IT TAKES THE LAST SPARE WORD, and the destination bound had to be derived rather than
|
|
* given one of its own: the L1 parameter slot is FT_TT_QUEUE_PARAM_WORDS = 16 for EVERY op
|
|
* family, so a seventeenth word is a change to the shared slot stride and to all six
|
|
* families, not to this one. kMemDstGroupStride is what carries the destination bound. */
|
|
enum { kMemSrcRowLimit = 15 };
|
|
enum { kMemWords = 16 };
|
|
|
|
// Written as literals, not as shifts: tests/test_memory_ops.c pins these by PARSING this
|
|
// file, and a parser that had to evaluate `1u << 2` would be a second thing to get wrong.
|
|
enum { kMemFlagSrcTiled = 0x1 };
|
|
enum { kMemFlagDstWholeTiles = 0x2 };
|
|
enum { kMemFlagGather = 0x4 };
|
|
|
|
/** One descriptor stays inside one source tile row and one destination tile row. The host
|
|
* splits; the kernel REFUSES rather than looping, because a wrap case inside face
|
|
* arithmetic is invisible in an output that is nearly right. */
|
|
enum { kMemMaxRows = 32 };
|
|
|
|
/**
|
|
* A NoC transfer length that may actually be issued.
|
|
*
|
|
* **A ZERO-LENGTH NoC READ IS SUSPECTED OF TAKING THE WHOLE CARD DOWN.** On 2026-08-12 the
|
|
* RoPE round's card-down happened at exactly the first descriptor that reached a branch
|
|
* computing a length of zero; every earlier stage had passed and the next access returned
|
|
* 0xFFFFFFFF everywhere. INFERRED, and deliberately NOT re-confirmed — confirming it costs
|
|
* the card for every worker on it.
|
|
*
|
|
* This family is the one most exposed to it: both ops are gathers built out of computed
|
|
* lengths, and a token count of zero, a collapsed range, or an underflowed `count - start`
|
|
* all produce a zero-length read by arithmetic rather than by a typo. So EVERY transfer
|
|
* here is guarded at the point of issue, in the KERNEL and not only on the host, and a zero
|
|
* length is a REFUSAL (kErrBadParameters) rather than a no-op — a no-op would advance the
|
|
* descriptor and publish completion for work nobody did.
|
|
*/
|
|
FTQ_INLINE bool ftm_length_is_issuable(uint32_t bytes)
|
|
{
|
|
return bytes != 0u;
|
|
}
|
|
|
|
/** Alignment a DRAM READ needs in address AND length (`ft_tt_noc_alignment()` host-side).
|
|
* Spelled here as well as in ft_tensix_transfer_guard.h because the COMPUTE images include
|
|
* this header and must not be made to include a data-movement one. */
|
|
enum { kMemDramReadAlignment = 64 };
|
|
|
|
/**
|
|
* A ROW-MAJOR source row this reader may actually fetch.
|
|
*
|
|
* TWO SEPARATE DEFECTS, and the second one is invisible half the time. The index arrives on
|
|
* the wire — for a GATHER it is a word the host wrote into CB_INDEX — and becomes a raw DRAM
|
|
* address as `base + index * stride`, so an unbounded index is an arbitrary DRAM read.
|
|
* And the STRIDE is never required to be 64-byte aligned, which is what a DRAM read needs in
|
|
* address as well as length: a 1500-wide bf16 table has a 3000-byte row stride, so
|
|
* `index * 3000` is 64-aligned for even indices and ILLEGAL for odd ones. Upstream's
|
|
* analogue does not hope — it rounds the stride up explicitly
|
|
* (`(page_size + 31u) & ~31u`).
|
|
*
|
|
* @param base kMemSrcBase, the table's own base — checked too, because an aligned
|
|
* stride off an unaligned base is still an unaligned read
|
|
* @param stride kMemSrcRowStride
|
|
* @param row_limit kMemSrcRowLimit; 0 refuses
|
|
* @param index the row this descriptor wants
|
|
* @param row_bytes bytes one row read moves
|
|
*/
|
|
FTQ_INLINE bool ftm_row_major_source_is_readable(uint32_t base, uint32_t stride,
|
|
uint32_t row_limit, uint32_t index,
|
|
uint32_t row_bytes)
|
|
{
|
|
if (row_limit == 0u || index >= row_limit)
|
|
return false;
|
|
if (!ftm_length_is_issuable(row_bytes) || (row_bytes % kMemDramReadAlignment) != 0u)
|
|
return false;
|
|
if ((base % kMemDramReadAlignment) != 0u || (stride % kMemDramReadAlignment) != 0u)
|
|
return false;
|
|
/* The address must not wrap, or the bound above proves nothing about where it lands. */
|
|
if (stride != 0u && index > 0xFFFFFFFFu / stride)
|
|
return false;
|
|
return base <= 0xFFFFFFFFu - index * stride;
|
|
}
|
|
|
|
/**
|
|
* A destination byte range this writer may actually put a group on, INSIDE THE GROUP'S OWN
|
|
* BYTES.
|
|
*
|
|
* `destination_row0` is an ABSOLUTE position, so `tile_row * group_tiles * tile_bytes` is an
|
|
* unbounded byte offset added to a raw base — past the end of the cache it wrote into
|
|
* whatever DRAM follows, at full length, with nothing faulting. The bound is
|
|
* kMemDstGroupStride, the bytes each group owns: for a KV region that is the head stride,
|
|
* so this refuses a position past `capacity_positions` exactly; for a one-group embedding it
|
|
* is the whole output. Checking the group rather than the region is the TIGHTER of the two
|
|
* statements — it also stops head h's append landing in head h+1.
|
|
*
|
|
* @param group_bytes_owned kMemDstGroupStride; 0 refuses
|
|
* @param offset bytes into this group the write starts at
|
|
* @param span_bytes bytes this group writes
|
|
*/
|
|
FTQ_INLINE bool ftm_destination_span_fits(uint32_t group_bytes_owned, uint32_t offset,
|
|
uint32_t span_bytes)
|
|
{
|
|
if (group_bytes_owned == 0u || !ftm_length_is_issuable(span_bytes))
|
|
return false;
|
|
if (offset > group_bytes_owned)
|
|
return false;
|
|
return span_bytes <= group_bytes_owned - offset;
|
|
}
|
|
|
|
enum { kMemTileRows = 32 };
|
|
enum { kMemFaceRows = 16 };
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Face arithmetic
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/** Bytes of one face row: 16 elements of one tile row, on one side of the face column
|
|
* split. 32 for bf16, 64 for fp32. */
|
|
FTQ_INLINE uint32_t ftm_chunk_bytes(uint32_t tile_bytes)
|
|
{
|
|
return tile_bytes / 64u;
|
|
}
|
|
|
|
/** Bytes of one face. */
|
|
FTQ_INLINE uint32_t ftm_face_bytes(uint32_t tile_bytes)
|
|
{
|
|
return tile_bytes / 4u;
|
|
}
|
|
|
|
/**
|
|
* Byte offset, inside a tile, of the face row holding tile row `row` and columns
|
|
* [16*half, 16*half+16).
|
|
*
|
|
* This is ft_tt_tile_face_index(row, 16*half) * element_bytes with the multiply folded
|
|
* away. tests/test_memory_ops.c checks it against ft_tt_tilize.c for all 64 (row, half)
|
|
* pairs rather than trusting the rewrite — a face-order mistake is invisible to any metric
|
|
* computed through this same code (docs/tt_kernel_model.md §9).
|
|
*/
|
|
FTQ_INLINE uint32_t ftm_chunk_offset(uint32_t row, uint32_t half, uint32_t tile_bytes)
|
|
{
|
|
return ((row / kMemFaceRows) * 2u + half) * ftm_face_bytes(tile_bytes) +
|
|
(row % kMemFaceRows) * ftm_chunk_bytes(tile_bytes);
|
|
}
|
|
|
|
/** Bytes one row of ONE group occupies in a ROW-MAJOR source: `group_tiles` tiles' worth
|
|
* of one row, i.e. two face rows per tile. Always a multiple of 64 for bf16 and fp32
|
|
* alike, which is what keeps a row read legal against the 64-byte DRAM read alignment. */
|
|
FTQ_INLINE uint32_t ftm_group_row_bytes(uint32_t group_tiles, uint32_t tile_bytes)
|
|
{
|
|
return group_tiles * ftm_chunk_bytes(tile_bytes) * 2u;
|
|
}
|
|
|
|
/** A word copy inside L1. L1 is ordinary memory to the RISC-V and its data cache is
|
|
* disabled (ft_tt_queue.h, THE ORDERING ASSUMPTION), so this is a plain loop; it is the
|
|
* cheapest way to move a 32-byte face row, an order below the ~50 ns a NoC transaction
|
|
* costs to issue. Both ends are 4-byte aligned by construction: a chunk offset is a
|
|
* multiple of `chunk_bytes`, itself a multiple of 4. */
|
|
FTQ_INLINE void ftm_copy_words(uint32_t* destination, const uint32_t* source, uint32_t words)
|
|
{
|
|
for (uint32_t index = 0; index < words; index++)
|
|
destination[index] = source[index];
|
|
}
|
|
|
|
FTQ_INLINE void ftm_zero_words(uint32_t* destination, uint32_t words)
|
|
{
|
|
for (uint32_t index = 0; index < words; index++)
|
|
destination[index] = 0u;
|
|
}
|
|
|
|
/**
|
|
* Place ONE row into a group of `group_tiles` face-ordered destination tiles.
|
|
*
|
|
* `source_row` is a pointer to `ftm_group_row_bytes()` bytes laid out ROW-MAJOR: face row
|
|
* (tile t, half h) at offset (t*2 + h) * chunk_bytes. That is the natural element order of
|
|
* a row-major table row, which is why an untilized embedding table needs no permutation
|
|
* beyond this scatter.
|
|
*/
|
|
FTQ_INLINE void ftm_place_row_from_row_major(uint32_t* destination_block,
|
|
const uint32_t* source_row,
|
|
uint32_t destination_row, uint32_t group_tiles,
|
|
uint32_t tile_bytes)
|
|
{
|
|
const uint32_t chunk = ftm_chunk_bytes(tile_bytes);
|
|
const uint32_t words = chunk / 4u;
|
|
|
|
for (uint32_t tile = 0; tile < group_tiles; tile++) {
|
|
for (uint32_t half = 0; half < 2u; half++) {
|
|
const uint32_t destination_offset =
|
|
tile * tile_bytes + ftm_chunk_offset(destination_row, half, tile_bytes);
|
|
const uint32_t source_offset = (tile * 2u + half) * chunk;
|
|
|
|
ftm_copy_words(destination_block + destination_offset / 4u,
|
|
source_row + source_offset / 4u, words);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Place ONE row of a group of face-ordered SOURCE tiles into the same-shaped destination
|
|
* group, moving it from `source_row` to `destination_row`.
|
|
*
|
|
* When the two rows are equal this is the identity, which is the prefill fast path the
|
|
* reader takes by skipping the staging buffer entirely.
|
|
*/
|
|
FTQ_INLINE void ftm_place_row_from_tiles(uint32_t* destination_block, const uint32_t* source_block,
|
|
uint32_t source_row, uint32_t destination_row,
|
|
uint32_t group_tiles, uint32_t tile_bytes)
|
|
{
|
|
const uint32_t words = ftm_chunk_bytes(tile_bytes) / 4u;
|
|
|
|
for (uint32_t tile = 0; tile < group_tiles; tile++) {
|
|
for (uint32_t half = 0; half < 2u; half++) {
|
|
const uint32_t destination_offset =
|
|
tile * tile_bytes + ftm_chunk_offset(destination_row, half, tile_bytes);
|
|
const uint32_t source_offset =
|
|
tile * tile_bytes + ftm_chunk_offset(source_row, half, tile_bytes);
|
|
|
|
ftm_copy_words(destination_block + destination_offset / 4u,
|
|
source_block + source_offset / 4u, words);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The destination runs a partial write is made of
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/** Most runs one tile's partial write needs: two face row groups x two column halves. */
|
|
enum { kMemMaxRuns = 4 };
|
|
|
|
typedef struct ftm_run {
|
|
uint32_t offset; /**< from the tile base */
|
|
uint32_t bytes;
|
|
} ftm_run;
|
|
|
|
/**
|
|
* The face-row runs covering destination rows [row0, row0 + row_count) of ONE tile.
|
|
*
|
|
* Rows inside one face row group and one column half are CONTIGUOUS, so a range collapses
|
|
* to at most four runs — and the caller checks `row0 == 0 && row_count == 32` first, which
|
|
* collapses it further to the whole tile and lets a whole GROUP of tiles go out as one
|
|
* transaction.
|
|
*
|
|
* @return how many runs were written, 0 for an out-of-range request
|
|
*/
|
|
FTQ_INLINE uint32_t ftm_destination_runs(uint32_t row0, uint32_t row_count, uint32_t tile_bytes,
|
|
ftm_run runs[kMemMaxRuns])
|
|
{
|
|
const uint32_t chunk = ftm_chunk_bytes(tile_bytes);
|
|
uint32_t count = 0u;
|
|
|
|
if (row_count == 0u || row0 + row_count > kMemTileRows)
|
|
return 0u;
|
|
|
|
for (uint32_t group = 0; group < 2u; group++) {
|
|
const uint32_t group_first = group * kMemFaceRows;
|
|
const uint32_t group_last = group_first + kMemFaceRows;
|
|
const uint32_t first = (row0 > group_first) ? row0 : group_first;
|
|
const uint32_t last = (row0 + row_count < group_last) ? row0 + row_count : group_last;
|
|
|
|
if (first >= last)
|
|
continue;
|
|
for (uint32_t half = 0; half < 2u; half++) {
|
|
runs[count].offset = ftm_chunk_offset(first, half, tile_bytes);
|
|
runs[count].bytes = (last - first) * chunk;
|
|
count++;
|
|
}
|
|
}
|
|
return count;
|
|
}
|
|
|
|
#endif // FOUNDRY_RESIDENT_MEMORY_PROTOCOL_H
|