183 lines
9.3 KiB
C
183 lines
9.3 KiB
C
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
//
|
|
// resident_vector_protocol.h
|
|
// The parameter block the VECTOR op family reads out of L1, shared by the resident
|
|
// elementwise triad, the resident softmax triad, and the ONE writer both of them use.
|
|
// The host half is ft_tt_op_vector.h; the two must agree word for word and
|
|
// tests/test_vector_ops.c pins the pair off-hardware.
|
|
//
|
|
// "Vector op family" means: one input tile stream (sometimes two), one output tile
|
|
// stream, the same tile count in as out, activations laid out [tokens x features] so a
|
|
// feature row lies along a tile's W. That covers SiLU, sigmoid, add, mul, sub, the
|
|
// SiLU-gated multiply, and softmax — and it is the shape most of the remaining ~35 ops
|
|
// have. Matmul is the exception, not the rule.
|
|
//
|
|
// ===========================================================================
|
|
// WHY ONE BLOCK LAYOUT ACROSS THE FAMILY, AND WHAT IT BUYS
|
|
// ===========================================================================
|
|
// docs/tt_bringup_log.md 17.8 note 2: "one binary per OPCODE SET, not per op", because a
|
|
// kernel's compile-time args, CB formats and DST_ACCUM_MODE are baked at build time while
|
|
// anything that varies per call has to be a runtime arg or a descriptor field. The 32-byte
|
|
// descriptor carries six free words; a shape that does not fit them points at a block.
|
|
//
|
|
// This header takes that one step further: the ELTWISE and SOFTMAX blocks are the SAME
|
|
// SIXTEEN WORDS with the same meanings, so
|
|
//
|
|
// 1. `resident_vector_writer.c` is written ONCE and serves both. A writer is
|
|
// "drain `tile_count` tiles from cb_out to (x, y, base)" and knows nothing about the
|
|
// op — which is a fact about all ~40 ops, not about these two, and is worth having
|
|
// in one file rather than forty. (resident_matmul_writer.c is the same function
|
|
// over a DIFFERENT block layout; unifying the two is a follow-up, not this round.)
|
|
// 2. A future combined binary — one program that answers both opcodes — is a source
|
|
// merge and not a re-layout, should the text budget allow it. Measured text sizes are
|
|
// in the report; the 12 KiB slab (ft_tt_op_l1_layout.h) is the ceiling that decides.
|
|
//
|
|
// A word a given op does not use is written as 0 by the host and never read by the kernel.
|
|
// That is deliberately NOT "reserved for that op to redefine": a block whose meaning
|
|
// depends on which kernel is resident is a wrong answer with no error anywhere, which is
|
|
// exactly what ft_tt_op_matmul.h's `weight_n_major` note warns about.
|
|
//
|
|
// ===========================================================================
|
|
// THE ARITY RULE
|
|
// ===========================================================================
|
|
// Whether a descriptor has a SECOND input operand is derived from the FUNCTION by
|
|
// ftv_function_is_binary() below, and by nothing else. The host does not send an arity
|
|
// field, because two fields that must agree are two fields that can disagree — and the
|
|
// disagreement here is "the reader fetches a B tile nobody wrote", i.e. a plausible wrong
|
|
// answer. The host sizes both input circular buffers unconditionally (a spare bf16 input
|
|
// CB is a few KiB of a 1.29 MiB arena), so a unary descriptor costs arena and nothing else.
|
|
//
|
|
#ifndef FOUNDRY_RESIDENT_VECTOR_PROTOCOL_H
|
|
#define FOUNDRY_RESIDENT_VECTOR_PROTOCOL_H
|
|
|
|
#include <stdint.h>
|
|
|
|
/* Header-only helper linkage, spelled the same way in both languages -- `inline` in C++ and
|
|
* `static inline` in C. NOT cosmetic: internal linkage changes GCC's inlining and cloning
|
|
* decisions, and the COMPUTE images that also include this header must not move for a
|
|
* change that is about the data-movement ones. */
|
|
#ifndef FTQ_INLINE
|
|
#ifdef __cplusplus
|
|
#define FTQ_INLINE inline
|
|
#else
|
|
#define FTQ_INLINE static inline
|
|
#endif
|
|
#endif
|
|
|
|
#ifndef __cplusplus
|
|
#include <stdbool.h>
|
|
#endif
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The sixteen words. Mirrors ft_tt_op_vector.h -- change both or neither.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/** Which function an ELTWISE descriptor applies. Unused (0) by SOFTMAX. */
|
|
enum { kVecFunction = 0 };
|
|
|
|
/** TOTAL tiles this descriptor moves, in and out. The writer reads this and nothing
|
|
* else about the shape, which is what lets one writer serve the family. */
|
|
enum { kVecTileCount = 1 };
|
|
|
|
/** Tiles across one softmax row (Wt). 0 for elementwise, which has no row structure.
|
|
* Row blocks are kVecTileCount / kVecTileCols -- derived, never sent twice. */
|
|
enum { kVecTileCols = 2 };
|
|
|
|
/** ELTWISE reuses word 2 (it has no row structure) as a flags word. kVecEltwiseRow0: the
|
|
* SFPU verbs (silu, sigmoid) compute ROW 0 of each tile only -- a decode strip's one live
|
|
* row (rows 0-3 are computed); rows 4-31 keep the copied input. FPU verbs and data movement are unchanged. */
|
|
enum { kVecEltwiseFlags = 2 };
|
|
enum { kVecEltwiseRow0 = 1u };
|
|
|
|
/** K-tiles per NoC barrier in the reader. Must DIVIDE the run it blocks over
|
|
* (kVecTileCount for elementwise, kVecTileCols for softmax) or a block straddles the CB
|
|
* wrap; the kernel refuses to guess and falls back to 1. See ft_tt_op_block_depth(). */
|
|
enum { kVecDepth = 3 };
|
|
|
|
/** E3 SPLIT PROBE (timing only — an armed bit's OUTPUT IS NOT EVIDENCE of anything): the
|
|
* top byte of word 3. The depth is the low 24 bits; every reader of word 3 masks. */
|
|
enum { kVecProbeShift = 24 };
|
|
enum { kVecProbeNoRead = 1u }; /* readers skip their NoC reads and barrier */
|
|
enum { kVecProbeNoWrite = 2u }; /* the vector writer skips its NoC writes and barrier */
|
|
enum { kVecProbeLegacyBarrier = 4u }; /* writer: the OLD per-tile ack barrier (the control) */
|
|
enum { kVecProbeNoCompute = 8u }; /* eltwise compute moves pages, no unpack/math/pack */
|
|
|
|
/** First input: the activation. */
|
|
enum { kVecAX = 4 };
|
|
enum { kVecAY = 5 };
|
|
enum { kVecABase = 6 };
|
|
|
|
/** Second operand. ELTWISE: the B tile stream, for a binary function. SOFTMAX: the ONE
|
|
* reduce scaler tile (1.0 everywhere), which is a single tile and not a stream. */
|
|
enum { kVecBX = 7 };
|
|
enum { kVecBY = 8 };
|
|
enum { kVecBBase = 9 };
|
|
|
|
/** Output. */
|
|
enum { kVecCX = 10 };
|
|
enum { kVecCY = 11 };
|
|
enum { kVecCBase = 12 };
|
|
|
|
/** Sixteen so a batch of blocks is one clean host write and a shape that grows moves
|
|
* nothing. Matches FT_TT_QUEUE_PARAM_WORDS. */
|
|
enum { kVecWords = 16 };
|
|
|
|
/** Word 3 split: the depth (low 24 bits) and the E3 probe bits (top byte). */
|
|
static inline uint32_t ftv_depth_word(uint32_t word3) { return word3 & 0x007FFFFFu; }
|
|
/** M3 (REDUCE only): bit 23 of word 3 = HALF TILE — the elementwise FPU passes (the map's
|
|
* square, the gamma multiply, the scale broadcast) unpack and compute faces 0-1 only (rows
|
|
* 0-15); the row reduce stays whole. A single-token descriptor's live row is row 0. */
|
|
static inline bool ftv_half_tile(uint32_t word3) { return ((word3 >> 23) & 1u) != 0u; }
|
|
static inline uint32_t ftv_probe_bits(uint32_t word3) { return word3 >> kVecProbeShift; }
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The elementwise function set -- ONE binary, six functions
|
|
// ---------------------------------------------------------------------------
|
|
//
|
|
// SiLU and sigmoid are the same kernel with a different SFPU verb; add, mul and sub are
|
|
// the same kernel with a different FPU opcode; and all five are the same kernel as each
|
|
// other because a resident binary is BOUND TO ONE SLAB, so two binaries cannot be resident
|
|
// on one core at once (docs/tt_bringup_log.md 17.8 note 4). Splitting silu from mul would
|
|
// mean reloading five images -- ~0.19 ms over the x1 link, measured (20.4) -- between the
|
|
// two halves of every SwiGLU. That is the argument, and it is a residency argument rather
|
|
// than a code-size one.
|
|
//
|
|
enum { kVecFnSilu = 1 }; // y = x * sigmoid(x) unary
|
|
enum { kVecFnSigmoid = 2 }; // y = sigmoid(x) unary
|
|
enum { kVecFnAdd = 3 }; // y = a + b binary
|
|
enum { kVecFnMul = 4 }; // y = a * b binary
|
|
enum { kVecFnSub = 5 }; // y = a - b binary
|
|
/** y = silu(a) * b -- the SwiGLU gate, and the ONE fusion 17.8 note 1 licenses here: as
|
|
* two descriptors it costs a full DRAM round trip of the intermediate, and as one it
|
|
* costs an L1 round trip through a scratch CB. It is NOT fused to save dispatch; there is
|
|
* no dispatch to save. */
|
|
enum { kVecFnSiluMul = 6 };
|
|
|
|
/** Whether `function` takes a second input stream. THE ONLY definition of arity; see the
|
|
* header note. */
|
|
FTQ_INLINE bool ftv_function_is_binary(uint32_t function)
|
|
{
|
|
return function >= kVecFnAdd;
|
|
}
|
|
|
|
/** Whether `function` needs the scratch CB (silu into L1, then multiply). */
|
|
FTQ_INLINE bool ftv_function_uses_scratch(uint32_t function)
|
|
{
|
|
return function == kVecFnSiluMul;
|
|
}
|
|
|
|
/**
|
|
* The depth a reader will actually block at.
|
|
*
|
|
* Transliterated from ft_tt_op_matmul_prefetch_depth() / ft_tt_op_block_depth(), and the
|
|
* kernel repeats the host's arithmetic rather than trusting it for the same reason
|
|
* resident_matmul_reader.c does: a depth that does not divide the run moves the CB write
|
|
* pointer off the block grid and every later block straddles the wrap. Falling back to 1 is always correct, just slower.
|
|
*/
|
|
FTQ_INLINE uint32_t ftv_block_depth(uint32_t run, uint32_t requested)
|
|
{
|
|
return (requested > 0u && run > 0u && run % requested == 0u) ? requested : 1u;
|
|
}
|
|
|
|
#endif // FOUNDRY_RESIDENT_VECTOR_PROTOCOL_H
|