585 lines
30 KiB
C
585 lines
30 KiB
C
/* SPDX-License-Identifier: AGPL-3.0-or-later */
|
|
/* Copyright (C) 2026 David (Corey) Herrera */
|
|
/**
|
|
* @file ft_tt_op_vector.h
|
|
* @brief Host driver for the VECTOR op family — the elementwise triad (SiLU, sigmoid,
|
|
* add, mul, sub, SiLU-gated multiply) and the REDUCE triad (softmax, RMS norm, layer
|
|
* norm, router normalise). Places their circular buffers, encodes their parameter
|
|
* blocks, builds the reduce family's constant tiles.
|
|
*
|
|
* The device half is `kernels/resident_vector_protocol.h` plus
|
|
* `kernels/resident_reduce_protocol.h`; the halves must agree word for word and
|
|
* `tests/test_vector_ops.c` pins the pair off-hardware.
|
|
*
|
|
* ---------------------------------------------------------------------------
|
|
* WHAT "THE VECTOR FAMILY" IS, AND WHY IT IS ONE MODULE
|
|
* ---------------------------------------------------------------------------
|
|
* One input tile stream (sometimes two), one output tile stream, the same tile count in as
|
|
* out, activations laid out [tokens x features] so a feature row lies along a tile's W —
|
|
* which is the orientation ft_tt_op_matmul.h settled and everything downstream inherits.
|
|
* That description covers most of what Phase 6 still owes: softmax, SiLU, sigmoid, add,
|
|
* mul, RoPE, the MoE weighted add, the gated norms. Matmul is the exception.
|
|
*
|
|
* They share this file because they share a parameter block and a WRITER, and because the
|
|
* interesting decision — which ops are one kernel and which are separate — is a statement
|
|
* about the family rather than about any member of it.
|
|
*
|
|
* ---------------------------------------------------------------------------
|
|
* THE DECISION: TWO BINARIES — SIX ELEMENTWISE FUNCTIONS, FOUR REDUCTIONS
|
|
* ---------------------------------------------------------------------------
|
|
* SiLU and sigmoid differ by one SFPU verb. Add, mul and sub differ by one FPU opcode.
|
|
* None of that is what decides it. **What decides it is that a resident binary is linked
|
|
* at a fixed per-processor slab, so only ONE program can be resident on a core at a time**
|
|
* (docs/tt_bringup_log.md §17.8 note 4). A core that runs a SwiGLU has to do silu and then
|
|
* mul; if those are two programs it reloads five kernel images between them, and image
|
|
* upload over the x1 link — not dispatch — is what the 0.245 ms fixed cost turned out to
|
|
* be (§20.4: ~0.19 ms of it). So the grouping question is "what does one core have to do
|
|
* in sequence", and for elementwise the answer is all of it.
|
|
*
|
|
* The same question answers the REDUCE family the same way. Softmax, RMS norm, layer norm
|
|
* and the router normalise are one skeleton — reduce along the row into DST column 0, one
|
|
* SFPU verb on that column, broadcast it back with *_tiles_bcast_cols — differing in a
|
|
* pass count and two verbs, and a transformer block does RMS norm, then attention's
|
|
* softmax, then RMS norm again ON ONE CORE. As separate binaries that is two image reloads
|
|
* per block. So they are ONE binary with the pass count and the verbs as descriptor
|
|
* fields, and RMS norm's Phase 4 one-shot triad is superseded by it.
|
|
*
|
|
* ELTWISE and REDUCE stay two binaries because nothing sequences a softmax between a silu
|
|
* and a mul, and because the reduce contract is different in kind (eleven CBs, three of
|
|
* them a whole row deep). They already share this file, the parameter block and the
|
|
* WRITER; merging the two compute sources would be a source change and not a re-layout,
|
|
* and the only question would be the 12 KiB text slab.
|
|
*
|
|
* ---------------------------------------------------------------------------
|
|
* THE ONE FUSION, AND WHY IT IS NOT THE K80's KIND
|
|
* ---------------------------------------------------------------------------
|
|
* FT_TT_OP_VECTOR_FN_SILU_MUL computes silu(a)*b in one descriptor. docs/tt_bringup_log.md
|
|
* §17.8 note 1 is explicit that fusing to save DISPATCH is porting a K80 workaround for a
|
|
* problem this backend does not have — dispatch is 0.39 µs of device time. This fusion is
|
|
* licensed by the other half of that sentence: as two descriptors it costs a full DRAM
|
|
* round trip of the intermediate, and as one it costs an L1 round trip through a scratch
|
|
* CB page. It avoids a round trip, which is the stated reason to fuse.
|
|
*
|
|
* @status Green on silicon, Tensix 20, 0 checks failed: six elementwise functions, and
|
|
* softmax / RMS norm / layer norm / router normalise on ONE reduce residency. Softmax
|
|
* reproduced its pre-merge numbers exactly (Wt=1 0.999983709, Wt=48 0.999985087,
|
|
* Wt=128 0.999980702) and RMS norm BEAT the Phase 4 one-shot it replaces (max-abs
|
|
* 0.01046 against 0.02381 on identical input), because the merged cbtable carries
|
|
* dst_accum_mode 1 and the one-shot did not.
|
|
* @issues None
|
|
* @todo None
|
|
*/
|
|
|
|
#ifndef FT_TT_OP_VECTOR_H
|
|
#define FT_TT_OP_VECTOR_H
|
|
|
|
#include <stdbool.h>
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#include "ft_tt_launch.h"
|
|
#include "ft_tt_op_dispatch.h"
|
|
#include "ft_tt_queue.h"
|
|
|
|
/* ================================================================
|
|
* The parameter block — mirrors kernels/resident_vector_protocol.h
|
|
* ================================================================ */
|
|
|
|
/** Words of one vector-family parameter block. A word an op does not use is written 0 by
|
|
* the host and never read by the kernel; it is NOT free for that op to redefine, because
|
|
* a block whose meaning depends on which kernel is resident is a wrong answer with no
|
|
* error anywhere. */
|
|
typedef enum ft_tt_op_vector_field {
|
|
FT_TT_OP_VECTOR_FUNCTION = 0, /**< ELTWISE: which function. SOFTMAX: 0 */
|
|
FT_TT_OP_VECTOR_TILE_COUNT = 1, /**< TOTAL tiles in and out — all the writer reads */
|
|
FT_TT_OP_VECTOR_TILE_COLS = 2, /**< softmax Wt; ELTWISE: flags (FT_TT_OP_ELTWISE_FLAG_*) */
|
|
FT_TT_OP_VECTOR_DEPTH = 3, /**< reader block depth; divides the run it blocks over */
|
|
FT_TT_OP_VECTOR_A_X = 4,
|
|
FT_TT_OP_VECTOR_A_Y = 5,
|
|
FT_TT_OP_VECTOR_A_BASE = 6,
|
|
FT_TT_OP_VECTOR_B_X = 7, /**< eltwise: B stream. softmax: the ONE scaler tile */
|
|
FT_TT_OP_VECTOR_B_Y = 8,
|
|
FT_TT_OP_VECTOR_B_BASE = 9,
|
|
FT_TT_OP_VECTOR_C_X = 10,
|
|
FT_TT_OP_VECTOR_C_Y = 11,
|
|
FT_TT_OP_VECTOR_C_BASE = 12,
|
|
FT_TT_OP_VECTOR_WORDS = 16
|
|
} ft_tt_op_vector_field;
|
|
|
|
/** Which function an ELTWISE descriptor applies. Values are wire values. */
|
|
typedef enum ft_tt_op_vector_function {
|
|
FT_TT_OP_VECTOR_FN_SILU = 1, /**< y = x * sigmoid(x) unary */
|
|
FT_TT_OP_VECTOR_FN_SIGMOID = 2, /**< y = sigmoid(x) unary */
|
|
FT_TT_OP_VECTOR_FN_ADD = 3, /**< y = a + b binary */
|
|
FT_TT_OP_VECTOR_FN_MUL = 4, /**< y = a * b binary */
|
|
FT_TT_OP_VECTOR_FN_SUB = 5, /**< y = a - b binary */
|
|
FT_TT_OP_VECTOR_FN_SILU_MUL = 6 /**< y = silu(a) * b binary, the SwiGLU gate */
|
|
} ft_tt_op_vector_function;
|
|
|
|
/**
|
|
* Does `function` take a second input stream?
|
|
*
|
|
* THE ONLY definition of arity, host side and device side both (the kernel has its own
|
|
* copy of this one line and tests/test_vector_ops.c asserts the two agree for every
|
|
* function). The host deliberately does not SEND an arity field: two fields that must
|
|
* agree are two fields that can disagree, and the disagreement here would be a reader
|
|
* fetching a B tile nobody wrote — a plausible wrong answer, not an error.
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: yes
|
|
*/
|
|
bool ft_tt_op_vector_function_is_binary(ft_tt_op_vector_function function);
|
|
|
|
/** "SILU", "ADD", … for logs. Never NULL. */
|
|
const char *ft_tt_op_vector_function_name(ft_tt_op_vector_function function);
|
|
|
|
/* ================================================================
|
|
* Elementwise
|
|
* ================================================================ */
|
|
|
|
/** CB indices, matching the compile-time args the elementwise triad is built with. */
|
|
#define FT_TT_OP_ELTWISE_CB_IN0 0u
|
|
#define FT_TT_OP_ELTWISE_CB_IN1 1u
|
|
#define FT_TT_OP_ELTWISE_CB_SCRATCH 2u
|
|
#define FT_TT_OP_ELTWISE_CB_OUT 16u
|
|
#define FT_TT_OP_ELTWISE_CB_COUNT 4u
|
|
|
|
/** One elementwise call: `tile_count` tiles of A (and of B, if binary) through `function`
|
|
* into `output`. Shape is a tile count and nothing else — an elementwise op has no row
|
|
* structure to get wrong. */
|
|
typedef struct ft_tt_op_eltwise_request {
|
|
ft_tt_op_vector_function function;
|
|
uint32_t tile_count;
|
|
/** K-tiles per NoC barrier in the reader. Shrunk to a divisor of tile_count by
|
|
* ft_tt_op_block_depth(); 0 or 1 is the per-tile barrier, which is always correct and
|
|
* is what cost 5.9x on the matmul. */
|
|
uint32_t prefetch_tiles;
|
|
ft_tt_op_operand input_a;
|
|
ft_tt_op_operand input_b; /**< ignored for a unary function */
|
|
ft_tt_op_operand output;
|
|
/** Nonzero: silu/sigmoid compute ROW 0 of each tile only (a decode strip's live row);
|
|
* rows 4-31 of the output are the input copied, not the function. FPU verbs ignore it. */
|
|
uint32_t row0;
|
|
} ft_tt_op_eltwise_request;
|
|
|
|
/** Word-2 flag of an ELTWISE descriptor. Mirrors kVecEltwiseRow0 -- change both or neither. */
|
|
#define FT_TT_OP_ELTWISE_FLAG_ROW0 1u
|
|
|
|
/**
|
|
* Encode `request` into the sixteen words a FT_TT_QUEUE_OP_ELTWISE descriptor points at.
|
|
*
|
|
* Pure: no device access, no allocation. That is what makes the wire format testable
|
|
* off-hardware, which matters more here than usual — the card is the expensive place to
|
|
* find a field in the wrong word.
|
|
*
|
|
* @param out_words FT_TT_OP_VECTOR_WORDS values, fully written (unused words zeroed)
|
|
* @return FT_TT_OK; FT_TT_EINVAL on a zero tile count, an unknown function, a
|
|
* zero tile size, or a binary function with a zero-sized B operand
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: yes
|
|
*/
|
|
int ft_tt_op_eltwise_parameters(const ft_tt_op_eltwise_request *request,
|
|
uint32_t out_words[FT_TT_OP_VECTOR_WORDS]);
|
|
|
|
/**
|
|
* Place the elementwise triad's four circular buffers in the arena.
|
|
*
|
|
* The two input CBs get TWO BLOCKS of the effective prefetch depth, so the reader fills
|
|
* one while the compute thread drains the other. cb_in1 is placed even for a unary
|
|
* function: a spare bf16 input CB is a few KiB of a 1.29 MiB arena, and the alternative is
|
|
* a host-side arity table that has to agree with the kernel's.
|
|
*
|
|
* @param out_entries FT_TT_OP_ELTWISE_CB_COUNT entries, with addresses filled in
|
|
* @param buffer receives the encoded CB descriptor array
|
|
* @param out_mask receives the launch message's local_cb_mask
|
|
* @return FT_TT_OK; FT_TT_EINVAL; FT_TT_ENOMEM if it does not fit the arena
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: yes
|
|
*/
|
|
int ft_tt_op_eltwise_place_cbs(const ft_tt_op_eltwise_request *request,
|
|
ft_tt_launch_cb_entry out_entries[FT_TT_OP_ELTWISE_CB_COUNT],
|
|
void *buffer, size_t buffer_bytes, uint64_t *out_mask);
|
|
|
|
/* ================================================================
|
|
* The REDUCE family — softmax, RMS norm, layer norm, router normalise
|
|
* ================================================================ */
|
|
|
|
/** CB indices, matching the compile-time args the reduce triad is built with. */
|
|
#define FT_TT_OP_REDUCE_CB_IN 0u
|
|
#define FT_TT_OP_REDUCE_CB_SCALER 1u
|
|
#define FT_TT_OP_REDUCE_CB_COL0 2u
|
|
#define FT_TT_OP_REDUCE_CB_COL1 3u
|
|
#define FT_TT_OP_REDUCE_CB_MID 4u
|
|
#define FT_TT_OP_REDUCE_CB_EPS 5u
|
|
#define FT_TT_OP_REDUCE_CB_GAMMA 6u
|
|
#define FT_TT_OP_REDUCE_CB_BETA 7u
|
|
#define FT_TT_OP_REDUCE_CB_STAGE0 8u
|
|
#define FT_TT_OP_REDUCE_CB_STAGE1 9u
|
|
#define FT_TT_OP_REDUCE_CB_OUT 16u
|
|
#define FT_TT_OP_REDUCE_CB_COUNT 11u
|
|
|
|
/** The three program words of a reduce parameter block, on top of the thirteen the vector
|
|
* family defines. Mirrors kernels/resident_reduce_protocol.h. */
|
|
#define FT_TT_OP_REDUCE_PASS0 13u
|
|
#define FT_TT_OP_REDUCE_PASS1 14u
|
|
#define FT_TT_OP_REDUCE_FINAL 15u
|
|
|
|
/** Byte positions inside a program word. */
|
|
#define FT_TT_OP_REDUCE_BYTE_POOL 0u
|
|
#define FT_TT_OP_REDUCE_BYTE_MAP 1u
|
|
#define FT_TT_OP_REDUCE_BYTE_VERB 2u
|
|
#define FT_TT_OP_REDUCE_BYTE_ADD_EPS 3u
|
|
#define FT_TT_OP_REDUCE_BYTE_PASS_COUNT 0u
|
|
#define FT_TT_OP_REDUCE_BYTE_SOURCE 1u
|
|
#define FT_TT_OP_REDUCE_BYTE_CHAIN 2u
|
|
|
|
/** Wire values. 0 is deliberately not a pool, so a zeroed block is an invalid program. */
|
|
#define FT_TT_OP_REDUCE_POOL_SUM 1u
|
|
#define FT_TT_OP_REDUCE_POOL_MAX 2u
|
|
|
|
#define FT_TT_OP_REDUCE_MAP_NONE 0u
|
|
#define FT_TT_OP_REDUCE_MAP_SQUARE_SELF 1u
|
|
#define FT_TT_OP_REDUCE_MAP_SUB_EXP 2u
|
|
#define FT_TT_OP_REDUCE_MAP_SUB_SQUARE 3u
|
|
|
|
#define FT_TT_OP_REDUCE_VERB_NONE 0u
|
|
#define FT_TT_OP_REDUCE_VERB_RECIP 1u
|
|
#define FT_TT_OP_REDUCE_VERB_RSQRT 2u
|
|
|
|
#define FT_TT_OP_REDUCE_SOURCE_IN 0u
|
|
#define FT_TT_OP_REDUCE_SOURCE_MID 1u
|
|
|
|
#define FT_TT_OP_REDUCE_CHAIN_CENTER 0x1u
|
|
#define FT_TT_OP_REDUCE_CHAIN_GAMMA 0x2u
|
|
#define FT_TT_OP_REDUCE_CHAIN_SCALE 0x4u
|
|
#define FT_TT_OP_REDUCE_CHAIN_BETA 0x8u
|
|
|
|
/** Constants-operand tile layout. Fixed for every member: a layout that moved with the
|
|
* flags would be a wrong address for one descriptor in four. */
|
|
#define FT_TT_OP_REDUCE_CONST_SCALER_TILE 0u
|
|
#define FT_TT_OP_REDUCE_CONST_EPS_TILE 1u
|
|
#define FT_TT_OP_REDUCE_CONST_STREAM_TILE 2u
|
|
|
|
/**
|
|
* Which member of the family a descriptor asks for.
|
|
*
|
|
* These values are NOT on the wire. What goes on the wire is the PROGRAM the member
|
|
* compiles to — a pass count, two pass words and a final-chain word — and the kernel reads
|
|
* nothing else, so there is no function id beside the program to disagree with it (dev
|
|
* notes §3 has the catalogue of what two fields that must agree have cost here).
|
|
*/
|
|
typedef enum ft_tt_op_reduce_function {
|
|
/** y = exp(x - max) / sum exp(x - max). Two passes: MAX, then SUM with a reciprocal. */
|
|
FT_TT_OP_REDUCE_FN_SOFTMAX = 1,
|
|
/** y = x * gamma * rsqrt(mean(x^2) + eps). One pass: SUM of squares with an rsqrt. */
|
|
FT_TT_OP_REDUCE_FN_RMS_NORM = 2,
|
|
/** y = (x - mean) * gamma * rsqrt(mean((x - mean)^2) + eps) + beta. Two passes. */
|
|
FT_TT_OP_REDUCE_FN_LAYER_NORM = 3,
|
|
/** y = x / sum x — the MoE router's renormalise over the selected experts' gates. One
|
|
* pass: SUM with a reciprocal. The cheapest member, and the one that shows the family
|
|
* is a family: it is softmax's second pass and nothing else. */
|
|
FT_TT_OP_REDUCE_FN_ROUTER_NORMALISE = 4,
|
|
/**
|
|
* y = x * rsqrt(sum(x^2) + eps) — the Gated-Delta-Net's per-head q/k norm. One pass,
|
|
* like RMS norm, and it differs from it in exactly two table cells: the reduce scaler
|
|
* is 1.0 rather than 1/W (a SUM, not a MEAN) and the final chain carries no gamma.
|
|
*
|
|
* **This is an L2 norm and NOT an RMS norm.** Pantheon's `pn_qwen35moe_l2_norm()` is the
|
|
* definition and divides by `max(||x||, eps)` — the vector magnitude with no 1/n factor,
|
|
* and a CLAMP rather than an epsilon under the root. Folding `eps*eps` into the sum is
|
|
* the same function to fp32 wherever `sum(x^2) >> eps*eps`, and lands on exactly `1/eps`
|
|
* where the clamp does, so **the caller passes `eps*eps` here, never `eps`**.
|
|
*/
|
|
FT_TT_OP_REDUCE_FN_L2_NORM = 5
|
|
} ft_tt_op_reduce_function;
|
|
|
|
/** "SOFTMAX", "RMS_NORM", … for logs. Never NULL. */
|
|
const char *ft_tt_op_reduce_function_name(ft_tt_op_reduce_function function);
|
|
|
|
/** How many reduce passes this member runs: 1 or 2, and 0 for an unknown function. */
|
|
uint32_t ft_tt_op_reduce_pass_count(ft_tt_op_reduce_function function);
|
|
|
|
/** Does this member multiply by a per-column gamma? (RMS norm, layer norm.) */
|
|
bool ft_tt_op_reduce_uses_gamma(ft_tt_op_reduce_function function);
|
|
|
|
/** Does this member add a per-column beta? (Layer norm.) */
|
|
bool ft_tt_op_reduce_uses_beta(ft_tt_op_reduce_function function);
|
|
|
|
/** Does this member fold eps into a reduce? (RMS norm, layer norm.) */
|
|
bool ft_tt_op_reduce_uses_eps(ft_tt_op_reduce_function function);
|
|
|
|
/** Is this member's reduce scaler 1/W rather than 1.0? True exactly when a pass wants a
|
|
* MEAN rather than a SUM or a MAX. */
|
|
bool ft_tt_op_reduce_scaler_is_mean(ft_tt_op_reduce_function function);
|
|
|
|
/**
|
|
* The three program words this member compiles to.
|
|
*
|
|
* Pure, and the single source of truth for what the kernel will do — which is why
|
|
* tests/test_vector_ops.c pins all four members' words as LITERALS rather than
|
|
* re-deriving them. A derived pin re-derives the mistake and passes (dev notes §3).
|
|
*
|
|
* @param out_pass0 the first pass's program word
|
|
* @param out_pass1 the second's, or 0 for a one-pass member
|
|
* @param out_final pass count, final-chain source and the chain's step flags
|
|
* @return FT_TT_OK, or FT_TT_EINVAL for an unknown function
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: yes
|
|
*/
|
|
int ft_tt_op_reduce_program(ft_tt_op_reduce_function function, uint32_t *out_pass0,
|
|
uint32_t *out_pass1, uint32_t *out_final);
|
|
|
|
/**
|
|
* What ONE resident reduce program is configured to serve.
|
|
*
|
|
* The circular buffers are placed once, at launch, and then serve every descriptor — so
|
|
* the residency, not the descriptor, is what fixes the deepest row and whether gamma and
|
|
* beta are carried at all. A residency sized for the union of the members a core will run
|
|
* is what makes the merge worth having: four ops, ONE image load.
|
|
*
|
|
* A stream the residency does not carry is still PLACED, at one page. A descriptor that
|
|
* asks for it then blocks in cb_reserve_back and is retired by its bound, instead of the
|
|
* reader writing into a circular buffer the host never configured — which would be a NoC
|
|
* write to an address nobody chose (spec §7.3).
|
|
*/
|
|
typedef struct ft_tt_op_reduce_residency {
|
|
uint32_t max_tile_cols; /**< the deepest Wt any descriptor will use */
|
|
uint32_t tile_bytes; /**< 2048 for Float16_b */
|
|
bool carries_gamma;
|
|
bool carries_beta;
|
|
} ft_tt_op_reduce_residency;
|
|
|
|
/** Fill a residency that serves every member in `functions`. */
|
|
int ft_tt_op_reduce_residency_for(const ft_tt_op_reduce_function *functions, size_t count,
|
|
uint32_t max_tile_cols, uint32_t tile_bytes,
|
|
ft_tt_op_reduce_residency *out_residency);
|
|
|
|
/** One reduce call: `row_blocks` blocks of 32 rows, each `tile_cols` tiles wide. W must be
|
|
* a whole number of tiles — there is no partial-tile mask here, and every dimension
|
|
* MiniCPM5-1B reduces over is a multiple of 32. */
|
|
typedef struct ft_tt_op_reduce_request {
|
|
ft_tt_op_reduce_function function;
|
|
uint32_t row_blocks; /**< rows / 32 */
|
|
uint32_t tile_cols; /**< Wt = W / 32 */
|
|
uint32_t prefetch_tiles;
|
|
ft_tt_op_operand input;
|
|
/** The constants buffer from ft_tt_op_reduce_build_constants(): the scaler tile, the
|
|
* eps tile, then gamma's Wt tiles and beta's Wt tiles. ONE operand rather than three
|
|
* because the parameter block has room for one and because rms_norm_reader.c
|
|
* already put its scaler and eps adjacent "so they cost one address rather than two". */
|
|
ft_tt_op_operand constants;
|
|
ft_tt_op_operand output;
|
|
} ft_tt_op_reduce_request;
|
|
|
|
/**
|
|
* The widest Wt that fits the CB arena for a given residency.
|
|
*
|
|
* This is a HARD CEILING and it is the reason flash-decode attention is its own op rather
|
|
* than "matmul, then softmax": a standalone softmax holds the whole row twice over (the
|
|
* activations, to subtract the row max from, and the exponentials, to divide by the row
|
|
* sum), so at 2048 bytes a tile it tops out around Wt = 326, i.e. W ≈ 10,432. A
|
|
* 32k-context attention row is Wt = 1024 and does not fit at any tile size. That is a
|
|
* capacity fact, not a performance preference. A residency that also carries gamma and
|
|
* beta holds FOUR rows and halves it again.
|
|
*
|
|
* @return the largest max_tile_cols ft_tt_op_reduce_place_cbs() will accept, or 0
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: yes
|
|
*/
|
|
uint32_t ft_tt_op_reduce_max_tile_cols(const ft_tt_op_reduce_residency *residency);
|
|
|
|
/**
|
|
* Encode `request` into the sixteen words a FT_TT_QUEUE_OP_REDUCE descriptor points at.
|
|
*
|
|
* Pure: no device access, no allocation. That is what makes the wire format testable
|
|
* off-hardware, which matters more here than usual — the card is the expensive place to
|
|
* find a field in the wrong word.
|
|
*
|
|
* @param out_words FT_TT_OP_VECTOR_WORDS values, fully written (unused words zeroed)
|
|
* @return FT_TT_OK; FT_TT_EINVAL on a zero row-block count, a zero tile_cols, an
|
|
* unknown function, a zero tile size, or a tile_cols past
|
|
* ft_tt_op_reduce_max_tile_cols() for a softmax-shaped residency
|
|
*/
|
|
int ft_tt_op_reduce_parameters(const ft_tt_op_reduce_request *request,
|
|
uint32_t out_words[FT_TT_OP_VECTOR_WORDS]);
|
|
|
|
/**
|
|
* Place the reduce triad's eleven circular buffers in the arena.
|
|
*
|
|
* cb_in and cb_mid are `max_tile_cols` pages each — a whole row block resident, twice
|
|
* over — and so are gamma and beta when the residency carries them. The rest are one or
|
|
* two pages because they are produced and consumed a tile at a time.
|
|
*
|
|
* @return FT_TT_OK; FT_TT_EINVAL; FT_TT_ENOMEM if it does not fit the arena
|
|
*/
|
|
int ft_tt_op_reduce_place_cbs(const ft_tt_op_reduce_residency *residency,
|
|
ft_tt_launch_cb_entry out_entries[FT_TT_OP_REDUCE_CB_COUNT],
|
|
void *buffer, size_t buffer_bytes, uint64_t *out_mask);
|
|
|
|
/** Bytes ft_tt_op_reduce_build_constants() writes for this member at this width. */
|
|
size_t ft_tt_op_reduce_constants_bytes(ft_tt_op_reduce_function function, uint32_t tile_cols,
|
|
uint32_t tile_bytes);
|
|
|
|
/**
|
|
* Build the constants buffer: the scaler tile, the eps tile, then gamma and beta expanded
|
|
* to full tiles.
|
|
*
|
|
* THE SCALER. ONE tile serves BOTH reduces of a descriptor. Upstream's scaler is 1.0 for
|
|
* SUM and for MAX (ttnn/kernel_lib/reduce_helpers_dataflow.hpp: "For SUM/MAX, the
|
|
* reduce_factor is ignored and the scaler is 1.0"), so the two separate scaler buffers
|
|
* moreh_softmax carries are two names for one tile; a member that wants a MEAN gets 1/W in
|
|
* the same slot. Only the first row of each face is read by the reduce LLK; filling the
|
|
* whole tile is a superset and costs nothing.
|
|
*
|
|
* THE EPS TILE. eps is one more accumulated row SUM against a tile whose every element is
|
|
* eps*W/32 — each row has 32 columns, so 32 * (eps*W/32) * (1/W) = eps. That buys eps for
|
|
* one extra reduce_tile instead of an add_tiles_bcast_scalar, a CB round trip and a pack.
|
|
* The slot exists even for a member that does not use it, so the layout never moves.
|
|
*
|
|
* GAMMA AND BETA ARRIVE AS FULL 32x32 TILES, not broadcast rows. A norm weight is 1-D and
|
|
* the FTTW artifact stores 1-D tensors untiled (bringup log §12.7); expanding here (or in
|
|
* the loader, ft_tt_fttw_upload_broadcast_rows()) turns the multiply into a plain mul_tiles
|
|
* instead of a bcast op, and costs 96 KiB of DRAM for a 1536-wide model.
|
|
*
|
|
* THE ELEMENT FORMAT IS THE PAGE SIZE'S, AND IT IS A PARAMETER FOR THE SAME REASON THE GDN
|
|
* family's page size had to become one. `reduce_float16b.cbtable` places every buffer at
|
|
* 2048 bytes and `reduce_float32.cbtable` at 4096; the reader issues `TILE_BYTES` reads
|
|
* either way and has no descriptor header to check them against. A constants buffer built at
|
|
* the wrong page size is refused nowhere — the scaler tile is simply half a tile of whatever
|
|
* followed it — so the caller passes the page size the residency was PLACED at, which is the
|
|
* same word `ft_tt_op_reduce_constants_bytes()` already takes.
|
|
*
|
|
* @param width W in ELEMENTS, not tiles — the eps tile's value depends on it
|
|
* @param eps ignored by a member that does not use eps
|
|
* @param gamma W floats, or NULL for a member that does not use gamma
|
|
* @param beta W floats, or NULL
|
|
* @param tile_bytes FT_TT_OP_TILE_BYTES_BF16 or _FP32; anything else is refused
|
|
* @param out_bytes at least ft_tt_op_reduce_constants_bytes() at the same `tile_bytes`
|
|
* @return FT_TT_OK, FT_TT_EINVAL
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: no
|
|
*/
|
|
int ft_tt_op_reduce_build_constants(ft_tt_op_reduce_function function, uint32_t width, float eps,
|
|
const float *gamma, const float *beta, uint32_t tile_bytes,
|
|
void *out, size_t out_bytes);
|
|
|
|
/* ================================================================
|
|
* A RAGGED W — served by PADDING, and only for the members it is exact for
|
|
* ================================================================ */
|
|
|
|
/**
|
|
* The value a padded lane carries when it must vanish under a MAX or an exp.
|
|
*
|
|
* The same constant attention masks a dead score with, for the same reason and with the
|
|
* same argument: large enough that `exp(pad - row_max)` is zero in bf16 at any plausible
|
|
* activation, small enough that the subtraction cannot overflow
|
|
* (FT_TT_OP_ATTENTION_MASK_VALUE, ft_tt_op_attention.h). It is on silicon there already, in
|
|
* the same SFPU `exp`, which is why this is not a new numerical claim.
|
|
*/
|
|
#define FT_TT_OP_REDUCE_PAD_MASK_VALUE (-1.0e30f)
|
|
|
|
/** The value a padded lane carries when it must vanish under a SUM. */
|
|
#define FT_TT_OP_REDUCE_PAD_ZERO_VALUE (0.0f)
|
|
|
|
/**
|
|
* Wt for a W that is not a whole number of tiles: ceil(width / 32).
|
|
*
|
|
* There is no ragged W on the wire — `ft_tt_op_reduce_request` carries `tile_cols` and the
|
|
* reduce runs over `tile_cols * 32` columns unconditionally (kernels/resident_reduce_
|
|
* reader.c). So a ragged W is served by making the padded width the real one and keeping
|
|
* the DIVISOR honest, which is what the two calls below do.
|
|
*
|
|
* @return ceil(width/32), or 0 for width 0
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: yes
|
|
*/
|
|
uint32_t ft_tt_op_reduce_padded_tile_cols(uint32_t width);
|
|
|
|
/**
|
|
* The constant every padded lane of this member's input must hold — or a refusal, because
|
|
* for one member of the four no constant exists.
|
|
*
|
|
* DERIVED PER PASS, NOT TABULATED, so a fifth member of the family gets the right answer
|
|
* from its own row in the definition table. A padded lane is harmless exactly when the
|
|
* value the pass MAPS it to is the pool's identity, and every pass of the member must want
|
|
* the same constant:
|
|
*
|
|
* | pass | padded lane maps to | vanishes when |
|
|
* |--------------------------|---------------------|-------------------|
|
|
* | SUM, map NONE | pad | pad == 0 |
|
|
* | SUM, map SQUARE_SELF | pad*pad | pad == 0 |
|
|
* | MAX, map NONE | pad | pad << row max |
|
|
* | SUM, map SUB_EXP | exp(pad - col0) | pad << col0 |
|
|
* | SUM, map SUB_SQUARE | (pad - col0)^2 | **pad == col0** |
|
|
*
|
|
* The last row is the whole finding, and it is worse than "no constant works". `col0` is
|
|
* the ROW MEAN, a different number for every row, so no constant can be it — and A PER-ROW
|
|
* PAD DOES NOT RESCUE IT EITHER: layer norm's FIRST pass is a plain SUM over the padded
|
|
* row, so a pad equal to that row's mean `m` makes pass 0 report `m * padded/W` instead of
|
|
* `m`. The two passes want different padding of the SAME buffer. **Nothing a caller writes
|
|
* into the input serves layer norm at a ragged W**, which is why it is refused here rather
|
|
* than shipped with a caveat, and why a mask tile is the only close. Measured off-hardware
|
|
* in tests/test_vector_ops.c: every constant pad AND the per-row mean land 23-24x the
|
|
* arrangement's own bf16 rounding floor.
|
|
*
|
|
* AND THE ERROR IS DATA-DEPENDENT, WHICH IS THE SECOND REASON IT IS A REFUSAL AND NOT A
|
|
* WARNING. On zero-centred rows the same hole measures AT the rounding floor and would pass
|
|
* any score anyone ran; on rows whose means spread over 0..31 it is 23x it. "It looked fine
|
|
* on our activations" is not evidence about this one.
|
|
*
|
|
* @param out_value receives FT_TT_OP_REDUCE_PAD_ZERO_VALUE or
|
|
* FT_TT_OP_REDUCE_PAD_MASK_VALUE
|
|
* @return FT_TT_OK; FT_TT_EINVAL for an unknown member, or for one whose passes
|
|
* cannot agree on a constant (layer norm) — ask
|
|
* ft_tt_op_reduce_padding_is_exact() to tell the two apart
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: yes
|
|
*/
|
|
int ft_tt_op_reduce_pad_value(ft_tt_op_reduce_function function, float *out_value);
|
|
|
|
/** Can this member be served at a ragged W by padding ALONE, exactly? True for softmax,
|
|
* RMS norm and the router normalise; FALSE for layer norm, and false for an unknown
|
|
* member. This is the named reason behind the refusal above, so a caller and a test can
|
|
* assert the reason rather than the return value (dev notes §3). */
|
|
bool ft_tt_op_reduce_padding_is_exact(ft_tt_op_reduce_function function);
|
|
|
|
/**
|
|
* Copy a [rows x width] fp32 activation into the [rows x padded_width] the wire needs,
|
|
* filling every padded lane with this member's pad value.
|
|
*
|
|
* @param in rows*width floats, row-major
|
|
* @param out rows * ft_tt_op_reduce_padded_tile_cols(width) * 32 floats
|
|
* @param out_count elements `out` can hold; a short buffer is refused, never truncated
|
|
* @return FT_TT_OK; FT_TT_EINVAL on a null buffer, a zero row or width count, a
|
|
* short output, or a member with no constant pad value
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: yes
|
|
*/
|
|
int ft_tt_op_reduce_pad_rows(ft_tt_op_reduce_function function, const float *in, uint32_t rows,
|
|
uint32_t width, float *out, size_t out_count);
|
|
|
|
/**
|
|
* ft_tt_op_reduce_build_constants() for a W that need not be a whole number of tiles.
|
|
*
|
|
* THE ONE THING THAT MAKES IT CORRECT IS THAT THE DIVISOR STAYS TRUE. The scaler tile is
|
|
* 1/W and the eps tile is eps*W/32 for the REAL W, while gamma and beta expand to
|
|
* ceil(W/32) tiles with the tail zeroed — so a mean is a mean over the columns that exist,
|
|
* and the padded lanes contribute nothing to it. Building the constants at the padded width
|
|
* instead is the silent wrong answer this call exists to prevent, and it is a SCALE error,
|
|
* which PCC cannot see (dev notes §3).
|
|
*
|
|
* @param width the REAL W in elements — not rounded, not the padded one
|
|
* @param tile_bytes FT_TT_OP_TILE_BYTES_BF16 or _FP32; anything else is refused
|
|
* @param out_bytes at least ft_tt_op_reduce_constants_bytes() at
|
|
* ft_tt_op_reduce_padded_tile_cols(width)
|
|
* @return FT_TT_OK; FT_TT_EINVAL for a ragged W on a member that padding cannot
|
|
* serve exactly (layer norm), and everything
|
|
* ft_tt_op_reduce_build_constants() refuses
|
|
*
|
|
* Thread-safe: yes Reentrant: yes Signal-safe: no
|
|
*/
|
|
int ft_tt_op_reduce_build_constants_ragged(ft_tt_op_reduce_function function, uint32_t width,
|
|
float eps, const float *gamma, const float *beta,
|
|
uint32_t tile_bytes, void *out, size_t out_bytes);
|
|
|
|
#endif /* FT_TT_OP_VECTOR_H */
|