1292 lines
62 KiB
C
1292 lines
62 KiB
C
/* SPDX-License-Identifier: AGPL-3.0-or-later */
|
|
/**
|
|
* @file resident_gdn_reader.c
|
|
* @brief Reader third of the RESIDENT GATED-DELTA-NET triad, and THE PROCESSOR THAT BUILDS
|
|
* EVERY BROADCAST SHAPE THE COMPUTE SURFACE CANNOT EXPRESS.
|
|
*
|
|
* The compute thread of this op never indexes an element. It is handed aligned tile streams
|
|
* and issues broadcast multiplies, row reduces, subs and adds. Everything that makes those
|
|
* legal -- the row-replicated k/q, the column-shaped v, the constant decay/beta/scale
|
|
* columns, the GQA repeat -- is ADDRESSING and L1 word copies, and it all lives here.
|
|
*
|
|
* ===========================================================================
|
|
* THE STAGING CONTRACT, AND THE ONE CORRECTION THAT SIMPLIFIES IT
|
|
* ===========================================================================
|
|
* The broadcast surface is BROADCAST_COL only: it reads column 0 of the second operand and
|
|
* applies that row's scalar across every column of the first operand's row. There is no row
|
|
* broadcast and no scalar broadcast. So:
|
|
*
|
|
* k, q ROW-REPLICATED -- [r][i] = k[i] for EVERY row r.
|
|
* v a TRUE COLUMN -- v[j] at [j][0].
|
|
* decay a CONSTANT COLUMN -- the same scalar in all 32 rows of column 0.
|
|
* beta likewise.
|
|
* scale likewise, splatted from parameters[kGdnScaleBits] WITHOUT arithmetic.
|
|
*
|
|
* ONE TILE SERVES BOTH USES OF k. The silicon probe that proved the reduce scaler is
|
|
* PER-POSITION built its scaler with tests/test_ops.c `build_column_scaler_tile()`, which
|
|
* writes `plain[row*32 + column] = s[column]` -- i.e. the value for reduce position c is
|
|
* present at EVERY [r][c]. The proven staging is therefore ROW-REPLICATED, not "the value at
|
|
* [i][0]". A row-replicated k tile is simultaneously the (b) reduce's scaler operand and the
|
|
* (d) rank-1 operand, so FT_TT_OP_GDN_CB_SCALER absorbs what the paper design called
|
|
* CB_REPLICA and index 4 is free for something else (see the CB map below).
|
|
* ft_tt_op_gdn.h now DOCUMENTS that collapse rather than contradicting it.
|
|
*
|
|
* THE NON-COLUMN-0 LANES OF v / decay / beta / scale ARE DELIBERATELY NOT WRITTEN. Only
|
|
* column 0 of any of them is ever consumed: `bcast_col` reads column 0, and the only full
|
|
* eltwise op they reach is `(c)`'s `v - pred`, whose result is itself consumed by a
|
|
* `bcast_col`. Zeroing 1,024 words a tile to make the other 31 columns tidy would roughly
|
|
* double this reader's L1 store count for no observable difference. If a future step ever
|
|
* consumes a lane other than column 0, this is the line that has to change.
|
|
*
|
|
* ===========================================================================
|
|
* THE GQA REPEAT IS NEVER MATERIALISED
|
|
* ===========================================================================
|
|
* `ftg_k_head_for(h, k_heads)` is one modulo. v-head h stages the k/q columns of k-head
|
|
* `h % k_heads`; the two v-heads that share a k-head rebuild the same replica tiles rather
|
|
* than a doubled tensor being read from DRAM. Materialising it is the 1.33x-WORSE row of the
|
|
* split-point table.
|
|
*
|
|
* ===========================================================================
|
|
* HOW A REPLICA IS BUILT: BULK READ, THEN PURE L1 WORD COPIES
|
|
* ===========================================================================
|
|
* The NoC cannot gather with stride, so this is resident_attention_reader.c's
|
|
* `read_and_pack_q()` idiom: ONE bulk `noc_read` of the source row block into a scratch CB
|
|
* that is reserved but NEVER pushed, then face-arithmetic word copies into the consumed CB's
|
|
* pages. A 32x32 Float32 tile is 4096 B: four 1024 B faces in row-major order of the 2x2
|
|
* face grid, sixteen 64 B rows inside each face.
|
|
*
|
|
* The cost is honest and worth stating: a replica tile is 1,024 word stores because every
|
|
* word of it is written, and one DELTA descriptor builds `2 * v_heads * state_tiles` of them.
|
|
* At the shipped shape that is 256 tiles, ~262 K word stores a layer. Against the 0.225 s a
|
|
* host recurrence costs it is noise; against the rest of this kernel it is the dominant term,
|
|
* and the named optimisation is to cache the staged k/q row block across the `v_heads /
|
|
* k_heads` heads that share it (deliberately not taken here -- it needs `2 * state_tiles`
|
|
* scratch pages held live, which is the whole scratch buffer).
|
|
*
|
|
* ===========================================================================
|
|
* THE CB MAP
|
|
* ===========================================================================
|
|
* cb 0 CB_IN CONV ONLY: the qkv stream. THIS PROCESSOR IS ITS ONLY PRODUCER, in
|
|
* either phase. PHASE DELTA's compute-private working state used to share
|
|
* this index; it is cb 8 now. "One producer per DESCRIPTOR" was the rule
|
|
* this file used to state and it is not strong enough -- a CB's books
|
|
* outlive a descriptor, and the dataflow push INCREMENTS the
|
|
* tiles_received register while the compute push STORES a local counter
|
|
* over it. See the compute kernel's cb_work note.
|
|
* cb 1 CB_STATE CONV: the rolling window. DELTA: one tile-row of the carried state.
|
|
* cb 2 CB_SCALER DELTA: this head's k replicas then its q replicas, `2*state_tiles`.
|
|
* cb 3 CB_VECTOR CONV: the conv1d weight taps. DELTA: v columns, then decay, beta, scale.
|
|
* cb 4 CB_STAGE READER-PRIVATE SCRATCH: never reserved, never pushed and never popped,
|
|
* by this processor or any other. That is exactly why it is safe to write
|
|
* through its write pointer -- the pointer can never have moved off the
|
|
* buffer base. THE ROUND'S SPEC NAMED cb 7 FOR THIS AND THAT IS WRONG:
|
|
* cb 7 is CB_PROD, which the compute thread pushes and pops inside the
|
|
* same descriptor, so the two would fight over the same L1 pages.
|
|
*
|
|
* THIS PROCESSOR OWNS `tail`, AND NOTHING ELSE IN THE CONTROL BLOCK
|
|
* (resident_queue_protocol.h rule 1). An aborted descriptor advances nothing.
|
|
*
|
|
* ONE BARRIER PER STREAM GROUP, NEVER PER TILE. A barrier is a per-CORE drain, and paying it
|
|
* per tile cost 5.9x on the matmul. CONV cannot put all of a block's reads under ONE barrier
|
|
* because the window's `taps-1` tiles cannot be reserved as one chunk -- `taps-1` = 3 does
|
|
* not divide the 8-page window buffer, and a chunk that does not divide the depth walks the
|
|
* write pointer off the end (ft_tensix_cb_push_grid_is_safe). So each window tap is its own
|
|
* reserve/read/barrier/push group and the qkv+weights are one more: `taps` groups a block,
|
|
* not `2*taps` transactions each with a barrier.
|
|
*
|
|
*
|
|
* C99 on Foundry's own `ft_tensix_dataflow.h` -- no tt-metal include (plan Phase 6.6).
|
|
* Compile-time args:
|
|
* 0: CB_IN index 1: CB_STATE index 2: CB_SCALER index 3: CB_VECTOR index
|
|
* 4: CB_STAGE index (reader-private staging; never pushed, never popped)
|
|
* 5: tile size in bytes 6: wait mode -- 0 upstream, 1 stoppable (ships), 2 bound-only
|
|
* 7: CONV block depth, SHARED with the compute kernel and with the host's
|
|
* FT_TT_OP_GDN_CONV_DEPTH. EIGHT args, not seven, and the count is load-bearing:
|
|
* `ft_tensix_compile_time_arg(7)` against a seven-arg line reads one word PAST the
|
|
* args array, and SFPI GCC 15.1.0 FOLDS THAT TO A LITERAL ZERO. Measured on a second card host
|
|
* 2026-08-18: a seven-arg build of this file is BYTE-IDENTICAL to one passing an
|
|
* explicit depth of 0, at -Os and at -O3, with no diagnostic even under
|
|
* `-Warray-bounds=2` -- so `-Werror` would not have caught it either, and the arg
|
|
* COUNT written beside this build line is the only guard that exists. Depth 0 makes
|
|
* conv_block_depth() return 0, which aborts every CONV descriptor with
|
|
* kErrBadParameters.
|
|
* 8: CB_WORK index -- the COMPUTE thread's private PHASE DELTA working buffer. This
|
|
* processor never touches it; the index is here so `delta_is_executable()` can refuse a
|
|
* placement the compute thread's indexed reads would walk off, which is the same
|
|
* three-sided refusal every other buffer gets. NINE args.
|
|
* 9: CB_PROD index -- the compute thread's rank-1 staging buffer, here for exactly the
|
|
* reason CB_WORK is. The batched (d) indexes it `batch` tiles wide instead of naming
|
|
* tile 0, so it acquired an indexed-read grid, and this processor is the only one of
|
|
* the three that can read a page count at all.
|
|
* 10: dst_batch_max -- PHASE DELTA's DST batch width before `ftg_dst_batch()` clamps it.
|
|
* SHARED with the compute kernel: the grid refused here is a function of it, so a pair
|
|
* that disagreed would refuse the wrong placements and admit the ones that matter.
|
|
* ELEVEN args.
|
|
* 11: reader extras, a bitmask -- see READER_EXTRAS. TWELVE args.
|
|
* Runtime args: the ten of resident_queue_protocol.h, identical on all five processors
|
|
*
|
|
* Build (the whole program; this file is the BRISC/NoC 0 third):
|
|
* RELOCATE_KERNEL_TEXT=1 \
|
|
* READER_CTARGS=0,1,2,3,4,4096,1,2,8,7,4,1 COMPUTE_CTARGS=0,1,2,3,5,6,7,16,1,2,8,4,1 \
|
|
* WRITER_CTARGS=16,4096,1,0 CB_TABLE=kernels/cb_tables/gdn_float32.cbtable \
|
|
* bash foundry/tt/build_tensix_program.sh \
|
|
* kernels/resident_gdn_reader.c kernels/resident_gdn_compute.c \
|
|
* kernels/resident_gdn_writer.c programs/gdn
|
|
*
|
|
* The round's spec wrote `READER_CTARGS=0,1,2,3,7,4096,1`. The 7 is corrected to 4 above for
|
|
* the reason in the CB map: index 7 is the compute thread's CB_PROD.
|
|
*
|
|
* @status EXECUTED ON SILICON 2026-08-18, and every address this file computes is CONFIRMED
|
|
* by readback rather than inferred: the host echoes the staged qkv row and all four conv
|
|
* weight tap planes out of DRAM with 0 of 8192 channels differing, and PASS B's newest
|
|
* window plane comes back equal to qkv tile for tile, which exercises the whole reader ->
|
|
* CB -> compute -> writer path on Float32 data with an answer that needs no arithmetic.
|
|
* @issues None. The DELTA input and CONV operand layouts were an INFERENCE living in this
|
|
* file's private address arithmetic; they are now STATED in resident_gdn_protocol.h, and
|
|
* every offset this file computes comes from the `ftg_*` accessors there, so the stager
|
|
* and the reader read one declaration. There is still no stager and no device check can
|
|
* catch a disagreement — `tests/test_gdn_ops.c` pins the arithmetic off-hardware instead.
|
|
*
|
|
* THE PUSH GRID WAS ONLY HALF THE RULE, and the missing half was this family's whole wrong
|
|
* answer (measured 2026-08-18). A consumer that indexes `n` tiles off its front needs
|
|
* `pages % n == 0` as well, and a RESIDENT op needs its pointers put back on the base at
|
|
* every descriptor because a descriptor's traffic is not a whole number of buffers.
|
|
* `index_grid_is_safe()` refuses the first and `rebase_produced_buffers()` supplies the
|
|
* second; see the section in resident_gdn_protocol.h.
|
|
* @todo None
|
|
*/
|
|
|
|
#include <stdbool.h>
|
|
#include <stdint.h>
|
|
|
|
#include "ft_tensix_dataflow.h"
|
|
#include "resident_gdn_protocol.h"
|
|
#include "resident_queue_protocol.h"
|
|
|
|
#define CB_IN ft_tensix_compile_time_arg(0)
|
|
#define CB_STATE ft_tensix_compile_time_arg(1)
|
|
#define CB_SCALER ft_tensix_compile_time_arg(2)
|
|
#define CB_VECTOR ft_tensix_compile_time_arg(3)
|
|
#define CB_STAGE ft_tensix_compile_time_arg(4)
|
|
#define TILE_BYTES ft_tensix_compile_time_arg(5)
|
|
#define WAIT_MODE ft_tensix_compile_time_arg(6)
|
|
/** Tile columns one CONV block covers. SHARED with the compute kernel, which is built with
|
|
* the same value and cannot derive it any other way — `cb_interface` does not exist on the
|
|
* MATH thread, so a depth read off the CB interface is an undefined reference there. Same
|
|
* two-sided compile-time contract TILE_BYTES has, and nothing checks it for you. */
|
|
#define CONV_DEPTH ft_tensix_compile_time_arg(7)
|
|
/** The COMPUTE thread's private PHASE DELTA working buffer. Named here ONLY so this
|
|
* processor's refusal covers it; nothing below reads or writes it. */
|
|
#define CB_WORK ft_tensix_compile_time_arg(8)
|
|
/** The COMPUTE thread's rank-1 staging buffer. Named here for the same reason CB_WORK is:
|
|
* the batched (d) indexes it, so it has a consumer grid, and this processor is the only one
|
|
* of the three that can see a page count. */
|
|
#define CB_PROD ft_tensix_compile_time_arg(9)
|
|
/** PHASE DELTA's DST batch width before the clamp. SHARED with the compute kernel; the grid
|
|
* refused below is a function of it. */
|
|
#define DST_BATCH_MAX ft_tensix_compile_time_arg(10)
|
|
/** Reader restructurings, as a BITMASK. Every bit is a measurement arm, and each produces
|
|
* BYTE-IDENTICAL circular-buffer contents to the arm below it -- what moves is how many
|
|
* scalar L1 accesses this processor makes to produce them, which is the term the DELTA
|
|
* descriptor may or may not be bound by. TWELVE args.
|
|
*
|
|
* bit 0 BUILD A ROW-REPLICATED TILE COLUMN-MAJOR. `push_replicated_tile` copies one 64-byte
|
|
* source row into all sixteen rows of each of four faces, and it does it row by row,
|
|
* re-reading the source every time -- 1,024 loads and 1,024 stores a tile, times
|
|
* `2 * v_heads * state_tiles` = 256 tiles a DELTA descriptor, which is ~262,000 word
|
|
* copies. Loading each source word once and storing it down the column drops the loads
|
|
* to 64 a tile. */
|
|
#define READER_EXTRAS ft_tensix_compile_time_arg(11)
|
|
|
|
enum { kReaderReplicaByColumn = 1 };
|
|
|
|
/** Rows of one face, and faces of one tile: fixed by gdn_float32.cbtable (face_r 16,
|
|
* num_faces 4). Named because a bare 16 in an offset expression is unreadable. */
|
|
enum { kGdnFaceRows = 16 };
|
|
enum { kGdnTileRows = 32 };
|
|
|
|
/** The five DRAM endpoints a descriptor names, decoded once. */
|
|
typedef struct gdn_endpoints {
|
|
uint32_t activation_x;
|
|
uint32_t activation_y;
|
|
uint32_t activation_in;
|
|
uint32_t constant_x;
|
|
uint32_t constant_y;
|
|
uint32_t constant_base;
|
|
uint32_t state_x;
|
|
uint32_t state_y;
|
|
uint32_t state_base;
|
|
} gdn_endpoints;
|
|
|
|
/* M5 GDN READER CLOCKS (control words 57-60, this processor's alone; zeroed at boot):
|
|
* 57 cycles >> 4 waiting in NoC read barriers, 58 building tiles in software (replicate /
|
|
* column / constant), 59 whole CONV descriptors, 60 whole DELTA descriptors. */
|
|
enum { kGdnClockBarrier = 57, kGdnClockBuild = 58, kGdnClockConv = 59, kGdnClockDelta = 60 };
|
|
/* M9: PHASE CONV's barriers go to word 56 (free on a gdn core), so 57 is DELTA's alone. */
|
|
enum { kGdnClockConvBarrier = 56 };
|
|
static bool gdn_conv_phase;
|
|
static bool gdn_conv_strip;
|
|
/** S2c: the row this descriptor's stream rows live in (kGdnRowShift). */
|
|
static uint32_t gdn_row;
|
|
static bool gdn_red_strip_half;
|
|
/** RED ONLY (kGdnFlagRedRowsRow0): a multi-row CONV convolves row 0's stream for every row. */
|
|
static bool gdn_red_rows_row0;
|
|
static volatile FTQ_L1_PTR uint32_t *gdn_clock_control;
|
|
|
|
static FT_BH_INLINE void gdn_clock_add(uint32_t word, uint32_t started)
|
|
{
|
|
if (gdn_clock_control)
|
|
gdn_clock_control[word] += (ft_bh_wall_clock_low() - started) >> 4;
|
|
}
|
|
|
|
static FT_BH_INLINE void gdn_read_barrier(void)
|
|
{
|
|
const uint32_t started = ft_bh_wall_clock_low();
|
|
|
|
ft_tensix_noc_read_barrier();
|
|
gdn_clock_add(gdn_conv_phase ? kGdnClockConvBarrier : kGdnClockBarrier, started);
|
|
}
|
|
|
|
static void load_endpoints(gdn_endpoints *endpoints, const ftg_shape *shape,
|
|
volatile FTQ_L1_PTR uint32_t *parameters)
|
|
{
|
|
endpoints->activation_x = ftg_xy_x(parameters[kGdnActivationXy]);
|
|
endpoints->activation_y = ftg_xy_y(parameters[kGdnActivationXy]);
|
|
endpoints->activation_in = parameters[kGdnActivationIn];
|
|
endpoints->constant_x = ftg_xy_x(parameters[kGdnConstantXy]);
|
|
endpoints->constant_y = ftg_xy_y(parameters[kGdnConstantXy]);
|
|
endpoints->state_x = ftg_xy_x(parameters[kGdnStateXy]);
|
|
endpoints->state_y = ftg_xy_y(parameters[kGdnStateXy]);
|
|
if (shape->phase == (uint32_t)kGdnPhaseConv) {
|
|
endpoints->constant_base = parameters[kGdnConstantWeight];
|
|
endpoints->state_base = parameters[kGdnStateConv];
|
|
} else {
|
|
endpoints->constant_base = parameters[kGdnConstantGate];
|
|
endpoints->state_base = parameters[kGdnStateRecurrent];
|
|
}
|
|
}
|
|
|
|
/* ===========================================================================
|
|
* Face arithmetic. Modelled on fta_chunk_offset(); spelled locally because
|
|
* resident_gdn_protocol.h is the wire contract and does not carry L1 layout.
|
|
* =========================================================================== */
|
|
|
|
/** Bytes of one face: a 2x2 face grid over the tile. */
|
|
static uint32_t ftg_face_bytes(void)
|
|
{
|
|
return TILE_BYTES / 4u;
|
|
}
|
|
|
|
/** Bytes of one face ROW: 16 elements of a 32-element row, i.e. tile_bytes/64. */
|
|
static uint32_t ftg_face_row_bytes(void)
|
|
{
|
|
return TILE_BYTES / 64u;
|
|
}
|
|
|
|
/** Words of one face row -- the unit every copy below moves. 16 for a Float32 tile. */
|
|
static uint32_t ftg_face_row_words(void)
|
|
{
|
|
return TILE_BYTES / 256u;
|
|
}
|
|
|
|
/** Byte offset, inside a tile, of the face row holding `row` and columns [16*half, +16). */
|
|
static uint32_t ftg_face_row_offset(uint32_t row, uint32_t half)
|
|
{
|
|
return ((row / (uint32_t)kGdnFaceRows) * 2u + half) * ftg_face_bytes() +
|
|
(row % (uint32_t)kGdnFaceRows) * ftg_face_row_bytes();
|
|
}
|
|
|
|
/** Byte offset, inside a tile, of element [row][column]. */
|
|
static uint32_t ftg_element_offset(uint32_t row, uint32_t column)
|
|
{
|
|
return ftg_face_row_offset(row, column / (uint32_t)kGdnFaceRows) +
|
|
(column % (uint32_t)kGdnFaceRows) * (TILE_BYTES / 1024u);
|
|
}
|
|
|
|
/**
|
|
* M9: `tiles` consecutive CONV tiles from (x, y, address) into L1 at `destination` — whole, or
|
|
* under kGdnFlagConvStrip only row 0 (faces 0-1: 2 x 64 B a tile). Barrier is the caller's.
|
|
*/
|
|
static void conv_read(uint32_t x, uint32_t y, uint32_t address, uint32_t destination,
|
|
uint32_t tiles)
|
|
{
|
|
uint32_t tile;
|
|
uint32_t half;
|
|
|
|
if (!gdn_conv_strip) {
|
|
ft_tensix_noc_read(ft_tensix_noc_address(x, y, address), destination,
|
|
tiles * TILE_BYTES);
|
|
return;
|
|
}
|
|
for (tile = 0; tile < tiles; tile++)
|
|
for (half = 0; half < (gdn_red_strip_half ? 1u : 2u); half++) {
|
|
const uint32_t offset = tile * TILE_BYTES + ftg_face_row_offset(0u, half);
|
|
|
|
ft_tensix_noc_read(ft_tensix_noc_address(x, y, address + offset),
|
|
destination + offset, ftg_face_row_bytes());
|
|
}
|
|
}
|
|
|
|
/** S2c: conv_read for the STREAM at DRAM row `row` into L1 row 0 (the strip-mode layout the
|
|
* compute thread reads). Row 0 is conv_read() itself. */
|
|
static void conv_read_stream_row(uint32_t x, uint32_t y, uint32_t address, uint32_t destination,
|
|
uint32_t tiles, uint32_t row)
|
|
{
|
|
uint32_t tile;
|
|
uint32_t half;
|
|
|
|
if (row == 0u || !gdn_conv_strip) {
|
|
conv_read(x, y, address, destination, tiles);
|
|
return;
|
|
}
|
|
for (tile = 0; tile < tiles; tile++)
|
|
for (half = 0; half < (gdn_red_strip_half ? 1u : 2u); half++)
|
|
ft_tensix_noc_read(ft_tensix_noc_address(x, y, address + tile * TILE_BYTES +
|
|
ftg_face_row_offset(row, half)),
|
|
destination + tile * TILE_BYTES + ftg_face_row_offset(0u, half),
|
|
ftg_face_row_bytes());
|
|
}
|
|
|
|
/** The row select's stream read: DRAM row `gdn_row`. */
|
|
static void conv_read_stream(uint32_t x, uint32_t y, uint32_t address, uint32_t destination,
|
|
uint32_t tiles)
|
|
{
|
|
conv_read_stream_row(x, y, address, destination, tiles, gdn_row);
|
|
}
|
|
|
|
/** A word copy inside L1. L1 is ordinary memory to the RISC-V; both ends are 4-byte aligned
|
|
* by construction, since every offset above is a multiple of the element size. */
|
|
static void ftg_copy_words(uint32_t *destination, const uint32_t *source, uint32_t words)
|
|
{
|
|
uint32_t index;
|
|
|
|
for (index = 0; index < words; index++)
|
|
destination[index] = source[index];
|
|
}
|
|
|
|
/* ===========================================================================
|
|
* PHASE DELTA -- staging
|
|
* =========================================================================== */
|
|
|
|
/** Pull `tiles` consecutive activation tiles into the scratch buffer. ONE transaction. */
|
|
static void stage_activation_tiles(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
uint32_t scratch, uint32_t first_tile, uint32_t tiles)
|
|
{
|
|
if (ctx->aborting)
|
|
return;
|
|
ft_tensix_noc_read(ft_tensix_noc_address(endpoints->activation_x, endpoints->activation_y,
|
|
endpoints->activation_in + first_tile * TILE_BYTES),
|
|
scratch, tiles * TILE_BYTES);
|
|
gdn_read_barrier();
|
|
}
|
|
|
|
/**
|
|
* One ROW-REPLICATED tile into CB_SCALER: [r][i] = source_row[i] for every row r.
|
|
*
|
|
* Serves BOTH the (b)/(e) reduce scaler and the (d) rank-1 update -- see the file header on
|
|
* why that is one tile rather than two.
|
|
*/
|
|
/* M6: build replicated tiles with the NoC instead of 1,024 RISC-V stores (kGdnFlagNocReplicate,
|
|
* set per descriptor by the host). Measured 2026-10-05: the software build is 3.56 ms/token of
|
|
* the DELTA reader (~4.6 us a tile, ~6 cycles a store). */
|
|
static bool gdn_noc_replicate;
|
|
static bool gdn_red_no_copy;
|
|
static uint32_t gdn_self_x, gdn_self_y;
|
|
|
|
/** Row 0 of faces 0 and 1 by RISC-V stores (32 words), then doubling inside each face with
|
|
* local NoC reads (64, 128, 256, 512 B), then faces 2-3 as one copy of faces 0-1. The bytes
|
|
* are the store loop's exactly: faces 0/2 repeat the source half for columns 0-15, faces
|
|
* 1/3 the half for 16-31. */
|
|
static void build_replicated_by_noc(uint32_t destination, uint32_t source, uint32_t source_row)
|
|
{
|
|
const uint32_t face = ftg_face_bytes();
|
|
const uint32_t row = ftg_face_row_bytes();
|
|
const uint32_t words = ftg_face_row_words();
|
|
uint32_t half, span;
|
|
|
|
for (half = 0; half < 2u; half++)
|
|
ftg_copy_words((uint32_t *)(uintptr_t)(destination + half * face),
|
|
(const uint32_t *)(uintptr_t)(source + ftg_face_row_offset(source_row, half)),
|
|
words);
|
|
for (span = row; span < face; span <<= 1) {
|
|
for (half = 0; half < 2u; half++) {
|
|
const uint32_t base = destination + half * face;
|
|
|
|
ft_tensix_noc_read(ft_tensix_noc_address(gdn_self_x, gdn_self_y, base), base + span,
|
|
span);
|
|
}
|
|
gdn_read_barrier();
|
|
}
|
|
if (!gdn_red_no_copy) { /* RED (kGdnFlagRedNoCopy): faces 2-3 stay stale; must DIFFER */
|
|
ft_tensix_noc_read(ft_tensix_noc_address(gdn_self_x, gdn_self_y, destination),
|
|
destination + 2u * face, 2u * face);
|
|
gdn_read_barrier();
|
|
}
|
|
}
|
|
|
|
static bool push_replicated_tile(ftq_context *ctx, uint32_t scratch, uint32_t source_row,
|
|
uint32_t source_tile)
|
|
{
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_SCALER, 1, WAIT_MODE))
|
|
return false;
|
|
|
|
if (!ctx->aborting && gdn_noc_replicate) {
|
|
const uint32_t build_started = ft_bh_wall_clock_low();
|
|
|
|
build_replicated_by_noc(ft_tensix_cb_write_pointer(CB_SCALER),
|
|
scratch + source_tile * TILE_BYTES, source_row);
|
|
gdn_clock_add(kGdnClockBuild, build_started);
|
|
} else if (!ctx->aborting) {
|
|
const uint32_t build_started = ft_bh_wall_clock_low();
|
|
|
|
const uint32_t source = scratch + source_tile * TILE_BYTES;
|
|
const uint32_t destination = ft_tensix_cb_write_pointer(CB_SCALER);
|
|
const uint32_t words = ftg_face_row_words();
|
|
uint32_t face;
|
|
|
|
/* Faces 0 and 2 hold columns 0..15, faces 1 and 3 columns 16..31, so a face takes
|
|
* the source half its column range names and repeats it down all 16 of its rows. */
|
|
for (face = 0; face < 4u; face++) {
|
|
const uint32_t from = source + ftg_face_row_offset(source_row, face & 1u);
|
|
uint32_t *const to =
|
|
(uint32_t *)(uintptr_t)(destination + face * ftg_face_bytes());
|
|
uint32_t line;
|
|
|
|
if ((READER_EXTRAS & (uint32_t)kReaderReplicaByColumn) != 0u) {
|
|
/* COLUMN-MAJOR: one source word is loaded ONCE and stored down all sixteen
|
|
* rows of the face, instead of the whole 64-byte row being re-read for every
|
|
* row. Same 1,024 stores, SIXTEEN loads instead of 1,024.
|
|
*
|
|
* The compiler cannot do this for you and the reason is aliasing: `source`
|
|
* and `destination` are both plain L1 addresses arrived at through
|
|
* `uintptr_t`, so a store through `to` may, as far as GCC knows, have landed
|
|
* on the very word `from` is about to be read from again. It must reload on
|
|
* every row. Hoisting the load into a local is the only way to say it cannot.
|
|
*
|
|
* The stores go out strided by a face row instead of sequentially, which
|
|
* costs nothing: L1 is SRAM with no line to fill and no prefetcher to
|
|
* confuse. */
|
|
const uint32_t stride = ftg_face_row_bytes() / 4u;
|
|
uint32_t word;
|
|
|
|
for (word = 0; word < words; word++) {
|
|
const uint32_t value = *(const uint32_t *)(uintptr_t)(from + word * 4u);
|
|
|
|
for (line = 0; line < (uint32_t)kGdnFaceRows; line++)
|
|
to[line * stride + word] = value;
|
|
}
|
|
continue;
|
|
}
|
|
for (line = 0; line < (uint32_t)kGdnFaceRows; line++)
|
|
ftg_copy_words((uint32_t *)(uintptr_t)(destination + face * ftg_face_bytes() +
|
|
line * ftg_face_row_bytes()),
|
|
(const uint32_t *)(uintptr_t)from, words);
|
|
}
|
|
gdn_clock_add(kGdnClockBuild, build_started);
|
|
}
|
|
ft_tensix_cb_push_back(CB_SCALER, 1);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* One TRUE COLUMN tile into CB_VECTOR: [j][0] = source_row[j], the other lanes untouched.
|
|
*
|
|
* This is a transpose of 32 elements, so it is 32 word copies and not a face-row memcpy.
|
|
*/
|
|
static bool push_column_tile(ftq_context *ctx, uint32_t scratch, uint32_t source_row,
|
|
uint32_t source_tile)
|
|
{
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_VECTOR, 1, WAIT_MODE))
|
|
return false;
|
|
|
|
if (!ctx->aborting) {
|
|
const uint32_t build_started = ft_bh_wall_clock_low();
|
|
|
|
const uint32_t source = scratch + source_tile * TILE_BYTES;
|
|
const uint32_t destination = ft_tensix_cb_write_pointer(CB_VECTOR);
|
|
uint32_t j;
|
|
|
|
for (j = 0; j < (uint32_t)kGdnTileRows; j++)
|
|
*(uint32_t *)(uintptr_t)(destination + ftg_element_offset(j, 0u)) =
|
|
*(const uint32_t *)(uintptr_t)(source + ftg_element_offset(source_row, j));
|
|
gdn_clock_add(kGdnClockBuild, build_started);
|
|
}
|
|
ft_tensix_cb_push_back(CB_VECTOR, 1);
|
|
return true;
|
|
}
|
|
|
|
/** One CONSTANT COLUMN tile into CB_VECTOR: the same fp32 bit pattern in all 32 rows of
|
|
* column 0, so `bcast_col` applies a per-head scalar to a whole tile. */
|
|
static bool push_constant_column(ftq_context *ctx, uint32_t bits)
|
|
{
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_VECTOR, 1, WAIT_MODE))
|
|
return false;
|
|
|
|
if (!ctx->aborting) {
|
|
const uint32_t build_started = ft_bh_wall_clock_low();
|
|
|
|
const uint32_t destination = ft_tensix_cb_write_pointer(CB_VECTOR);
|
|
uint32_t j;
|
|
|
|
for (j = 0; j < (uint32_t)kGdnTileRows; j++)
|
|
*(uint32_t *)(uintptr_t)(destination + ftg_element_offset(j, 0u)) = bits;
|
|
gdn_clock_add(kGdnClockBuild, build_started);
|
|
}
|
|
ft_tensix_cb_push_back(CB_VECTOR, 1);
|
|
return true;
|
|
}
|
|
|
|
/** One tile-row of one head's carried state: `state_tiles` CONSECUTIVE tiles, one read. */
|
|
static bool push_state_row(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, uint32_t head, uint32_t tile_row)
|
|
{
|
|
const uint32_t tiles = shape->state_tiles;
|
|
const uint32_t first = ftg_state_tile(shape, head, tile_row, 0u);
|
|
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_STATE, (int32_t)tiles, WAIT_MODE))
|
|
return false;
|
|
|
|
if (!ctx->aborting) {
|
|
ft_tensix_noc_read(ft_tensix_noc_address(endpoints->state_x, endpoints->state_y,
|
|
endpoints->state_base + first * TILE_BYTES),
|
|
ft_tensix_cb_write_pointer(CB_STATE), tiles * TILE_BYTES);
|
|
gdn_read_barrier();
|
|
}
|
|
ft_tensix_cb_push_back(CB_STATE, tiles);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Everything one v-head needs, in the order the compute thread consumes it.
|
|
*
|
|
* CB_SCALER: `state_tiles` k replicas then `state_tiles` q replicas -- the whole buffer, held
|
|
* for the head's every tile-row, because (b) and (d) want k and (e) wants q at every one.
|
|
* CB_VECTOR: `state_tiles` v columns then decay, beta and scale.
|
|
*/
|
|
static bool read_one_head(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, uint32_t scratch, uint32_t gate,
|
|
uint32_t head)
|
|
{
|
|
const uint32_t tiles = shape->state_tiles;
|
|
const uint32_t k_head = ftg_k_head_for(head, shape->k_heads);
|
|
uint32_t index;
|
|
|
|
/* THE THREE STAGED STREAMS ARE ROW-0 STRIPS, HEAD-MAJOR. A head's S values are
|
|
* `state_tiles` CONSECUTIVE tiles at `ftg_delta_head_tile()` and the row inside each of
|
|
* them is always 0. They used to be one row per head inside a shared row block, which
|
|
* read the SAME `state_tiles` tiles for every head and picked row `head % 32` out of
|
|
* them -- identical read volume, denser DRAM. What the strip buys is that PHASE CONV's
|
|
* output IS this buffer, so nothing crosses PCIe to be re-tilized between the phases. */
|
|
stage_activation_tiles(ctx, endpoints, scratch,
|
|
ftg_delta_head_tile(shape, (uint32_t)kGdnStreamK, k_head), tiles);
|
|
for (index = 0; index < tiles; index++)
|
|
if (!push_replicated_tile(ctx, scratch, gdn_row, index))
|
|
return false;
|
|
|
|
stage_activation_tiles(ctx, endpoints, scratch,
|
|
ftg_delta_head_tile(shape, (uint32_t)kGdnStreamQ, k_head), tiles);
|
|
for (index = 0; index < tiles; index++)
|
|
if (!push_replicated_tile(ctx, scratch, gdn_row, index))
|
|
return false;
|
|
|
|
stage_activation_tiles(ctx, endpoints, scratch,
|
|
ftg_delta_head_tile(shape, (uint32_t)kGdnStreamV, head), tiles);
|
|
for (index = 0; index < tiles; index++)
|
|
if (!push_column_tile(ctx, scratch, gdn_row, index))
|
|
return false;
|
|
|
|
/* decay[h] then beta[h], out of the gate block this descriptor staged once. */
|
|
if (!push_constant_column(ctx, *(const uint32_t *)(uintptr_t)(gate + head * 4u)))
|
|
return false;
|
|
if (!push_constant_column(ctx, *(const uint32_t *)(uintptr_t)(gate +
|
|
(shape->v_heads + head) * 4u)))
|
|
return false;
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* The reader half of one DELTA descriptor.
|
|
*
|
|
* The gate block -- `decay[v_heads]` then `beta[v_heads]`, fp32 -- is read ONCE into the
|
|
* scratch page ABOVE the row-block staging area, because it is 2 KiB at the widest shape this
|
|
* binary accepts and re-reading four bytes a head would be 64 sub-alignment transactions.
|
|
*/
|
|
static bool read_one_delta(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, volatile FTQ_L1_PTR uint32_t *parameters)
|
|
{
|
|
const uint32_t scratch = ft_tensix_cb_write_pointer(CB_STAGE);
|
|
const uint32_t gate = scratch + shape->state_tiles * TILE_BYTES;
|
|
const uint32_t scale_bits = parameters[kGdnScaleBits];
|
|
/* Rounded UP to the NoC's 64-byte read granule: the gate block is 2*v_heads fp32 words,
|
|
* which is only a multiple of 64 B for an even head count, and an under-aligned length is
|
|
* a refused transaction rather than a short one. 2048 B at the widest shape this binary
|
|
* accepts, so it always fits the one scratch page reserved for it. */
|
|
const uint32_t gate_bytes = (2u * shape->v_heads * 4u + 63u) & ~63u;
|
|
uint32_t head;
|
|
|
|
if (!ctx->aborting) {
|
|
ft_tensix_noc_read(ft_tensix_noc_address(endpoints->constant_x, endpoints->constant_y,
|
|
endpoints->constant_base),
|
|
gate, gate_bytes);
|
|
gdn_read_barrier();
|
|
}
|
|
|
|
/* THIS MEMBER'S HEADS, not every head. The gate block above is read WHOLE regardless --
|
|
* it is 2*v_heads fp32 words, 256 B at Ornith, and one 64 B-granule read of the whole
|
|
* thing is cheaper than computing a sliced offset that has to stay 64 B-aligned. */
|
|
for (head = shape->head_first; head < shape->head_first + shape->head_count; head++) {
|
|
uint32_t tile_row;
|
|
|
|
if (!read_one_head(ctx, endpoints, shape, scratch, gate, head))
|
|
return false;
|
|
if (!push_constant_column(ctx, scale_bits))
|
|
return false;
|
|
for (tile_row = 0; tile_row < shape->state_tiles; tile_row++)
|
|
if (!push_state_row(ctx, endpoints, shape, head, tile_row))
|
|
return false;
|
|
}
|
|
return !ctx->aborting;
|
|
}
|
|
|
|
/* ===========================================================================
|
|
* PHASE DELTA -- MULTI-ROW (PM 5)
|
|
* =========================================================================== */
|
|
|
|
/** One head's k, q and v row blocks into scratch at [0, t), [t, 2t), [2t, 3t): three reads,
|
|
* ONE barrier. Every verify row's operands are then built from these 12 tiles. */
|
|
static void stage_head_streams(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, uint32_t scratch, uint32_t head)
|
|
{
|
|
const uint32_t tiles = shape->state_tiles;
|
|
const uint32_t k_head = ftg_k_head_for(head, shape->k_heads);
|
|
const uint32_t first[3] = {
|
|
ftg_delta_head_tile(shape, (uint32_t)kGdnStreamK, k_head),
|
|
ftg_delta_head_tile(shape, (uint32_t)kGdnStreamQ, k_head),
|
|
ftg_delta_head_tile(shape, (uint32_t)kGdnStreamV, head) };
|
|
uint32_t block;
|
|
|
|
if (ctx->aborting)
|
|
return;
|
|
for (block = 0; block < 3u; block++)
|
|
ft_tensix_noc_read(ft_tensix_noc_address(endpoints->activation_x, endpoints->activation_y,
|
|
endpoints->activation_in +
|
|
first[block] * TILE_BYTES),
|
|
scratch + block * tiles * TILE_BYTES, tiles * TILE_BYTES);
|
|
gdn_read_barrier();
|
|
}
|
|
|
|
/** One verify row's operands, in the one-row walk's order: k replicas, q replicas, v columns,
|
|
* decay, beta, scale. `gate` is THIS row's gate block. */
|
|
static bool push_row_operands(ftq_context *ctx, const ftg_shape *shape, uint32_t scratch,
|
|
uint32_t gate, uint32_t head, uint32_t row, uint32_t scale_bits)
|
|
{
|
|
const uint32_t tiles = shape->state_tiles;
|
|
uint32_t index;
|
|
|
|
for (index = 0; index < tiles; index++)
|
|
if (!push_replicated_tile(ctx, scratch, row, index))
|
|
return false;
|
|
for (index = 0; index < tiles; index++)
|
|
if (!push_replicated_tile(ctx, scratch + tiles * TILE_BYTES, row, index))
|
|
return false;
|
|
for (index = 0; index < tiles; index++)
|
|
if (!push_column_tile(ctx, scratch + 2u * tiles * TILE_BYTES, row, index))
|
|
return false;
|
|
return push_constant_column(ctx, *(const uint32_t *)(uintptr_t)(gate + head * 4u)) &&
|
|
push_constant_column(ctx, *(const uint32_t *)(uintptr_t)(gate +
|
|
(shape->v_heads + head) * 4u)) &&
|
|
push_constant_column(ctx, scale_bits);
|
|
}
|
|
|
|
/**
|
|
* The reader half of a MULTI-ROW DELTA descriptor: per head, q/k/v and the state read ONCE,
|
|
* then every row's operands built from them. The state goes to the compute thread for row 0
|
|
* only; later rows decay the compute thread's own carry.
|
|
*/
|
|
static bool read_one_delta_rows(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, volatile FTQ_L1_PTR uint32_t *parameters)
|
|
{
|
|
const uint32_t scratch = ft_tensix_cb_write_pointer(CB_STAGE);
|
|
const uint32_t gate = scratch + 3u * shape->state_tiles * TILE_BYTES;
|
|
const uint32_t stride = ftg_gate_stride(shape->v_heads);
|
|
const uint32_t scale_bits = parameters[kGdnScaleBits];
|
|
const bool red_rows_row0 = (parameters[kGdnFlags] & (uint32_t)kGdnFlagRedRowsRow0) != 0u;
|
|
uint32_t head;
|
|
|
|
if (!ctx->aborting) {
|
|
ft_tensix_noc_read(ft_tensix_noc_address(endpoints->constant_x, endpoints->constant_y,
|
|
endpoints->constant_base),
|
|
gate, shape->row_count * stride);
|
|
gdn_read_barrier();
|
|
}
|
|
for (head = shape->head_first; head < shape->head_first + shape->head_count; head++) {
|
|
uint32_t row, tile_row;
|
|
|
|
stage_head_streams(ctx, endpoints, shape, scratch, head);
|
|
for (row = 0; row < shape->row_count; row++) {
|
|
if (!push_row_operands(ctx, shape, scratch, gate + row * stride, head,
|
|
shape->row_first + (red_rows_row0 ? 0u : row), scale_bits))
|
|
return false;
|
|
if (row != 0u)
|
|
continue;
|
|
for (tile_row = 0; tile_row < shape->state_tiles; tile_row++)
|
|
if (!push_state_row(ctx, endpoints, shape, head, tile_row))
|
|
return false;
|
|
}
|
|
}
|
|
return !ctx->aborting;
|
|
}
|
|
|
|
/* ===========================================================================
|
|
* PHASE CONV -- streaming
|
|
* =========================================================================== */
|
|
|
|
/**
|
|
* Tile columns one CONV block covers.
|
|
*
|
|
* Bounded by three things at once: half of each buffer, so the reader can fill one block
|
|
* while the compute thread drains the other; the PUSH GRID, because a chunk that does not
|
|
* divide the buffer depth walks the write pointer off the limit; and the stream width, so
|
|
* there is no short last block. Zero means "no depth satisfies all three", which is a
|
|
* refusal, not a fallback.
|
|
*/
|
|
static uint32_t conv_block_depth(uint32_t tile_cols, uint32_t taps)
|
|
{
|
|
const uint32_t in_pages = ft_tensix_cb(CB_IN)->fifo_num_pages;
|
|
const uint32_t weight_pages = ft_tensix_cb(CB_VECTOR)->fifo_num_pages;
|
|
const uint32_t window_pages = ft_tensix_cb(CB_STATE)->fifo_num_pages;
|
|
uint32_t depth = CONV_DEPTH;
|
|
|
|
if (taps < 2u || tile_cols == 0u || depth == 0u)
|
|
return 0u;
|
|
/* THE DEPTH ITSELF IS THE SHARED COMPILE-TIME ARG, not a page count, because the COMPUTE
|
|
* kernel has to reach the identical number and cannot read `cb_interface` at all — it
|
|
* does not exist on the MATH thread (link failure, measured 2026-08-18), and rule 3
|
|
* forbids a trip count taken from runtime buffer state regardless. What the page counts
|
|
* are still good for here is a REFUSAL: this processor can see whether the host actually
|
|
* placed buffers deep enough for that depth, and a descriptor that does not fit is
|
|
* refused by name instead of walking the FIFO write pointer off its limit. */
|
|
if (depth > tile_cols)
|
|
depth = tile_cols;
|
|
while (depth > 0u && (tile_cols % depth) != 0u)
|
|
depth--;
|
|
if (depth == 0u)
|
|
return 0u;
|
|
if (in_pages < 2u * depth || weight_pages < 2u * depth * taps ||
|
|
window_pages < 2u * depth * (taps - 1u))
|
|
return 0u;
|
|
if ((in_pages % depth) != 0u || (weight_pages % (depth * taps)) != 0u ||
|
|
(window_pages % depth) != 0u)
|
|
return 0u;
|
|
return depth;
|
|
}
|
|
|
|
/** One window tap for one block: its own reserve/read/barrier/push group -- see the header on
|
|
* why `taps-1` tiles cannot share one reservation. */
|
|
static bool read_window_tap(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, uint32_t depth, uint32_t first_col,
|
|
uint32_t tap)
|
|
{
|
|
const uint32_t first = ftg_conv_plane_tile(shape, tap, first_col);
|
|
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_STATE, (int32_t)depth, WAIT_MODE))
|
|
return false;
|
|
if (!ctx->aborting) {
|
|
conv_read(endpoints->state_x, endpoints->state_y,
|
|
endpoints->state_base + first * TILE_BYTES,
|
|
ft_tensix_cb_write_pointer(CB_STATE), depth);
|
|
gdn_read_barrier();
|
|
}
|
|
ft_tensix_cb_push_back(CB_STATE, depth);
|
|
return true;
|
|
}
|
|
|
|
/** qkv and every weight tap for one block: `1 + taps` reads under ONE barrier. */
|
|
static bool read_conv_operands(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, uint32_t depth, uint32_t first_col)
|
|
{
|
|
const uint32_t taps = shape->conv_taps;
|
|
uint32_t weight_write;
|
|
uint32_t tap;
|
|
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_IN, (int32_t)depth, WAIT_MODE))
|
|
return false;
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_VECTOR, (int32_t)(depth * taps),
|
|
WAIT_MODE))
|
|
return false;
|
|
weight_write = ft_tensix_cb_write_pointer(CB_VECTOR);
|
|
|
|
if (!ctx->aborting) {
|
|
conv_read_stream(endpoints->activation_x, endpoints->activation_y,
|
|
endpoints->activation_in + first_col * TILE_BYTES,
|
|
ft_tensix_cb_write_pointer(CB_IN), depth);
|
|
for (tap = 0; tap < taps; tap++) {
|
|
const uint32_t first = ftg_conv_plane_tile(shape, tap, first_col);
|
|
|
|
conv_read(endpoints->constant_x, endpoints->constant_y,
|
|
endpoints->constant_base + first * TILE_BYTES,
|
|
weight_write + tap * depth * TILE_BYTES, depth);
|
|
}
|
|
gdn_read_barrier();
|
|
}
|
|
ft_tensix_cb_push_back(CB_IN, depth);
|
|
ft_tensix_cb_push_back(CB_VECTOR, depth * taps);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* The reader half of one CONV descriptor: TWO passes over the stream.
|
|
*
|
|
* Pass A feeds the convolution itself. Pass B feeds the ROLLED WINDOW, and it is a second
|
|
* pass rather than an interleave because CB_OUT is a FIFO and the writer's split puts every
|
|
* conv_out tile before every window tile. The rolled window is `window_new[t] =
|
|
* window_old[t+1]` for t < taps-2 and `window_new[taps-2] = qkv`, so pass B re-reads the
|
|
* window one tap higher and finally re-reads qkv.
|
|
*/
|
|
static bool read_one_conv(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, uint32_t depth)
|
|
{
|
|
/* THIS MEMBER'S COLUMNS, at the full plane stride: `ftg_conv_plane_tile()` and the qkv
|
|
* address both take an ABSOLUTE column, so the slice is only the loop bounds here. */
|
|
const uint32_t start = shape->col_first;
|
|
const uint32_t end = shape->col_first + shape->col_count;
|
|
const uint32_t taps = shape->conv_taps;
|
|
uint32_t first_col;
|
|
uint32_t tap;
|
|
|
|
for (first_col = start; first_col < end; first_col += depth) {
|
|
if (!read_conv_operands(ctx, endpoints, shape, depth, first_col))
|
|
return false;
|
|
for (tap = 0; tap + 1u < taps; tap++)
|
|
if (!read_window_tap(ctx, endpoints, shape, depth, first_col, tap))
|
|
return false;
|
|
}
|
|
|
|
for (tap = 0; tap + 1u < taps; tap++) {
|
|
for (first_col = start; first_col < end; first_col += depth) {
|
|
if (tap + 2u < taps) {
|
|
if (!read_window_tap(ctx, endpoints, shape, depth, first_col, tap + 1u))
|
|
return false;
|
|
continue;
|
|
}
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_IN, (int32_t)depth, WAIT_MODE))
|
|
return false;
|
|
if (!ctx->aborting) {
|
|
conv_read_stream(endpoints->activation_x, endpoints->activation_y,
|
|
endpoints->activation_in + first_col * TILE_BYTES,
|
|
ft_tensix_cb_write_pointer(CB_IN), depth);
|
|
gdn_read_barrier();
|
|
}
|
|
ft_tensix_cb_push_back(CB_IN, depth);
|
|
}
|
|
}
|
|
return !ctx->aborting;
|
|
}
|
|
|
|
static bool rebase_produced_buffers(ftq_context *ctx, const ftg_shape *shape);
|
|
|
|
/**
|
|
* MULTI-ROW CONV (PM 6): element `j` of row-sequence [live window planes 0..taps-2, stream rows
|
|
* row_first, row_first+1, ...] into CB_STATE for one block — a window plane, or a stream row
|
|
* read into row 0 (strip mode). Row r's tap t is element t + r: exactly the window the one-row
|
|
* descriptor of row r reads, after rows row_first..r-1 rolled it.
|
|
*/
|
|
static bool read_conv_sequence_tap(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, uint32_t depth, uint32_t first_col,
|
|
uint32_t element)
|
|
{
|
|
if (element + 1u < shape->conv_taps)
|
|
return read_window_tap(ctx, endpoints, shape, depth, first_col, element);
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_STATE, (int32_t)depth, WAIT_MODE))
|
|
return false;
|
|
if (!ctx->aborting) {
|
|
conv_read_stream_row(endpoints->activation_x, endpoints->activation_y,
|
|
endpoints->activation_in + first_col * TILE_BYTES,
|
|
ft_tensix_cb_write_pointer(CB_STATE), depth,
|
|
shape->row_first + element - (shape->conv_taps - 1u));
|
|
gdn_read_barrier();
|
|
}
|
|
ft_tensix_cb_push_back(CB_STATE, depth);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* MULTI-ROW CONV: the one-row reader's two passes once a row, rows in order. Pass A is
|
|
* read_one_conv()'s with the window taps taken from the row sequence; pass B's rolled window
|
|
* is elements r+1 .. r+taps-1 of the same sequence, its newest tap the row's own stream row
|
|
* (through CB_IN, exactly as the one-row pass B re-reads qkv).
|
|
*/
|
|
static bool read_one_conv_rows(ftq_context *ctx, const gdn_endpoints *endpoints,
|
|
const ftg_shape *shape, uint32_t depth)
|
|
{
|
|
const uint32_t start = shape->col_first;
|
|
const uint32_t end = shape->col_first + shape->col_count;
|
|
const uint32_t taps = shape->conv_taps;
|
|
uint32_t row, first_col, tap;
|
|
|
|
for (row = 0; row < shape->row_count; row++) {
|
|
/* EVERY ROW STARTS ON THE BUFFERS' BASES, as every descriptor does: pass B leaves
|
|
* CB_STATE off the `depth*(taps-1)` grid pass A indexes (2 taps x the columns), and
|
|
* the next row's window would then be read across the wrap. The compute thread
|
|
* rebases its read pointers at the same row boundary. */
|
|
if (row > 0u && !rebase_produced_buffers(ctx, shape))
|
|
return false;
|
|
gdn_row = shape->row_first + (gdn_red_rows_row0 ? 0u : row);
|
|
for (first_col = start; first_col < end; first_col += depth) {
|
|
if (!read_conv_operands(ctx, endpoints, shape, depth, first_col))
|
|
return false;
|
|
for (tap = 0; tap + 1u < taps; tap++)
|
|
if (!read_conv_sequence_tap(ctx, endpoints, shape, depth, first_col, tap + row))
|
|
return false;
|
|
}
|
|
for (tap = 0; tap + 1u < taps; tap++) {
|
|
for (first_col = start; first_col < end; first_col += depth) {
|
|
if (tap + 2u < taps) {
|
|
if (!read_conv_sequence_tap(ctx, endpoints, shape, depth, first_col,
|
|
tap + 1u + row))
|
|
return false;
|
|
continue;
|
|
}
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_IN, (int32_t)depth,
|
|
WAIT_MODE))
|
|
return false;
|
|
if (!ctx->aborting) {
|
|
conv_read_stream(endpoints->activation_x, endpoints->activation_y,
|
|
endpoints->activation_in + first_col * TILE_BYTES,
|
|
ft_tensix_cb_write_pointer(CB_IN), depth);
|
|
gdn_read_barrier();
|
|
}
|
|
ft_tensix_cb_push_back(CB_IN, depth);
|
|
}
|
|
}
|
|
}
|
|
gdn_row = shape->row_first;
|
|
return !ctx->aborting;
|
|
}
|
|
|
|
/* ===========================================================================
|
|
* Dispatch
|
|
* =========================================================================== */
|
|
|
|
/**
|
|
* Refuse a descriptor the circular buffers this residency was STAGED with cannot serve.
|
|
*
|
|
* The page counts are fixed when the program is staged; the shape is a per-descriptor word,
|
|
* so the host CAN ask for a block bigger than the buffer -- it did, on another op's first
|
|
* hardware run, and the result was ten seconds of blocked polling reported as an idle
|
|
* timeout. Every chunk this reader pushes is checked against the push grid as well as
|
|
* against capacity, because the FIFO write pointer wraps on an EXACT hit of the limit and
|
|
* nothing else.
|
|
*/
|
|
/**
|
|
* Does `cb_index` hold a whole number of `chunk_pages` blocks?
|
|
*
|
|
* THE GRID A CONSUMER INDEXES ACROSS, which is a DIFFERENT rule from the push grid and the
|
|
* one this family did not have. `ft_tensix_unpack_ab(cb, ..., tile)` resolves to
|
|
* `read_pointer + tile * page_size` with no wrap, so a compute thread that waits for `n`
|
|
* pages and then names tiles `0..n-1` needs the front low enough that the whole window fits.
|
|
* The front is only ever a multiple of the chunk if the chunk divides the buffer — and the
|
|
* three kernels rebase to the base at every descriptor so the multiple starts at zero.
|
|
*
|
|
* The pointer clause of `ft_tensix_cb_push_grid_is_safe()` is deliberately not reused here:
|
|
* it tests the PRODUCER's pointer, and this is a statement about the CONSUMER's.
|
|
*/
|
|
static bool index_grid_is_safe(uint32_t cb_index, uint32_t chunk_pages)
|
|
{
|
|
const uint32_t pages = ft_tensix_cb(cb_index)->fifo_num_pages;
|
|
|
|
return chunk_pages != 0u && pages >= chunk_pages && (pages % chunk_pages) == 0u;
|
|
}
|
|
|
|
static bool delta_is_executable(const ftg_shape *shape)
|
|
{
|
|
const uint32_t tiles = shape->state_tiles;
|
|
|
|
if (ft_tensix_cb(CB_SCALER)->fifo_num_pages < 2u * tiles)
|
|
return false;
|
|
if (ft_tensix_cb(CB_VECTOR)->fifo_num_pages < tiles + (uint32_t)kGdnVectorConstants)
|
|
return false;
|
|
/* `state_tiles` staging tiles for a row block, plus ONE page above them for the gate
|
|
* block this descriptor reads once. */
|
|
if (ft_tensix_cb(CB_STAGE)->fifo_num_pages < tiles + 1u)
|
|
return false;
|
|
/* CB_WORK holds the decayed tile-row AND the updated one: (b) indexes off the first and
|
|
* (e) off the second, so the front cycles 0 -> tiles -> 0 inside a tile-row and the
|
|
* buffer needs both tile-rows live. */
|
|
if (ft_tensix_cb(CB_WORK)->fifo_num_pages < 2u * tiles)
|
|
return false;
|
|
if (!ft_tensix_cb_push_grid_is_safe(CB_SCALER, 1u) ||
|
|
!ft_tensix_cb_push_grid_is_safe(CB_VECTOR, 1u) ||
|
|
!ft_tensix_cb_push_grid_is_safe(CB_STATE, tiles))
|
|
return false;
|
|
/* What the compute thread INDEXES ACROSS in this phase: a whole state tile-row out of
|
|
* CB_WORK and CB_STATE, this head's k AND q replicas out of CB_SCALER, and its v columns
|
|
* plus decay/beta/scale out of CB_VECTOR. CB_IN is NOT named: this phase does not touch
|
|
* it on either processor. */
|
|
if (!index_grid_is_safe(CB_WORK, tiles) || !index_grid_is_safe(CB_STATE, tiles) ||
|
|
!index_grid_is_safe(CB_SCALER, 2u * tiles) ||
|
|
!index_grid_is_safe(CB_VECTOR, tiles + (uint32_t)kGdnVectorConstants))
|
|
return false;
|
|
/* THE BATCHED (d)'s OWN GRIDS. It stages `batch` rank-1 products in CB_PROD and then
|
|
* indexes all of them off its front, and it rewrites CB_WORK a whole block at a time --
|
|
* so both need `batch` to divide them, on top of the tile-row grid above. `batch` divides
|
|
* `tiles` by construction (`ftg_dst_batch`), so CB_WORK's clause is implied by the one
|
|
* above it and CB_PROD's is not: nothing else in this function names CB_PROD, and before
|
|
* the batch it had no grid at all because every access to it named tile 0. */
|
|
{
|
|
const uint32_t batch = ftg_dst_batch(tiles, DST_BATCH_MAX);
|
|
|
|
if (ft_tensix_cb(CB_PROD)->fifo_num_pages < 2u * batch)
|
|
return false;
|
|
if (!index_grid_is_safe(CB_PROD, batch))
|
|
return false;
|
|
if (!index_grid_is_safe(CB_WORK, batch))
|
|
return false;
|
|
}
|
|
/* MULTI-ROW: three staged row blocks plus the gate page, every row's gate block inside
|
|
* that one page, and the compute thread's carry able to hold a head's whole state on its
|
|
* tile-row grid. The carry is the compute thread's, but only this processor can see a
|
|
* page count. */
|
|
if (shape->row_count > 1u &&
|
|
(ft_tensix_cb(CB_STAGE)->fifo_num_pages < 3u * tiles + 1u ||
|
|
shape->row_count * ftg_gate_stride(shape->v_heads) > TILE_BYTES ||
|
|
ft_tensix_cb((uint32_t)kGdnCbCarry)->fifo_num_pages < tiles * tiles ||
|
|
!index_grid_is_safe((uint32_t)kGdnCbCarry, tiles)))
|
|
return false;
|
|
return shape->tile_count >= shape->head_count * ftg_state_tiles_per_head(shape);
|
|
}
|
|
|
|
static bool conv_is_executable(const ftg_shape *shape, uint32_t depth)
|
|
{
|
|
if (depth == 0u)
|
|
return false;
|
|
/* THE THIRD SIDE OF THE THREE-SIDED CONTRACT. Every check below asks whether the buffers
|
|
* the HOST placed can SERVE this image's depth, and a host that placed for a LARGER depth
|
|
* passes all of them -- so the drift that silently costs a depth sweep its number is
|
|
* invisible to capacity. This clause compares the two numbers themselves. See
|
|
* kGdnByteConvDepth in resident_gdn_protocol.h. */
|
|
if (!ftg_conv_depth_agrees(shape->declared_conv_depth, (uint32_t)CONV_DEPTH))
|
|
return false;
|
|
/* THE PUSH GRID IS CHECKED AFTER THE REBASE (conv_push_grid_is_safe(), read_one_gdn()),
|
|
* not here. It reads the CURRENT write pointers, and `rebase_produced_buffers()` is about
|
|
* to put every one of them back on its base — so checking first refused a CONV after a
|
|
* DELTA that left CB_VECTOR off the CONV grid, a pointer the rebase was about to fix.
|
|
* MEASURED 2026-10-04: a DELTA over 8 members pushes (32/8)*7 = 28 pages, off the 8-page
|
|
* CONV grid, and every CONV refused and replayed (1,151 of 1,152). */
|
|
/* `conv_one_column` names EVERY tap of a column, so the whole block's window and weights
|
|
* are one indexed window even though the window is pushed a tap at a time. */
|
|
if (!index_grid_is_safe(CB_IN, depth) ||
|
|
!index_grid_is_safe(CB_STATE, depth * (shape->conv_taps - 1u)) ||
|
|
!index_grid_is_safe(CB_VECTOR, depth * shape->conv_taps))
|
|
return false;
|
|
return shape->tile_count >= shape->conv_tile_cols * (shape->conv_taps - 1u);
|
|
}
|
|
|
|
/**
|
|
* Put this processor's write pointers back on their buffers' bases, for the buffers it
|
|
* produces into IN THIS PHASE.
|
|
*
|
|
* The whole-buffer reservation is the precondition `ft_tensix_cb_rebase_write_pointer()`
|
|
* states: it does not move a pointer, it waits until nothing live is behind it. Only the
|
|
* buffers this phase writes are named — reserving a buffer this phase does not touch would
|
|
* block on a consumer that has no reason to drain it.
|
|
*
|
|
* @return false if a wait was abandoned, in which case the descriptor is not retired
|
|
*/
|
|
static bool rebase_produced_buffers(ftq_context *ctx, const ftg_shape *shape)
|
|
{
|
|
const uint32_t delta[3] = { CB_STATE, CB_SCALER, CB_VECTOR };
|
|
const uint32_t conv[3] = { CB_IN, CB_STATE, CB_VECTOR };
|
|
const uint32_t *buffers = (shape->phase == (uint32_t)kGdnPhaseDelta) ? delta : conv;
|
|
uint32_t index;
|
|
|
|
for (index = 0; index < 3u; index++) {
|
|
const uint32_t cb = buffers[index];
|
|
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)cb,
|
|
(int32_t)ft_tensix_cb(cb)->fifo_num_pages,
|
|
WAIT_MODE))
|
|
return false;
|
|
ft_tensix_cb_rebase_write_pointer(cb);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static bool descriptor_is_executable(ftq_context *ctx, const ftg_shape *shape, uint32_t depth)
|
|
{
|
|
bool executable;
|
|
|
|
if (TILE_BYTES == 0u || shape->tile_bytes != TILE_BYTES)
|
|
executable = false;
|
|
else if (shape->phase == (uint32_t)kGdnPhaseDelta)
|
|
executable = delta_is_executable(shape);
|
|
else
|
|
executable = conv_is_executable(shape, depth);
|
|
|
|
if (!executable)
|
|
ctx->error = kErrBadParameters;
|
|
return executable;
|
|
}
|
|
|
|
/**
|
|
* The reader half of one GDN descriptor.
|
|
*
|
|
* @return true if the whole descriptor was read; false if a wait was abandoned, in which case
|
|
* the caller must not retire it (rule 5). A failed reservation returns here rather
|
|
* than pushing anyway, exactly as resident_rope_reader.c and
|
|
* resident_attention_reader.c do: a push with no satisfied reservation advances a
|
|
* page counter past the buffer's capacity, which is corruption rather than a drain.
|
|
*/
|
|
static bool read_one_gdn(ftq_context *ctx, volatile FTQ_L1_PTR uint32_t *parameters)
|
|
{
|
|
const ftg_shape shape = ftg_decode(parameters);
|
|
gdn_endpoints endpoints;
|
|
uint32_t depth = 0u;
|
|
|
|
if (shape.phase == (uint32_t)kGdnPhaseConv)
|
|
depth = conv_block_depth(shape.col_count, shape.conv_taps);
|
|
if (!ftg_shape_is_supported(&shape) || !descriptor_is_executable(ctx, &shape, depth)) {
|
|
ftq_abort(ctx, kErrBadParameters);
|
|
return false;
|
|
}
|
|
|
|
load_endpoints(&endpoints, &shape, parameters);
|
|
if (!rebase_produced_buffers(ctx, &shape))
|
|
return false;
|
|
gdn_noc_replicate = (parameters[kGdnFlags] & (uint32_t)kGdnFlagNocReplicate) != 0u;
|
|
gdn_red_no_copy = (parameters[kGdnFlags] & (uint32_t)kGdnFlagRedNoCopy) != 0u;
|
|
gdn_conv_strip = (parameters[kGdnFlags] & (uint32_t)kGdnFlagConvStrip) != 0u;
|
|
gdn_red_strip_half = (parameters[kGdnFlags] & (uint32_t)kGdnFlagRedStripHalf) != 0u;
|
|
gdn_row = (parameters[kGdnFlags] >> kGdnRowShift) & (uint32_t)kGdnRowMask;
|
|
gdn_red_rows_row0 = (parameters[kGdnFlags] & (uint32_t)kGdnFlagRedRowsRow0) != 0u;
|
|
/* The CONV row select — and so the multi-row CONV — exists in strip mode only (a
|
|
* whole-tile read has no row). */
|
|
if ((gdn_row != 0u || shape.row_count > 1u) && shape.phase == (uint32_t)kGdnPhaseConv &&
|
|
!gdn_conv_strip) {
|
|
ftq_abort(ctx, kErrBadParameters);
|
|
return false;
|
|
}
|
|
if (shape.phase == (uint32_t)kGdnPhaseDelta) {
|
|
const uint32_t started = ft_bh_wall_clock_low();
|
|
const bool done = shape.row_count > 1u
|
|
? read_one_delta_rows(ctx, &endpoints, &shape, parameters)
|
|
: read_one_delta(ctx, &endpoints, &shape, parameters);
|
|
|
|
gdn_clock_add(kGdnClockDelta, started);
|
|
return done;
|
|
}
|
|
/* THE PUSH GRID, on the REBASED pointers — where it is a property of the placement, which
|
|
* is the thing it exists to check, instead of of the previous descriptor. */
|
|
if (!ft_tensix_cb_push_grid_is_safe(CB_IN, depth) ||
|
|
!ft_tensix_cb_push_grid_is_safe(CB_VECTOR, depth * shape.conv_taps) ||
|
|
!ft_tensix_cb_push_grid_is_safe(CB_STATE, depth)) {
|
|
ftq_abort(ctx, kErrBadParameters);
|
|
return false;
|
|
}
|
|
{
|
|
const uint32_t started = ft_bh_wall_clock_low();
|
|
bool done;
|
|
|
|
gdn_conv_phase = true;
|
|
done = shape.row_count > 1u ? read_one_conv_rows(ctx, &endpoints, &shape, depth)
|
|
: read_one_conv(ctx, &endpoints, &shape, depth);
|
|
gdn_conv_phase = false;
|
|
gdn_clock_add(kGdnClockConv, started);
|
|
return done;
|
|
}
|
|
}
|
|
|
|
void kernel_main(void)
|
|
{
|
|
ftq_context ctx;
|
|
bool running = true;
|
|
|
|
ftq_context_init(&ctx);
|
|
ftq_status_set(&ctx, kStateBoot);
|
|
ctx.control[kFieldState] = kStateBoot;
|
|
|
|
if (!ftq_check_control(&ctx)) {
|
|
ctx.control[kFieldError] = ctx.error;
|
|
ctx.control[kFieldState] = kStateExit;
|
|
ftq_status_set(&ctx, kStateExit);
|
|
return;
|
|
}
|
|
|
|
// RESUME rather than restart. All five processors resume from completed_count, the ONE
|
|
// number that means "every stage finished this descriptor" — resident_matmul_reader.c
|
|
// carries the full argument and the refusal that enforces it.
|
|
if (ctx.control[kFieldTail] != ctx.control[kFieldCompletedCount]) {
|
|
ctx.error = kErrBadParameters;
|
|
ctx.control[kFieldError] = ctx.error;
|
|
ctx.control[kFieldState] = kStateExit;
|
|
ftq_status_set(&ctx, kStateExit);
|
|
return;
|
|
}
|
|
ctx.cursor = ctx.control[kFieldCompletedCount];
|
|
gdn_clock_control = ctx.control;
|
|
{
|
|
/* This core's own NoC coordinate, for the local copies (the stock firmware derives it
|
|
* from NOC_NODE_ID). */
|
|
const uint32_t id = ft_bh_noc_cmd_read(NOC_INDEX, 0u,
|
|
FT_BH_NOC_REG_BASE + 0x44u);
|
|
|
|
gdn_self_x = id & FT_BH_NOC_NODE_ID_MASK;
|
|
gdn_self_y = (id >> FT_BH_NOC_NODE_ID_BITS) & FT_BH_NOC_NODE_ID_MASK;
|
|
}
|
|
ctx.control[kGdnClockBarrier] = 0u;
|
|
ctx.control[kGdnClockBuild] = 0u;
|
|
ctx.control[kGdnClockConv] = 0u;
|
|
ctx.control[kGdnClockConvBarrier] = 0u;
|
|
ctx.control[kGdnClockDelta] = 0u;
|
|
|
|
while (running) {
|
|
if (!ftq_wait_for_work(&ctx, kFieldHead))
|
|
break;
|
|
|
|
ctx.aborting = false;
|
|
ftq_status_set(&ctx, kStateExec);
|
|
|
|
volatile FTQ_L1_PTR uint32_t *descriptor = ftq_descriptor(&ctx);
|
|
const uint32_t opcode = descriptor[kDescOpcode];
|
|
bool done;
|
|
|
|
ctx.control[kFieldLastOpcode] = opcode;
|
|
ftq_status_opcode(&ctx, opcode);
|
|
|
|
if (opcode == kOpGdn) {
|
|
ftq_status_set(&ctx, kStateCbReserveBack);
|
|
done = read_one_gdn(&ctx, ftq_parameters(descriptor));
|
|
ftq_status_set(&ctx, kStateExec);
|
|
if (!done)
|
|
running = false;
|
|
} else if (opcode == kOpNop) {
|
|
done = true;
|
|
} else if (opcode == kOpStop) {
|
|
done = true;
|
|
running = false;
|
|
} else {
|
|
// A binary is bound to one slab, so an opcode another program implements is
|
|
// genuinely not available here. Refusing names that; executing nothing would
|
|
// retire work that never happened.
|
|
ctx.error = kErrBadOpcode;
|
|
done = false;
|
|
running = false;
|
|
}
|
|
|
|
// Rule 5. An abandoned descriptor was handed to nobody, so it advances NOTHING and
|
|
// the next launch re-attempts it.
|
|
if (done) {
|
|
ctx.cursor++;
|
|
ctx.control[kFieldTail] = ctx.cursor;
|
|
}
|
|
}
|
|
|
|
// Nothing is outstanding here: the drain path issues no transactions. The barrier is
|
|
// the cheap statement of that invariant rather than a fix for a case that can happen.
|
|
gdn_read_barrier();
|
|
ft_tensix_noc_write_barrier();
|
|
ctx.control[kFieldError] = ctx.error;
|
|
ctx.control[kFieldState] = kStateExit;
|
|
ftq_status_set(&ctx, kStateExit);
|
|
}
|