968 lines
50 KiB
C
968 lines
50 KiB
C
/* SPDX-License-Identifier: AGPL-3.0-or-later */
|
|
/**
|
|
* @file resident_matmul_reader.c
|
|
* @brief Reader third of the RESIDENT matmul triad -- the first Foundry kernel that is both
|
|
* resident (it does not return between ops) and part of a reader/compute/writer triad
|
|
* with real circular buffers. Phase 5's resident kernel was BRISC-only and CB-free
|
|
* because `cb_wait_front` was the one wait the host could not break; this kernel and its
|
|
* two peers are what closes that (resident_queue_protocol.h, rules 2-4).
|
|
*
|
|
* IT IS matmul_reader.c'S INNER LOOP INSIDE A QUEUE LOOP, PLUS A BLOCKED `nt`. The
|
|
* one-transaction A block, the N-major B block and the depth-divides-Kt contract are all
|
|
* exactly as that file has them and its header documents them; `cb_reserve_back` became
|
|
* ftq_cb_reserve_back_stoppable(, WAIT_MODE) and the shapes arrive in a parameter block
|
|
* instead of a runtime-arg block.
|
|
*
|
|
* THE ONE ARITHMETIC CHANGE IS `kMatmulColumnBlock`. At Mt = 1 the A offset is
|
|
* `(mt*Kt + kt)` and does not depend on `nt`, so the unblocked loop re-reads the whole A
|
|
* column ONCE PER OUTPUT COLUMN — measured at 15.871 of the 24.303 GB a token reads, 65.3%,
|
|
* nearly all of it out of the ONE bank the activation strip lives in. With NB columns per
|
|
* pass it is fetched Nt/NB times. B's bytes and B's transaction size are untouched: the
|
|
* panel is N-major, so each column is still one read of `depth` contiguous tiles, and the
|
|
* block lands COLUMN-MAJOR in L1 (column j at tile `j*depth`) which is what the compute
|
|
* thread's SrcA stride is set for. NB = 0 or 1 is the unblocked shape, byte for byte.
|
|
*
|
|
* AND ITS MIRROR IS THE ROW BLOCK, IN THE HIGH HALF OF THE SAME WORD. `MB` A tile-rows are
|
|
* held in L1 and B is streamed past ALL of them, so the WEIGHT column is fetched
|
|
* `ceil(Mt/MB)` times instead of `Mt`. The two are exactly symmetric —
|
|
* `total = Mt*Nt*Kt*(1/NB + 1/MB)` — and they SHARE the DST section, which is four tiles:
|
|
* `MB * NB <= 4`, and the minimum of that sum is at MB = NB = 2, not at either extreme.
|
|
* resident_matmul_block_protocol.h carries the derivation and the upstream citation. At
|
|
* MB = 1 every loop below steps exactly as it did before the field existed.
|
|
*
|
|
* THIS PROCESSOR OWNS `tail`, AND NOTHING ELSE IN THE CONTROL BLOCK. `tail` advances
|
|
* when the reader has finished its half of a descriptor and handed it to the writer. An
|
|
* ABORTED descriptor advances nothing at all -- it was handed to nobody, and the next
|
|
* launch re-attempts it.
|
|
*
|
|
*
|
|
* C99 on Foundry's own `ft_tensix_dataflow.h` — no tt-metal include (plan Phase 6.6).
|
|
* The wait mode, which used to be a `template <uint32_t kMode>` argument, is now an
|
|
* ordinary parameter carrying the same compile-time value; `-Os` folds it identically.
|
|
* Compile-time args:
|
|
* 0: CB_IN0 index 1: CB_IN1 index
|
|
* 2: in0 tile size in bytes 3: in1 tile size in bytes
|
|
* 4: wait mode -- 0 upstream (the cost arm), 1 stoppable (ships), 2 bound-only (the
|
|
* red proof for the ask). resident_queue_protocol.h says why all three exist
|
|
* Runtime args: the ten of resident_queue_protocol.h, identical on all five processors,
|
|
* plus the COOPERATIVE tail of resident_matmul_multicast_protocol.h (args 10..22), which
|
|
* ONLY this processor reads. `ftq_context_init()` is untouched, so the other four
|
|
* resident families' images are unaffected by that tail existing.
|
|
*
|
|
* AND IT IS ALSO THE COOPERATIVE READER. The shared A block is 65% of the bytes a token
|
|
* reads, and every member of a split group reads it from the ONE bank holding that strip --
|
|
* which is why four cores buy 3.44x and sixteen buy only 4.80x. With `role != SOLO` the
|
|
* leader reads A once and MULTICASTS it as ceil(receivers / 4) packets; receivers read only
|
|
* their own B panel. The mechanism is `cooperative_matmul_reader.c`'s, moved off a ONE-SHOT
|
|
* program whose per-op image upload cost 218 us per member per op -- 147 ms a token at N=4
|
|
* -- and onto the resident queue, where it costs a descriptor.
|
|
*
|
|
* Build (BRISC, NoC 0):
|
|
* CTARGS=0,1,2048,2048,1 NOC=0 KERNEL_TEXT_BASE=0x15000 bash \
|
|
* foundry/tt/build_tensix_dataflow_kernel.sh \
|
|
* foundry/tt/kernels/resident_matmul_reader.c brisc /tmp/ttk_rtriad
|
|
*
|
|
* @status See docs/tt_bringup_log.md. The cooperative path is new; the solo path is
|
|
* unchanged and is what a member_count == 1 residency runs.
|
|
* @issues A cooperative descriptor arriving at a SOLO member -- or a solo one arriving at a
|
|
* cooperating member -- is REFUSED, and a refused descriptor strands the triad until the
|
|
* host re-arms it. That is the cheap outcome against a group whose CB_IN0 write pointers
|
|
* have drifted apart, which is silent and produces a wrong answer with every kernel
|
|
* completing and every status word clean.
|
|
* @todo None
|
|
*/
|
|
|
|
#include <stdbool.h>
|
|
#include <stdint.h>
|
|
|
|
#include "ft_tensix_dataflow.h"
|
|
#include "ft_tensix_noc_multicast.h"
|
|
#include "resident_matmul_block_protocol.h"
|
|
#include "resident_matmul_multicast_protocol.h"
|
|
#include "resident_queue_protocol.h"
|
|
|
|
#define CB_IN0 ft_tensix_compile_time_arg(0)
|
|
#define CB_IN1 ft_tensix_compile_time_arg(1)
|
|
#define IN0_TILE_BYTES ft_tensix_compile_time_arg(2)
|
|
#define IN1_TILE_BYTES ft_tensix_compile_time_arg(3)
|
|
#define WAIT_MODE ft_tensix_compile_time_arg(4)
|
|
|
|
/* =========================================================================
|
|
* The COOPERATIVE tail: the shared A block, multicast once instead of read N times.
|
|
*
|
|
* All of it is gated on `role != SOLO`, which is a runtime arg fixed for the residency,
|
|
* so a member_count == 1 residency executes the same instruction stream it always has
|
|
* apart from one predictable branch per block. resident_matmul_multicast_protocol.h is
|
|
* the contract and carries the reasoning; what is here is the loop body.
|
|
* ========================================================================= */
|
|
|
|
/** Everything the cooperative roles share, read once at launch so a block indexes a
|
|
* struct instead of re-reading L1 through the runtime-arg accessor. `counter` is the
|
|
* member's own count of cooperative blocks and is what the rendezvous agrees on. */
|
|
typedef struct coop_context {
|
|
uint32_t role;
|
|
uint32_t leader_x;
|
|
uint32_t leader_y;
|
|
uint32_t ready_semaphore_address;
|
|
uint32_t arrival_semaphore_address;
|
|
uint32_t counter_word_address;
|
|
uint32_t receiver_count;
|
|
uint32_t group_count;
|
|
uint32_t group[FT_TT_RESIDENT_COOP_MAX_GROUPS];
|
|
uint32_t spin_limit;
|
|
uint32_t counter;
|
|
/** The queue's own stop word, reached as a plain pointer because that is what the
|
|
* multicast waits take. It is the SAME L1 word `ftq_stop_requested()` reads — rule 4
|
|
* is that every wait here is askable, and a rendezvous wait is no exception. */
|
|
volatile const uint32_t *stop_flag;
|
|
} coop_context;
|
|
|
|
static void coop_read_arguments(coop_context *coop)
|
|
{
|
|
uint32_t index;
|
|
|
|
coop->stop_flag = (volatile const uint32_t *)(uintptr_t)(FTQ_RUNTIME_ARG(kArgControl) +
|
|
(uint32_t)kFieldStopRequest * 4u);
|
|
coop->role = FTQ_RUNTIME_ARG(kArgCoopRole);
|
|
coop->leader_x = FTQ_RUNTIME_ARG(kArgCoopLeaderX);
|
|
coop->leader_y = FTQ_RUNTIME_ARG(kArgCoopLeaderY);
|
|
coop->ready_semaphore_address = FTQ_RUNTIME_ARG(kArgCoopReadySemaphore);
|
|
coop->arrival_semaphore_address = FTQ_RUNTIME_ARG(kArgCoopArrivalSemaphore);
|
|
coop->counter_word_address = FTQ_RUNTIME_ARG(kArgCoopCounterWord);
|
|
coop->receiver_count = FTQ_RUNTIME_ARG(kArgCoopReceiverCount);
|
|
coop->group_count = FTQ_RUNTIME_ARG(kArgCoopGroupCount);
|
|
for (index = 0u; index < FT_TT_RESIDENT_COOP_MAX_GROUPS; index++)
|
|
coop->group[index] = FTQ_RUNTIME_ARG((int)(kArgCoopGroup0 + index));
|
|
coop->spin_limit = FTQ_RUNTIME_ARG(kArgCoopSpinLimit);
|
|
coop->counter = FT_TT_RESIDENT_COOP_FIRST_COUNTER;
|
|
}
|
|
|
|
/**
|
|
* Refuse a cooperative CONFIGURATION at launch, before any descriptor runs.
|
|
*
|
|
* Checked here rather than trusted from the host for the reason DEV_NOTES.md §1 gives for
|
|
* every other kernel-side guard: `ft_tt_noc_check_transfer()` runs host-side and cannot
|
|
* see a kernel-issued transfer, and a multicast at a wrong coordinate is not a rejected
|
|
* transaction — it is a write into a core another residency owns, or a hung NoC.
|
|
*/
|
|
static bool coop_configuration_is_legal(const coop_context *coop)
|
|
{
|
|
uint32_t index;
|
|
uint32_t destinations = 0u;
|
|
|
|
if (coop->role == FT_TT_RESIDENT_COOP_ROLE_SOLO)
|
|
return true;
|
|
if (coop->role != FT_TT_RESIDENT_COOP_ROLE_LEADER &&
|
|
coop->role != FT_TT_RESIDENT_COOP_ROLE_RECEIVER)
|
|
return false;
|
|
if (coop->ready_semaphore_address == 0u || coop->arrival_semaphore_address == 0u ||
|
|
coop->counter_word_address == 0u || coop->spin_limit == 0u)
|
|
return false;
|
|
if (coop->receiver_count == 0u)
|
|
/* The zero-length NoC transfer in multicast clothing. A group of one has no
|
|
* receivers, and refusing is the only answer: "optimise it into a solo run" turns
|
|
* a mis-planned group into a wrong answer that looks like a right one. */
|
|
return false;
|
|
if (coop->role != FT_TT_RESIDENT_COOP_ROLE_LEADER)
|
|
return true;
|
|
if (coop->group_count == 0u || coop->group_count > FT_TT_RESIDENT_COOP_MAX_GROUPS)
|
|
return false;
|
|
for (index = 0u; index < coop->group_count; index++) {
|
|
const uint32_t packed = coop->group[index];
|
|
const uint32_t count = ftcoop_rectangle_destinations(packed);
|
|
|
|
if (count == 0u || count > FT_TT_RESIDENT_COOP_MAX_DESTINATIONS)
|
|
return false;
|
|
if (ftcoop_rectangle_x_low(packed) > ftcoop_rectangle_x_high(packed) ||
|
|
ftcoop_rectangle_y_low(packed) > ftcoop_rectangle_y_high(packed))
|
|
return false;
|
|
destinations += count;
|
|
}
|
|
/* The barrier's acknowledgement accounting is this sum. Too many and the write
|
|
* barrier never returns; too few and it returns while a payload is still landing in
|
|
* L1 the kernel is about to reuse. */
|
|
return destinations == coop->receiver_count;
|
|
}
|
|
|
|
/**
|
|
* Where block `counter` of this residency MUST land in CB_IN0, from the wrap rule alone.
|
|
*
|
|
* A multicast writes ONE address into every destination, so the A block has to land at the
|
|
* same L1 address on all N members. It does, because every member is given the same CB
|
|
* placement and pushes the same pages in the same order — but "does" is a claim about host
|
|
* code that no hardware would refuse if it stopped being true. The pointer resets by
|
|
* subtraction on an EXACT hit of the limit and no other wrap is legal
|
|
* (ft_tensix_circular_buffer.h), so the position is a pure function of how many blocks
|
|
* this member has pushed. Everything is in the firmware's own pointer unit, whatever that
|
|
* is: only the ratio is used.
|
|
*/
|
|
static uint32_t coop_expected_write_pointer(uint32_t cb_index, uint32_t chunk_pages,
|
|
uint32_t counter)
|
|
{
|
|
const ft_tensix_cb_interface *interface = ft_tensix_cb(cb_index);
|
|
const uint32_t block = chunk_pages * interface->fifo_page_size;
|
|
const uint32_t base = interface->fifo_limit - interface->fifo_size;
|
|
|
|
if (block == 0u || interface->fifo_size < block)
|
|
return 0u;
|
|
return base + ((counter - FT_TT_RESIDENT_COOP_FIRST_COUNTER) % (interface->fifo_size / block)) *
|
|
block;
|
|
}
|
|
|
|
/**
|
|
* LEADER: collect every receiver, fetch A, then hand it to the group as `group_count`
|
|
* packets of at most four destinations each.
|
|
*
|
|
* The order of the last four steps is the protocol and none of it is rearrangeable —
|
|
* ft_tensix_noc_multicast.h's header has the full account, including why the flush between
|
|
* the payload and the semaphore is mandatory on Blackhole and nowhere else. ONE flush
|
|
* covers every piece: it waits for requests SENT, and once every payload request has left,
|
|
* ordering behind it on the shared static VC is the network's job.
|
|
*
|
|
* The payload's SOURCE is this core's own CB_IN0 block, so there is no staging buffer.
|
|
* Reusing that block next round is safe for exactly one reason: the flush guarantees the
|
|
* request has LEFT, which is what frees the source.
|
|
*/
|
|
static void coop_leader_block(ftq_context *ctx, coop_context *coop, uint32_t a_write,
|
|
uint32_t payload_bytes, bool skip_payload)
|
|
{
|
|
volatile uint32_t *ready_semaphore =
|
|
(volatile uint32_t *)(uintptr_t)coop->ready_semaphore_address;
|
|
volatile uint32_t *counter_word = (volatile uint32_t *)(uintptr_t)coop->counter_word_address;
|
|
uint32_t index;
|
|
int code;
|
|
|
|
code = ft_tensix_wait_semaphore_at_least(ready_semaphore, coop->receiver_count,
|
|
coop->spin_limit, coop->stop_flag);
|
|
if (code != FT_TENSIX_MULTICAST_OK) {
|
|
/* A host that ASKED is not a fault, and reporting it as one would make every
|
|
* retirement that lands mid-block look like a broken group. A bound that expired
|
|
* is a different diagnosis and keeps its own code. */
|
|
ftq_abort(ctx, code == FT_TENSIX_MULTICAST_ERROR_STOPPED ? kErrNone
|
|
: kErrCoopRendezvous);
|
|
return;
|
|
}
|
|
/* Safe here and only here: a receiver cannot increment again until it has been told
|
|
* this round landed, which is the last thing this function does. */
|
|
ft_tensix_set_local_semaphore(ready_semaphore, 0u);
|
|
/* Staged BEFORE the payload so the value the semaphore multicast reads is already in
|
|
* place, and re-staged every block because it is the block counter and not a constant
|
|
* VALID — the receivers wait for the number THEY expect, so a group that has drifted
|
|
* times out instead of computing from the wrong block. */
|
|
ft_tensix_set_local_semaphore(counter_word, coop->counter);
|
|
|
|
/* RED PROOF: skipping the payload leaves every receiver computing from the PREVIOUS
|
|
* block while still being told this one arrived. The barrier accounting stays exact,
|
|
* because the semaphore multicasts below advance it for themselves alone. */
|
|
for (index = 0u; index < coop->group_count && !skip_payload; index++) {
|
|
const uint32_t packed = coop->group[index];
|
|
uint64_t destination;
|
|
|
|
code = ft_tensix_multicast_address(ftcoop_rectangle_x_low(packed),
|
|
ftcoop_rectangle_y_low(packed),
|
|
ftcoop_rectangle_x_high(packed),
|
|
ftcoop_rectangle_y_high(packed), a_write,
|
|
&destination);
|
|
if (code == FT_TENSIX_MULTICAST_OK)
|
|
code = ft_tensix_noc_write_multicast(a_write, destination, payload_bytes,
|
|
ftcoop_rectangle_destinations(packed));
|
|
if (code != FT_TENSIX_MULTICAST_OK) {
|
|
ftq_abort(ctx, kErrCoopIssue);
|
|
return;
|
|
}
|
|
}
|
|
|
|
/* BLACKHOLE ONLY, and load-bearing: the payload multicasts use the WRITE command
|
|
* buffer and the semaphore multicasts the WRITE_REG one; the two FIFOs are not
|
|
* guaranteed to drain in issue order, so without this a receiver can be told the block
|
|
* arrived before it did and read the PREVIOUS block's bytes with no error anywhere. */
|
|
ft_tensix_noc_writes_flushed_barrier();
|
|
|
|
for (index = 0u; index < coop->group_count; index++) {
|
|
const uint32_t packed = coop->group[index];
|
|
uint64_t destination;
|
|
|
|
code = ft_tensix_multicast_address(ftcoop_rectangle_x_low(packed),
|
|
ftcoop_rectangle_y_low(packed),
|
|
ftcoop_rectangle_x_high(packed),
|
|
ftcoop_rectangle_y_high(packed),
|
|
coop->arrival_semaphore_address, &destination);
|
|
if (code == FT_TENSIX_MULTICAST_OK)
|
|
code = ft_tensix_noc_semaphore_set_multicast(
|
|
coop->counter_word_address, destination,
|
|
ftcoop_rectangle_destinations(packed));
|
|
if (code != FT_TENSIX_MULTICAST_OK) {
|
|
ftq_abort(ctx, kErrCoopIssue);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* RECEIVER: clear the arrival word, start its own B read, announce readiness, wait for A.
|
|
*
|
|
* The local clear comes FIRST. The leader cannot set the word until it has seen this
|
|
* core's atomic, and this core issues that atomic after the clear — so an arrival can
|
|
* never be erased by the clear that was meant to precede it.
|
|
*
|
|
* The B read is issued by the CALLER before this runs, so it overlaps the whole rendezvous:
|
|
* this core's own DRAM traffic and the leader's multicast are then in flight together,
|
|
* which is the only reason a receiver's block costs less than the sum of the two.
|
|
*/
|
|
static void coop_receiver_block(ftq_context *ctx, const coop_context *coop)
|
|
{
|
|
volatile uint32_t *arrival_semaphore =
|
|
(volatile uint32_t *)(uintptr_t)coop->arrival_semaphore_address;
|
|
int code;
|
|
|
|
ft_tensix_set_local_semaphore(arrival_semaphore, FT_TENSIX_SEMAPHORE_INVALID);
|
|
ft_tensix_noc_semaphore_increment(
|
|
ft_tensix_noc_address(coop->leader_x, coop->leader_y, coop->ready_semaphore_address), 1u);
|
|
code = ft_tensix_wait_semaphore_equals(arrival_semaphore, coop->counter, coop->spin_limit,
|
|
coop->stop_flag);
|
|
if (code != FT_TENSIX_MULTICAST_OK)
|
|
ftq_abort(ctx, code == FT_TENSIX_MULTICAST_ERROR_STOPPED ? kErrNone
|
|
: kErrCoopRendezvous);
|
|
}
|
|
|
|
/**
|
|
* Refuse a descriptor this program cannot execute, INSTEAD of writing outside the circular
|
|
* buffers.
|
|
*
|
|
* THE BLOCK-PREFETCH INVARIANT IS NOT "DEPTH DIVIDES Kt" ON A RESIDENT OP. That is the
|
|
* ONE-SHOT rule (docs/tt_kernel_model.md §12), where the FIFO write pointer starts at the
|
|
* base. This kernel runs many descriptors, and `cb_push_back` wraps only on an EXACT hit of
|
|
* the limit — so a descriptor that leaves the pointer part-way through the buffer makes the
|
|
* next one's blocks step over the limit and walk up L1.
|
|
*
|
|
* It is reachable here with nothing exotic: at Mt=1, Nt=1, Kt=48, depth=48 the descriptor
|
|
* pushes 48 pages into a 96-page (2*depth) buffer and leaves the pointer half way. The next
|
|
* descriptor is then off the grid for good. `ft_tensix_cb_push_grid_is_safe()` carries the
|
|
* derivation and the measured incident; resident_topk_reader.c refuses on exactly this and
|
|
* this file had none of it.
|
|
*
|
|
* Both input CBs are tested, not just CB_IN0: the host places them independently and only
|
|
* their page counts are supposed to match.
|
|
*/
|
|
static bool descriptor_is_executable(ftq_context *ctx, const coop_context *coop,
|
|
bool cooperative, uint32_t tile_rows, uint32_t tile_cols,
|
|
uint32_t tile_inner, uint32_t depth, uint32_t column_block,
|
|
uint32_t row_block)
|
|
{
|
|
if (tile_rows == 0u || tile_cols == 0u || tile_inner == 0u || depth == 0u ||
|
|
column_block == 0u || row_block == 0u || IN0_TILE_BYTES == 0u || IN1_TILE_BYTES == 0u)
|
|
ctx->error = kErrBadParameters;
|
|
else if (!ftmm_block_fits_dst(row_block, column_block))
|
|
/* MB * NB OUTPUT TILES ARE LIVE IN DST ACROSS THE WHOLE K LOOP, and DST has no
|
|
* bounds check: a pass that asks for five would accumulate into the OTHER half of
|
|
* the section, which the packer is reading, and every kernel would retire clean
|
|
* with a wrong answer. Refused here for the same reason the host also refuses it —
|
|
* the two failures cost differently, and only this one runs on silicon. */
|
|
ctx->error = kErrBadParameters;
|
|
else if (!ft_tensix_cb_push_grid_is_safe(CB_IN0, depth * row_block) ||
|
|
!ft_tensix_cb_push_grid_is_safe(CB_IN1, depth * column_block))
|
|
/* THE BLOCKS ARE PART OF THE CHUNK SIZE, and the chunk size is what this guard is
|
|
* about. CB_IN0 moves `depth * MB` pages per pass and CB_IN1 `depth * NB`, so a
|
|
* residency whose CBs were sized for a different pair — or a host that changed
|
|
* either mid-residency — is refused here rather than walking the write pointer off
|
|
* the end of the buffer. */
|
|
ctx->error = kErrBadParameters;
|
|
else if (row_block > 1u && cooperative)
|
|
/* The rendezvous counts A BLOCKS and a row block changes what an A block IS: the
|
|
* leader would multicast `depth * MB` tiles where the receivers expect `depth`.
|
|
* Same reason as the column block below, one operand over. */
|
|
ctx->error = kErrBadParameters;
|
|
else if (column_block > 1u && cooperative)
|
|
/* The rendezvous counts A BLOCKS, and blocking `nt` changes how many there are. Two
|
|
* mechanisms that both move A's supply, agreed separately, is a wrong answer with
|
|
* every status word clean; cooperation is disarmed on this model anyway (its
|
|
* `Nt = 1` weights break the all-or-nothing rule), so this costs nothing. */
|
|
ctx->error = kErrBadParameters;
|
|
else if (cooperative != (coop->role != FT_TT_RESIDENT_COOP_ROLE_SOLO))
|
|
/* COOPERATION IS ALL-OR-NOTHING FOR A RESIDENCY, and this is the line that makes
|
|
* it so. A non-SOLO member that ran a solo descriptor would push CB_IN0 pages its
|
|
* peers did not, and from then on every multicast would land in the wrong half of
|
|
* their buffers with every kernel completing and every status word clean. Refusing
|
|
* strands the triad and costs a re-arm, which is the cheap outcome. */
|
|
ctx->error = kErrCoopRoleMismatch;
|
|
else
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* M4 PIPELINED READER (kMatmulBNMajor bit 8; N-major, solo, no starve only). Issues block k+1
|
|
* into the CB's other half under its own NoC transaction id BEFORE waiting on block k's id,
|
|
* so the DRAM never sees the per-block drain. CB_IN0/CB_IN1 hold exactly two chunks
|
|
* (pn_tt_matmul_group.c: `depth * rows * 2`, `depth * block * 2`), so "the other half" is
|
|
* the write pointer plus one chunk, wrapped. DRAM harness sweep M: 16 cores x 9 reads of
|
|
* 4352 B = 407 GB/s with a barrier per block, 463 pipelined. Bit 9 = RED: never wait.
|
|
*/
|
|
static FT_BH_INLINE uint32_t ftmm_other_half(uint32_t cb, uint32_t address, uint32_t pages)
|
|
{
|
|
uint32_t next = address + pages * ft_tensix_cb(cb)->fifo_page_size;
|
|
|
|
if (next >= ft_tensix_cb(cb)->fifo_limit)
|
|
next -= ft_tensix_cb(cb)->fifo_size;
|
|
return next;
|
|
}
|
|
|
|
static void ftmm_issue_block(uint32_t mt, uint32_t rows, uint32_t nt, uint32_t columns,
|
|
uint32_t kt, uint32_t tile_inner, uint32_t depth,
|
|
uint32_t a_x, uint32_t a_y, uint32_t a_base, uint32_t b_x,
|
|
uint32_t b_y, uint32_t b_base, uint32_t a_write, uint32_t b_write,
|
|
bool a_half, bool red_face0, bool diag_face1_512, bool skip_a)
|
|
{
|
|
/* BYTES A-REUSE: the CB_IN0 ring already holds this pass's A chunk (see
|
|
* read_matmul_pipelined), so only B is fetched. */
|
|
for (uint32_t row = 0; row < (skip_a ? 0u : rows); row++) {
|
|
const uint32_t a_offset = ((mt + row) * tile_inner + kt) * IN0_TILE_BYTES;
|
|
|
|
if (a_half) {
|
|
/* M5: faces 0-1 (rows 0-15, bytes 0..1023) of each A tile — the half tile the
|
|
* compute thread multiplies. RED (bit 11): face 1 never lands; must DIFFER. */
|
|
for (uint32_t t = 0; t < depth; t++) {
|
|
const uint32_t src = a_base + a_offset + t * IN0_TILE_BYTES;
|
|
const uint32_t dst = a_write + (row * depth + t) * IN0_TILE_BYTES;
|
|
|
|
ft_tensix_noc_read(ft_tensix_noc_address(a_x, a_y, src), dst,
|
|
red_face0 ? 512u : 1024u);
|
|
}
|
|
(void)diag_face1_512;
|
|
continue;
|
|
}
|
|
ft_tensix_noc_read(ft_tensix_noc_address(a_x, a_y, a_base + a_offset),
|
|
a_write + row * depth * IN0_TILE_BYTES, depth * IN0_TILE_BYTES);
|
|
}
|
|
for (uint32_t column = 0; column < columns; column++) {
|
|
const uint32_t b_offset = ((nt + column) * tile_inner + kt) * IN1_TILE_BYTES;
|
|
|
|
ft_tensix_noc_read(ft_tensix_noc_address(b_x, b_y, b_base + b_offset),
|
|
b_write + column * depth * IN1_TILE_BYTES, depth * IN1_TILE_BYTES);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* PHASE G0 WEIGHT STASH (descriptor a1 bit 31; bits 0..23 = L1 address >> 4): the member's
|
|
* whole B slice (Nt_local x Kt tiles, N-major, contiguous) is read from DRAM ONCE into L1 at
|
|
* the stash address, and every B read of every row block then comes from that L1 copy
|
|
* through a LOCAL NoC read instead of DRAM. Same bytes, same order into CB_IN1: the compute
|
|
* thread cannot tell. RED (a1 bit 30): the slice is loaded ONE TILE LATE (from b_base + one
|
|
* tile) — every B tile shifts by one, and the answer must DIFFER. (Skipping the load is not a
|
|
* red: L1 survives a re-boot, so a previous run's identical stash would still be there.)
|
|
*/
|
|
static bool ftmm_load_stash(ftq_context *ctx, uint32_t stash, uint32_t bytes, uint32_t b_x,
|
|
uint32_t b_y, uint32_t b_base, bool red_shift)
|
|
{
|
|
if (ctx->aborting)
|
|
return false;
|
|
ft_tensix_noc_read(ft_tensix_noc_address(b_x, b_y, b_base + (red_shift ? IN1_TILE_BYTES : 0u)),
|
|
stash, bytes);
|
|
ft_tensix_noc_read_barrier();
|
|
return true;
|
|
}
|
|
|
|
static uint32_t ftmm_self_x, ftmm_self_y;
|
|
|
|
static bool read_matmul_pipelined(ftq_context *ctx, volatile FTQ_L1_PTR uint32_t *parameters,
|
|
uint32_t depth, uint32_t row_block, uint32_t column_block,
|
|
bool red_no_wait, uint32_t a1)
|
|
{
|
|
const uint32_t tile_rows = parameters[kMatmulTileRows];
|
|
const uint32_t tile_inner = parameters[kMatmulTileInner];
|
|
const uint32_t tile_cols = parameters[kMatmulTileCols];
|
|
const uint32_t a_x = parameters[kMatmulAX];
|
|
const uint32_t a_y = parameters[kMatmulAY];
|
|
const uint32_t a_base = parameters[kMatmulABase];
|
|
const bool stash = (a1 & 0x80000000u) != 0u;
|
|
const uint32_t b_x = stash ? ftmm_self_x : parameters[kMatmulBX];
|
|
const uint32_t b_y = stash ? ftmm_self_y : parameters[kMatmulBY];
|
|
const uint32_t b_base = stash ? (a1 & 0x00FFFFFFu) << 4 : parameters[kMatmulBBase];
|
|
const uint32_t a_pages = depth * row_block;
|
|
const uint32_t b_pages = depth * column_block;
|
|
const uint32_t blocks_per_tile = tile_inner / depth;
|
|
const uint32_t column_blocks = (tile_cols + column_block - 1u) / column_block;
|
|
const uint32_t row_blocks = (tile_rows + row_block - 1u) / row_block;
|
|
const uint32_t total = row_blocks * column_blocks * blocks_per_tile;
|
|
uint32_t a_write, b_write, block;
|
|
uint32_t trid = 1u;
|
|
bool ok = true;
|
|
const bool no_read = (parameters[kMatmulBNMajor] & kMatmulProbeNoRead) != 0u;
|
|
/* The same condition the compute thread tests, from the same words. */
|
|
const bool a_half = (parameters[kMatmulBNMajor] & kMatmulRow0) != 0u && tile_rows == 1u &&
|
|
row_block == 1u;
|
|
const bool red_face0 = (parameters[kMatmulBNMajor] & (1u << 11)) != 0u;
|
|
const bool diag_face1_512 = (parameters[kMatmulBNMajor] & (1u << 16)) != 0u;
|
|
/* BYTES A-REUSE (word 13 bit 18): when CB_IN0's ring is EXACTLY Kt pages and MB = 1, the
|
|
* chunk for (pass, kt) lands at the same L1 address on every pass over one A row, and
|
|
* nothing but this reader writes the ring — so on every pass after a row's first (nt != 0)
|
|
* pass 0's bytes are still there and the A read is skipped; the pages are pushed as
|
|
* usual. A ring of any other size is refused here, not trusted. RED (bit 19): skip
|
|
* regardless of the ring, so the compute sees another kt's A and the answer must DIFFER. */
|
|
const uint32_t in0_ring_pages = ft_tensix_cb(CB_IN0)->fifo_size /
|
|
ft_tensix_cb(CB_IN0)->fifo_page_size;
|
|
const bool a_reuse = ((parameters[kMatmulBNMajor] & kMatmulAReuseRed) != 0u) ||
|
|
((parameters[kMatmulBNMajor] & kMatmulAReuse) != 0u && row_block == 1u &&
|
|
in0_ring_pages == tile_inner);
|
|
|
|
if (stash && !no_read &&
|
|
!ftmm_load_stash(ctx, b_base, tile_cols * tile_inner * IN1_TILE_BYTES,
|
|
parameters[kMatmulBX], parameters[kMatmulBY], parameters[kMatmulBBase],
|
|
(a1 & 0x40000000u) != 0u))
|
|
return false;
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_IN0, (int32_t)a_pages, WAIT_MODE) ||
|
|
!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_IN1, (int32_t)b_pages, WAIT_MODE))
|
|
return false;
|
|
a_write = ft_tensix_cb_write_pointer(CB_IN0);
|
|
b_write = ft_tensix_cb_write_pointer(CB_IN1);
|
|
|
|
for (block = 0u; block < total; block++) {
|
|
/* Block `block` -> (mt, nt, kt), the same order the unpipelined loop walks. */
|
|
const uint32_t kt = (block % blocks_per_tile) * depth;
|
|
const uint32_t pass = block / blocks_per_tile;
|
|
const uint32_t mt = (pass / column_blocks) * row_block;
|
|
const uint32_t nt = (pass % column_blocks) * column_block;
|
|
const uint32_t rows = (tile_rows - mt) < row_block ? (tile_rows - mt) : row_block;
|
|
const uint32_t columns = (tile_cols - nt) < column_block ? (tile_cols - nt) : column_block;
|
|
|
|
if (block == 0u && !ctx->aborting && !no_read) {
|
|
ft_tensix_noc_read_set_trid(trid);
|
|
ftmm_issue_block(mt, rows, nt, columns, kt, tile_inner, depth, a_x, a_y, a_base,
|
|
b_x, b_y, b_base, a_write, b_write, a_half, red_face0,
|
|
diag_face1_512, a_reuse && nt != 0u);
|
|
}
|
|
if (block + 1u < total) {
|
|
const uint32_t next = block + 1u;
|
|
const uint32_t nkt = (next % blocks_per_tile) * depth;
|
|
const uint32_t npass = next / blocks_per_tile;
|
|
const uint32_t nmt = (npass / column_blocks) * row_block;
|
|
const uint32_t nnt = (npass % column_blocks) * column_block;
|
|
const uint32_t nrows = (tile_rows - nmt) < row_block ? (tile_rows - nmt) : row_block;
|
|
const uint32_t ncolumns =
|
|
(tile_cols - nnt) < column_block ? (tile_cols - nnt) : column_block;
|
|
|
|
/* BOTH halves free = the compute thread has popped block `block - 1`. */
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_IN0, (int32_t)(2u * a_pages),
|
|
WAIT_MODE) ||
|
|
!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_IN1, (int32_t)(2u * b_pages),
|
|
WAIT_MODE))
|
|
ok = false;
|
|
if (ok && !ctx->aborting && !no_read) {
|
|
ft_tensix_noc_read_set_trid(trid ^ 3u);
|
|
ftmm_issue_block(nmt, nrows, nnt, ncolumns, nkt, tile_inner, depth, a_x, a_y,
|
|
a_base, b_x, b_y, b_base,
|
|
ftmm_other_half(CB_IN0, a_write, a_pages),
|
|
ftmm_other_half(CB_IN1, b_write, b_pages), a_half,
|
|
red_face0, diag_face1_512, a_reuse && nnt != 0u);
|
|
}
|
|
}
|
|
if (!red_no_wait)
|
|
ft_tensix_noc_read_barrier_trid(trid);
|
|
if (!ok) {
|
|
ft_tensix_noc_read_barrier();
|
|
ft_tensix_noc_read_set_trid(0u);
|
|
return false;
|
|
}
|
|
ft_tensix_cb_push_back(CB_IN0, (int32_t)a_pages);
|
|
ft_tensix_cb_push_back(CB_IN1, (int32_t)b_pages);
|
|
a_write = ft_tensix_cb_write_pointer(CB_IN0);
|
|
b_write = ft_tensix_cb_write_pointer(CB_IN1);
|
|
trid ^= 3u;
|
|
}
|
|
ft_tensix_noc_read_barrier();
|
|
ft_tensix_noc_read_set_trid(0u);
|
|
return !ctx->aborting;
|
|
}
|
|
|
|
/**
|
|
* The reader half of one matmul descriptor: matmul_reader.c's loop, with a stoppable
|
|
* reserve and the drain rule.
|
|
*
|
|
* `starve_blocks` is a DEBUG field and the only thing here that is not in the Phase 4
|
|
* kernel: it makes the reader skip that many trailing K-blocks of the LAST output tile, so
|
|
* the compute thread is left genuinely blocked in the real `cb_wait_front` of the real op
|
|
* rather than in a synthetic wait built to be easy to break. That distinction is the whole
|
|
* point of the acceptance test -- a stop protocol proven against a wait nobody would ever
|
|
* write is not proven.
|
|
*
|
|
* @return true if the whole descriptor was read; false if a wait was abandoned, in which
|
|
* case the loop still ran to completion (rule 3) and the caller must not retire it
|
|
*/
|
|
static bool read_one_matmul(ftq_context *ctx, coop_context *coop,
|
|
volatile FTQ_L1_PTR uint32_t *parameters, uint32_t starve_blocks)
|
|
{
|
|
const uint32_t cooperative = parameters[kMatmulCooperative];
|
|
const uint32_t tile_rows = parameters[kMatmulTileRows];
|
|
const uint32_t tile_inner = parameters[kMatmulTileInner];
|
|
const uint32_t tile_cols = parameters[kMatmulTileCols];
|
|
const uint32_t a_x = parameters[kMatmulAX];
|
|
const uint32_t a_y = parameters[kMatmulAY];
|
|
const uint32_t a_base = parameters[kMatmulABase];
|
|
const uint32_t b_x = parameters[kMatmulBX];
|
|
const uint32_t b_y = parameters[kMatmulBY];
|
|
const uint32_t b_base = parameters[kMatmulBBase];
|
|
const uint32_t requested_depth = parameters[kMatmulDepth];
|
|
const uint32_t b_n_major = parameters[kMatmulBNMajor] & 1u;
|
|
const bool pipelined = (parameters[kMatmulBNMajor] & (1u << 8)) != 0u;
|
|
const bool red_no_wait = (parameters[kMatmulBNMajor] & (1u << 9)) != 0u;
|
|
const uint32_t block_word = parameters[kMatmulColumnBlock];
|
|
|
|
// A depth that does not divide Kt would make a later block straddle the CB wrap, so the
|
|
// kernel refuses to guess: anything the host did not make legal collapses to the
|
|
// per-pair behaviour, which is always correct (matmul_reader.c).
|
|
const uint32_t depth =
|
|
(requested_depth > 0u && tile_inner % requested_depth == 0u) ? requested_depth : 1u;
|
|
// 0 is what every descriptor written before the field existed says, and it means one --
|
|
// on BOTH halves of the word (resident_matmul_block_protocol.h).
|
|
const uint32_t column_block = ftmm_column_block(block_word);
|
|
const uint32_t row_block = ftmm_row_block(block_word);
|
|
if (!descriptor_is_executable(ctx, coop, cooperative != 0u, tile_rows, tile_cols, tile_inner,
|
|
depth, column_block, row_block))
|
|
return false;
|
|
|
|
if (pipelined && b_n_major && cooperative == 0u &&
|
|
(starve_blocks == 0u || (starve_blocks & 0x80000000u) != 0u))
|
|
return read_matmul_pipelined(ctx, parameters, depth, row_block, column_block,
|
|
red_no_wait, starve_blocks);
|
|
|
|
const uint32_t blocks_per_tile = tile_inner / depth;
|
|
// The LAST block of either axis is short when the block does not divide the extent. The
|
|
// CHUNK does not shorten with it — only the fill does — because a chunk size that
|
|
// changes walks the write pointer off the grid the next descriptor needs
|
|
// (ft_tensix_cb_block_grid_is_safe).
|
|
const uint32_t column_blocks = (tile_cols + column_block - 1u) / column_block;
|
|
const uint32_t row_blocks = (tile_rows + row_block - 1u) / row_block;
|
|
const uint32_t passes = row_blocks * column_blocks;
|
|
uint32_t block_index = 0u;
|
|
const uint32_t starve_from = (starve_blocks == 0u || starve_blocks > blocks_per_tile)
|
|
? 0xFFFFFFFFu
|
|
: passes * blocks_per_tile - starve_blocks;
|
|
|
|
for (uint32_t mt = 0; mt < tile_rows; mt += row_block) {
|
|
const uint32_t rows = (tile_rows - mt) < row_block ? (tile_rows - mt) : row_block;
|
|
|
|
for (uint32_t nt = 0; nt < tile_cols; nt += column_block) {
|
|
const uint32_t columns =
|
|
(tile_cols - nt) < column_block ? (tile_cols - nt) : column_block;
|
|
|
|
for (uint32_t kt = 0; kt < tile_inner; kt += depth, block_index++) {
|
|
uint32_t a_write;
|
|
uint32_t b_write;
|
|
const bool starved = block_index >= starve_from;
|
|
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_IN0,
|
|
(int32_t)(depth * row_block), WAIT_MODE))
|
|
return false;
|
|
if (!ftq_cb_reserve_back_stoppable(ctx, (int32_t)CB_IN1,
|
|
(int32_t)(depth * column_block), WAIT_MODE))
|
|
return false;
|
|
a_write = ft_tensix_cb_write_pointer(CB_IN0);
|
|
b_write = ft_tensix_cb_write_pointer(CB_IN1);
|
|
|
|
// Rule 3: while draining, issue NOTHING. The pushes below still happen so
|
|
// the page counters stay balanced, but a transaction issued during a drain
|
|
// would still be in flight when the kernel returns.
|
|
if (!ctx->aborting && !starved) {
|
|
const bool receiving = cooperative != 0u &&
|
|
coop->role == FT_TT_RESIDENT_COOP_ROLE_RECEIVER;
|
|
|
|
if (cooperative != 0u &&
|
|
a_write != coop_expected_write_pointer(CB_IN0, depth * row_block,
|
|
coop->counter)) {
|
|
// The address the LEADER is about to multicast into is its own
|
|
// write pointer, so a member whose pointer has drifted would take
|
|
// the block in the wrong half of its buffer. Caught before
|
|
// anything is issued; see the protocol header on why this is the
|
|
// local half of the check and the counter is the other half.
|
|
ftq_abort(ctx, kErrCoopWritePointer);
|
|
} else {
|
|
// ONE A READ PER ROW FOR THE WHOLE COLUMN BLOCK, and that is the
|
|
// point of the column blocking: the offset never depended on `nt`,
|
|
// so at NB columns per pass the A column is fetched Nt/NB times
|
|
// instead of Nt.
|
|
//
|
|
// ROW-MAJOR IN L1: row i's `depth` tiles land at `i * depth`, which
|
|
// is the layout the compute thread's `tile_index_a` walks and the
|
|
// mirror of what the reader does with B. A row is one read of
|
|
// `depth` contiguous tiles, exactly as it was at MB = 1 — the row
|
|
// block adds reads, never shrinks one.
|
|
if (!receiving) {
|
|
for (uint32_t row = 0; row < rows; row++) {
|
|
const uint32_t a_offset =
|
|
((mt + row) * tile_inner + kt) * IN0_TILE_BYTES;
|
|
|
|
ft_tensix_noc_read(
|
|
ft_tensix_noc_address(a_x, a_y, a_base + a_offset),
|
|
a_write + row * depth * IN0_TILE_BYTES,
|
|
depth * IN0_TILE_BYTES);
|
|
}
|
|
}
|
|
// COLUMN-MAJOR IN L1: column j's `depth` tiles land at `j * depth`.
|
|
// That keeps the N-major panel's per-column transaction whole — one
|
|
// read of `depth` contiguous tiles, the size every recorded number
|
|
// was taken at — and the compute thread's MOP walks the columns with
|
|
// an SrcA stride of `depth` tiles instead of one.
|
|
for (uint32_t column = 0; column < columns; column++) {
|
|
const uint32_t b_column = nt + column;
|
|
const uint32_t b_land = b_write + column * depth * IN1_TILE_BYTES;
|
|
|
|
if (b_n_major) {
|
|
const uint32_t b_offset =
|
|
(b_column * tile_inner + kt) * IN1_TILE_BYTES;
|
|
|
|
ft_tensix_noc_read(
|
|
ft_tensix_noc_address(b_x, b_y, b_base + b_offset), b_land,
|
|
depth * IN1_TILE_BYTES);
|
|
} else {
|
|
for (uint32_t step = 0; step < depth; step++) {
|
|
const uint32_t b_tile = (kt + step) * tile_cols + b_column;
|
|
const uint32_t b_offset = b_tile * IN1_TILE_BYTES;
|
|
|
|
ft_tensix_noc_read(
|
|
ft_tensix_noc_address(b_x, b_y, b_base + b_offset),
|
|
b_land + step * IN1_TILE_BYTES, IN1_TILE_BYTES);
|
|
}
|
|
}
|
|
}
|
|
if (receiving) {
|
|
// The B read is already ISSUED and not yet waited on, which is
|
|
// the whole point: this core's own DRAM traffic and the
|
|
// leader's multicast are in flight together. A receiver reads
|
|
// no A at all, which is why the row loop above is skipped
|
|
// wholesale rather than per row.
|
|
coop_receiver_block(ctx, coop);
|
|
// Taken whether or not the rendezvous succeeded — the B read
|
|
// was issued, and a transaction left outstanding is how L1 the
|
|
// next kernel is about to use gets written underneath it.
|
|
ft_tensix_noc_read_barrier();
|
|
} else {
|
|
// ONE barrier for the whole block: it is per-core, not
|
|
// per-transaction.
|
|
ft_tensix_noc_read_barrier();
|
|
if (cooperative != 0u)
|
|
coop_leader_block(
|
|
ctx, coop, a_write, depth * row_block * IN0_TILE_BYTES,
|
|
cooperative ==
|
|
FT_TT_RESIDENT_COOP_DESCRIPTOR_RED_SKIP_PAYLOAD);
|
|
}
|
|
}
|
|
}
|
|
// Advanced on EVERY cooperative block, including one that aborted, so two
|
|
// members that gave up at different points still agree on what block
|
|
// number they are at. A launch resets it, and a launch is what the host
|
|
// does after any refusal.
|
|
if (cooperative != 0u)
|
|
coop->counter = ftcoop_next_counter(coop->counter);
|
|
|
|
if (starved) {
|
|
// The starved block is never pushed, which is precisely what leaves the
|
|
// compute thread in cb_wait_front. Everything else about the descriptor
|
|
// is a normal, correct matmul.
|
|
return false;
|
|
}
|
|
// The FULL chunk on BOTH inputs, even on a short last block of either axis:
|
|
// the pad pages carry whatever L1 held and the compute thread never names
|
|
// them, but the grid closes.
|
|
ft_tensix_cb_push_back(CB_IN0, depth * row_block);
|
|
ft_tensix_cb_push_back(CB_IN1, depth * column_block);
|
|
}
|
|
}
|
|
}
|
|
if (cooperative != 0u) {
|
|
/* NOTHING OUTSTANDING ACROSS A DESCRIPTOR BOUNDARY. The next descriptor may be a
|
|
* different shape reusing the same circular buffers, and an abort leaves whatever
|
|
* the failing block had already issued in flight. The leader's multicasts are
|
|
* non-posted writes and the receivers' announcements are remote atomics, so both
|
|
* counters are drained here — cheap, because in the normal case the rendezvous has
|
|
* already forced them to have landed. */
|
|
ft_tensix_noc_write_barrier();
|
|
ft_tensix_noc_atomic_barrier();
|
|
}
|
|
return !ctx->aborting;
|
|
}
|
|
|
|
/**
|
|
* The opcodes that need no circular buffer. Identical in meaning to resident_queue.c's,
|
|
* and the reader is the processor that executes them because it is the one that owns
|
|
* `tail` -- the writer's cursor follows `tail`, so a writer cannot retire one of these
|
|
* before the reader has performed it.
|
|
*/
|
|
static bool execute_simple_opcode(ftq_context *ctx, uint32_t opcode, uint32_t a0, uint32_t a1,
|
|
uint32_t a2,
|
|
uint32_t a3, uint32_t a4, bool *keep_running)
|
|
{
|
|
switch (opcode) {
|
|
case kOpNop:
|
|
return true;
|
|
|
|
case kOpWriteWord:
|
|
*(volatile FTQ_L1_PTR uint32_t *)(uintptr_t)(a0) = a1;
|
|
return true;
|
|
|
|
case kOpL1Copy:
|
|
/* Every word of this transfer arrived on the wire — see
|
|
* ft_tensix_l1_copy_is_issuable(). Refuse, never issue-and-hope: a zero length is
|
|
* the card-down shape and a low destination overwrites the other core's mailboxes. */
|
|
if (!ft_tensix_l1_copy_is_issuable(a3, a2, a4)) {
|
|
ctx->error = kErrBadParameters;
|
|
*keep_running = false;
|
|
return false;
|
|
}
|
|
ft_tensix_noc_write(a3, ft_tensix_noc_address(a0, a1, a2), a4);
|
|
ft_tensix_noc_write_barrier();
|
|
return true;
|
|
|
|
case kOpSemPost:
|
|
ft_tensix_noc_semaphore_increment(ft_tensix_noc_address(a0, a1, a2), a3);
|
|
return true;
|
|
|
|
case kOpSemWait: {
|
|
volatile FTQ_L1_PTR uint32_t *semaphore = (volatile FTQ_L1_PTR uint32_t *)(uintptr_t)(a0);
|
|
const uint32_t bound = (a2 != 0u) ? a2 : ctx->idle_spin_limit;
|
|
uint32_t spins = 0u;
|
|
|
|
ftq_status_set(ctx, kStateSemWait);
|
|
while (*semaphore < a1) {
|
|
ctx->heartbeat++;
|
|
if (ctx->status)
|
|
ctx->status[kStatusHeartbeat] = ctx->heartbeat;
|
|
if (ftq_stop_requested(ctx)) {
|
|
ftq_abort(ctx, kErrNone);
|
|
*keep_running = false;
|
|
return false;
|
|
}
|
|
if (++spins >= bound) {
|
|
ftq_abort(ctx, kErrSemWaitTimeout);
|
|
*keep_running = false;
|
|
return false;
|
|
}
|
|
}
|
|
ftq_status_set(ctx, kStateExec);
|
|
return true;
|
|
}
|
|
|
|
case kOpDelay: {
|
|
volatile uint32_t counter = 0;
|
|
|
|
for (uint32_t iteration = 0; iteration < a0; iteration++)
|
|
counter = counter + 1;
|
|
return true;
|
|
}
|
|
|
|
case kOpStop:
|
|
*keep_running = false;
|
|
return true;
|
|
|
|
default:
|
|
ctx->error = kErrBadOpcode;
|
|
*keep_running = false;
|
|
return false;
|
|
}
|
|
}
|
|
|
|
void kernel_main(void)
|
|
{
|
|
ftq_context ctx;
|
|
coop_context coop;
|
|
bool running = true;
|
|
|
|
ftq_context_init(&ctx);
|
|
coop_read_arguments(&coop);
|
|
ftq_status_set(&ctx, kStateBoot);
|
|
ctx.control[kFieldState] = kStateBoot;
|
|
|
|
if (!ftq_check_control(&ctx)) {
|
|
ctx.control[kFieldError] = ctx.error;
|
|
ctx.control[kFieldState] = kStateExit;
|
|
ftq_status_set(&ctx, kStateExit);
|
|
return;
|
|
}
|
|
|
|
// RESUME rather than restart, so a retired triad can be relaunched. All five processors
|
|
// resume from `completed_count`, the ONE number that means "every stage finished this
|
|
// descriptor" -- resuming from `tail` would let the reader and the writer restart at
|
|
// different descriptors, because the writer legitimately trails the reader by the CB
|
|
// pipeline's depth. A relaunch is therefore only defined over a queue whose head, tail
|
|
// and completed_count agree: a fresh one, or one that was drained before it was
|
|
// retired. Anything else is refused here rather than run.
|
|
if (ctx.control[kFieldTail] != ctx.control[kFieldCompletedCount]) {
|
|
ctx.error = kErrBadParameters;
|
|
ctx.control[kFieldError] = ctx.error;
|
|
ctx.control[kFieldState] = kStateExit;
|
|
ftq_status_set(&ctx, kStateExit);
|
|
return;
|
|
}
|
|
/* THE COOPERATIVE CONFIGURATION IS REFUSED AT LAUNCH, not at the first descriptor.
|
|
* A rectangle with a coordinate that does not fit the 6-bit node-id field does not
|
|
* fail: it TRUNCATES onto a different live endpoint, and on this board a wrong
|
|
* endpoint is a hung NoC and a walk to the machine. So the geometry is checked once,
|
|
* here, before anything can be issued. */
|
|
if (!coop_configuration_is_legal(&coop)) {
|
|
ctx.error = kErrCoopBadPartition;
|
|
ctx.control[kFieldError] = ctx.error;
|
|
ctx.control[kFieldState] = kStateExit;
|
|
ftq_status_set(&ctx, kStateExit);
|
|
return;
|
|
}
|
|
ctx.cursor = ctx.control[kFieldCompletedCount];
|
|
{
|
|
/* G0 stash: this core's own NoC coordinate (upstream my_x/my_y from NOC_NODE_ID). */
|
|
const uint32_t id = ft_bh_noc_cmd_read(NOC_INDEX, 0u,
|
|
FT_BH_NOC_REG_BASE + 0x44u);
|
|
|
|
ftmm_self_x = id & FT_BH_NOC_NODE_ID_MASK;
|
|
ftmm_self_y = (id >> FT_BH_NOC_NODE_ID_BITS) & FT_BH_NOC_NODE_ID_MASK;
|
|
}
|
|
|
|
while (running) {
|
|
if (!ftq_wait_for_work(&ctx, kFieldHead))
|
|
break;
|
|
|
|
ctx.aborting = false;
|
|
ftq_status_set(&ctx, kStateExec);
|
|
|
|
volatile FTQ_L1_PTR uint32_t *descriptor = ftq_descriptor(&ctx);
|
|
const uint32_t opcode = descriptor[kDescOpcode];
|
|
const uint32_t a0 = descriptor[kDescA0];
|
|
const uint32_t a1 = descriptor[kDescA1];
|
|
const uint32_t a2 = descriptor[kDescA2];
|
|
const uint32_t a3 = descriptor[kDescA3];
|
|
const uint32_t a4 = descriptor[kDescA4];
|
|
bool done;
|
|
|
|
ctx.control[kFieldLastOpcode] = opcode;
|
|
ftq_status_opcode(&ctx, opcode);
|
|
|
|
if (opcode == kOpMatmul) {
|
|
ftq_status_set(&ctx, kStateCbReserveBack);
|
|
done = read_one_matmul(&ctx, &coop,
|
|
(volatile FTQ_L1_PTR uint32_t *)(uintptr_t)(a0), a1);
|
|
ftq_status_set(&ctx, kStateExec);
|
|
if (!done)
|
|
running = false;
|
|
} else {
|
|
done = execute_simple_opcode(&ctx, opcode, a0, a1, a2, a3, a4, &running);
|
|
}
|
|
|
|
// Rule 5. An abandoned descriptor was handed to nobody, so it advances NOTHING and
|
|
// the next launch re-attempts it. Only a descriptor the reader finished becomes
|
|
// visible to the writer.
|
|
if (done) {
|
|
ctx.cursor++;
|
|
ctx.control[kFieldTail] = ctx.cursor;
|
|
}
|
|
}
|
|
|
|
// Nothing is outstanding here: the drain path issues no transactions, and a
|
|
// cooperative descriptor drains its own multicasts and atomics before it returns. The
|
|
// barriers are the cheap statement of that invariant rather than a fix for a case that
|
|
// can happen. The ATOMIC one is here because a receiver's readiness announcement is a
|
|
// remote atomic, and the stock BRISC firmware asserts every NoC counter is settled after this returns.
|
|
ft_tensix_noc_read_barrier();
|
|
ft_tensix_noc_write_barrier();
|
|
ft_tensix_noc_atomic_barrier();
|
|
ctx.control[kFieldError] = ctx.error;
|
|
ctx.control[kFieldState] = kStateExit;
|
|
ftq_status_set(&ctx, kStateExit);
|
|
}
|