David
/
Foundry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Foundry/tt/kernels/ft_tensix_compute_matmul.h

375 lines
19 KiB
C
Raw Blame History

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
/**
* @file ft_tensix_compute_matmul.h
* @brief Tile matrix multiply for the MATH and UNPACK threads: address-modifier programming,
* replay-buffer recording, MOP templates and per-tile issue.
*
* MATH: a 32x32 by 32x32 tile product is 16 MVMULs (WormholeB0/TensixTile/TensixCoprocessor/
* MVMUL.md), each consuming 8 rows; their SrcA/SrcB/Dst walk is encoded in address-modifier
* slots 0, 1, 2, 4 and 5 (RWCs.md, slot 5 also advances the fidelity phase). The 16 MVMULs
* are recorded once into the replay buffer (REPLAY.md) and the MOP loop instruction replays
* them (MOPExpander.md). A 16-row in0 variant uses 8 MVMULs.
*
* UNPACK: a 12-instruction program is recorded into the replay buffer, two 6-instruction
* halves, one per unpacker config context. Each half unpacks SrcA, then advances unpacker 0's
* L1 base address by the A tile size (GPR 36) so the MOP can walk the K dimension. The
* operands are deliberately crossed: in0 goes to SrcB (unpacker 1) and in1 to SrcA
* (unpacker 0), since the FPU computes SrcB x SrcA.
*
* Replay slots: [0, 16) belong to the UNPACK program and the SFPU, [16, 32) to the MATH
* thread's FPU sequences (Foundry convention). GPR numbers are Foundry conventions.
*
* @status Clean-room rewrite (plan Phase 4, step 4.5b).
* @issues None
* @todo None
*/
#ifndef FT_TENSIX_COMPUTE_MATMUL_H
#define FT_TENSIX_COMPUTE_MATMUL_H
#include <stdint.h>
#include "ft_tensix_circular_buffer.h"
#include "ft_tensix_compute.h"
#include "../fw/ft_bh_isa.h"
/* The SFPI compiler's replay-record intrinsic. Its usual declaration cannot be included from C,
* so it is declared here (ft_tensix_compute_reduce.h declares the same prototype). With
* record = 1 the next `length` Tensix instructions of this thread are stored into replay slots
* [start, start + length) instead of executed. */
void __builtin_rvtt_ttreplay(volatile void *instruction_buffer, unsigned length, unsigned unused0,
unsigned unused1, unsigned start, unsigned execute_while_loading,
unsigned record);
#define FT_TENSIX_REPLAY_RECORD(start, length) \
__builtin_rvtt_ttreplay((volatile void *)__instrn_buffer, (length), 0, 0, (start), 0, 1)
/* REPLAY word that plays slots [start, start + length) (a data value for the MOP). */
#define FT_TENSIX_REPLAY_INSN(start, length) FT_BH_ENC_REPLAY((start), (length), 0, 0)
/* Replay layout (Foundry convention). */
#define FT_TENSIX_MATH_REPLAY_BUF_OFFSET 16u
#define FT_TENSIX_MATH_MATMUL_REPLAY_LEN 16u
#define FT_TENSIX_MATH_MATMUL_HALF_REPLAY_LEN 8u
#define FT_TENSIX_UNPACK_MATMUL_REPLAY_PROG_LEN 12u
#define FT_TENSIX_UNPACK_MATMUL_REPLAY_RUN_LEN 6u
/* Unpack-thread GPRs (Foundry convention). */
#ifndef FT_TENSIX_GPR_UNPACK_TMP0
#define FT_TENSIX_GPR_UNPACK_TMP0 12u
#endif
#ifndef FT_TENSIX_GPR_UNPACK_KT_DIM
#define FT_TENSIX_GPR_UNPACK_KT_DIM 38u
#endif
/* ===== Math thread ======================================================================= */
#ifdef FT_TRISC_MATH
/** Address modifiers of the full-tile walk. `transpose` swaps the SrcA steps of slots 1, 4. */
static FT_BH_INLINE void ft_tensix_math_matmul_configure_addrmod(uint32_t transpose,
uint32_t fidelity_increment)
{
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_0, FT_TENSIX_ADDR_MOD_SRC_VAL(0, 0, 0),
FT_TENSIX_ADDR_MOD_SRC_VAL(8, 0, 0),
FT_TENSIX_ADDR_MOD_DEST_VAL(8, 0, 0, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_5, FT_TENSIX_ADDR_MOD_SRC_VAL(0, 1, 1),
FT_TENSIX_ADDR_MOD_SRC_VAL(0, 1, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(0, 1, 1, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(fidelity_increment, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
if (transpose != 0) {
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_1, FT_TENSIX_ADDR_MOD_SRC_VAL(32, 0, 0),
FT_TENSIX_ADDR_MOD_SRC_VAL(0, 0, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(8, 0, 0, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
} else {
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_1, FT_TENSIX_ADDR_MOD_SRC_VAL(16, 0, 0),
FT_TENSIX_ADDR_MOD_SRC_VAL(0, 0, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(8, 0, 0, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
}
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_2, FT_TENSIX_ADDR_MOD_SRC_VAL(0, 0, 1),
FT_TENSIX_ADDR_MOD_SRC_VAL(32, 0, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(8, 0, 0, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
if (transpose != 0) {
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_4, FT_TENSIX_ADDR_MOD_SRC_VAL(16, 0, 1),
FT_TENSIX_ADDR_MOD_SRC_VAL(48, 0, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(0, 0, 1, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
} else {
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_4, FT_TENSIX_ADDR_MOD_SRC_VAL(32, 0, 1),
FT_TENSIX_ADDR_MOD_SRC_VAL(48, 0, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(0, 0, 1, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
}
}
/** Record the 16 MVMULs of one fidelity phase and program a MOP that replays them
* `inner_loops` times (once per fidelity phase at high fidelity). At low fidelity the last
* MVMUL releases SrcA; at high fidelity the MOP end op does, after all phases. */
static FT_BH_INLINE void ft_tensix_math_matmul_configure_mop(uint32_t inner_loops,
int high_fidelity)
{
ft_tensix_mop_template t;
FT_TENSIX_REPLAY_RECORD(FT_TENSIX_MATH_REPLAY_BUF_OFFSET, FT_TENSIX_MATH_MATMUL_REPLAY_LEN);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_1, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_2, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_1, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_4, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_1, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_2, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_1, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
if (high_fidelity) {
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_5, 0);
} else {
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_A, 0, FT_BH_ADDR_MOD_5, 0);
}
t = ft_tensix_mop_template_1(1u, inner_loops,
FT_TENSIX_REPLAY_INSN(FT_TENSIX_MATH_REPLAY_BUF_OFFSET,
FT_TENSIX_MATH_MATMUL_REPLAY_LEN));
if (high_fidelity) {
t.end_op0 = FT_BH_ENC_SETRWC(FT_BH_SETRWC_CLR_A, 0, 0, 0, 0, FT_BH_SETRWC_SET_ABD_F);
t.end_op1 = FT_BH_ENC_NOP;
}
ft_tensix_mop_template_program(&t);
}
static FT_BH_INLINE void ft_tensix_math_matmul_init(uint32_t transpose,
uint32_t fidelity_increment,
uint32_t inner_loops, int high_fidelity)
{
ft_tensix_math_matmul_configure_addrmod(transpose, fidelity_increment);
ft_tensix_math_matmul_configure_mop(inner_loops, high_fidelity);
FT_BH_TTINSN_SETRWC(FT_BH_SETRWC_CLR_NONE, 0, 0, 0, 0, FT_BH_SETRWC_SET_ABD_F);
}
/** Accumulate SrcB x SrcA into Dst tile `dst_index` (Dst is not cleared here). */
static FT_BH_INLINE void ft_tensix_math_matmul(uint32_t dst_index)
{
uint32_t dst_offset = (dst_index << 6) + ((ft_tensix_dest_offset_id != 0)
? FT_TENSIX_DEST_REGISTER_HALF_SIZE
: 0u);
FT_BH_PUSH_SETC16(FT_BH_CFG_MATH_DEST_BASE_OFFSET_WORD, dst_offset);
ft_tensix_mop_template_run();
FT_BH_TTINSN_SETRWC(FT_BH_SETRWC_CLR_B, 0, 0, 0, 0, FT_BH_SETRWC_SET_ABD_F);
}
/** One row of `ct_dim` output tiles from Dst tile `dst_index` on, then release SrcB. */
static FT_BH_INLINE void ft_tensix_math_matmul_block(uint32_t dst_index, uint32_t ct_dim)
{
uint32_t column;
for (column = 0u; column < ct_dim; column++) {
uint32_t dst_offset = ((dst_index + column) << 6) +
((ft_tensix_dest_offset_id != 0) ? FT_TENSIX_DEST_REGISTER_HALF_SIZE
: 0u);
FT_BH_PUSH_SETC16(FT_BH_CFG_MATH_DEST_BASE_OFFSET_WORD, dst_offset);
ft_tensix_mop_template_run();
}
FT_BH_TTINSN_SETRWC(FT_BH_SETRWC_CLR_B, 0, 0, 0, 0, FT_BH_SETRWC_SET_ABD_F);
}
/** Address modifiers of the 16-row in0 walk. */
static FT_BH_INLINE void ft_tensix_math_matmul_configure_addrmod_in0_16row(
uint32_t fidelity_increment)
{
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_0, FT_TENSIX_ADDR_MOD_SRC_VAL(0, 0, 0),
FT_TENSIX_ADDR_MOD_SRC_VAL(8, 0, 0),
FT_TENSIX_ADDR_MOD_DEST_VAL(8, 0, 0, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_5, FT_TENSIX_ADDR_MOD_SRC_VAL(0, 1, 1),
FT_TENSIX_ADDR_MOD_SRC_VAL(0, 1, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(0, 1, 1, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(fidelity_increment, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_1, FT_TENSIX_ADDR_MOD_SRC_VAL(16, 0, 0),
FT_TENSIX_ADDR_MOD_SRC_VAL(0, 0, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(8, 0, 0, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_2, FT_TENSIX_ADDR_MOD_SRC_VAL(16, 0, 0),
FT_TENSIX_ADDR_MOD_SRC_VAL(0, 0, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(8, 0, 0, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
FT_TENSIX_ADDR_MOD_SET(FT_BH_ADDR_MOD_4, FT_TENSIX_ADDR_MOD_SRC_VAL(16, 0, 0),
FT_TENSIX_ADDR_MOD_SRC_VAL(16, 0, 1),
FT_TENSIX_ADDR_MOD_DEST_VAL(0, 0, 1, 0),
FT_TENSIX_ADDR_MOD_FIDELITY_VAL(0, 0),
FT_TENSIX_ADDR_MOD_BIAS_VAL(0, 0));
}
/** As ft_tensix_math_matmul_configure_mop with 8 MVMULs per phase. */
static FT_BH_INLINE void ft_tensix_math_matmul_configure_mop_in0_16row(uint32_t inner_loops,
int high_fidelity)
{
ft_tensix_mop_template t;
FT_TENSIX_REPLAY_RECORD(FT_TENSIX_MATH_REPLAY_BUF_OFFSET,
FT_TENSIX_MATH_MATMUL_HALF_REPLAY_LEN);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_2, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_4, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_1, 0);
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_0, 0);
if (high_fidelity) {
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_NONE, 0, FT_BH_ADDR_MOD_5, 0);
} else {
FT_BH_TTINSN_MVMUL(FT_BH_SETRWC_CLR_A, 0, FT_BH_ADDR_MOD_5, 0);
}
t = ft_tensix_mop_template_1(1u, inner_loops,
FT_TENSIX_REPLAY_INSN(FT_TENSIX_MATH_REPLAY_BUF_OFFSET,
FT_TENSIX_MATH_MATMUL_HALF_REPLAY_LEN));
if (high_fidelity) {
t.end_op0 = FT_BH_ENC_SETRWC(FT_BH_SETRWC_CLR_A, 0, 0, 0, 0, FT_BH_SETRWC_SET_ABD_F);
t.end_op1 = FT_BH_ENC_NOP;
}
ft_tensix_mop_template_program(&t);
}
static FT_BH_INLINE void ft_tensix_math_matmul_init_in0_16row(uint32_t fidelity_increment,
uint32_t inner_loops,
int high_fidelity)
{
ft_tensix_math_matmul_configure_addrmod_in0_16row(fidelity_increment);
ft_tensix_math_matmul_configure_mop_in0_16row(inner_loops, high_fidelity);
FT_BH_TTINSN_SETRWC(FT_BH_SETRWC_CLR_NONE, 0, 0, 0, 0, FT_BH_SETRWC_SET_ABD_F);
}
#endif /* FT_TRISC_MATH */
/* ===== Unpack thread ===================================================================== */
#ifdef FT_TRISC_UNPACK
/** Record the two-context unpack program and program the single-loop unpack MOP to replay
* the half of the current context (MOPExpander.md, unpack template registers 1..8). */
static FT_BH_INLINE void ft_tensix_unpack_ab_matmul_mop_config(void)
{
volatile uint32_t *mop_cfg = FT_TENSIX_MOP_CFG;
FT_TENSIX_REPLAY_RECORD(0u, FT_TENSIX_UNPACK_MATMUL_REPLAY_PROG_LEN);
/* Context 0: unpack SrcA, then unpacker 0 base address += A tile size. */
FT_BH_TTINSN_UNPACR(FT_BH_UNPACR_SRC_A, 0, 0, 0, 0, 1, 1, FT_BH_UNPACR_RAREFYB_DISABLE, 0, 0,
0, 0, 1);
FT_BH_TTINSN_RDCFG(FT_TENSIX_GPR_UNPACK_TMP0, FT_BH_CFG_UNPACKER0_L1_BASE_CONTEXT0_WORD);
FT_BH_TTINSN_ADDDMAREG(0, FT_TENSIX_GPR_UNPACK_TMP0, FT_TENSIX_GPR_UNPACK_TMP0,
FT_TENSIX_GPR_UNPACK_TILE_SIZE_A);
FT_BH_TTINSN_STALLWAIT(FT_BH_STALLWAIT_BLOCK_CFG, FT_BH_STALLWAIT_COND_THCON);
FT_BH_TTINSN_WRCFG(FT_TENSIX_GPR_UNPACK_TMP0, FT_BH_WRCFG_32B,
FT_BH_CFG_UNPACKER0_L1_BASE_CONTEXT0_WORD);
FT_BH_TTINSN_NOP;
/* Context 1: the same on the context-1 base address. */
FT_BH_TTINSN_UNPACR(FT_BH_UNPACR_SRC_A, 0, 0, 0, 0, 1, 1, FT_BH_UNPACR_RAREFYB_DISABLE, 0, 0,
0, 0, 1);
FT_BH_TTINSN_RDCFG(FT_TENSIX_GPR_UNPACK_TMP0, FT_BH_CFG_UNPACKER0_L1_BASE_CONTEXT1_WORD);
FT_BH_TTINSN_ADDDMAREG(0, FT_TENSIX_GPR_UNPACK_TMP0, FT_TENSIX_GPR_UNPACK_TMP0,
FT_TENSIX_GPR_UNPACK_TILE_SIZE_A);
FT_BH_TTINSN_STALLWAIT(FT_BH_STALLWAIT_BLOCK_CFG, FT_BH_STALLWAIT_COND_THCON);
FT_BH_TTINSN_WRCFG(FT_TENSIX_GPR_UNPACK_TMP0, FT_BH_WRCFG_32B,
FT_BH_CFG_UNPACKER0_L1_BASE_CONTEXT1_WORD);
FT_BH_TTINSN_NOP;
ft_tensix_mop_sync();
/* Unpack template: [1] flags, [2] B instruction, [3] A instruction, [4..6] A1..A3,
* [7] skip-A instruction (zmask set: context 1), [8] skip-B instruction. */
mop_cfg[1] = 0u;
mop_cfg[2] = 0u;
mop_cfg[3] = FT_TENSIX_REPLAY_INSN(0u, FT_TENSIX_UNPACK_MATMUL_REPLAY_RUN_LEN);
mop_cfg[4] = 0u;
mop_cfg[5] = 0u;
mop_cfg[6] = 0u;
mop_cfg[7] = FT_TENSIX_REPLAY_INSN(FT_TENSIX_UNPACK_MATMUL_REPLAY_RUN_LEN,
FT_TENSIX_UNPACK_MATMUL_REPLAY_RUN_LEN);
mop_cfg[8] = 0u;
}
/** Configure the matmul unpack: in-face transpose of SrcA, row lengths of both unpackers,
* K tile count (GPR 38). */
static FT_BH_INLINE void ft_tensix_unpack_ab_matmul_init(uint32_t transpose, uint32_t unpa_x_end,
uint32_t unpb_x_end, uint32_t kt_dim)
{
FT_TENSIX_CFG_RMW_FIELD(UNPACKER0_HALOIZE_MODE, transpose);
FT_BH_TTINSN_SETADCZW(FT_BH_SETADC_UNP_AB, 0, 0, 0, 0, 0xF);
FT_BH_PUSH_SETADCXX(FT_BH_SETADC_UNP_A, unpa_x_end, 0x0);
FT_BH_PUSH_SETADCXX(FT_BH_SETADC_UNP_B, unpb_x_end, 0x0);
FT_BH_PUSH_SETDMAREG(0, FT_TENSIX_LOWER_HALFWORD(kt_dim), 0,
FT_TENSIX_LO_16(FT_TENSIX_GPR_UNPACK_KT_DIM));
ft_tensix_unpack_ab_matmul_mop_config();
}
/** Set the SrcA step between K tiles (GPR 36, low half). */
static FT_BH_INLINE void ft_tensix_unpack_matmul_set_srca_tile_stride(uint32_t tile_stride)
{
FT_BH_PUSH_SETDMAREG(0, FT_TENSIX_LOWER_HALFWORD(tile_stride), 0,
FT_TENSIX_LO_16(FT_TENSIX_GPR_UNPACK_TILE_SIZE_A));
}
/** Unpack in0 tile `tile_index_a` of CB `operand_a` to SrcB and `ct_dim` in1 tiles from
* `tile_index_b` of CB `operand_b` to SrcA, then flip the config context. */
static FT_BH_INLINE void ft_tensix_unpack_ab_matmul(uint32_t operand_a, uint32_t operand_b,
uint32_t tile_index_a, uint32_t tile_index_b,
uint32_t ct_dim)
{
uint32_t base_address_a = ft_tensix_cb(operand_a)->fifo_read_pointer - 1u;
uint32_t base_address_b = ft_tensix_cb(operand_b)->fifo_read_pointer - 1u;
uint32_t tile_size_a = ft_tensix_cb(operand_a)->fifo_page_size;
uint32_t tile_size_b = ft_tensix_cb(operand_b)->fifo_page_size;
volatile uint32_t FT_BH_REG_PTR *cfg = ft_tensix_cfg_pointer();
uint32_t address_a = base_address_a + tile_size_a * tile_index_a;
uint32_t address_b = base_address_b + tile_size_b * tile_index_b;
uint32_t context;
/* Wait for a free config context (two exist). */
while (ft_tensix_semaphore_read(FT_TENSIX_SEM_UNPACK_SYNC) >= 2u) {
}
context = unp_cfg_context;
if (context != 0) {
cfg[FT_BH_CFG_UNPACKER0_L1_BASE_CONTEXT1_WORD] = address_b;
cfg[FT_BH_CFG_UNPACKER1_L1_BASE_CONTEXT1_WORD] = address_a;
} else {
cfg[FT_BH_CFG_UNPACKER0_L1_BASE_CONTEXT0_WORD] = address_b;
cfg[FT_BH_CFG_UNPACKER1_L1_BASE_CONTEXT0_WORD] = address_a;
}
ft_tensix_semaphore_post(FT_TENSIX_SEM_UNPACK_SYNC);
FT_BH_TTINSN_STALLWAIT(FT_BH_STALLWAIT_BLOCK_UNPACK, FT_BH_STALLWAIT_COND_TRISC_CFG);
FT_BH_TTINSN_UNPACR(FT_BH_UNPACR_SRC_B, 0, 0, 0, 0, 1, 1, FT_BH_UNPACR_RAREFYB_DISABLE, 0, 0,
0, 0, 1);
FT_BH_PUSH_MOP(0, ct_dim - 1u, (context == 0) ? 0u : 0xffu);
FT_BH_TTINSN_SEMGET(FT_TENSIX_T6_SEM(FT_TENSIX_SEM_UNPACK_SYNC));
unp_cfg_context = 1u - context;
if (context != 0) {
FT_BH_TTINSN_SETC16(FT_BH_CFG_UNPACKER0_CONTEXT_OFFSET_WORD, 0x0000);
} else {
FT_BH_TTINSN_SETC16(FT_BH_CFG_UNPACKER0_CONTEXT_OFFSET_WORD, 0x0101);
}
}
#endif /* FT_TRISC_UNPACK */
#endif /* FT_TENSIX_COMPUTE_MATMUL_H */
Reference in New Issue View Git Blame Copy Permalink