358 lines
16 KiB
C
358 lines
16 KiB
C
/* SPDX-License-Identifier: AGPL-3.0-or-later */
|
|
/* Copyright (C) 2026 David (Corey) Herrera */
|
|
/**
|
|
* @file ft_tensix_sfpu.h
|
|
* @brief C substrate for Vector Unit (SFPU) code on the math thread: the 32-lane vector type,
|
|
* Dst load/store/advance, immediates and constant registers, float/integer/exponent
|
|
* primitives, comparisons and the lane-predication (condition-code) stack.
|
|
*
|
|
* Every primitive is one SFPI compiler intrinsic (`__builtin_rvtt_*`) with fixed modifier
|
|
* arguments. Those intrinsics are known to the SFPI GCC (-mcpu=tt-bh-tensix) without any
|
|
* declaration; declaring one by hand turns it into a real call, so none is declared here.
|
|
* Intrinsics that take an instruction-buffer argument take it first; it only orders and
|
|
* aliases the issue and is never referenced by the emitted code.
|
|
*
|
|
* Modifier values follow the Vector Unit instruction pages (BlackholeA0/TensixTile/
|
|
* TensixCoprocessor/SFPLOAD.md, SFPSTORE.md, SFPLOADI.md, SFPMOV.md, SFPIADD.md, SFPSHFT.md,
|
|
* SFPEXEXP.md, SFPDIVP2.md, SFPSTOCHRND.md, SFPCAST.md, SFPABS.md, SFPARECIP.md, SFPPUSHC.md,
|
|
* SFPPOPC.md and the WormholeB0 pages they defer to). Only math-thread code includes this
|
|
* header, through ft_tensix_sfpu_math.h.
|
|
*
|
|
* @status Clean-room rewrite (plan Phase 4, step 4.5b).
|
|
* @issues None
|
|
* @todo None
|
|
*/
|
|
#ifndef FT_TENSIX_SFPU_H
|
|
#define FT_TENSIX_SFPU_H
|
|
|
|
#include <stdint.h>
|
|
|
|
#include "ft_tensix_compute.h"
|
|
#include "../fw/ft_bh_isa.h"
|
|
|
|
/* ===== Vector type ======================================================================= */
|
|
|
|
/* One SFPU register: 32 lanes of 32 bits. This exact GCC vector type is the one the SFPI
|
|
* intrinsics accept and return. */
|
|
typedef unsigned __attribute__((vector_size(sizeof(unsigned) * 32))) ft_sfpu_vector;
|
|
|
|
/* Ordering token passed first to intrinsics that issue through the instruction buffer. */
|
|
#define FT_SFPU_INSTRUCTION_BUFFER ((volatile void *)FT_BH_INSTRN_BUF)
|
|
|
|
/* ===== Modifier fields =================================================================== */
|
|
|
|
/* SFPLOAD / SFPSTORE: format "as stored in Dst" and the Blackhole "no increment" address mode
|
|
* (slot 7, which Foundry programs to all-zero increments; Wormhole uses slot 3). One Dst
|
|
* "register" of the SFPU view is two address units. */
|
|
#define FT_SFPU_LOAD_FMT_SRCB 0u
|
|
#define FT_SFPU_STORE_FMT_SRCB 0u
|
|
#define FT_SFPU_LOAD_ADDR_MODE_NOINC 7u
|
|
#define FT_SFPU_STORE_ADDR_MODE_NOINC 7u
|
|
#define FT_SFPU_DST_REGISTER_STRIDE 2u
|
|
|
|
/* SFPLOADI: immediate is a bf16 (upper half of an fp32). */
|
|
#define FT_SFPU_LOADI_FMT_FLOAT16B 0u
|
|
|
|
/* SFPMOV: copy with the sign bit complemented. */
|
|
#define FT_SFPU_MOV_COMPLEMENT_SIGN 1u
|
|
|
|
/* SFPIADD: condition-code handling and operand negation. */
|
|
#define FT_SFPU_IADD_CC_NONE 4u
|
|
#define FT_SFPU_IADD_CC_LESS_THAN_ZERO 0u
|
|
#define FT_SFPU_IADD_ARG_2SCOMP_LREG_DST 2u
|
|
|
|
/* SFPSHFT: logical shift. */
|
|
#define FT_SFPU_SHIFT_LOGICAL 0u
|
|
|
|
/* SFPEXEXP: exponent with or without the bias removed. */
|
|
#define FT_SFPU_EXEXP_DEBIAS 0u
|
|
#define FT_SFPU_EXEXP_NODEBIAS 1u
|
|
|
|
/* SFPDIVP2: add the immediate to the exponent. */
|
|
#define FT_SFPU_DIVP2_ADD 1u
|
|
|
|
/* SFPSTOCHRND: rounding mode and conversion selectors. */
|
|
#define FT_SFPU_STOCHRND_ROUND_EVEN 0u
|
|
#define FT_SFPU_STOCHRND_FP32_TO_FP16B 1u
|
|
#define FT_SFPU_STOCHRND_FP32_TO_SMAG16 7u
|
|
|
|
/* SFPCAST: sign-magnitude int32 to fp32, round to nearest even. */
|
|
#define FT_SFPU_CAST_SMAG32_TO_FP32_ROUND_EVEN 0u
|
|
|
|
/* SFPABS: integer absolute value. */
|
|
#define FT_SFPU_ABS_INT 0u
|
|
|
|
/* SFPARECIP (Blackhole only): approximate reciprocal. */
|
|
#define FT_SFPU_ARECIP_RECIPROCAL 0u
|
|
|
|
/* SFPPUSHC / SFPPOPC: plain push and pop of the condition-code stack. */
|
|
#define FT_SFPU_PUSHC_PUSH 0u
|
|
#define FT_SFPU_POPC_POP 0u
|
|
|
|
/* Comparison codes and flags of the compiler's compare intrinsics. */
|
|
#define FT_SFPU_COMPARE_LT 1u
|
|
#define FT_SFPU_COMPARE_NE 4u
|
|
#define FT_SFPU_COMPARE_IMMEDIATE_IS_FLOAT 32u
|
|
#define FT_SFPU_COMPARE_INT_SIGNED 8u
|
|
|
|
/* Boolean combiner code for two condition handles. */
|
|
#define FT_SFPU_BOOL_AND 2u
|
|
|
|
/* Programmable constant registers (LREG indices 12..14). */
|
|
#define FT_SFPU_CONST_REG_PROGRAMMABLE_0 12u
|
|
#define FT_SFPU_CONST_REG_PROGRAMMABLE_1 13u
|
|
#define FT_SFPU_CONST_REG_PROGRAMMABLE_2 14u
|
|
|
|
/* ===== Bits and constants ================================================================ */
|
|
|
|
/** IEEE-754 bits of `value` (folds to a constant for literals). */
|
|
static FT_BH_INLINE uint32_t ft_sfpu_float_bits(float value)
|
|
{
|
|
union {
|
|
float as_float;
|
|
uint32_t as_bits;
|
|
} pun;
|
|
pun.as_float = value;
|
|
return pun.as_bits;
|
|
}
|
|
|
|
/** Full 32-bit float immediate, as an opaque bit pattern. */
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_float_constant(float value)
|
|
{
|
|
return __builtin_rvtt_sfpxloadi(FT_SFPU_INSTRUCTION_BUFFER, ft_sfpu_float_bits(value), 0, 0,
|
|
-32);
|
|
}
|
|
|
|
/** Full 32-bit unsigned immediate, as an opaque bit pattern. */
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_uint_constant(uint32_t value)
|
|
{
|
|
return __builtin_rvtt_sfpxloadi(FT_SFPU_INSTRUCTION_BUFFER, value, 0, 0, -32);
|
|
}
|
|
|
|
/** Signed integer immediate; the compiler may narrow it to a shorter load. */
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_int_constant(int32_t value)
|
|
{
|
|
return __builtin_rvtt_sfpxloadi(FT_SFPU_INSTRUCTION_BUFFER, (uint32_t)value, 0, 0, 31);
|
|
}
|
|
|
|
/** bf16 immediate: the float with its low 16 mantissa bits dropped (one SFPLOADI). */
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_float16b_constant(float value)
|
|
{
|
|
return __builtin_rvtt_sfploadi(FT_SFPU_INSTRUCTION_BUFFER, ft_sfpu_float_bits(value) >> 16, 0,
|
|
0, FT_SFPU_LOADI_FMT_FLOAT16B);
|
|
}
|
|
|
|
/** Read constant register `index`. */
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_read_constant_register(unsigned index)
|
|
{
|
|
return __builtin_rvtt_sfpreadlreg(index);
|
|
}
|
|
|
|
/** Program constant register `index` with lane values `value`. */
|
|
static FT_BH_INLINE void ft_sfpu_write_constant_register(unsigned index, ft_sfpu_vector value)
|
|
{
|
|
__builtin_rvtt_sfpwriteconfig_v(value, index);
|
|
}
|
|
|
|
/* ===== Dst register file ================================================================= */
|
|
|
|
/* Load / store SFPU-view Dst register `index` (compile-time constant) in its stored format. */
|
|
#define FT_SFPU_DST_LOAD_FLOAT(index) \
|
|
__builtin_rvtt_sfpload(FT_SFPU_INSTRUCTION_BUFFER, (index) * FT_SFPU_DST_REGISTER_STRIDE, 0, \
|
|
0, FT_SFPU_LOAD_FMT_SRCB, FT_SFPU_LOAD_ADDR_MODE_NOINC)
|
|
#define FT_SFPU_DST_STORE_FLOAT(index, value) \
|
|
__builtin_rvtt_sfpstore(FT_SFPU_INSTRUCTION_BUFFER, (value), \
|
|
(index) * FT_SFPU_DST_REGISTER_STRIDE, 0, 0, FT_SFPU_STORE_FMT_SRCB, \
|
|
FT_SFPU_STORE_ADDR_MODE_NOINC)
|
|
|
|
/** Advance the Dst read/write counter by `registers` SFPU-view registers. */
|
|
static FT_BH_INLINE void ft_sfpu_dst_advance(int registers)
|
|
{
|
|
__builtin_rvtt_ttincrwc(0, (int)FT_SFPU_DST_REGISTER_STRIDE * registers, 0, 0);
|
|
}
|
|
|
|
/* ===== Assignment ======================================================================== */
|
|
|
|
/* Re-assign an already-live vector variable (a fresh declaration uses a plain initialiser). */
|
|
#define FT_SFPU_ASSIGN(destination, value) \
|
|
((destination) = __builtin_rvtt_sfpassign_lv((destination), (value)))
|
|
|
|
/* A declared vector with no value yet. */
|
|
#define FT_SFPU_UNINITIALISED() __builtin_rvtt_sfpnovalue()
|
|
|
|
/* ===== Float arithmetic ================================================================== */
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_float_mul(ft_sfpu_vector a, ft_sfpu_vector b)
|
|
{
|
|
return __builtin_rvtt_sfpmul(a, b, 0);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_float_add(ft_sfpu_vector a, ft_sfpu_vector b)
|
|
{
|
|
return __builtin_rvtt_sfpadd(a, b, 0);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_float_neg(ft_sfpu_vector a)
|
|
{
|
|
return __builtin_rvtt_sfpmov(a, FT_SFPU_MOV_COMPLEMENT_SIGN);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_float_sub(ft_sfpu_vector a, ft_sfpu_vector b)
|
|
{
|
|
return __builtin_rvtt_sfpadd(a, ft_sfpu_float_neg(b), 0);
|
|
}
|
|
|
|
/* ===== Integer arithmetic ================================================================ */
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_int_add(ft_sfpu_vector a, ft_sfpu_vector b)
|
|
{
|
|
return __builtin_rvtt_sfpiadd_v(a, b, FT_SFPU_IADD_CC_NONE);
|
|
}
|
|
|
|
/** a - b: the intrinsic negates its first operand, so the operands go in reversed. */
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_int_sub(ft_sfpu_vector a, ft_sfpu_vector b)
|
|
{
|
|
return __builtin_rvtt_sfpiadd_v(b, a, FT_SFPU_IADD_CC_NONE | FT_SFPU_IADD_ARG_2SCOMP_LREG_DST);
|
|
}
|
|
|
|
#define FT_SFPU_INT_ADD_IMMEDIATE(a, immediate) \
|
|
__builtin_rvtt_sfpiadd_i(FT_SFPU_INSTRUCTION_BUFFER, (a), (unsigned)(immediate), 0, 0, \
|
|
FT_SFPU_IADD_CC_NONE)
|
|
|
|
/* As above, and the lane condition code is set where the sum is negative. */
|
|
#define FT_SFPU_INT_ADD_IMMEDIATE_SET_CC_NEGATIVE(a, immediate) \
|
|
__builtin_rvtt_sfpiadd_i(FT_SFPU_INSTRUCTION_BUFFER, (a), (unsigned)(immediate), 0, 0, \
|
|
FT_SFPU_IADD_CC_LESS_THAN_ZERO)
|
|
|
|
/* A right shift is a shift by the negated amount. */
|
|
#define FT_SFPU_SHIFT_RIGHT_LOGICAL(a, amount) \
|
|
__builtin_rvtt_sfpshft_i(FT_SFPU_INSTRUCTION_BUFFER, (a), (unsigned)(-(amount)), 0, 0, \
|
|
FT_SFPU_SHIFT_LOGICAL)
|
|
|
|
/* ===== Exponent, sign and conversion ===================================================== */
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_extract_exponent(ft_sfpu_vector v)
|
|
{
|
|
return __builtin_rvtt_sfpexexp(v, FT_SFPU_EXEXP_DEBIAS);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_extract_exponent_biased(ft_sfpu_vector v)
|
|
{
|
|
return __builtin_rvtt_sfpexexp(v, FT_SFPU_EXEXP_NODEBIAS);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_set_exponent(ft_sfpu_vector v, ft_sfpu_vector e)
|
|
{
|
|
return __builtin_rvtt_sfpsetexp_v(v, e, 0);
|
|
}
|
|
|
|
#define FT_SFPU_SET_EXPONENT_IMMEDIATE(value, exponent) \
|
|
__builtin_rvtt_sfpsetexp_i(FT_SFPU_INSTRUCTION_BUFFER, (value), (unsigned)(exponent), 0, 0, 0)
|
|
|
|
#define FT_SFPU_SET_SIGN_IMMEDIATE(value, sign) \
|
|
__builtin_rvtt_sfpsetsgn_i(FT_SFPU_INSTRUCTION_BUFFER, (value), (unsigned)(sign), 0, 0, 0)
|
|
|
|
/* Multiply by 2^exponent by adding to the exponent field. */
|
|
#define FT_SFPU_ADD_EXPONENT(value, exponent) \
|
|
__builtin_rvtt_sfpdivp2(FT_SFPU_INSTRUCTION_BUFFER, (value), (unsigned)(exponent), 0, 0, \
|
|
FT_SFPU_DIVP2_ADD)
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_round_to_float16b(ft_sfpu_vector v)
|
|
{
|
|
return __builtin_rvtt_sfpstochrnd_i(FT_SFPU_INSTRUCTION_BUFFER, v, 0, 0, 0,
|
|
FT_SFPU_STOCHRND_FP32_TO_FP16B,
|
|
FT_SFPU_STOCHRND_ROUND_EVEN);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_round_to_signed_magnitude16(ft_sfpu_vector v)
|
|
{
|
|
return __builtin_rvtt_sfpstochrnd_i(FT_SFPU_INSTRUCTION_BUFFER, v, 0, 0, 0,
|
|
FT_SFPU_STOCHRND_FP32_TO_SMAG16,
|
|
FT_SFPU_STOCHRND_ROUND_EVEN);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_convert_signed_magnitude_to_float(ft_sfpu_vector v)
|
|
{
|
|
return __builtin_rvtt_sfpcast(v, FT_SFPU_CAST_SMAG32_TO_FP32_ROUND_EVEN);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_int_abs(ft_sfpu_vector v)
|
|
{
|
|
return __builtin_rvtt_sfpabs(v, FT_SFPU_ABS_INT);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_copy_sign(ft_sfpu_vector value,
|
|
ft_sfpu_vector sign_source)
|
|
{
|
|
return __builtin_rvtt_sfpsetsgn_v(value, sign_source, 0);
|
|
}
|
|
|
|
static FT_BH_INLINE ft_sfpu_vector ft_sfpu_approximate_reciprocal(ft_sfpu_vector v)
|
|
{
|
|
return __builtin_rvtt_sfparecip(v, FT_SFPU_ARECIP_RECIPROCAL);
|
|
}
|
|
|
|
/* ===== Comparisons ======================================================================= */
|
|
/* Each yields an opaque condition handle for FT_SFPU_IF or ft_sfpu_and. */
|
|
|
|
#define FT_SFPU_COMPARE_FLOAT_IMMEDIATE(a, comparison, literal) \
|
|
__builtin_rvtt_sfpxfcmps(FT_SFPU_INSTRUCTION_BUFFER, (a), ft_sfpu_float_bits(literal), 0, 0, \
|
|
(comparison) | FT_SFPU_COMPARE_IMMEDIATE_IS_FLOAT)
|
|
|
|
#define FT_SFPU_COMPARE_INT_IMMEDIATE(a, comparison, literal) \
|
|
__builtin_rvtt_sfpxicmps(FT_SFPU_INSTRUCTION_BUFFER, (a), (unsigned)(literal), 0, 0, \
|
|
(comparison) | FT_SFPU_COMPARE_INT_SIGNED)
|
|
|
|
/** Both conditions. A function so each handle is evaluated exactly once. */
|
|
static FT_BH_INLINE unsigned ft_sfpu_and(unsigned a, unsigned b)
|
|
{
|
|
return __builtin_rvtt_sfpxbool(FT_SFPU_BOOL_AND, a, b);
|
|
}
|
|
|
|
/* ===== Predication ======================================================================= */
|
|
/*
|
|
* Usage: FT_SFPU_IF(c) { ... } FT_SFPU_ELSE { ... } FT_SFPU_ENDIF, and
|
|
* FT_SFPU_BLOCK { ... } FT_SFPU_ENDBLOCK. Each opener pushes the condition-code stack once and
|
|
* records it in a block-scoped counter; the closer pops that many times (the loop folds away).
|
|
* A nested use shadows the outer counter on purpose.
|
|
*/
|
|
|
|
/* Narrow the active lanes by `expression`. The three statements must stay separate and in this
|
|
* order: the compare has to be emitted after the dependency marker. */
|
|
#define FT_SFPU_CONDITION(expression) \
|
|
do { \
|
|
unsigned ft_sfpu_cc_dependency = __builtin_rvtt_sfpxvif(); \
|
|
unsigned ft_sfpu_cc_result = (expression); \
|
|
__builtin_rvtt_sfpxcondb(ft_sfpu_cc_result, ft_sfpu_cc_dependency); \
|
|
} while (0)
|
|
|
|
#define FT_SFPU_IF(expression) \
|
|
{ \
|
|
unsigned ft_sfpu_cc_depth = 1u; \
|
|
__builtin_rvtt_sfppushc(FT_SFPU_PUSHC_PUSH); \
|
|
FT_SFPU_CONDITION(expression); \
|
|
{
|
|
|
|
#define FT_SFPU_ELSE \
|
|
} \
|
|
__builtin_rvtt_sfpcompc(); \
|
|
{
|
|
|
|
#define FT_SFPU_ENDIF \
|
|
} \
|
|
for (unsigned ft_sfpu_cc_pop = ft_sfpu_cc_depth; ft_sfpu_cc_pop--;) \
|
|
__builtin_rvtt_sfppopc(FT_SFPU_POPC_POP); \
|
|
}
|
|
|
|
#define FT_SFPU_BLOCK \
|
|
{ \
|
|
unsigned ft_sfpu_cc_depth = 1u; \
|
|
__builtin_rvtt_sfppushc(FT_SFPU_PUSHC_PUSH);
|
|
|
|
#define FT_SFPU_ENDBLOCK \
|
|
for (unsigned ft_sfpu_cc_pop = ft_sfpu_cc_depth; ft_sfpu_cc_pop--;) \
|
|
__builtin_rvtt_sfppopc(FT_SFPU_POPC_POP); \
|
|
}
|
|
|
|
#endif /* FT_TENSIX_SFPU_H */
|