David
/
Foundry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Foundry/tt/kernels/ft_tensix_noc.h

290 lines
13 KiB
C
Raw Permalink Blame History

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
/**
* @file ft_tensix_noc.h
* @brief Unicast NoC transfers, a remote atomic increment and the NIU-counter barriers for a
* Blackhole data-movement core (BRISC or NCRISC).
*
* A transfer is programmed into one of the NIU's command buffers (target, return, length,
* packet control) and launched by a store to the command-control register. Completion is
* observed by comparing the NIU's status counters with software counters that every issue
* path here bumps. Those software counters belong to the firmware: it defines them, and it
* checks them against the hardware after the kernel returns, so each issue path must move
* exactly the counters it documents.
*
* Linkage: functions marked "forced" are always inlined. The six plain `static inline`
* functions (read, write, semaphore increment, read barrier, writes-flushed barrier, inline
* word write) may be kept out of line by the compiler; their definition order is kept fixed
* because it can decide the order of those copies in .text.
*
* Register layout: BlackholeA0/NoC/MemoryMap.md and the NIU counter list in the same tree.
*
* @status Complete for the transfers Foundry kernels issue.
* @issues None
* @todo None
*/
#ifndef FT_TENSIX_NOC_H
#define FT_TENSIX_NOC_H
#include <stdint.h>
#include "../fw/ft_bh_regs.h"
/* ---- Software transfer counters (defined by the firmware image, one per NoC) ---- */
extern uint32_t noc_reads_num_issued[NUM_NOCS];
extern uint32_t noc_nonposted_writes_num_issued[NUM_NOCS];
extern uint32_t noc_nonposted_writes_acked[NUM_NOCS];
extern uint32_t noc_nonposted_atomics_acked[NUM_NOCS];
extern uint32_t noc_posted_writes_num_issued[NUM_NOCS];
/** Size of one Tensix core's L1 (1.5 MiB). */
#define FT_TENSIX_L1_SIZE_BYTES (1536u * 1024u)
/** Result of ft_tensix_noc_inline_dword_write_l1 (Foundry convention). */
enum {
FT_TENSIX_NOC_ISSUED = 0,
FT_TENSIX_NOC_REFUSED_ALIGNMENT = 1,
FT_TENSIX_NOC_REFUSED_NOT_L1 = 2
};
/** Packet-tag register of a command buffer; the transaction id is 4 bits at bit 10. */
#define FT_TENSIX_NOC_PACKET_TAG (FT_BH_NOC_REG_BASE + 0x18u)
#define FT_TENSIX_NOC_PACKET_TAG_TRID_SHIFT 10u
#define FT_TENSIX_NOC_PACKET_TAG_TRID_MASK 0xFu
/** NIU status counter "requests outstanding with transaction id `id`" (one per id 0..15). */
#define FT_TENSIX_NIU_MST_REQS_OUTSTANDING_ID(id) (0x10u + (id))
/** 64-bit NoC address of `address` in the core at (noc_x, noc_y). Forced. */
static FT_BH_INLINE uint64_t ft_tensix_noc_address(uint32_t noc_x, uint32_t noc_y,
uint32_t address)
{
return FT_BH_NOC_XY_ADDR(noc_x, noc_y, address);
}
/** Program and launch one read burst; the caller has waited for the buffer. Forced.
* The packet-control register is left alone: the firmware's setting stands for reads. */
static FT_BH_INLINE void ft_tensix_noc_issue_read(uint32_t noc, uint32_t command_buffer,
uint64_t source_address,
uint32_t destination_address,
uint32_t length_bytes,
uint32_t read_request_vc)
{
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_RETURN_ADDR_LOW, destination_address);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_TARGET_ADDR_LOW,
(uint32_t)source_address);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_TARGET_ADDR_MID,
(uint32_t)(source_address >> 32) & FT_BH_NOC_PCIE_HIGH_ADDR_MASK);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_TARGET_COORDINATE,
(uint32_t)(source_address >> FT_BH_NOC_COORDINATE_SHIFT) &
FT_BH_NOC_COORDINATE_MASK);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_ATOMIC_LEN_BE, length_bytes);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_COMMAND_CONTROL,
FT_BH_NOC_COMMAND_SEND);
noc_reads_num_issued[noc] += 1;
(void)read_request_vc;
}
/** Program and launch one non-posted unicast write burst (one acknowledgement). Forced. */
static FT_BH_INLINE void ft_tensix_noc_issue_write(uint32_t noc, uint32_t command_buffer,
uint32_t source_address,
uint64_t destination_address,
uint32_t length_bytes, uint32_t vc)
{
uint32_t packet_control = FT_BH_NOC_PACKET_COPY | FT_BH_NOC_PACKET_WRITE |
FT_BH_NOC_PACKET_STATIC_VC_ENABLE | FT_BH_NOC_PACKET_STATIC_VC(vc) |
FT_BH_NOC_PACKET_RESPONSE_MARKED;
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_PACKET_CONTROL, packet_control);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_TARGET_ADDR_LOW, source_address);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_RETURN_ADDR_LOW,
(uint32_t)destination_address);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_RETURN_ADDR_MID,
(uint32_t)(destination_address >> 32) & FT_BH_NOC_PCIE_HIGH_ADDR_MASK);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_RETURN_COORDINATE,
(uint32_t)(destination_address >> 36) & FT_BH_NOC_COORDINATE_MASK);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_ATOMIC_LEN_BE, length_bytes);
ft_bh_noc_cmd_write(noc, command_buffer, FT_BH_NOC_REG_COMMAND_CONTROL,
FT_BH_NOC_COMMAND_SEND);
noc_nonposted_writes_num_issued[noc] += 1;
noc_nonposted_writes_acked[noc] += 1;
}
/* ---- Counter predicates: hardware counter equals the software one. Forced. ---- */
static FT_BH_INLINE int ft_tensix_noc_reads_flushed(uint32_t noc)
{
return ft_bh_noc_status_read(noc, FT_BH_NOC_COUNTER_READ_RESPONSES_RECEIVED) ==
noc_reads_num_issued[noc];
}
static FT_BH_INLINE int ft_tensix_noc_writes_flushed(uint32_t noc)
{
return ft_bh_noc_status_read(noc, FT_BH_NOC_COUNTER_WRITE_ACKS_RECEIVED) ==
noc_nonposted_writes_acked[noc];
}
static FT_BH_INLINE int ft_tensix_noc_writes_sent(uint32_t noc)
{
return ft_bh_noc_status_read(noc, FT_BH_NOC_COUNTER_NONPOSTED_WRITES_SENT) ==
noc_nonposted_writes_num_issued[noc];
}
static FT_BH_INLINE int ft_tensix_noc_atomics_flushed(uint32_t noc)
{
return ft_bh_noc_status_read(noc, FT_BH_NOC_COUNTER_ATOMIC_RESPONSES_RECEIVED) ==
noc_nonposted_atomics_acked[noc];
}
static FT_BH_INLINE int ft_tensix_noc_posted_writes_sent(uint32_t noc)
{
return ft_bh_noc_status_read(noc, FT_BH_NOC_COUNTER_POSTED_WRITES_SENT) ==
noc_posted_writes_num_issued[noc];
}
/** Read `length_bytes` (non-zero) from NoC address `source` into local L1 `destination`,
* split into maximum-size bursts. Plain. */
static inline void ft_tensix_noc_read(uint64_t source, uint32_t destination,
uint32_t length_bytes)
{
while (length_bytes > FT_BH_NOC_MAX_BURST_BYTES) {
while (!ft_bh_noc_cmd_buf_ready(NOC_INDEX, FT_BH_READ_CMD_BUF))
;
ft_tensix_noc_issue_read(NOC_INDEX, FT_BH_READ_CMD_BUF, source, destination,
FT_BH_NOC_MAX_BURST_BYTES, 1);
source += FT_BH_NOC_MAX_BURST_BYTES;
destination += FT_BH_NOC_MAX_BURST_BYTES;
length_bytes -= FT_BH_NOC_MAX_BURST_BYTES;
}
while (!ft_bh_noc_cmd_buf_ready(NOC_INDEX, FT_BH_READ_CMD_BUF))
;
ft_tensix_noc_issue_read(NOC_INDEX, FT_BH_READ_CMD_BUF, source, destination, length_bytes, 1);
}
/** Write `length_bytes` (non-zero) of local L1 `source` to NoC address `destination`,
* split into maximum-size non-posted bursts. Plain. */
static inline void ft_tensix_noc_write(uint32_t source, uint64_t destination,
uint32_t length_bytes)
{
while (length_bytes > FT_BH_NOC_MAX_BURST_BYTES) {
while (!ft_bh_noc_cmd_buf_ready(NOC_INDEX, FT_BH_WRITE_CMD_BUF))
;
ft_tensix_noc_issue_write(NOC_INDEX, FT_BH_WRITE_CMD_BUF, source, destination,
FT_BH_NOC_MAX_BURST_BYTES, FT_BH_NOC_UNICAST_WRITE_VC);
source += FT_BH_NOC_MAX_BURST_BYTES;
destination += FT_BH_NOC_MAX_BURST_BYTES;
length_bytes -= FT_BH_NOC_MAX_BURST_BYTES;
}
while (!ft_bh_noc_cmd_buf_ready(NOC_INDEX, FT_BH_WRITE_CMD_BUF))
;
ft_tensix_noc_issue_write(NOC_INDEX, FT_BH_WRITE_CMD_BUF, source, destination, length_bytes,
FT_BH_NOC_UNICAST_WRITE_VC);
}
/** Non-posted remote atomic: add `increment` to the 32-bit word at `destination`. Plain. */
static inline void ft_tensix_noc_semaphore_increment(uint64_t destination, uint32_t increment)
{
while (!ft_bh_noc_cmd_buf_ready(NOC_INDEX, FT_BH_ATOMIC_CMD_BUF))
;
ft_bh_noc_cmd_write(NOC_INDEX, FT_BH_ATOMIC_CMD_BUF, FT_BH_NOC_REG_TARGET_ADDR_LOW,
(uint32_t)destination);
ft_bh_noc_cmd_write(NOC_INDEX, FT_BH_ATOMIC_CMD_BUF, FT_BH_NOC_REG_TARGET_ADDR_MID,
(uint32_t)(destination >> 32) & FT_BH_NOC_PCIE_HIGH_ADDR_MASK);
ft_bh_noc_cmd_write(NOC_INDEX, FT_BH_ATOMIC_CMD_BUF, FT_BH_NOC_REG_TARGET_COORDINATE,
(uint32_t)(destination >> 36) & FT_BH_NOC_COORDINATE_MASK);
ft_bh_noc_cmd_write(NOC_INDEX, FT_BH_ATOMIC_CMD_BUF, FT_BH_NOC_REG_PACKET_CONTROL,
FT_BH_NOC_PACKET_STATIC_VC_ENABLE |
FT_BH_NOC_PACKET_STATIC_VC(FT_BH_NOC_UNICAST_WRITE_VC) |
FT_BH_NOC_PACKET_RESPONSE_MARKED | FT_BH_NOC_PACKET_ATOMIC);
ft_bh_noc_cmd_write(NOC_INDEX, FT_BH_ATOMIC_CMD_BUF, FT_BH_NOC_REG_ATOMIC_LEN_BE,
FT_BH_NOC_ATOMIC_OPCODE(FT_BH_NOC_ATOMIC_OP_INCREMENT_GET) |
FT_BH_NOC_ATOMIC_WRAP(31u) |
FT_BH_NOC_ATOMIC_WORD_INDEX((((uint32_t)destination) >> 2) & 0x3u) |
FT_BH_NOC_ATOMIC_SOURCE_INDEX(0u));
ft_bh_noc_cmd_write(NOC_INDEX, FT_BH_ATOMIC_CMD_BUF, FT_BH_NOC_REG_ATOMIC_DATA, increment);
ft_bh_noc_cmd_write(NOC_INDEX, FT_BH_ATOMIC_CMD_BUF, FT_BH_NOC_REG_COMMAND_CONTROL,
FT_BH_NOC_COMMAND_SEND);
noc_nonposted_atomics_acked[NOC_INDEX] += 1;
}
/** Wait until every issued read has landed, then drop stale cached lines. Plain. */
static inline void ft_tensix_noc_read_barrier(void)
{
while (!ft_tensix_noc_reads_flushed(NOC_INDEX))
;
ft_bh_l1_cache_invalidate();
}
/** Wait until every non-posted write is acknowledged. Forced. */
static FT_BH_INLINE void ft_tensix_noc_write_barrier(void)
{
while (!ft_tensix_noc_writes_flushed(NOC_INDEX))
;
ft_bh_l1_cache_invalidate();
}
/** Wait until every non-posted write has left this core (sources reusable). Plain. */
static inline void ft_tensix_noc_writes_flushed_barrier(void)
{
while (!ft_tensix_noc_writes_sent(NOC_INDEX))
;
}
/** Wait until every issued atomic has its response. Forced. */
static FT_BH_INLINE void ft_tensix_noc_atomic_barrier(void)
{
while (!ft_tensix_noc_atomics_flushed(NOC_INDEX))
;
}
/** Write one 32-bit word into another core's L1 by staging it in this processor's own slot
* (Foundry L1 layout: one 16-byte slot per processor and NoC, word at the destination's
* offset within 16 bytes) and issuing a 4-byte non-posted write. Refusals issue nothing.
* Plain. */
static inline int ft_tensix_noc_inline_dword_write_l1(uint64_t destination, uint32_t value)
{
const uint32_t local = (uint32_t)destination;
uint32_t staging;
volatile FT_BH_L1_PTR uint32_t *staging_word;
if ((local & 0x3u) != 0u)
return FT_TENSIX_NOC_REFUSED_ALIGNMENT;
if (local >= FT_TENSIX_L1_SIZE_BYTES)
return FT_TENSIX_NOC_REFUSED_NOT_L1;
staging = FT_BH_L1_INLINE_STAGING_BASE +
(2u * FT_BH_L1_INLINE_STAGING_PER_NOC) * FT_BH_PROCESSOR_TYPE +
NOC_INDEX * FT_BH_L1_INLINE_STAGING_PER_NOC + (local & 0xFu);
while (!ft_tensix_noc_writes_sent(NOC_INDEX))
;
while (!ft_tensix_noc_posted_writes_sent(NOC_INDEX))
;
staging_word = (volatile FT_BH_L1_PTR uint32_t *)(uintptr_t)staging;
staging_word[0] = value;
while (!ft_bh_noc_cmd_buf_ready(NOC_INDEX, FT_BH_WRITE_CMD_BUF))
;
ft_tensix_noc_issue_write(NOC_INDEX, FT_BH_WRITE_CMD_BUF, staging, destination, 4u,
FT_BH_NOC_UNICAST_WRITE_VC);
return FT_TENSIX_NOC_ISSUED;
}
/** Tag later reads on the read command buffer with transaction id `trid` (sticky). Forced. */
static FT_BH_INLINE void ft_tensix_noc_read_set_trid(uint32_t trid)
{
while (!ft_bh_noc_cmd_buf_ready(NOC_INDEX, FT_BH_READ_CMD_BUF))
;
ft_bh_noc_cmd_write(NOC_INDEX, FT_BH_READ_CMD_BUF, FT_TENSIX_NOC_PACKET_TAG,
(trid & FT_TENSIX_NOC_PACKET_TAG_TRID_MASK)
<< FT_TENSIX_NOC_PACKET_TAG_TRID_SHIFT);
}
/** Wait until no read tagged `trid` is outstanding, then drop stale cached lines. Forced. */
static FT_BH_INLINE void ft_tensix_noc_read_barrier_trid(uint32_t trid)
{
while (ft_bh_noc_status_read(NOC_INDEX, FT_TENSIX_NIU_MST_REQS_OUTSTANDING_ID(trid)) != 0u)
;
ft_bh_l1_cache_invalidate();
}
#endif /* FT_TENSIX_NOC_H */
Reference in New Issue View Git Blame Copy Permalink