David
/
Foundry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Foundry/tt/ft_tt_queue.c

850 lines
35 KiB
C
Raw Blame History

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
/**
* @file ft_tt_queue.c
* @brief Host half of the Phase 5 work-queue protocol — see ft_tt_queue.h for the
* contract and the three rules it is shaped by.
*
* @status Green off-hardware in tests/test_queue.c and on the card in
* tests/test_phase5_resident.c, on two disjoint core sets, 2026-08-12.
* @issues None
* @todo None
*/
/* clock_gettime and CLOCK_MONOTONIC are POSIX.1-2008; -std=c99 turns off glibc's
* default feature set, so ask for them by name or the Linux card host's gcc hides them. */
#ifndef _POSIX_C_SOURCE
#define _POSIX_C_SOURCE 200809L
#endif
#include "ft_tt_queue.h"
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include "ft_tt_device.h"
/** Descriptors written in one device access. A batch is contiguous in the ring, so this
* bounds a stack buffer rather than the protocol. */
#define QUEUE_MAX_BATCH 1024u
/* LAYOUT PINS. Deliberately literal, not derived: a derived pin re-derives the mistake and
* passes, which is exactly how eleven self-consistent asserts on ft_tt_mailbox.h missed a
* 16-byte insertion (dev notes §3). These are the offsets the kernels have compiled
* against on silicon. Moving one is a rebuild of every resident kernel, not a header edit. */
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_CONTROL_BYTES == 64u, control_block_is_64_bytes);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_STATUS_OFFSET == 0x40u, status_at_0x40);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_STATUS_BYTES == 32u, status_slot_is_32_bytes);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_STATUS_OFFSET + FT_TT_PROC_COUNT * FT_TT_QUEUE_STATUS_BYTES <=
FT_TT_QUEUE_RING_OFFSET,
status_array_clears_the_ring);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_RING_OFFSET == 0x100u, ring_at_0x100);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_DESCRIPTOR_BYTES == 32u, descriptor_is_32_bytes);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_MATMUL_PARAM_OFFSET == 0x9000u, matmul_params_at_0x9000);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_GROUP_OFFSET == 0xF000u, group_block_at_0xF000);
/* The ring at its largest must not reach the parameter blocks, and the parameter blocks
* must not reach the group block. */
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_RING_OFFSET + QUEUE_MAX_BATCH * FT_TT_QUEUE_DESCRIPTOR_BYTES <=
FT_TT_QUEUE_MATMUL_PARAM_OFFSET,
max_ring_clears_the_parameter_blocks);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_MATMUL_PARAM_OFFSET +
FT_TT_QUEUE_MATMUL_PARAM_SLOTS * FT_TT_QUEUE_MATMUL_PARAM_BYTES <=
FT_TT_QUEUE_GROUP_OFFSET,
parameter_blocks_clear_the_group_block);
/* And the whole thing must fit the region ft_tt_op_l1_layout.h reserved for it. */
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_GROUP_OFFSET +
(FT_TT_QUEUE_GROUP_HEADER_WORDS +
FT_TT_QUEUE_GROUP_MAX_MEMBERS * FT_TT_QUEUE_GROUP_SLOT_WORDS) * 4u <=
FT_TT_OP_QUEUE_BYTES,
group_block_fits_the_queue_region);
/* ------------------------------------------------------------------------- */
/* Small helpers */
/* ------------------------------------------------------------------------- */
static bool is_power_of_two(uint32_t value)
{
return value != 0u && (value & (value - 1u)) == 0u;
}
static uint32_t control_field_address(const ft_tt_queue *queue, ft_tt_queue_control_field field)
{
return queue->control_address + (uint32_t)field * 4u;
}
static int write_control_word(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
ft_tt_queue_control_field field, uint32_t value)
{
return noc->write(noc->user, queue->x, queue->y, control_field_address(queue, field), &value,
sizeof(value));
}
static uint64_t monotonic_microseconds(void)
{
struct timespec now;
if (clock_gettime(CLOCK_MONOTONIC, &now) != 0)
return 0u;
return (uint64_t)now.tv_sec * 1000000u + (uint64_t)(now.tv_nsec / 1000);
}
/* ------------------------------------------------------------------------- */
/* Setup */
/* ------------------------------------------------------------------------- */
uint32_t ft_tt_queue_make_nonce(void)
{
uint64_t mixed = monotonic_microseconds();
uint32_t nonce;
mixed ^= (uint64_t)getpid() << 32;
mixed ^= mixed >> 29;
mixed *= 0xBF58476D1CE4E5B9u;
mixed ^= mixed >> 32;
nonce = (uint32_t)mixed;
/* Never 0 and never the poison: both are values a completion check must be able to
* tell a real token APART from. */
if (nonce == 0u || nonce == (uint32_t)FT_TT_QUEUE_STATE_HOST_POISON)
nonce = 0x5150C0DEu;
return nonce;
}
uint32_t ft_tt_queue_token_at(const ft_tt_queue *queue, uint32_t index)
{
uint32_t token;
if (!queue)
return 0u;
/* Index is mixed in as an XOR of a rotated counter rather than added, so two runs
* whose nonces differ by a small amount cannot produce overlapping token sets. */
token = queue->run_nonce ^ ((index + 1u) * 0x9E3779B9u);
if (token == 0u || token == (uint32_t)FT_TT_QUEUE_STATE_HOST_POISON)
token = queue->run_nonce ^ 0xA5A5A5A5u;
return token;
}
int ft_tt_queue_init(const ft_tt_noc_access *noc, uint16_t x, uint16_t y,
uint32_t control_address, uint32_t capacity, uint32_t run_nonce,
ft_tt_queue *out_queue)
{
uint32_t block[FT_TT_QUEUE_FIELD_COUNT];
ft_tt_queue queue = {0};
int rc;
if (!noc || !noc->write || !out_queue)
return FT_TT_EINVAL;
if ((control_address & 3u) != 0u)
return FT_TT_EINVAL;
if (!is_power_of_two(capacity) || capacity < 2u)
return FT_TT_EINVAL;
if (run_nonce == 0u || run_nonce == (uint32_t)FT_TT_QUEUE_STATE_HOST_POISON)
return FT_TT_EINVAL;
queue.x = x;
queue.y = y;
queue.control_address = control_address;
queue.ring_address = control_address + FT_TT_QUEUE_RING_OFFSET;
queue.capacity = capacity;
queue.head = 0u;
queue.run_nonce = run_nonce;
queue.issued = 0u;
memset(block, 0, sizeof(block));
block[FT_TT_QUEUE_FIELD_MAGIC] = FT_TT_QUEUE_MAGIC;
block[FT_TT_QUEUE_FIELD_CAPACITY] = capacity;
block[FT_TT_QUEUE_FIELD_STATE] = (uint32_t)FT_TT_QUEUE_STATE_HOST_POISON;
/* The token slot is poisoned too. A zero there would be indistinguishable from a
* kernel that completed an item whose token happened to be zero — which is why
* ft_tt_queue_token_at() never issues one. */
block[FT_TT_QUEUE_FIELD_COMPLETED_TOKEN] = (uint32_t)FT_TT_QUEUE_STATE_HOST_POISON;
rc = noc->write(noc->user, x, y, control_address, block, sizeof(block));
if (rc != FT_TT_OK) {
ft_tt_log_error("queue: writing the control block at 0x%08x failed (%d)",
control_address, rc);
return rc;
}
*out_queue = queue;
return FT_TT_OK;
}
int ft_tt_queue_kernel_args(const ft_tt_queue *queue, const ft_tt_queue_kernel_binding *binding,
uint32_t out_args[FT_TT_QUEUE_KERNEL_ARG_COUNT])
{
ft_tt_queue_kernel_binding defaults = {0};
if (!queue || !out_args)
return FT_TT_EINVAL;
if (!is_power_of_two(queue->capacity))
return FT_TT_EINVAL;
if (!binding) {
defaults.idle_spin_limit = FT_TT_QUEUE_DEFAULT_IDLE_SPINS;
defaults.cb_wait_spin_limit = FT_TT_QUEUE_DEFAULT_IDLE_SPINS;
binding = &defaults;
}
/* 0 would have to mean "spin forever" on either bound, and a resident kernel with an
* unbounded wait is the one thing this phase may not ship (ft_tt_queue.h rule 3, and
* kernels/resident_queue_protocol.h rule 4). */
if (binding->idle_spin_limit == 0u || binding->cb_wait_spin_limit == 0u)
return FT_TT_EINVAL;
if (binding->group_address != 0u && binding->group_member >= FT_TT_QUEUE_GROUP_MAX_MEMBERS)
return FT_TT_EINVAL;
out_args[0] = queue->control_address;
out_args[1] = queue->capacity - 1u;
out_args[2] = queue->ring_address;
out_args[3] = binding->idle_spin_limit;
out_args[4] = binding->status_address;
out_args[5] = binding->cb_wait_spin_limit;
out_args[6] = binding->group_x;
out_args[7] = binding->group_y;
out_args[8] = binding->group_address;
out_args[9] = binding->group_member;
return FT_TT_OK;
}
int ft_tt_queue_read_status(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
unsigned processor, uint32_t out_fields[FT_TT_QUEUE_STATUS_WORDS])
{
if (!noc || !noc->read || !queue || !out_fields)
return FT_TT_EINVAL;
if (processor >= FT_TT_PROC_COUNT)
return FT_TT_EINVAL;
return noc->read(noc->user, queue->x, queue->y,
FT_TT_QUEUE_STATUS_ADDRESS(queue->control_address, processor), out_fields,
FT_TT_QUEUE_STATUS_BYTES);
}
/* ------------------------------------------------------------------------- */
/* The group completion block, addressed by COORDINATE */
/* ------------------------------------------------------------------------- */
/** L1 address of slot 0 — where the hot-path read starts, past the header. */
static uint32_t group_slots_address(uint32_t group_address)
{
return group_address + FT_TT_QUEUE_GROUP_HEADER_WORDS * 4u;
}
uint32_t ft_tt_queue_group_block_words(uint32_t members)
{
return FT_TT_QUEUE_GROUP_HEADER_WORDS + members * FT_TT_QUEUE_GROUP_SLOT_WORDS;
}
int ft_tt_queue_group_init(const ft_tt_noc_access *noc, uint16_t x, uint16_t y,
uint32_t group_address, uint32_t members)
{
uint32_t block[FT_TT_QUEUE_GROUP_HEADER_WORDS +
FT_TT_QUEUE_GROUP_MAX_MEMBERS * FT_TT_QUEUE_GROUP_SLOT_WORDS];
if (!noc || !noc->write || group_address == 0u || (group_address & 3u) != 0u)
return FT_TT_EINVAL;
if (members == 0u)
return FT_TT_EINVAL;
if (members > FT_TT_QUEUE_GROUP_MAX_MEMBERS)
return FT_TT_ENOMEM;
memset(block, 0, sizeof(block));
block[FT_TT_QUEUE_GROUP_MAGIC] = FT_TT_QUEUE_GROUP_MAGIC_VALUE;
block[FT_TT_QUEUE_GROUP_MEMBERS] = members;
return noc->write(noc->user, x, y, group_address, block,
(size_t)ft_tt_queue_group_block_words(members) * 4u);
}
int ft_tt_queue_group_read(const ft_tt_noc_access *noc, uint16_t x, uint16_t y,
uint32_t group_address, uint32_t members,
ft_tt_queue_group_slot *out_slots)
{
uint32_t block[FT_TT_QUEUE_GROUP_HEADER_WORDS +
FT_TT_QUEUE_GROUP_MAX_MEMBERS * FT_TT_QUEUE_GROUP_SLOT_WORDS];
uint32_t slot;
int rc;
if (!noc || !noc->read || !out_slots || group_address == 0u || (group_address & 3u) != 0u)
return FT_TT_EINVAL;
if (members == 0u || members > FT_TT_QUEUE_GROUP_MAX_MEMBERS)
return FT_TT_EINVAL;
rc = noc->read(noc->user, x, y, group_address, block,
(size_t)ft_tt_queue_group_block_words(members) * 4u);
if (rc != FT_TT_OK)
return rc;
if (block[FT_TT_QUEUE_GROUP_MAGIC] != FT_TT_QUEUE_GROUP_MAGIC_VALUE)
return FT_TT_ERROR;
for (slot = 0u; slot < members; slot++) {
const uint32_t base =
FT_TT_QUEUE_GROUP_HEADER_WORDS + slot * FT_TT_QUEUE_GROUP_SLOT_WORDS;
out_slots[slot].sequence = block[base];
out_slots[slot].token = block[base + 1u];
}
return FT_TT_OK;
}
int ft_tt_queue_group_read_slots(const ft_tt_noc_access *noc, uint16_t x, uint16_t y,
uint32_t group_address, uint32_t members,
ft_tt_queue_group_slot *out_slots)
{
/* One slot of slack: an ODD member count is rounded UP to the 16-byte unit below. */
uint32_t words[(FT_TT_QUEUE_GROUP_MAX_MEMBERS + 1u) * FT_TT_QUEUE_GROUP_SLOT_WORDS];
size_t bytes;
uint32_t slot;
int rc;
if (!noc || !noc->read || !out_slots || group_address == 0u || (group_address & 3u) != 0u)
return FT_TT_EINVAL;
if (members == 0u || members > FT_TT_QUEUE_GROUP_MAX_MEMBERS)
return FT_TT_EINVAL;
/* THE 16-BYTE UNIT IS THE POINT OF THIS CALL, so the length has to respect it: a slot is
* 8 bytes and an ODD member count is not a multiple of 16. Reading one slot further is
* free — the block is 4 KiB and the header keeps the slots 16-byte aligned — and the
* extra slot is simply not looked at. Getting this wrong is not a wrong answer: the L1
* accessor REFUSES the length, and here that refusal reached the forward as a failed
* projection, which retired and re-armed the residency three times in a row. */
bytes = ((size_t)members * FT_TT_QUEUE_GROUP_SLOT_WORDS * 4u + 15u) & ~(size_t)15u;
rc = noc->read(noc->user, x, y, group_slots_address(group_address), words, bytes);
if (rc != FT_TT_OK)
return rc;
for (slot = 0u; slot < members; slot++) {
out_slots[slot].sequence = words[slot * FT_TT_QUEUE_GROUP_SLOT_WORDS];
out_slots[slot].token = words[slot * FT_TT_QUEUE_GROUP_SLOT_WORDS + 1u];
}
return FT_TT_OK;
}
int ft_tt_queue_all_processors_exited(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t processor_mask, uint32_t *out_pending)
{
uint32_t pending = 0u;
unsigned processor;
if (!noc || !noc->read || !queue)
return FT_TT_EINVAL;
if (processor_mask == 0u) {
/* The BRISC-only kernel: the control block's own state IS the whole answer, because
* there is exactly one processor and it is the writer of that word. */
uint32_t state = 0u;
int rc = ft_tt_queue_read_field(noc, queue, FT_TT_QUEUE_FIELD_STATE, &state);
if (rc != FT_TT_OK)
return rc;
if (out_pending)
*out_pending = (state == (uint32_t)FT_TT_QUEUE_STATE_EXIT) ? 0u : 1u;
return (state == (uint32_t)FT_TT_QUEUE_STATE_EXIT) ? FT_TT_OK : FT_TT_EIO;
}
for (processor = 0; processor < FT_TT_PROC_COUNT; processor++) {
uint32_t fields[FT_TT_QUEUE_STATUS_WORDS];
int rc;
if ((processor_mask & (1u << processor)) == 0u)
continue;
rc = ft_tt_queue_read_status(noc, queue, processor, fields);
if (rc != FT_TT_OK)
return rc;
if (fields[FT_TT_QUEUE_STATUS_STATE] != (uint32_t)FT_TT_QUEUE_STATE_EXIT)
pending |= 1u << processor;
}
if (out_pending)
*out_pending = pending;
return pending ? FT_TT_EIO : FT_TT_OK;
}
int ft_tt_queue_write_op_parameters(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t slot, const uint32_t words[FT_TT_QUEUE_PARAM_WORDS],
uint32_t *out_addr)
{
return ft_tt_queue_write_matmul_parameters(noc, queue, slot, words, out_addr);
}
int ft_tt_queue_write_matmul_parameters(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t slot,
const uint32_t words[FT_TT_QUEUE_MATMUL_WORDS],
uint32_t *out_addr)
{
uint32_t address;
int rc;
if (!noc || !noc->write || !queue || !words)
return FT_TT_EINVAL;
if (slot >= FT_TT_QUEUE_MATMUL_PARAM_SLOTS)
return FT_TT_EINVAL;
address = FT_TT_QUEUE_MATMUL_PARAM_ADDRESS(queue->control_address, slot);
rc = noc->write(noc->user, queue->x, queue->y, address, words, FT_TT_QUEUE_MATMUL_PARAM_BYTES);
if (rc != FT_TT_OK)
return rc;
if (out_addr)
*out_addr = address;
return FT_TT_OK;
}
int ft_tt_queue_write_param_cache(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t slot, const uint32_t words[FT_TT_QUEUE_PARAM_WORDS],
uint32_t *out_addr)
{
uint32_t address;
int rc;
if (!noc || !noc->write || !queue || !words)
return FT_TT_EINVAL;
if (slot >= FT_TT_QUEUE_PARAM_CACHE_SLOTS)
return FT_TT_EINVAL;
address = FT_TT_QUEUE_PARAM_CACHE_ADDRESS(queue->control_address, slot);
rc = noc->write(noc->user, queue->x, queue->y, address, words,
FT_TT_QUEUE_MATMUL_PARAM_BYTES);
if (rc != FT_TT_OK)
return rc;
if (out_addr)
*out_addr = address;
return FT_TT_OK;
}
/* The cache must start where the rotating slots END and end where the group block BEGINS.
* The point of the region is that it is the GAP, so a pin that re-derived the gap could not
* notice the gap closing — these compare the three offsets against each other directly. */
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_MATMUL_PARAM_OFFSET +
FT_TT_QUEUE_MATMUL_PARAM_SLOTS * FT_TT_QUEUE_MATMUL_PARAM_BYTES <=
FT_TT_QUEUE_PARAM_CACHE_OFFSET,
param_cache_clears_the_rotating_slots);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_PARAM_CACHE_OFFSET +
FT_TT_QUEUE_PARAM_CACHE_SLOTS * FT_TT_QUEUE_MATMUL_PARAM_BYTES <=
FT_TT_QUEUE_GROUP_OFFSET,
param_cache_clears_the_group_block);
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_PARAM_CACHE_OFFSET % 16u == 0u, param_cache_is_l1_aligned);
/* ------------------------------------------------------------------------- */
/* Enqueue — the whole per-op cost of the model */
/* ------------------------------------------------------------------------- */
/** Write a run of descriptors that does not wrap. */
static int write_ring_run(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t first_slot, const ft_tt_queue_descriptor *descriptors,
uint32_t count)
{
uint32_t address = queue->ring_address + first_slot * FT_TT_QUEUE_DESCRIPTOR_BYTES;
return noc->write(noc->user, queue->x, queue->y, address, descriptors,
(size_t)count * FT_TT_QUEUE_DESCRIPTOR_BYTES);
}
int ft_tt_queue_stage(const ft_tt_noc_access *noc, ft_tt_queue *queue,
ft_tt_queue_descriptor *descriptors, uint32_t count)
{
uint32_t first_slot;
uint32_t contiguous;
uint32_t index;
int rc;
if (!noc || !noc->write || !queue || !descriptors || count == 0u)
return FT_TT_EINVAL;
if (count > queue->capacity || count > QUEUE_MAX_BATCH)
return FT_TT_ENOMEM;
/* Stamp the tokens here rather than trusting the caller. A caller that supplies its
* own token can supply one a previous run also used, and then the completion check
* passes on residue. */
for (index = 0u; index < count; index++)
descriptors[index].token = ft_tt_queue_token_at(queue, queue->issued + index);
first_slot = (queue->head + queue->staged) & (queue->capacity - 1u);
contiguous = queue->capacity - first_slot;
if (contiguous > count)
contiguous = count;
rc = write_ring_run(noc, queue, first_slot, descriptors, contiguous);
if (rc != FT_TT_OK)
return rc;
if (contiguous < count) {
rc = write_ring_run(noc, queue, 0u, descriptors + contiguous, count - contiguous);
if (rc != FT_TT_OK)
return rc;
}
queue->staged += count;
queue->issued += count;
return FT_TT_OK;
}
int ft_tt_queue_publish(const ft_tt_noc_access *noc, ft_tt_queue *queue)
{
uint32_t published;
int rc;
if (!noc || !noc->write || !queue)
return FT_TT_EINVAL;
if (queue->staged == 0u)
return FT_TT_OK;
published = queue->head + queue->staged;
rc = write_control_word(noc, queue, FT_TT_QUEUE_FIELD_HEAD, published);
if (rc != FT_TT_OK)
return rc;
queue->head = published;
queue->staged = 0u;
return FT_TT_OK;
}
bool ft_tt_queue_issue_is_wide_addressable(const ft_tt_queue *queue)
{
if (!queue)
return false;
/* ASKED OF THE SHIPPED RULE, not re-derived from it. `ft_tt_noc_check_transfer()` is the
* function the BLOCK16 L1 accessor itself calls, so this predicate and the refusal it
* predicts cannot drift apart. Slot 0 answers for every slot: both parameter regions
* stride by 64 bytes and the ring by 32, and every one of those is a multiple of 16. */
return ft_tt_noc_check_transfer(FT_TT_MEM_SPACE_L1, false,
FT_TT_QUEUE_PARAM_ADDRESS(queue->control_address, 0u),
FT_TT_QUEUE_MATMUL_PARAM_BYTES) == FT_TT_OK &&
ft_tt_noc_check_transfer(FT_TT_MEM_SPACE_L1, false,
FT_TT_QUEUE_PARAM_CACHE_ADDRESS(queue->control_address, 0u),
FT_TT_QUEUE_MATMUL_PARAM_BYTES) == FT_TT_OK &&
ft_tt_noc_check_transfer(FT_TT_MEM_SPACE_L1, false, queue->ring_address,
FT_TT_QUEUE_DESCRIPTOR_BYTES) == FT_TT_OK;
}
int ft_tt_queue_write_head(const ft_tt_noc_access *noc, const ft_tt_queue *queue, uint32_t head)
{
if (!noc || !noc->write || !queue)
return FT_TT_EINVAL;
return write_control_word(noc, queue, FT_TT_QUEUE_FIELD_HEAD, head);
}
int ft_tt_queue_enqueue(const ft_tt_noc_access *noc, ft_tt_queue *queue,
ft_tt_queue_descriptor *descriptors, uint32_t count)
{
int rc = ft_tt_queue_stage(noc, queue, descriptors, count);
if (rc != FT_TT_OK)
return rc;
/* The head bump is LAST and is one word. */
return ft_tt_queue_publish(noc, queue);
}
/* ------------------------------------------------------------------------- */
/* Observation */
/* ------------------------------------------------------------------------- */
int ft_tt_queue_read_control(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t out_fields[FT_TT_QUEUE_FIELD_COUNT])
{
if (!noc || !noc->read || !queue || !out_fields)
return FT_TT_EINVAL;
return noc->read(noc->user, queue->x, queue->y, queue->control_address, out_fields,
FT_TT_QUEUE_CONTROL_BYTES);
}
int ft_tt_queue_read_completion(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t out_fields[FT_TT_QUEUE_FIELD_COUNT])
{
if (!noc || !noc->read || !queue || !out_fields)
return FT_TT_EINVAL;
/* The unread fields are ZEROED rather than left indeterminate. A caller that reaches for
* HEAD out of this buffer then reads a 0 it can see, instead of whatever the stack had —
* and the four the completion check names are all inside the window. */
memset(out_fields, 0, FT_TT_QUEUE_FIELD_COUNT * sizeof(out_fields[0]));
return noc->read(noc->user, queue->x, queue->y,
control_field_address(queue, FT_TT_QUEUE_COMPLETION_FIRST_FIELD),
&out_fields[FT_TT_QUEUE_COMPLETION_FIRST_FIELD],
FT_TT_QUEUE_COMPLETION_FIELD_COUNT * 4u);
}
/** First field of the 16-byte-aligned span that CONTAINS the completion window, and how
* many fields that span holds. Word 5 does not start on a 16-byte boundary; word 4 does,
* and eight fields from there is exactly two BLOCK16 transactions. */
#define QUEUE_WIDE_FIRST_FIELD 4u
#define QUEUE_WIDE_FIELD_COUNT 8u
int ft_tt_queue_read_completion_wide(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t out_fields[FT_TT_QUEUE_FIELD_COUNT])
{
if (!noc || !noc->read || !queue || !out_fields)
return FT_TT_EINVAL;
memset(out_fields, 0, FT_TT_QUEUE_FIELD_COUNT * sizeof(out_fields[0]));
return noc->read(noc->user, queue->x, queue->y,
queue->control_address + QUEUE_WIDE_FIRST_FIELD * 4u,
&out_fields[QUEUE_WIDE_FIRST_FIELD], QUEUE_WIDE_FIELD_COUNT * 4u);
}
/** The ONE 16-byte line that holds count, token and state. Word 4 starts it; four fields is
* exactly one BLOCK16 transaction. `error` is word 9 and is deliberately outside it. */
#define QUEUE_LINE_FIRST_FIELD 4u
#define QUEUE_LINE_FIELD_COUNT 4u
int ft_tt_queue_read_completion_line(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t out_fields[FT_TT_QUEUE_FIELD_COUNT])
{
if (!noc || !noc->read || !queue || !out_fields)
return FT_TT_EINVAL;
memset(out_fields, 0, FT_TT_QUEUE_FIELD_COUNT * sizeof(out_fields[0]));
return noc->read(noc->user, queue->x, queue->y,
queue->control_address + QUEUE_LINE_FIRST_FIELD * 4u,
&out_fields[QUEUE_LINE_FIRST_FIELD], QUEUE_LINE_FIELD_COUNT * 4u);
}
int ft_tt_queue_check_completion_line(const ft_tt_queue *queue,
const uint32_t fields[FT_TT_QUEUE_FIELD_COUNT],
uint32_t expected, const char **out_reason)
{
const char *reason = NULL;
if (!queue || !fields || expected == 0u) {
if (out_reason)
*out_reason = "bad argument";
return FT_TT_EINVAL;
}
/* THE SAME THREE TESTS IN THE SAME ORDER as ft_tt_queue_check_completion(), with the
* `error` clause removed rather than reordered — so a divergence between the two is a
* missing clause and not a different verdict. */
if (fields[FT_TT_QUEUE_FIELD_STATE] == (uint32_t)FT_TT_QUEUE_STATE_HOST_POISON)
reason = "the kernel never ran: state is still the host poison";
else if (fields[FT_TT_QUEUE_FIELD_COMPLETED_COUNT] != expected)
reason = "completed_count does not match the number enqueued";
else if (fields[FT_TT_QUEUE_FIELD_COMPLETED_TOKEN] !=
ft_tt_queue_token_at(queue, expected - 1u))
reason = "completed_token is not this run's last token";
if (out_reason)
*out_reason = reason;
return reason ? FT_TT_ERROR : FT_TT_OK;
}
bool ft_tt_queue_completion_line_is_addressable(const ft_tt_queue *queue)
{
if (!queue)
return false;
/* The SPAN, not just the base: the length is a compile-time 16 and is pinned by the
* assert below, so the only runtime variable is where the control block was placed. */
return ((queue->control_address + QUEUE_LINE_FIRST_FIELD * 4u) % 16u) == 0u;
}
FT_TT_STATIC_ASSERT(QUEUE_LINE_FIRST_FIELD * 4u % 16u == 0u, line_span_is_16_byte_aligned);
FT_TT_STATIC_ASSERT(QUEUE_LINE_FIELD_COUNT * 4u == 16u, line_span_is_exactly_one_transaction);
FT_TT_STATIC_ASSERT(QUEUE_LINE_FIRST_FIELD <= FT_TT_QUEUE_FIELD_COMPLETED_COUNT &&
QUEUE_LINE_FIRST_FIELD + QUEUE_LINE_FIELD_COUNT >
FT_TT_QUEUE_FIELD_COMPLETED_COUNT,
line_span_holds_completed_count);
FT_TT_STATIC_ASSERT(QUEUE_LINE_FIRST_FIELD + QUEUE_LINE_FIELD_COUNT >
FT_TT_QUEUE_FIELD_COMPLETED_TOKEN,
line_span_holds_completed_token);
FT_TT_STATIC_ASSERT(QUEUE_LINE_FIRST_FIELD + QUEUE_LINE_FIELD_COUNT > FT_TT_QUEUE_FIELD_STATE,
line_span_holds_state);
/* AND THE ONE THAT IS THE POINT: `error` is NOT in the line. If a later renumbering brings
* it in, this assert fires and whoever moved it must delete the line/wide split rather than
* leave a check whose documentation is a lie. */
FT_TT_STATIC_ASSERT(FT_TT_QUEUE_FIELD_ERROR >= QUEUE_LINE_FIRST_FIELD + QUEUE_LINE_FIELD_COUNT,
line_span_deliberately_excludes_error);
FT_TT_STATIC_ASSERT(QUEUE_WIDE_FIRST_FIELD * 4u % 16u == 0u, wide_span_is_16_byte_aligned);
FT_TT_STATIC_ASSERT(QUEUE_WIDE_FIELD_COUNT * 4u % 16u == 0u, wide_span_is_a_16_byte_multiple);
FT_TT_STATIC_ASSERT(QUEUE_WIDE_FIRST_FIELD <= FT_TT_QUEUE_COMPLETION_FIRST_FIELD &&
QUEUE_WIDE_FIRST_FIELD + QUEUE_WIDE_FIELD_COUNT >=
FT_TT_QUEUE_COMPLETION_FIRST_FIELD +
FT_TT_QUEUE_COMPLETION_FIELD_COUNT,
wide_span_contains_the_completion_window);
int ft_tt_queue_read_field(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
ft_tt_queue_control_field field, uint32_t *out_value)
{
if (!noc || !noc->read || !queue || !out_value)
return FT_TT_EINVAL;
if ((unsigned)field >= FT_TT_QUEUE_FIELD_COUNT)
return FT_TT_EINVAL;
return noc->read(noc->user, queue->x, queue->y, control_field_address(queue, field), out_value,
sizeof(*out_value));
}
int ft_tt_queue_request_stop(const ft_tt_noc_access *noc, const ft_tt_queue *queue)
{
if (!noc || !noc->write || !queue)
return FT_TT_EINVAL;
return write_control_word(noc, queue, FT_TT_QUEUE_FIELD_STOP_REQUEST, FT_TT_QUEUE_STOP_REQUESTED);
}
uint32_t ft_tt_queue_stop_request_address(const ft_tt_queue *queue)
{
/* Deliberately the SAME derivation ft_tt_queue_request_stop() uses, so the address a
* DYING process hands the driver and the address a LIVING one writes cannot drift
* apart — two spellings of one address is how a cleanup silently stops firing. */
return queue ? control_field_address(queue, FT_TT_QUEUE_FIELD_STOP_REQUEST) : 0u;
}
int ft_tt_queue_wait_completed(const ft_tt_noc_access *noc, const ft_tt_queue *queue,
uint32_t target, uint32_t timeout_ms, uint32_t *out_completed)
{
uint64_t deadline;
uint32_t completed = 0u;
if (!noc || !noc->read || !queue)
return FT_TT_EINVAL;
/* Same rule as ft_tt_launch_wait_done(): there is no untimed path on a board where a
* hang costs a physical power cycle. */
if (timeout_ms == 0u)
return FT_TT_EINVAL;
deadline = monotonic_microseconds() + (uint64_t)timeout_ms * 1000u;
for (;;) {
int rc = ft_tt_queue_read_field(noc, queue, FT_TT_QUEUE_FIELD_COMPLETED_COUNT, &completed);
if (rc != FT_TT_OK)
return rc;
if (completed >= target) {
if (out_completed)
*out_completed = completed;
return FT_TT_OK;
}
if (monotonic_microseconds() >= deadline) {
if (out_completed)
*out_completed = completed;
return FT_TT_EIO;
}
}
}
int ft_tt_queue_check_completion(const ft_tt_queue *queue,
const uint32_t fields[FT_TT_QUEUE_FIELD_COUNT],
uint32_t expected, const char **out_reason)
{
const char *reason = NULL;
if (!queue || !fields || expected == 0u) {
if (out_reason)
*out_reason = "bad argument";
return FT_TT_EINVAL;
}
if (fields[FT_TT_QUEUE_FIELD_STATE] == (uint32_t)FT_TT_QUEUE_STATE_HOST_POISON)
reason = "the kernel never ran: state is still the host poison";
else if (fields[FT_TT_QUEUE_FIELD_ERROR] != (uint32_t)FT_TT_QUEUE_KERR_NONE)
reason = ft_tt_queue_kernel_error_string(fields[FT_TT_QUEUE_FIELD_ERROR]);
else if (fields[FT_TT_QUEUE_FIELD_COMPLETED_COUNT] != expected)
reason = "completed_count does not match the number enqueued";
else if (fields[FT_TT_QUEUE_FIELD_COMPLETED_TOKEN] !=
ft_tt_queue_token_at(queue, expected - 1u))
reason = "completed_token is not this run's last token";
if (out_reason)
*out_reason = reason;
return reason ? FT_TT_ERROR : FT_TT_OK;
}
/* ------------------------------------------------------------------------- */
/* Strings */
/* ------------------------------------------------------------------------- */
const char *ft_tt_queue_state_string(uint32_t state)
{
switch (state) {
case FT_TT_QUEUE_STATE_HOST_POISON:
return "HOST_POISON (the kernel never ran)";
case FT_TT_QUEUE_STATE_BOOT:
return "BOOT";
case FT_TT_QUEUE_STATE_IDLE:
return "IDLE";
case FT_TT_QUEUE_STATE_EXEC:
return "EXEC";
case FT_TT_QUEUE_STATE_SEM_WAIT:
return "SEM_WAIT";
case FT_TT_QUEUE_STATE_EXIT:
return "EXIT";
case FT_TT_QUEUE_STATE_CB_WAIT_FRONT:
return "CB_WAIT_FRONT";
case FT_TT_QUEUE_STATE_CB_RESERVE_BACK:
return "CB_RESERVE_BACK";
default:
return "unknown";
}
}
const char *ft_tt_queue_kernel_error_string(uint32_t error)
{
switch (error) {
case FT_TT_QUEUE_KERR_NONE:
return "none";
case FT_TT_QUEUE_KERR_BAD_MAGIC:
return "the control block did not carry the queue magic";
case FT_TT_QUEUE_KERR_BAD_OPCODE:
return "a descriptor named an opcode the kernel does not implement";
case FT_TT_QUEUE_KERR_IDLE_TIMEOUT:
return "the idle spin limit expired: the host stopped feeding the queue";
case FT_TT_QUEUE_KERR_SEM_WAIT_TIMEOUT:
return "a semaphore wait exhausted its spin bound";
case FT_TT_QUEUE_KERR_BAD_CAPACITY:
return "the capacity mask is not one less than a power of two";
case FT_TT_QUEUE_KERR_CB_WAIT_TIMEOUT:
return "a circular-buffer wait exhausted its spin bound";
case FT_TT_QUEUE_KERR_BAD_PARAMETERS:
return "a runtime argument was refused: an unbounded wait, or a queue whose tail "
"and completed_count disagree";
case FT_TT_QUEUE_KERR_COOP_ROLE_MISMATCH:
return "a descriptor's cooperate word disagreed with this member's role: "
"cooperation is all-or-nothing for a residency";
case FT_TT_QUEUE_KERR_COOP_WRITE_POINTER:
return "CB_IN0's write pointer is not where this member's own block count says: "
"the group's circular buffers have drifted apart";
case FT_TT_QUEUE_KERR_COOP_RENDEZVOUS:
return "a multicast rendezvous wait was abandoned: the group is not in lockstep, "
"or the host asked it to stop";
case FT_TT_QUEUE_KERR_COOP_ISSUE:
return "a multicast was refused at the point of issue: a zero length, zero "
"destinations, a misalignment or an inverted rectangle";
case FT_TT_QUEUE_KERR_COOP_BAD_PARTITION:
return "the cooperative runtime args are illegal: a piece over four destinations, "
"an inverted rectangle, or a destination total that is not the receiver count";
default:
return "unknown";
}
}
const char *ft_tt_queue_opcode_string(uint32_t opcode)
{
switch (opcode) {
case FT_TT_QUEUE_OP_NOP:
return "NOP";
case FT_TT_QUEUE_OP_WRITE_WORD:
return "WRITE_WORD";
case FT_TT_QUEUE_OP_L1_COPY:
return "L1_COPY";
case FT_TT_QUEUE_OP_SEM_POST:
return "SEM_POST";
case FT_TT_QUEUE_OP_SEM_WAIT:
return "SEM_WAIT";
case FT_TT_QUEUE_OP_DELAY:
return "DELAY";
case FT_TT_QUEUE_OP_STOP:
return "STOP";
case FT_TT_QUEUE_OP_MATMUL:
return "MATMUL";
case FT_TT_QUEUE_OP_ELTWISE:
return "ELTWISE";
case FT_TT_QUEUE_OP_SOFTMAX:
return "SOFTMAX";
case FT_TT_QUEUE_OP_REDUCE:
return "REDUCE";
case FT_TT_QUEUE_OP_EMBEDDING:
return "EMBEDDING";
case FT_TT_QUEUE_OP_KV_APPEND:
return "KV_APPEND";
case FT_TT_QUEUE_OP_ROPE:
return "ROPE";
case FT_TT_QUEUE_OP_ATTENTION:
return "ATTENTION";
case FT_TT_QUEUE_OP_GDN:
return "GDN";
case FT_TT_QUEUE_OP_GDN_CHUNK:
return "GDN_CHUNK";
case FT_TT_QUEUE_OP_GDN_GATE:
return "GDN_GATE";
default:
return "unknown";
}
}
Reference in New Issue View Git Blame Copy Permalink