David
/
Foundry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Foundry/tt/ft_tt_udrv_noc_bank.c

787 lines
33 KiB
C
Raw Blame History

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
/**
* @file ft_tt_udrv_noc_bank.c
* @brief Implementation of the multi-window accessor (see the header for the
* whole argument, including the audit and the mis-aim red arm).
*
* Everything below the window CHOICE is unchanged: once a window is chosen
* this calls `ft_tt_udrv_noc_transfer()` on it, so the slice arithmetic, the
* copy shape, the store fence and the aim verification are the ones already
* on silicon. This file only decides WHICH window.
*
* ---------------------------------------------------------------------------
* WHY THE LOOKUP COMPARES THREE INTEGERS AND NOT THE WHOLE CONFIG
* ---------------------------------------------------------------------------
* An aim is `template + endpoint`. The bank stamps EVERY path with the same
* template at open and nothing else in the tree changes a bank path's template,
* so two paths' aims can differ only in the endpoint triple. Comparing
* (x_end, y_end, addr) is therefore exactly equivalent to comparing the full
* config, at a fraction of the cost — and the cost matters, because this scan
* runs on every window slice of every descriptor.
*
* ---------------------------------------------------------------------------
* THIS LAYER RETURNS CODES; ITS INSTRUMENTS PRINT
* ---------------------------------------------------------------------------
* The rest of the userspace driver logs nothing and leaves the message to its
* caller, which is why it has no dependency on `ft_tt_device.h`. That is kept.
* The two things that DO print here are instruments, not error paths: the
* histogram dump, and the audit's refusal — which has to name the two endpoints
* it disagrees about at the moment it sees them, because by the time a code has
* travelled up through three layers the coordinates are gone.
*
* @status On silicon 2026-08-26.
* @issues None
* @todo None
*/
#define _POSIX_C_SOURCE 200809L
#include "ft_tt_udrv_noc_bank.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "ft_tt_udrv.h"
/** Windows a bank asks for when `FT_TT_UDRV_WINDOWS` says nothing.
*
* THIRTY-TWO, ON A MEASURED WORKING SET — AND THE WORKING SET IS A FUNCTION OF
* A KNOB. The dispatch path aims at exactly 26 distinct (x, y) on both banks at
* EIGHT matmul members, and the ladder saturates there: aims a token run
* 4,329.5 / 3,449.1 / 1,220.7 / 262.2 / 130.8 / 111.8 at N = 1 / 8 / 16 / 24 /
* 28 / 32, a cliff exactly where the bank grows past the working set, because
* LRU over a CYCLIC access pattern is worst-case until it fits.
*
* **THE MEMBER COUNT MOVES THE SET, AND THE 32 WAS SIZED AT ONE POINT OF IT.**
* Raising the matmul group to twelve and sixteen members adds cores the
* dispatch path talks to every descriptor, and the bulk bank's hit rate falls
* 99.74% / 99.50% / 98.20% at N = 8 / 12 / 16 while the aims a token go
* 829.2 / 1,939.9 / 7,566.1. A cache that is one entry short of a cyclic
* working set misses on nearly every reference, which is why the last step is
* 4x and not 20%.
*
* **SIZE IT FROM `FT_TT_UDRV_AIM_REUSE`, NOT FROM A CORE COUNT.** The routing
* key is (x, y, base), not (x, y): one core reached at two 2 MiB bases is two
* entries. The reuse-distance ladder that instrument prints is the hit rate
* this bank would have had at every size, measured on the stream the run
* actually produced, and it is what this default is set from.
*
* **`FT_TT_UDRV_WINDOWS=1` is the control**: one window per kind is the
* single-window transport this replaced, in the same binary with the same
* counters, which is why the ladder is a ladder and not a comparison of two
* builds. */
/* SIXTY-FOUR SINCE 2026-10, flipped with the 16-member matmul placement: at
* N = 16 the working set is 41 register / 65 bulk keys and the knee is 40; 64
* is the knee plus margin and overflows only at N ~ 40. */
#define BANK_WINDOWS_DEFAULT 64u
/* ------------------------------------------------------------------------- */
/* Knobs */
/* ------------------------------------------------------------------------- */
uint32_t ft_tt_udrv_noc_bank_windows_selected(void)
{
static long cached = -1;
if (cached < 0) {
const char *value = getenv("FT_TT_UDRV_WINDOWS");
char *end = NULL;
long parsed;
cached = (long)BANK_WINDOWS_DEFAULT;
if (value && *value) {
parsed = strtol(value, &end, 10);
if (end && *end == '\0' && parsed >= 1 &&
parsed <= (long)FT_TT_UDRV_NOC_BANK_MAX_WINDOWS)
cached = parsed;
else
fprintf(stderr,
"ft_tt_udrv_noc_bank: FT_TT_UDRV_WINDOWS=%s is not 1..%u — using %u\n",
value, FT_TT_UDRV_NOC_BANK_MAX_WINDOWS, BANK_WINDOWS_DEFAULT);
}
}
return (uint32_t)cached;
}
/** Whether an environment flag is set to something other than "0". */
static bool env_flag(const char *name)
{
const char *value = getenv(name);
return value && *value && strcmp(value, "0") != 0;
}
/** The second key `misaim` needs. See env_red(). */
#define BANK_RED_CONFIRM "i-know-this-can-wedge-the-card"
/**
* Which deliberate defect is armed.
*
* ---------------------------------------------------------------------------
* `misaim` TAKES TWO KEYS, AND THAT IS NOT CEREMONY — IT COST A CARD
* ---------------------------------------------------------------------------
* The arm sends a READ to whatever endpoint another window happens to hold, and
* on a live model those windows include the ARC tile at (8, 0), whose address
* map is not a Tensix's. **On 2026-08-26 a read aimed there did not complete,
* the NoC wedged, the next process could not bring the ARC up, and the host
* went off the network and needed a physical power cycle.** "Reads only" makes
* the arm safe for OTHER PEOPLE'S DATA; it does not make it safe for the fabric.
*
* So: arm it through `tests/test_udrv_card_bank.c`, which drives two DRAM banks
* that always decode and compares the bytes, and NOT against a live model. The
* confirmation string exists so nobody reaches this by adding one word to an
* env line.
*/
static ft_tt_udrv_noc_bank_red env_red(void)
{
const char *value = getenv("FT_TT_UDRV_RED");
const char *confirm;
if (!value || !*value)
return FT_TT_UDRV_NOC_BANK_RED_NONE;
if (strcmp(value, "misaim") != 0) {
fprintf(stderr, "ft_tt_udrv_noc_bank: FT_TT_UDRV_RED=%s names no arm here — none armed\n",
value);
return FT_TT_UDRV_NOC_BANK_RED_NONE;
}
confirm = getenv("FT_TT_UDRV_RED_CONFIRM");
if (!confirm || strcmp(confirm, BANK_RED_CONFIRM) != 0) {
fprintf(stderr,
"ft_tt_udrv_noc_bank: FT_TT_UDRV_RED=misaim NOT ARMED — it also needs "
"FT_TT_UDRV_RED_CONFIRM=%s. It reads from whatever endpoint another window "
"holds, which on a live model includes the ARC tile; one such read did not "
"complete on 2026-08-26 and the box needed a power cycle. Use "
"tests/test_udrv_card_bank.c instead\n",
BANK_RED_CONFIRM);
return FT_TT_UDRV_NOC_BANK_RED_NONE;
}
fprintf(stderr,
"ft_tt_udrv_noc_bank: FT_TT_UDRV_RED=misaim IS ARMED. Every read that hits may be "
"served by a window pointed at another endpoint. THIS CAN WEDGE THE NoC\n");
return FT_TT_UDRV_NOC_BANK_RED_MISAIM;
}
/* ------------------------------------------------------------------------- */
/* Open and close */
/* ------------------------------------------------------------------------- */
/**
* Claim one window, walking DOWN from `*cursor` and skipping the index another
* driver owns. Same policy and same reasoning as the single-window claim in
* ft_tt_runtime_udrv.c, kept here so a bank is self-contained.
*
* THE CURSOR IS WHAT MAKES A LARGE BANK CHEAP TO OPEN. Without it every claim
* restarts at the top of the pool and re-tries — open, flock, fail, close — on
* every window THIS BANK ALREADY HOLDS, so opening W windows costs O(W^2)
* syscalls and a second bank pays the first bank's windows too. Two banks of 96
* is ~18,000 futile open/flock pairs; with the cursor it is one attempt per
* index. Passing a window we skipped can only mean losing one that a DIFFERENT
* process released mid-open, which costs a cache entry and never correctness.
*
* The walk STOPS at `FT_TT_UDRV_NOC_BANK_MIN_WINDOW_INDEX` — see the header for
* why a window below the strided line is a more EXPENSIVE window and not merely
* a lower-numbered one, and why running SHORT is the right answer to wanting
* one.
*/
static int bank_claim_one(const ft_tt_udrv_claim_namespace *ns, uint32_t reserved_window,
const char *reason, uint32_t *cursor, ft_tt_udrv_claim **out_claim)
{
int last_other = FT_TT_UDRV_OK;
while (*cursor > FT_TT_UDRV_NOC_BANK_MIN_WINDOW_INDEX) {
uint32_t index;
int rc;
(*cursor)--;
index = *cursor;
if (index == reserved_window)
continue;
rc = ft_tt_udrv_claim_acquire(ns, index, reason, out_claim);
if (rc == FT_TT_UDRV_OK)
return FT_TT_UDRV_OK;
if (rc != FT_TT_UDRV_EAGAIN)
last_other = rc;
}
return last_other != FT_TT_UDRV_OK ? last_other : FT_TT_UDRV_EAGAIN;
}
int ft_tt_udrv_noc_bank_open(ft_tt_udrv_noc_device *device,
const ft_tt_udrv_claim_namespace *ns, uint32_t reserved_window,
bool write_combine, uint32_t windows, const char *name,
ft_tt_udrv_noc_bank *out_bank)
{
uint32_t cursor;
uint32_t opened;
if (!device || !device->registers || !ns || !out_bank || !name)
return FT_TT_UDRV_EINVAL;
if (windows == 0u || windows > FT_TT_UDRV_NOC_BANK_MAX_WINDOWS)
return FT_TT_UDRV_EINVAL;
memset(out_bank, 0, sizeof(*out_bank));
out_bank->name = name;
out_bank->windows_requested = windows;
out_bank->histogram_enabled = env_flag("FT_TT_UDRV_AIM_HISTOGRAM");
out_bank->audit_enabled = env_flag("FT_TT_UDRV_AIM_AUDIT");
out_bank->reuse_enabled = env_flag("FT_TT_UDRV_AIM_REUSE");
out_bank->red = env_red();
cursor = ns->count;
for (opened = 0u; opened < windows; opened++) {
uint32_t id;
int rc = bank_claim_one(ns, reserved_window, name, &cursor, &out_bank->claim[opened]);
if (rc != FT_TT_UDRV_OK)
break;
id = ft_tt_udrv_claim_index(out_bank->claim[opened]);
rc = ft_tt_udrv_noc_path_open(device, id, write_combine, &out_bank->path[opened]);
if (rc != FT_TT_UDRV_OK) {
(void)ft_tt_udrv_claim_release(out_bank->claim[opened]);
out_bank->claim[opened] = NULL;
break;
}
/* THE TEMPLATE IS THE BANK'S INVARIANT, NOT A DEFAULT IT INHERITS. The
* three-integer lookup below is only equivalent to a full config compare
* because every path here carries the same non-endpoint fields. Stamped
* explicitly so the invariant is code rather than an assumption about
* what path_open() memset. */
(void)ft_tt_udrv_noc_path_set_aim_template(&out_bank->path[opened], 0u, 0u, 0u, 0u, 0u,
0u);
out_bank->window_count = opened + 1u;
}
if (out_bank->window_count == 0u) {
fprintf(stderr,
"ft_tt_udrv_noc_bank: %s bank could claim NO window of %u — refusing\n", name,
ns->count);
return FT_TT_UDRV_EAGAIN;
}
if (out_bank->window_count < windows)
fprintf(stderr,
"ft_tt_udrv_noc_bank: %s bank asked for %u windows and got %u — running SHORT. "
"Every aim that misses this smaller set costs 8.9 us; nothing is shared. ANY "
"FIGURE SCORED AGAINST %u IS VOID — the bank that ran is %u\n",
name, windows, out_bank->window_count, windows, out_bank->window_count);
return FT_TT_UDRV_OK;
}
int ft_tt_udrv_noc_bank_close(ft_tt_udrv_noc_bank *bank)
{
uint32_t i;
int rc = FT_TT_UDRV_OK;
if (!bank)
return FT_TT_UDRV_EINVAL;
for (i = 0u; i < bank->window_count; i++) {
int one = ft_tt_udrv_noc_path_close(&bank->path[i]);
if (one != FT_TT_UDRV_OK)
rc = one;
}
for (i = 0u; i < FT_TT_UDRV_NOC_BANK_MAX_WINDOWS; i++) {
if (bank->claim[i])
(void)ft_tt_udrv_claim_release(bank->claim[i]);
bank->claim[i] = NULL;
}
memset(bank, 0, sizeof(*bank));
return rc;
}
/* ------------------------------------------------------------------------- */
/* The histogram */
/* ------------------------------------------------------------------------- */
static void histogram_record(ft_tt_udrv_noc_bank *bank, uint16_t x, uint16_t y, uint64_t base,
bool was_hit)
{
ft_tt_udrv_noc_bank_endpoint *row = NULL;
uint32_t i;
for (i = 0u; i < bank->histogram_used; i++)
if (bank->histogram[i].x == x && bank->histogram[i].y == y) {
row = &bank->histogram[i];
break;
}
if (!row) {
if (bank->histogram_used >= FT_TT_UDRV_NOC_BANK_HISTOGRAM_MAX) {
bank->histogram_overflow++;
return;
}
row = &bank->histogram[bank->histogram_used++];
row->x = x;
row->y = y;
}
row->selections++;
if (!was_hit)
row->aims++;
if (row->base_uncounted)
return;
for (i = 0u; i < row->base_count; i++)
if (row->base[i] == base)
return;
/* THE LIST IS THE COUNTER. Incrementing past it would count SELECTIONS and
* print them as a cardinality, which is exactly the defect this replaced. */
if (row->base_count >= FT_TT_UDRV_NOC_BANK_BASES_PER_ROW) {
row->base_uncounted = true;
return;
}
row->base[row->base_count++] = base;
}
uint32_t ft_tt_udrv_noc_bank_endpoint_count(const ft_tt_udrv_noc_bank *bank)
{
return bank ? bank->histogram_used : 0u;
}
void ft_tt_udrv_noc_bank_histogram_log(const ft_tt_udrv_noc_bank *bank)
{
uint32_t order[FT_TT_UDRV_NOC_BANK_HISTOGRAM_MAX];
uint64_t total = 0u;
uint64_t running = 0u;
uint32_t i;
if (!bank || !bank->histogram_enabled)
return;
for (i = 0u; i < bank->histogram_used; i++) {
order[i] = i;
total += bank->histogram[i].selections;
}
for (i = 0u; i + 1u < bank->histogram_used; i++) {
uint32_t j;
for (j = i + 1u; j < bank->histogram_used; j++)
if (bank->histogram[order[j]].selections > bank->histogram[order[i]].selections) {
uint32_t swap = order[i];
order[i] = order[j];
order[j] = swap;
}
}
fprintf(stderr,
"ft_tt_udrv_noc_bank: %s HISTOGRAM — %u distinct (x,y), %llu selections, %llu past "
"the table, %u windows\n",
bank->name, bank->histogram_used, (unsigned long long)total,
(unsigned long long)bank->histogram_overflow, bank->window_count);
for (i = 0u; i < bank->histogram_used; i++) {
const ft_tt_udrv_noc_bank_endpoint *row = &bank->histogram[order[i]];
running += row->selections;
fprintf(stderr,
"ft_tt_udrv_noc_bank: %s #%-3u (%2u,%2u) — %10llu selections (%5.2f%%, "
"cumulative %6.2f%%), %llu aims, %s%u bases from 0x%09llx\n",
bank->name, i, row->x, row->y, (unsigned long long)row->selections,
total ? 100.0 * (double)row->selections / (double)total : 0.0,
total ? 100.0 * (double)running / (double)total : 0.0,
(unsigned long long)row->aims, row->base_uncounted ? ">=" : "",
row->base_count, (unsigned long long)row->base[0]);
}
}
/* ------------------------------------------------------------------------- */
/* The reuse-distance ladder — what EVERY bank size would have done */
/* ------------------------------------------------------------------------- */
/**
* Record one routing decision against an LRU stack of routing keys.
*
* ---------------------------------------------------------------------------
* WHY A REUSE DISTANCE IS THE WHOLE LADDER
* ---------------------------------------------------------------------------
* An LRU cache of `W` entries holds exactly the `W` most recently used distinct
* keys. So a reference hits it if and only if fewer than `W` DISTINCT other keys
* were touched since this key last was — its reuse distance. Bucket every
* reference by that distance once and the hit rate at every size falls out of
* one run, instead of one run per rung. The recorded window ladder cost six
* model loads to produce six points; this produces every point up to
* `FT_TT_UDRV_NOC_BANK_REUSE_MAX` from one.
*
* ---------------------------------------------------------------------------
* THE KEY IS THE ROUTING KEY, WHICH IS NOT THE ENDPOINT
* ---------------------------------------------------------------------------
* `path_holds()` compares (x, y, base). The endpoint histogram beside this is
* keyed on (x, y) and deliberately so — keyed on the triple it is FLOODED by
* the weight upload's thousands of DRAM bases. Both keys are right for their own
* question and neither answers the other's: a core reached at two bases is ONE
* endpoint and TWO windows.
*
* IT IS O(depth) A REFERENCE. That is why it is off by default and why a run
* carrying it is measuring the pattern, not the wall.
*/
static void reuse_record(ft_tt_udrv_noc_bank *bank, uint16_t x, uint16_t y, uint64_t base)
{
uint32_t found = FT_TT_UDRV_NOC_BANK_REUSE_MAX;
uint32_t i;
bank->reuse_references++;
for (i = 0u; i < bank->reuse_depth; i++)
if (bank->reuse_stack[i].x == x && bank->reuse_stack[i].y == y &&
bank->reuse_stack[i].base == base) {
found = i;
break;
}
if (found < FT_TT_UDRV_NOC_BANK_REUSE_MAX)
bank->reuse_hist[found]++;
else
bank->reuse_cold++;
/* Move to front. A miss pushes and drops the tail, which is exactly what an
* LRU cache of REUSE_MAX entries does; anything that fell off the tail is
* already counted as cold and cannot be double counted. */
if (found == FT_TT_UDRV_NOC_BANK_REUSE_MAX) {
if (bank->reuse_depth < FT_TT_UDRV_NOC_BANK_REUSE_MAX)
bank->reuse_depth++;
found = bank->reuse_depth - 1u;
}
for (i = found; i > 0u; i--)
bank->reuse_stack[i] = bank->reuse_stack[i - 1u];
bank->reuse_stack[0].x = x;
bank->reuse_stack[0].y = y;
bank->reuse_stack[0].base = base;
}
uint64_t ft_tt_udrv_noc_bank_reuse_hits_at(const ft_tt_udrv_noc_bank *bank, uint32_t windows)
{
uint64_t total = 0u;
uint32_t i;
if (!bank || !bank->reuse_enabled)
return 0u;
if (windows > FT_TT_UDRV_NOC_BANK_REUSE_MAX)
windows = FT_TT_UDRV_NOC_BANK_REUSE_MAX;
for (i = 0u; i < windows; i++)
total += bank->reuse_hist[i];
return total;
}
void ft_tt_udrv_noc_bank_reuse_log(const ft_tt_udrv_noc_bank *bank)
{
static const uint32_t rungs[] = {1u, 2u, 4u, 8u, 12u, 16u, 20u, 24u, 28u,
32u, 40u, 48u, 56u, 64u, 80u, 96u, 128u, 192u};
uint64_t predicted;
uint64_t measured;
double predicted_rate;
double measured_rate;
size_t r;
if (!bank || !bank->reuse_enabled)
return;
if (bank->reuse_references == 0u) {
fprintf(stderr,
"ft_tt_udrv_noc_bank: %s REUSE — the instrument was ARMED and saw ZERO "
"references. It measured nothing; do not read a ladder off it\n",
bank->name);
return;
}
fprintf(stderr,
"ft_tt_udrv_noc_bank: %s REUSE — %llu routing decisions over (x,y,base) keys, "
"%llu of them cold or past a %u-deep stack. THIS RUN'S WALL IS NOT COMPARABLE: the "
"instrument costs a stack search per decision\n",
bank->name, (unsigned long long)bank->reuse_references,
(unsigned long long)bank->reuse_cold, FT_TT_UDRV_NOC_BANK_REUSE_MAX);
for (r = 0u; r < sizeof(rungs) / sizeof(rungs[0]); r++) {
uint64_t hits = ft_tt_udrv_noc_bank_reuse_hits_at(bank, rungs[r]);
fprintf(stderr,
"ft_tt_udrv_noc_bank: %s windows %3u -> %10llu hits, %10llu aims, %7.4f%% hit\n",
bank->name, rungs[r], (unsigned long long)hits,
(unsigned long long)(bank->reuse_references - hits),
100.0 * (double)hits / (double)bank->reuse_references);
}
/* THE FALSIFIER, PRINTED BESIDE THE LADDER. The ladder's entry at the bank's
* OWN size and the bank's OWN hit counter are the same quantity computed two
* ways. They agree or the simulation does not describe this cache. */
predicted = ft_tt_udrv_noc_bank_reuse_hits_at(bank, bank->window_count);
measured = bank->hits;
predicted_rate = 100.0 * (double)predicted / (double)bank->reuse_references;
measured_rate = bank->selections
? 100.0 * (double)measured / (double)bank->selections
: 0.0;
fprintf(stderr,
"ft_tt_udrv_noc_bank: %s SELF-CHECK at the live size %u — ladder says %llu hits "
"of %llu (%.4f%%), the bank counted %llu of %llu (%.4f%%). %s\n",
bank->name, bank->window_count, (unsigned long long)predicted,
(unsigned long long)bank->reuse_references, predicted_rate,
(unsigned long long)measured, (unsigned long long)bank->selections, measured_rate,
(predicted == measured && bank->reuse_references == bank->selections)
? "IDENTICAL — the ladder describes this cache"
: "THEY DISAGREE — THE LADDER IS VOID, do not size a bank from it");
}
/* ------------------------------------------------------------------------- */
/* Routing */
/* ------------------------------------------------------------------------- */
/** True when this path's window is already pointed at (x, y, base). See the
* file header for why three integers are the whole comparison. */
static bool path_holds(const ft_tt_udrv_noc_path *path, uint16_t x, uint16_t y, uint64_t base)
{
return path->aimed && path->aim.x_end == (uint8_t)x && path->aim.y_end == (uint8_t)y &&
path->aim.addr == base;
}
/** The window to displace on a miss: an unused one if there is one, otherwise
* the least recently used. */
static uint32_t bank_choose_victim(const ft_tt_udrv_noc_bank *bank)
{
uint32_t best = 0u;
uint32_t i;
for (i = 0u; i < bank->window_count; i++)
if (!bank->path[i].aimed)
return i;
for (i = 1u; i < bank->window_count; i++)
if (bank->last_used[i] < bank->last_used[best])
best = i;
return best;
}
/**
* Deliberately make one window LIE about where it points.
*
* ---------------------------------------------------------------------------
* WHY THIS FALSIFIES THE PATH'S BELIEF AND NOT JUST THE ROUTING
* ---------------------------------------------------------------------------
* The first version of this arm simply returned a window aimed somewhere else,
* and it was VACUOUS — measured on silicon 2026-08-26, 176,406 "injections" and
* a completion byte-identical to its own control. The reason is that
* `ft_tt_udrv_noc_transfer()` calls `aim_if_needed()`, which compares the path's
* recorded aim against the endpoint it was given and RE-AIMS. The arm handed the
* transfer a wrong window and the transfer quietly corrected it; all that moved
* was the cache statistics.
*
* **The real failure mode of a bank is a path whose RECORDED aim disagrees with
* its register**, because that is the one state in which `aim_if_needed()` skips
* and the aperture is dereferenced while pointed at another core. So the arm
* writes the wanted endpoint into `path->aim` WITHOUT touching the hardware:
* the belief now says (x, y, base), the register still holds the old endpoint,
* the re-aim is skipped, and the read lands on the wrong core. That is a STATE
* the whole transfer sees, not a race it might.
*
* READS ONLY, and the belief is restored the moment the read is done
* (`ft_tt_udrv_noc_bank_transfer()` invalidates the path). A poisoned belief
* left in place would eventually carry a WRITE into a live core's L1 and corrupt
* a resident kernel, and a red arm on a card nobody can power-cycle has to be
* survivable.
*
* @return the index to use instead, or `chosen` when no other window is aimed
* anywhere different — in which case nothing was injected and the counter does
* not move, so an arm that could not fire reads as a zero.
*/
static uint32_t bank_inject_misaim(ft_tt_udrv_noc_bank *bank, uint32_t chosen, uint16_t x,
uint16_t y, uint64_t base)
{
uint32_t i;
for (i = 0u; i < bank->window_count; i++) {
if (i == chosen || !bank->path[i].aimed)
continue;
if (path_holds(&bank->path[i], x, y, base))
continue;
bank->path[i].aim.x_end = (uint8_t)x;
bank->path[i].aim.y_end = (uint8_t)y;
bank->path[i].aim.addr = base;
bank->misaims_injected++;
return i;
}
return chosen;
}
/**
* Read the chosen window's register triple back from the card and refuse if it
* does not decode to the endpoint this transfer asked for.
*
* This is four non-posted reads — the very cost the bank exists to remove — so
* it is an audit and never the shipped path. What it buys is an INDEPENDENT
* reading: the routing above believes `path->aim`, and this believes the
* register.
*/
static int bank_audit(ft_tt_udrv_noc_bank *bank, uint32_t index, uint16_t x, uint16_t y,
uint64_t base)
{
ft_tt_udrv_tlb2m_config held;
uint32_t words[3] = {0u, 0u, 0u};
int rc = ft_tt_udrv_noc_window_read(bank->path[index].device, bank->path[index].window_id,
words);
if (rc != FT_TT_UDRV_OK)
return rc;
rc = ft_tt_udrv_tlb2m_decode(words, &held);
if (rc != FT_TT_UDRV_OK)
return rc;
if (held.x_end == (uint8_t)x && held.y_end == (uint8_t)y && held.addr == base)
return FT_TT_UDRV_OK;
bank->audit_failures++;
fprintf(stderr,
"ft_tt_udrv_noc_bank: %s AUDIT REFUSED — window %u holds (%u,%u) base 0x%09llx and "
"this transfer wants (%u,%u) base 0x%09llx. A wrong window is a wrong ADDRESS, so "
"nothing was dereferenced\n",
bank->name, bank->path[index].window_id, held.x_end, held.y_end,
(unsigned long long)held.addr, x, y, (unsigned long long)base);
return FT_TT_UDRV_EIO;
}
/**
* Pick the window for (x, y, base), recording the hit or miss.
*
* On a miss the victim is returned WITHOUT being aimed — the aim happens inside
* `ft_tt_udrv_noc_transfer()`, which is the one place in the tree that aims and
* verifies, and duplicating it here is how the two would drift.
*/
static int bank_select(ft_tt_udrv_noc_bank *bank, uint16_t x, uint16_t y, uint64_t base,
bool is_read, uint32_t *out_index, bool *out_poisoned)
{
uint32_t index = bank->window_count;
bool hit = false;
uint32_t i;
*out_poisoned = false;
for (i = 0u; i < bank->window_count; i++)
if (path_holds(&bank->path[i], x, y, base)) {
index = i;
hit = true;
break;
}
if (!hit) {
index = bank_choose_victim(bank);
if (bank->path[index].aimed)
bank->evictions++;
bank->misses++;
} else {
bank->hits++;
}
bank->selections++;
bank->last_used[index] = ++bank->clock;
if (bank->histogram_enabled)
histogram_record(bank, x, y, base, hit);
if (bank->reuse_enabled)
reuse_record(bank, x, y, base);
if (hit && bank->red == FT_TT_UDRV_NOC_BANK_RED_MISAIM && is_read) {
uint32_t poisoned = bank_inject_misaim(bank, index, x, y, base);
*out_poisoned = poisoned != index;
index = poisoned;
}
if (hit && bank->audit_enabled) {
int rc = bank_audit(bank, index, x, y, base);
if (rc != FT_TT_UDRV_OK) {
/* The transfer will not run, so nothing downstream restores a
* poisoned belief. Undo it here or the arm outlives its own read. */
if (*out_poisoned)
bank->path[index].aimed = false;
return rc;
}
}
*out_index = index;
return FT_TT_UDRV_OK;
}
int ft_tt_udrv_noc_bank_transfer(ft_tt_udrv_noc_bank *bank, uint16_t x, uint16_t y,
uint64_t address, void *host, size_t bytes, bool is_read,
ft_tt_noc_transfer_width width)
{
unsigned char *host_bytes = (unsigned char *)host;
if (!bank || bank->window_count == 0u)
return FT_TT_UDRV_EINVAL;
if (!host && bytes != 0u)
return FT_TT_UDRV_EINVAL;
while (bytes > 0u) {
ft_tt_noc_window_slice slice;
uint32_t index = 0u;
bool poisoned = false;
int rc;
if (!ft_tt_noc_window_slice_compute(address, bytes, bank->path[0].window_bytes, &slice))
return FT_TT_UDRV_EINVAL;
rc = bank_select(bank, x, y, slice.window_base, is_read, &index, &poisoned);
if (rc != FT_TT_UDRV_OK)
return rc;
/* The chosen path re-derives this same slice and finds its window either
* already there (a hit costs no aim) or in need of one. Everything about
* the move itself is that function's, unchanged. */
rc = ft_tt_udrv_noc_transfer(&bank->path[index], x, y, address, host_bytes, slice.bytes,
is_read, width);
/* A red arm's poisoned belief lives for exactly one read. Left in place
* it would eventually carry a WRITE to the wrong core. */
if (poisoned)
bank->path[index].aimed = false;
if (rc != FT_TT_UDRV_OK)
return rc;
address += slice.bytes;
host_bytes += slice.bytes;
bytes -= slice.bytes;
}
return FT_TT_UDRV_OK;
}
int ft_tt_udrv_noc_bank_access_write(void *user, uint16_t x, uint16_t y, uint64_t address,
const void *src, size_t bytes)
{
/* Const is dropped so one walk serves both directions; with is_read false
* nothing is ever written through this pointer. */
return ft_tt_udrv_noc_bank_transfer((ft_tt_udrv_noc_bank *)user, x, y, address,
(void *)(uintptr_t)src, bytes, false,
FT_TT_NOC_WIDTH_WORD32);
}
int ft_tt_udrv_noc_bank_access_read(void *user, uint16_t x, uint16_t y, uint64_t address,
void *dst, size_t bytes)
{
return ft_tt_udrv_noc_bank_transfer((ft_tt_udrv_noc_bank *)user, x, y, address, dst, bytes,
true, FT_TT_NOC_WIDTH_WORD32);
}
/* ------------------------------------------------------------------------- */
/* Totals */
/* ------------------------------------------------------------------------- */
#define BANK_SUM(field, type) \
type total = (type)0; \
uint32_t i; \
if (!bank) \
return (type)0; \
for (i = 0u; i < bank->window_count; i++) \
total += field; \
return total
uint64_t ft_tt_udrv_noc_bank_aim_count(const ft_tt_udrv_noc_bank *bank)
{
BANK_SUM(ft_tt_udrv_noc_path_aim_count(&bank->path[i]), uint64_t);
}
uint64_t ft_tt_udrv_noc_bank_aim_failures(const ft_tt_udrv_noc_bank *bank)
{
BANK_SUM(ft_tt_udrv_noc_path_aim_failures(&bank->path[i]), uint64_t);
}
double ft_tt_udrv_noc_bank_aim_seconds(const ft_tt_udrv_noc_bank *bank)
{
BANK_SUM(ft_tt_udrv_noc_path_aim_seconds(&bank->path[i]), double);
}
double ft_tt_udrv_noc_bank_aim_verify_seconds(const ft_tt_udrv_noc_bank *bank)
{
BANK_SUM(ft_tt_udrv_noc_path_aim_verify_seconds(&bank->path[i]), double);
}
uint64_t ft_tt_udrv_noc_bank_bytes_read(const ft_tt_udrv_noc_bank *bank)
{
BANK_SUM(bank->path[i].bytes_read, uint64_t);
}
uint64_t ft_tt_udrv_noc_bank_bytes_written(const ft_tt_udrv_noc_bank *bank)
{
BANK_SUM(bank->path[i].bytes_written, uint64_t);
}
Reference in New Issue View Git Blame Copy Permalink