David
/
Foundry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Foundry/tt/tests/test_cooperative_matmul_plan.c

846 lines
46 KiB
C
Raw Permalink Blame History

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
/**
* test_cooperative_matmul_plan.c — the cooperative matmul's WORK SPLIT and its refusals,
* off hardware.
*
* Everything ft_tt_op_matmul_cooperative.c does is arithmetic, and that is deliberate: a wrong
* multicast rectangle is not a failed transaction, it is a write into a core somebody else is
* using or a coordinate that hangs the chip, and the iteration loop after a NoC hang is
* minutes plus physical access to the machine (DEV_NOTES.md §1). So every claim the planner
* makes is provable here, on the Mac, while somebody else owns the card.
*
* What this file does NOT cover, and cannot: whether the multicast actually lands, whether the
* CB write pointer really is where the kernel computes it will be, and whether the answer is
* right. Those need silicon and are tests/test_cooperative_matmul.c's.
*
* make -f Makefile.cooperative check
*/
#include <stdio.h>
#include <string.h>
#include "../ft_tt_dram_shard.h"
#include "../ft_tt_op_matmul_cooperative.h"
#include "ft_tt_test_report.h"
#define TILE_BYTES FT_TT_OP_TILE_BYTES_BF16
/** A 2x2 block of TRANSLATED coordinates that really is a rectangle: NoC x 3,4 and y 2,3.
* Written out rather than derived, so this file has no grid and cannot inherit a grid bug. */
static const ft_tt_noc_coord g_square[4] = {{3u, 2u}, {4u, 2u}, {3u, 3u}, {4u, 3u}};
/** The same four cores with one moved across the non-Tensix column gap. Its bounding box then
* contains NoC x 8 and 9 — the ARC/L2CPU spine and the east DRAM column — and a multicast
* writes EVERY node in its bounding box. */
static const ft_tt_noc_coord g_across_the_gap[4] = {{3u, 2u}, {10u, 2u}, {3u, 3u}, {10u, 3u}};
static void fill_request(ft_tt_op_matmul_cooperative_request *request, uint32_t tile_cols,
ft_tt_op_matmul_cooperative_mode mode)
{
memset(request, 0, sizeof(*request));
request->tile_rows = 1u; /* 32 tokens, which is every decode shape on record */
request->tile_inner = 48u; /* Kt of ffn_gate/up */
request->tile_cols = tile_cols;
request->prefetch_tiles = 48u;
request->weight_n_major = 1u;
request->mode = mode;
request->spin_limit = 1000u;
request->activation.x = 17u;
request->activation.y = 12u;
request->activation.address = 0x00100000u;
request->activation.tile_bytes = TILE_BYTES;
request->weight = request->activation;
request->output = request->activation;
request->output.address = 0x00200000u;
}
/* ------------------------------------------------------------------------- */
/** The split itself: who owns which columns, and where their output goes. */
static void test_the_split(void)
{
ft_tt_op_matmul_cooperative_request request;
ft_tt_op_matmul_cooperative_plan plan;
bool columns_partition = true;
bool outputs_are_consecutive = true;
uint32_t index;
int rc;
ft_tt_test_step("the split — 4 cores over Nt = 144");
fill_request(&request, 144u, FT_TT_OP_MATMUL_COOPERATIVE);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &plan);
ft_tt_test_check_fmt("a 2x2 group over Nt = 144 plans", rc == FT_TT_OK, "rc=%d (%s)", rc,
ft_tt_error_string(rc));
if (rc != FT_TT_OK)
return;
for (index = 0u; index < 4u; index++) {
if (plan.member[index].column_count != 36u ||
plan.member[index].column_first != index * 36u)
columns_partition = false;
if (plan.member[index].output_address !=
request.output.address + index * 36u * TILE_BYTES)
outputs_are_consecutive = false;
}
ft_tt_test_check_fmt("the output columns are partitioned, not overlapped or dropped",
columns_partition,
"member first/count: %u/%u %u/%u %u/%u %u/%u (wanted 0/36 36/36 "
"72/36 108/36)",
plan.member[0].column_first, plan.member[0].column_count,
plan.member[1].column_first, plan.member[1].column_count,
plan.member[2].column_first, plan.member[2].column_count,
plan.member[3].column_first, plan.member[3].column_count);
/* At Mt = 1 the panels concatenate into exactly the single-core output, which is the whole
* reason the on-card gate can be a byte comparison rather than a PCC. */
ft_tt_test_check_fmt("at Mt = 1 the panels concatenate into the single-core layout",
outputs_are_consecutive,
"member 1 writes at 0x%08x, expected 0x%08x",
plan.member[1].output_address,
request.output.address + 36u * TILE_BYTES);
ft_tt_test_check_fmt("the leader is inside the rectangle, so it receives nothing",
plan.shared_valid && plan.shared.rectangle.sender_inside &&
plan.shared.rectangle.destination_count == 3u,
"sender_inside %d, destination_count %u (wanted 3 = members - 1), "
"node_count %u",
(int)plan.shared.rectangle.sender_inside,
plan.shared.rectangle.destination_count,
plan.shared.rectangle.node_count);
ft_tt_test_check_fmt("the shared operand is one whole prefetch block of A",
plan.shared.layout.operand_bytes == 48u * TILE_BYTES,
"%u bytes (wanted %u = depth 48 x %u)",
plan.shared.layout.operand_bytes, 48u * TILE_BYTES, TILE_BYTES);
ft_tt_test_check_fmt("the shared operand lands in CB_IN0, which is what the receivers "
"reserve", plan.shared.layout.operand_address == plan.cb[0].address,
"operand at 0x%08x, CB_IN0 at 0x%08x",
plan.shared.layout.operand_address, plan.cb[0].address);
}
/**
* THE RAGGED SPLIT — an Nt the member count does not divide, in SPLIT mode.
*
* COOPERATIVE refuses this and must keep refusing it: its per-block multicast barrier is
* correct only because every member walks the same number of blocks. SPLIT has no barrier
* at all, so refusing there cost a whole tensor its split — MEASURED on MiniCPM5-1B, where
* N=5 left 3,864 of 3,887 projections on one core and N=7 left every one of them.
*
* The property that has to hold is a PARTITION: the column ranges are consecutive, cover
* [0, Nt) exactly, differ in width by at most one, and every output address is the running
* sum. A check that only added the counts up would pass a plan whose members overlap.
*/
static void test_the_ragged_split(void)
{
ft_tt_op_matmul_cooperative_request request;
ft_tt_op_matmul_cooperative_plan plan;
bool partitions = true;
bool balanced = true;
bool outputs_follow = true;
uint32_t expected_first = 0u;
uint32_t total = 0u;
uint32_t index;
int rc;
ft_tt_test_step("the ragged split — 7 cores over Nt = 45, which 7 does not divide");
fill_request(&request, 45u, FT_TT_OP_MATMUL_SPLIT);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 7u, 0u, &request, &plan);
ft_tt_test_check_fmt("SPLIT plans a ragged Nt rather than refusing it", rc == FT_TT_OK,
"Nt 45 over 7 members gave rc=%d (%s)", rc, ft_tt_error_string(rc));
if (rc != FT_TT_OK)
return;
for (index = 0u; index < 7u; index++) {
const ft_tt_op_matmul_cooperative_member *member = &plan.member[index];
if (member->column_first != expected_first)
partitions = false;
if (member->column_count != 6u && member->column_count != 7u)
balanced = false;
if (member->output_address !=
request.output.address + expected_first * request.tile_rows * TILE_BYTES)
outputs_follow = false;
expected_first += member->column_count;
total += member->column_count;
}
ft_tt_test_check_fmt("the ranges are consecutive, so nothing is computed twice or dropped",
partitions,
"member first/count: %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u",
plan.member[0].column_first, plan.member[0].column_count,
plan.member[1].column_first, plan.member[1].column_count,
plan.member[2].column_first, plan.member[2].column_count,
plan.member[3].column_first, plan.member[3].column_count,
plan.member[4].column_first, plan.member[4].column_count,
plan.member[5].column_first, plan.member[5].column_count,
plan.member[6].column_first, plan.member[6].column_count);
ft_tt_test_check_fmt("and they COVER Nt exactly", total == 45u, "%u columns of 45", total);
ft_tt_test_check_fmt("the widths differ by at most one, which is the balance a column "
"split can reach", balanced, "3 members of 7 should hold 7 columns");
ft_tt_test_check_fmt("the output addresses are the RUNNING SUM, not index x width",
outputs_follow, "member 4 writes at 0x%08x, expected 0x%08x",
plan.member[4].output_address,
request.output.address +
(plan.member[0].column_count + plan.member[1].column_count +
plan.member[2].column_count + plan.member[3].column_count) *
request.tile_rows * TILE_BYTES);
/* And the COOPERATIVE refusal is unchanged, because its reason is the RENDEZVOUS and not
* the arithmetic. Without this the change reads as "ragged splits are fine now". */
fill_request(&request, 45u, FT_TT_OP_MATMUL_COOPERATIVE);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 7u, 0u, &request, &plan);
ft_tt_test_check_fmt("COOPERATIVE still REFUSES the same shape", rc == FT_TT_EINVAL,
"rc=%d (%s)", rc, ft_tt_error_string(rc));
/* Fewer columns than members would give one of them ZERO, which the resident reader
* refuses on the card — and a refusal there strands a triad. */
fill_request(&request, 3u, FT_TT_OP_MATMUL_SPLIT);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 7u, 0u, &request, &plan);
ft_tt_test_check_fmt("an Nt SMALLER than the member count is REFUSED, because a member "
"with zero columns is a refusal on the card",
rc == FT_TT_EINVAL, "Nt 3 over 7 members gave rc=%d (%s)", rc,
ft_tt_error_string(rc));
}
/** Nothing about the CB placement may depend on how many members there are — the multicast
* writes ONE L1 address into every node of the rectangle, so a placement that moved with the
* group size would be a different address on a differently-sized group. */
static void test_placement_is_group_independent(void)
{
ft_tt_op_matmul_cooperative_request request;
ft_tt_op_matmul_cooperative_plan two;
ft_tt_op_matmul_cooperative_plan four;
ft_tt_op_matmul_cooperative_plan solo;
int rc_two;
int rc_four;
int rc_solo;
ft_tt_test_step("the circular-buffer placement does not move with the group size");
fill_request(&request, 144u, FT_TT_OP_MATMUL_COOPERATIVE);
rc_two = ft_tt_op_matmul_cooperative_plan_build(g_square, 2u, 0u, &request, &two);
rc_four = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &four);
request.mode = FT_TT_OP_MATMUL_INDEPENDENT;
rc_solo = ft_tt_op_matmul_cooperative_plan_build(NULL, 1u, 0u, &request, &solo);
ft_tt_test_check_fmt("all three plans built", rc_two == FT_TT_OK && rc_four == FT_TT_OK &&
rc_solo == FT_TT_OK,
"rc %d / %d / %d", rc_two, rc_four, rc_solo);
if (rc_two != FT_TT_OK || rc_four != FT_TT_OK || rc_solo != FT_TT_OK)
return;
ft_tt_test_check_fmt("CB_IN0 is at the same L1 address for 1, 2 and 4 members",
two.cb[0].address == four.cb[0].address &&
two.cb[0].address == solo.cb[0].address,
"0x%08x / 0x%08x / 0x%08x", solo.cb[0].address, two.cb[0].address,
four.cb[0].address);
ft_tt_test_check_fmt("CB_IN0 is 64-byte aligned, which is what a DRAM read into it needs",
(four.cb[0].address % FT_TT_DRAM_ALIGNMENT) == 0u, "0x%08x",
four.cb[0].address);
ft_tt_test_check_fmt("the input CBs hold exactly two prefetch blocks",
four.cb[0].page_count == 2u * four.depth &&
four.cb[1].page_count == 2u * four.depth,
"%u and %u pages at depth %u", four.cb[0].page_count,
four.cb[1].page_count, four.depth);
ft_tt_test_check_fmt("the control region clears the circular-buffer arena",
FT_TT_OP_COOP_CONTROL_BASE + FT_TT_OP_COOP_CONTROL_BYTES <=
FT_TT_OP_CB_ARENA_BASE,
"control 0x%08x + %u vs arena 0x%08x",
(unsigned)FT_TT_OP_COOP_CONTROL_BASE,
(unsigned)FT_TT_OP_COOP_CONTROL_BYTES,
(unsigned)FT_TT_OP_CB_ARENA_BASE);
}
/** The refusals. Each one is a shape that produces no error on the card. */
static void test_the_refusals(void)
{
ft_tt_op_matmul_cooperative_request request;
ft_tt_op_matmul_cooperative_plan plan;
int rc;
ft_tt_test_step("the refusals — every one of these is silent on the card");
/* A ragged split gives members different BLOCK COUNTS, and the per-block rendezvous is
* only correct because every member walks the same number. It would deadlock on the last
* blocks: the leader waiting for members that have already finished. */
fill_request(&request, 145u, FT_TT_OP_MATMUL_COOPERATIVE);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &plan);
ft_tt_test_check_fmt("an Nt that does not divide by the member count is REFUSED",
rc == FT_TT_EINVAL, "Nt 145 over 4 members gave rc=%d (%s)", rc,
ft_tt_error_string(rc));
/* One member leaves the rectangle with zero destinations, which is the zero-length NoC
* transfer in different clothes — the shape that took this card from working to
* `Mem- BusMaster-` (DEV_NOTES.md §1). */
fill_request(&request, 144u, FT_TT_OP_MATMUL_COOPERATIVE);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 1u, 0u, &request, &plan);
ft_tt_test_check_fmt("a COOPERATIVE group of ONE is REFUSED", rc == FT_TT_EINVAL,
"rc=%d (%s)", rc, ft_tt_error_string(rc));
/* A multicast writes EVERY node in its bounding box, and x = 8, 9 are the ARC/L2CPU spine
* and a DRAM column. This is the single most expensive mistake available here. */
fill_request(&request, 144u, FT_TT_OP_MATMUL_COOPERATIVE);
rc = ft_tt_op_matmul_cooperative_plan_build(g_across_the_gap, 4u, 0u, &request, &plan);
ft_tt_test_check_fmt("a group whose bounding box CROSSES the non-Tensix column gap is "
"REFUSED", rc != FT_TT_OK,
"x 3 and 10 with x 8, 9 between them gave rc=%d (%s)", rc,
ft_tt_error_string(rc));
fill_request(&request, 144u, FT_TT_OP_MATMUL_COOPERATIVE);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 4u, &request, &plan);
ft_tt_test_check_fmt("a leader outside the group is REFUSED", rc == FT_TT_EINVAL,
"leader 4 of 4 members gave rc=%d (%s)", rc, ft_tt_error_string(rc));
fill_request(&request, 0u, FT_TT_OP_MATMUL_COOPERATIVE);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &plan);
ft_tt_test_check_fmt("a zero Nt is REFUSED", rc == FT_TT_EINVAL, "rc=%d (%s)", rc,
ft_tt_error_string(rc));
}
/** The runtime-arg blocks, which are the whole of what the kernel sees. */
static void test_the_argument_blocks(void)
{
ft_tt_op_matmul_cooperative_request request;
ft_tt_op_matmul_cooperative_plan plan;
uint32_t leader_args[FTCM_ARG_COUNT];
uint32_t receiver_args[FTCM_ARG_COUNT];
uint32_t compute_args[3];
uint32_t writer_args[4];
int rc;
ft_tt_test_step("the runtime-arg blocks");
fill_request(&request, 144u, FT_TT_OP_MATMUL_COOPERATIVE);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &plan);
if (rc != FT_TT_OK) {
ft_tt_test_check("the plan for the argument blocks built", false,
ft_tt_error_string(rc));
return;
}
rc = ft_tt_op_matmul_cooperative_reader_arguments(&plan, &request, g_square, 0u,
leader_args, FTCM_ARG_COUNT);
if (rc == FT_TT_OK)
rc = ft_tt_op_matmul_cooperative_reader_arguments(&plan, &request, g_square, 2u,
receiver_args, FTCM_ARG_COUNT);
if (rc == FT_TT_OK)
rc = ft_tt_op_matmul_cooperative_compute_arguments(&plan, &request, 2u, compute_args,
3u);
if (rc == FT_TT_OK)
rc = ft_tt_op_matmul_cooperative_writer_arguments(&plan, &request, 2u, writer_args, 4u);
ft_tt_test_check_fmt("every argument block built", rc == FT_TT_OK, "rc=%d (%s)", rc,
ft_tt_error_string(rc));
if (rc != FT_TT_OK)
return;
ft_tt_test_check_fmt("exactly one member is the LEADER and the rest RECEIVE",
leader_args[FTCM_ARG_ROLE] == FTCM_ROLE_LEADER &&
receiver_args[FTCM_ARG_ROLE] == FTCM_ROLE_RECEIVER,
"roles %u and %u (wanted %u and %u)", leader_args[FTCM_ARG_ROLE],
receiver_args[FTCM_ARG_ROLE], FTCM_ROLE_LEADER, FTCM_ROLE_RECEIVER);
ft_tt_test_check_fmt("every member is told the SAME leader, rectangle and semaphores",
leader_args[FTCM_ARG_LEADER_X] == receiver_args[FTCM_ARG_LEADER_X] &&
leader_args[FTCM_ARG_LEADER_Y] ==
receiver_args[FTCM_ARG_LEADER_Y] &&
leader_args[FTCM_ARG_RECTANGLE_X_LOW] ==
receiver_args[FTCM_ARG_RECTANGLE_X_LOW] &&
leader_args[FTCM_ARG_ARRIVAL_SEMAPHORE] ==
receiver_args[FTCM_ARG_ARRIVAL_SEMAPHORE] &&
leader_args[FTCM_ARG_READY_SEMAPHORE] ==
receiver_args[FTCM_ARG_READY_SEMAPHORE],
"leader (%u,%u) vs (%u,%u), arrival 0x%08x vs 0x%08x",
leader_args[FTCM_ARG_LEADER_X], leader_args[FTCM_ARG_LEADER_Y],
receiver_args[FTCM_ARG_LEADER_X], receiver_args[FTCM_ARG_LEADER_Y],
leader_args[FTCM_ARG_ARRIVAL_SEMAPHORE],
receiver_args[FTCM_ARG_ARRIVAL_SEMAPHORE]);
ft_tt_test_check_fmt("every member is told the SAME CB_IN0 base, which is the address the "
"multicast lands at",
leader_args[FTCM_ARG_CB_IN0_BASE] ==
receiver_args[FTCM_ARG_CB_IN0_BASE] &&
leader_args[FTCM_ARG_CB_IN0_BASE] == plan.cb[0].address,
"0x%08x vs 0x%08x, CB_IN0 at 0x%08x",
leader_args[FTCM_ARG_CB_IN0_BASE],
receiver_args[FTCM_ARG_CB_IN0_BASE], plan.cb[0].address);
ft_tt_test_check_fmt("the destination count EXCLUDES the leader",
leader_args[FTCM_ARG_DESTINATION_COUNT] == 3u &&
receiver_args[FTCM_ARG_DESTINATION_COUNT] == 3u,
"%u and %u for a group of 4 with the leader inside (wanted 3): too "
"many and the barrier never returns, too few and it returns while "
"writes are still landing",
leader_args[FTCM_ARG_DESTINATION_COUNT],
receiver_args[FTCM_ARG_DESTINATION_COUNT]);
ft_tt_test_check_fmt("a member is told its own column slice and the FULL width",
receiver_args[FTCM_ARG_B_COLUMN_FIRST] == 72u &&
receiver_args[FTCM_ARG_TILE_COLS] == 36u &&
receiver_args[FTCM_ARG_B_FULL_COLS] == 144u,
"first %u, count %u, full %u (wanted 72, 36, 144)",
receiver_args[FTCM_ARG_B_COLUMN_FIRST],
receiver_args[FTCM_ARG_TILE_COLS],
receiver_args[FTCM_ARG_B_FULL_COLS]);
ft_tt_test_check_fmt("the compute third is told its own width, and nothing else changes",
compute_args[0] == 1u && compute_args[1] == 48u &&
compute_args[2] == 36u,
"Mt %u, Kt %u, Nt %u (wanted 1, 48, 36)", compute_args[0],
compute_args[1], compute_args[2]);
ft_tt_test_check_fmt("the writer third streams this member's tiles to its own panel",
writer_args[0] == 36u &&
writer_args[3] == request.output.address + 72u * TILE_BYTES,
"%u tiles to 0x%08x (wanted 36 to 0x%08x)", writer_args[0],
writer_args[3],
(unsigned)(request.output.address + 72u * TILE_BYTES));
/* A SOLO member gets NO rectangle. A zeroed one is (0,0)..(0,0), a real live endpoint of
* the wrong type, and leaving destination_count at zero is what makes the kernel refuse to
* issue rather than issue somewhere plausible-looking. */
request.mode = FT_TT_OP_MATMUL_SPLIT;
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &plan);
if (rc == FT_TT_OK)
rc = ft_tt_op_matmul_cooperative_reader_arguments(&plan, &request, g_square, 0u,
leader_args, FTCM_ARG_COUNT);
ft_tt_test_check_fmt("a SPLIT member is given NO rectangle and zero destinations",
rc == FT_TT_OK && leader_args[FTCM_ARG_ROLE] == FTCM_ROLE_SOLO &&
leader_args[FTCM_ARG_DESTINATION_COUNT] == 0u &&
leader_args[FTCM_ARG_RECTANGLE_X_HIGH] == 0u,
"rc=%d, role %u, destinations %u, x_high %u", rc,
leader_args[FTCM_ARG_ROLE], leader_args[FTCM_ARG_DESTINATION_COUNT],
leader_args[FTCM_ARG_RECTANGLE_X_HIGH]);
/* INDEPENDENT is not a split: every member does the whole width into its own result. */
request.mode = FT_TT_OP_MATMUL_INDEPENDENT;
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &plan);
ft_tt_test_check_fmt("INDEPENDENT gives every member the WHOLE width and its own output",
rc == FT_TT_OK && plan.member[3].column_first == 0u &&
plan.member[3].column_count == 144u &&
plan.member[3].output_address ==
request.output.address + 3u * 144u * TILE_BYTES,
"rc=%d, member 3 first %u count %u at 0x%08x", rc,
plan.member[3].column_first, plan.member[3].column_count,
plan.member[3].output_address);
}
/* ------------------------------------------------------------------------- */
/** Doors of one bank, spelled out rather than derived, for the same reason g_square is: this
* file has no grid and must not be able to inherit a grid bug. `coord` is a stand-in
* translated alias; `noc_row` is the NoC 0 y the row-sharing rule is expressed in. */
static void fill_port_set(ft_tt_dram_port_set *set, uint32_t bank, const uint32_t *rows,
uint32_t port_count)
{
uint32_t port;
memset(set, 0, sizeof(*set));
set->logical_bank = bank;
set->port_count = port_count;
for (port = 0u; port < port_count; port++) {
set->port[port].logical_bank = bank;
set->port[port].port = port;
set->port[port].coord.x = 17u;
set->port[port].coord.y = bank * 3u + port;
set->port[port].noc_row = rows[port];
}
}
/**
* The port rotation: which DOOR each member reads through, that NO other reader arg moves,
* and that a bad bank pairing is COUNTED rather than silently accepted.
*
* The last is the one worth the most: two DRAM endpoints on one NoC row share ~60 GB/s where
* two in one column do not interact at all (plan Phase 6.7), and the pairing that produces it
* — the same port index of a bank and of bank+4 — looks completely innocent.
*/
static void test_the_port_rotation(void)
{
/* Blackhole's real rows, ft_tt_arch_blackhole.c:93. Bank 5 (east) and bank 1 (west) carry
* the SAME three y, which is the collision this must catch; bank 6 carries three others. */
static const uint32_t bank5_rows[3] = {2u, 10u, 3u};
static const uint32_t bank6_rows[3] = {9u, 4u, 8u};
static const uint32_t bank1_rows[3] = {2u, 10u, 3u};
ft_tt_op_matmul_cooperative_request request;
ft_tt_op_matmul_cooperative_plan striped;
ft_tt_op_matmul_cooperative_plan flat;
uint32_t striped_args[FTCM_ARG_COUNT];
uint32_t flat_args[FTCM_ARG_COUNT];
uint32_t member;
uint32_t endpoint_args_moved = 0u;
uint32_t other_args_moved = 0u;
uint32_t doors_not_in_the_args = 0u;
int rc;
ft_tt_test_step("the DRAM port rotation — the doors, and the row collision it must count");
fill_request(&request, 144u, FT_TT_OP_MATMUL_SPLIT);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &flat);
ft_tt_test_check_fmt("a request with NO port sets plans exactly as it always did",
rc == FT_TT_OK && flat.distinct_endpoints == 0u &&
flat.row_collisions == 0u &&
flat.member[3].weight_x == request.weight.x &&
flat.member[3].weight_y == request.weight.y,
"rc=%d, %u endpoints, %u collisions, member 3 reads B at (%u,%u) "
"against the request's (%u,%u)",
rc, flat.distinct_endpoints, flat.row_collisions,
flat.member[3].weight_x, flat.member[3].weight_y, request.weight.x,
request.weight.y);
fill_port_set(&request.activation_ports, 5u, bank5_rows, 3u);
fill_port_set(&request.weight_ports, 6u, bank6_rows, 3u);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &striped);
ft_tt_test_check_fmt("four members over three doors take ports 0,1,2,0 — the balanced "
"assignment, since the outstanding-read budget is PER ENDPOINT",
rc == FT_TT_OK && striped.member[0].weight_y == 18u &&
striped.member[1].weight_y == 19u &&
striped.member[2].weight_y == 20u &&
striped.member[3].weight_y == 18u,
"rc=%d, B doors y %u %u %u %u (wanted 18 19 20 18)", rc,
striped.member[0].weight_y, striped.member[1].weight_y,
striped.member[2].weight_y, striped.member[3].weight_y);
ft_tt_test_check_fmt("three doors of each of two banks is six distinct endpoints, on six "
"distinct rows",
rc == FT_TT_OK && striped.distinct_endpoints == 6u &&
striped.row_collisions == 0u,
"%u endpoints, %u row collisions", striped.distinct_endpoints,
striped.row_collisions);
/* The whole claim of the rotation is that ONLY the endpoint moves. Every other reader arg
* has to be bit-identical to the un-striped plan's, or the byte comparison on the card
* would be scoring two different programs rather than two addressings of one. */
for (member = 0u; member < 4u && rc == FT_TT_OK; member++) {
uint32_t index;
rc = ft_tt_op_matmul_cooperative_reader_arguments(&striped, &request, g_square, member,
striped_args, FTCM_ARG_COUNT);
if (rc == FT_TT_OK)
rc = ft_tt_op_matmul_cooperative_reader_arguments(&flat, &request, g_square, member,
flat_args, FTCM_ARG_COUNT);
if (rc != FT_TT_OK)
break;
/* Named individually, because "some endpoint word moved" is satisfied by A alone and
* would pass a build that dropped B's door entirely — the mutation that goes green
* against the weaker form. */
if (striped_args[FTCM_ARG_A_X] != striped.member[member].activation_x ||
striped_args[FTCM_ARG_A_Y] != striped.member[member].activation_y ||
striped_args[FTCM_ARG_B_X] != striped.member[member].weight_x ||
striped_args[FTCM_ARG_B_Y] != striped.member[member].weight_y)
doors_not_in_the_args++;
for (index = 0u; index < FTCM_ARG_COUNT; index++) {
if (striped_args[index] == flat_args[index])
continue;
if (index == FTCM_ARG_A_X || index == FTCM_ARG_A_Y || index == FTCM_ARG_B_X ||
index == FTCM_ARG_B_Y)
endpoint_args_moved++;
else
other_args_moved++;
}
}
ft_tt_test_check_fmt("every member's reader block carries THAT member's four door words",
rc == FT_TT_OK && doors_not_in_the_args == 0u,
"rc=%d, %u of 4 members' arg blocks disagree with the plan's doors",
rc, doors_not_in_the_args);
/* Absolute, not differential: the ADDRESSES are the claim striping makes — one memory,
* several doors — and a differential check cannot see a change made to both arms. */
ft_tt_test_check_fmt("the operand ADDRESSES are untouched by striping, on every member",
rc == FT_TT_OK &&
striped_args[FTCM_ARG_A_BASE] == request.activation.address &&
striped_args[FTCM_ARG_B_BASE] == request.weight.address,
"A base 0x%08x (wanted 0x%08x), B base 0x%08x (wanted 0x%08x)",
striped_args[FTCM_ARG_A_BASE], request.activation.address,
striped_args[FTCM_ARG_B_BASE], request.weight.address);
ft_tt_test_check_fmt("striping moves ENDPOINT args and nothing else — the bases, the "
"column slice, the depth and the CB all stay",
rc == FT_TT_OK && endpoint_args_moved > 0u && other_args_moved == 0u,
"rc=%d, %u endpoint arg words moved over 4 members, %u OTHER arg "
"words moved (0 endpoint words would mean the rotation did nothing; "
"any other word is the failure this counts)",
rc, endpoint_args_moved, other_args_moved);
/* THE RED HALF, off hardware: bank 1 west and bank 5 east share a NoC y per port index, so
* each of the three doors collides with its opposite number. */
fill_port_set(&request.activation_ports, 1u, bank1_rows, 3u);
fill_port_set(&request.weight_ports, 5u, bank5_rows, 3u);
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &striped);
ft_tt_test_check_fmt("a bank pairing whose ports share NoC rows is COUNTED, not silently "
"accepted",
rc == FT_TT_OK && striped.row_collisions == 3u &&
striped.distinct_endpoints == 6u,
"rc=%d, %u collisions (wanted 3, one per port index), %u endpoints — "
"a 0 here would mean the check cannot fail",
rc, striped.row_collisions, striped.distinct_endpoints);
}
/**
* PART B — the per-member SHARDS: the member reads ITS OWN panel, from column 0 of a grid its
* own width, at its own bank's door.
*
* The three arg words this changes are the ones a bank-striped plan lives or dies on, and each
* of them is in range whatever it holds — a member reading the full grid's column offset out
* of its own panel would run to completion and answer wrongly. So they are checked by value.
*/
static void test_the_bank_shards(void)
{
static const uint32_t bank2_rows[3] = {9u, 4u, 8u};
static const uint32_t bank3_rows[3] = {5u, 7u, 6u};
ft_tt_op_matmul_cooperative_request request;
ft_tt_op_matmul_cooperative_plan plan;
ft_tt_dram_port_set banks[2];
uint32_t args[FTCM_ARG_COUNT];
uint32_t member;
uint32_t wrong_grid = 0u;
int rc;
ft_tt_test_step("the per-member bank shards — own panel, own grid, own door");
fill_request(&request, 144u, FT_TT_OP_MATMUL_SPLIT);
fill_port_set(&banks[0], 2u, bank2_rows, 3u);
fill_port_set(&banks[1], 3u, bank3_rows, 3u);
for (member = 0u; member < 4u; member++) {
request.weight_shards[member].endpoint = banks[member % 2u].port[member % 3u];
request.weight_shards[member].address = 0x10000000u + member * 0x1000u;
request.activation_shards[member].endpoint = banks[(member + 1u) % 2u].port[member % 3u];
request.activation_shards[member].address = 0x20000000u;
}
/* A shard list that covers SOME members is refused, not partly applied: the covered ones
* would read their own panel from column 0 while the rest read the full grid at the full
* offset, and both are in range. */
request.weight_shard_count = 3u;
request.activation_shard_count = 4u;
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &plan);
ft_tt_test_check_fmt("a shard list that does not cover every member is REFUSED",
rc == FT_TT_EINVAL, "rc=%d (%s), wanted FT_TT_EINVAL", rc,
ft_tt_error_string(rc));
request.weight_shard_count = 4u;
rc = ft_tt_op_matmul_cooperative_plan_build(g_square, 4u, 0u, &request, &plan);
ft_tt_test_check_fmt("a full shard list plans", rc == FT_TT_OK, "rc=%d (%s)", rc,
ft_tt_error_string(rc));
if (rc != FT_TT_OK)
return;
for (member = 0u; member < 4u; member++) {
rc = ft_tt_op_matmul_cooperative_reader_arguments(&plan, &request, g_square, member,
args, FTCM_ARG_COUNT);
if (rc != FT_TT_OK)
break;
if (args[FTCM_ARG_B_BASE] != request.weight_shards[member].address ||
args[FTCM_ARG_B_COLUMN_FIRST] != 0u || args[FTCM_ARG_B_FULL_COLS] != 36u ||
args[FTCM_ARG_A_BASE] != request.activation_shards[member].address ||
args[FTCM_ARG_B_X] != request.weight_shards[member].endpoint.coord.x ||
args[FTCM_ARG_B_Y] != request.weight_shards[member].endpoint.coord.y)
wrong_grid++;
}
ft_tt_test_check_fmt("every member reads ITS OWN panel from column 0 of a 36-wide grid at "
"its own door",
rc == FT_TT_OK && wrong_grid == 0u,
"rc=%d, %u of 4 members disagree. Member 3 got base 0x%08x, first "
"column %u, full cols %u (wanted 0x%08x, 0, 36)",
rc, wrong_grid, args[FTCM_ARG_B_BASE], args[FTCM_ARG_B_COLUMN_FIRST],
args[FTCM_ARG_B_FULL_COLS], request.weight_shards[3].address);
ft_tt_test_check_fmt("the member still knows which columns of the FULL answer it owns",
plan.member[3].column_first == 108u && plan.member[3].column_count == 36u,
"member 3 owns columns %u..%u of 144 — the OUTPUT panel depends on it, "
"and sharding must not fold it away",
plan.member[3].column_first,
plan.member[3].column_first + plan.member[3].column_count - 1u);
}
/* ------------------------------------------------------------------------- */
/** This card's telemetry masks, written out rather than read from a device, so the shard
* placement is provable on the Mac. All 14 Tensix columns, all 8 GDDR banks, translation on
* — the grid tests/test_coord.c is an oracle for. */
static ft_tt_grid_info card_grid(void)
{
ft_tt_grid_info grid;
memset(&grid, 0, sizeof(grid));
grid.valid = true;
grid.arch = ft_tt_arch_blackhole();
grid.tensix_enabled_col = 0x3FFFu;
grid.enabled_gddr = 0xFFu;
grid.enabled_eth = 0x3FFFu;
grid.enabled_l2cpu = 0xFu;
grid.noc_translation_enabled = 1u;
return grid;
}
/** One operand pair to place: a PARTITIONED weight and a REPLICATED activation, which is the
* shape every bank-striped matmul arm asks for. */
static void fill_shard_operands(ft_tt_dram_shard_operand *operands, uint32_t weight_bytes,
uint32_t activation_bytes)
{
memset(operands, 0, sizeof(operands[0]) * 2u);
operands[0].spread = FT_TT_DRAM_SHARD_PARTITIONED;
operands[0].bank_base_address = 0x10000000u;
operands[0].member_bytes = weight_bytes;
operands[1].spread = FT_TT_DRAM_SHARD_REPLICATED;
operands[1].bank_base_address = 0x20000000u;
operands[1].member_bytes = activation_bytes;
}
/**
* LOAD-TIME placement — the fork Phase 11 settled, proved without silicon.
*
* The claim is that the ARTIFACT STAYS CANONICAL: nothing about it changes with N, and all
* the group size decides is which contiguous run of its bytes goes where. So the checks are
* about the MAP, not about any layout:
*
* COVERAGE at every N the partitioned source ranges tile the artifact exactly once — no
* gap, no overlap. A placement that dropped or duplicated a column slice would
* still complete on silicon and still read live endpoints.
* DISJOINT no two members get the same (bank, address). This is the bug the byte
* comparison against single-core caught at N=12 and N=16 and NOTHING ELSE did:
* two members wrote panels to one base and silently overwrote each other, every
* kernel completed, every status word was clean and the GB/s did not move. It is
* an ARITHMETIC property, so it belongs here, where finding it costs a second
* instead of a run on the card.
* BALANCE busiest and quietest bank differ by at most one stream at every N. The
* outstanding-read budget is per endpoint and an unbalanced map measured 30%
* worse than either neighbouring balanced count (plan Phase 6.7).
* SEPARATION no member pulls its two operands through one bank, which serves ~64 GB/s
* however many of its doors are used.
*/
static void test_the_shard_placement(void)
{
static const uint32_t group_sizes[6] = {1u, 2u, 4u, 8u, 12u, 16u};
const ft_tt_grid_info grid = card_grid();
/* 48 columns of weight in the canonical artifact: divisible by every group size below, so
* a member's slice is always a whole number of them. */
const uint32_t canonical_weight_bytes = 48u * 0x4000u;
ft_tt_dram_shard_operand operands[2];
uint32_t entry;
uint32_t coverage_wrong = 0u;
uint32_t collisions = 0u;
uint32_t unbalanced = 0u;
uint32_t same_bank = 0u;
uint32_t worst_gap = 0u;
int rc = FT_TT_OK;
ft_tt_test_step("the LOAD-TIME shard placement — canonical bytes in, per-member banks out");
for (entry = 0u; entry < 6u && rc == FT_TT_OK; entry++) {
const uint32_t members = group_sizes[entry];
ft_tt_dram_shard_plan plan;
uint32_t member;
fill_shard_operands(operands, canonical_weight_bytes / members, 0x1000u);
rc = ft_tt_dram_shard_plan_build(&grid, operands, 2u, members, FT_TT_COORD_TRANSLATED,
&plan);
if (rc != FT_TT_OK)
break;
if (plan.busiest_bank_streams - plan.quietest_bank_streams > 1u)
unbalanced++;
if (plan.busiest_bank_streams - plan.quietest_bank_streams > worst_gap)
worst_gap = plan.busiest_bank_streams - plan.quietest_bank_streams;
for (member = 0u; member < members; member++) {
const ft_tt_dram_shard_placement *weight =
ft_tt_dram_shard_placement_of(&plan, 0u, member);
const ft_tt_dram_shard_placement *activation =
ft_tt_dram_shard_placement_of(&plan, 1u, member);
uint32_t other;
if (weight == NULL || activation == NULL) {
coverage_wrong++;
continue;
}
/* Member c's slice starts exactly where c-1's ended and the last ends at the
* artifact's end — coverage without gap or overlap, stated once. */
if (weight->source_offset != member * weight->bytes ||
weight->bytes * members != canonical_weight_bytes ||
activation->source_offset != 0u)
coverage_wrong++;
if (weight->endpoint.logical_bank == activation->endpoint.logical_bank)
same_bank++;
for (other = 0u; other < member; other++) {
const ft_tt_dram_shard_placement *earlier =
ft_tt_dram_shard_placement_of(&plan, 0u, other);
if (earlier != NULL &&
earlier->endpoint.logical_bank == weight->endpoint.logical_bank &&
earlier->address == weight->address)
collisions++;
}
}
}
ft_tt_test_check_fmt("every group size plans", rc == FT_TT_OK, "rc=%d (%s)", rc,
ft_tt_error_string(rc));
ft_tt_test_check_fmt("a member's slice is a CONTIGUOUS run of the canonical bytes, and the "
"slices tile it exactly once at N = 1,2,4,8,12,16",
coverage_wrong == 0u,
"%u placements disagree — the artifact is meant to be untouched and "
"N-independent, so a wrong offset here is a converter's decision that "
"has leaked into the loader",
coverage_wrong);
ft_tt_test_check_fmt("no two members are given the SAME bank and address", collisions == 0u,
"%u colliding pairs. This is the failure that shows ONLY as BYTES "
"DIFFER at N=12 and N=16 on silicon: both members read a valid panel "
"from a live endpoint and complete",
collisions);
ft_tt_test_check_fmt("the map is BALANCED at every group size — busiest and quietest bank "
"differ by at most one stream",
unbalanced == 0u, "%u group sizes unbalanced, worst gap %u streams",
unbalanced, worst_gap);
ft_tt_test_check_fmt("no member pulls both operands through one bank", same_bank == 0u,
"%u members do; a bank serves ~64 GB/s however many doors are used",
same_bank);
}
/**
* The refusals. Each one is a wrong DRAM ADDRESS if it is let through, and a wrong DRAM
* address is a wrong answer with no error anywhere on the card.
*/
static void test_the_shard_refusals(void)
{
const ft_tt_grid_info grid = card_grid();
ft_tt_dram_shard_operand operands[2];
ft_tt_dram_shard_plan plan;
int rc;
ft_tt_test_step("the shard placement's refusals");
fill_shard_operands(operands, 0x4000u, 0x1000u);
operands[0].member_bytes = 0u;
rc = ft_tt_dram_shard_plan_build(&grid, operands, 2u, 4u, FT_TT_COORD_TRANSLATED, &plan);
ft_tt_test_check_fmt("an operand that places ZERO bytes per member is refused",
rc == FT_TT_EINVAL, "rc=%d (%s), wanted FT_TT_EINVAL", rc,
ft_tt_error_string(rc));
fill_shard_operands(operands, 0x4000u, 0x1000u);
operands[1].bank_base_address = operands[0].bank_base_address;
rc = ft_tt_dram_shard_plan_build(&grid, operands, 2u, 4u, FT_TT_COORD_TRANSLATED, &plan);
ft_tt_test_check_fmt("two operands at ONE bank base are refused — their members would "
"stack over each other and both would still complete",
rc == FT_TT_EINVAL, "rc=%d (%s), wanted FT_TT_EINVAL", rc,
ft_tt_error_string(rc));
fill_shard_operands(operands, 0x4000u, 0x1000u);
rc = ft_tt_dram_shard_plan_build(&grid, operands, 2u, FT_TT_DRAM_SHARD_MAX_MEMBERS + 1u,
FT_TT_COORD_TRANSLATED, &plan);
ft_tt_test_check_fmt("more members than the plan can hold is refused, not truncated",
rc == FT_TT_EINVAL, "rc=%d (%s), wanted FT_TT_EINVAL", rc,
ft_tt_error_string(rc));
/* 16 members over 8 banks is two 1.875 GiB pieces per bank on top of a 256 MiB base —
* 4 GiB, past this part's 0xFF000000. The arithmetic is done in 64 bits on purpose: in 32
* it would WRAP and place the second piece at a low, live address instead of refusing. */
fill_shard_operands(operands, 0x78000000u, 0x1000u);
rc = ft_tt_dram_shard_plan_build(&grid, operands, 2u, 16u, FT_TT_COORD_TRANSLATED, &plan);
ft_tt_test_check_fmt("a placement that would run past the END of a bank is refused",
rc == FT_TT_ENOMEM, "rc=%d (%s), wanted FT_TT_ENOMEM", rc,
ft_tt_error_string(rc));
}
int main(void)
{
test_the_split();
test_the_ragged_split();
test_placement_is_group_independent();
test_the_refusals();
test_the_argument_blocks();
test_the_port_rotation();
test_the_bank_shards();
test_the_shard_placement();
test_the_shard_refusals();
return ft_tt_test_finish();
}
Reference in New Issue View Git Blame Copy Permalink