/* SPDX-License-Identifier: AGPL-3.0-or-later */ /* Copyright (C) 2026 David (Corey) Herrera */ /** * @file ft_tt_dram_shard.c * @brief Implementation of ft_tt_dram_shard.h — see that header for why the placement is the * loader's decision and which measured rules it respects. * * @status Off-hardware in tests/test_cooperative_matmul_plan.c. * @issues None * @todo None */ #include "ft_tt_dram_shard.h" #include #include "ft_tt_device.h" /** * The bank member `member` of operand `operand` reads from. * * Round-robin over members, with each operand offset a whole FRACTION of the bank list from * the last. The offset is what keeps a member's two operands in different banks — member c * would otherwise pull both through one bank's ~64 GB/s and the second door would buy it * nothing. */ static uint32_t bank_of(uint32_t bank_count, uint32_t operand_count, uint32_t operand, uint32_t member) { return (member + operand * (bank_count / operand_count)) % bank_count; } /** * Does every operand fit in the bank that will hold the most of it? * * Checked BEFORE anything is placed so the plan is written only on success, and computable in * closed form because the map is round-robin: the busiest bank holds exactly * `ceil(member_count / bank_count)` pieces of each operand. * * The arithmetic is 64-bit deliberately. In 32 it would WRAP and place a piece at a low, live * DRAM address — which is not an error anywhere, it is a wrong answer. */ static int check_extents(const ft_tt_dram_shard_operand *operands, uint32_t operand_count, uint32_t member_count, uint32_t bank_count, uint64_t bank_bytes) { const uint32_t busiest = (member_count + bank_count - 1u) / bank_count; uint32_t operand; for (operand = 0u; operand < operand_count; operand++) { const uint64_t end = (uint64_t)operands[operand].bank_base_address + (uint64_t)busiest * operands[operand].member_bytes; if (end > bank_bytes) { ft_tt_log_error("dram shard: operand %u needs %u pieces of %u bytes from 0x%08x in " "the busiest bank, ending at %llu of %llu — %u members do not fit " "at this base", operand, busiest, operands[operand].member_bytes, operands[operand].bank_base_address, (unsigned long long)end, (unsigned long long)bank_bytes, member_count); return FT_TT_ENOMEM; } } return FT_TT_OK; } /** Refuse before anything is computed: a bad count here becomes a wrong DRAM address, and a * wrong DRAM address is a wrong answer with no error anywhere. */ static int check_arguments(const ft_tt_grid_info *grid, const ft_tt_dram_shard_operand *operands, uint32_t operand_count, uint32_t member_count, const ft_tt_dram_shard_plan *out) { uint32_t operand; if (grid == NULL || operands == NULL || out == NULL) { ft_tt_log_error("dram shard: NULL grid/operands/out"); return FT_TT_EINVAL; } if (operand_count == 0u || operand_count > FT_TT_DRAM_SHARD_MAX_OPERANDS) { ft_tt_log_error("dram shard: %u operands is outside 1..%u", operand_count, FT_TT_DRAM_SHARD_MAX_OPERANDS); return FT_TT_EINVAL; } if (member_count == 0u || member_count > FT_TT_DRAM_SHARD_MAX_MEMBERS) { ft_tt_log_error("dram shard: %u members is outside 1..%u", member_count, FT_TT_DRAM_SHARD_MAX_MEMBERS); return FT_TT_EINVAL; } for (operand = 0u; operand < operand_count; operand++) { uint32_t earlier; if (operands[operand].member_bytes == 0u) { ft_tt_log_error("dram shard: operand %u places 0 bytes per member", operand); return FT_TT_EINVAL; } /* Two operands sharing a base would stack their members over each other and both * would still read a live endpoint and complete. That exact shape — two pieces at * one base, everything green — is what the byte comparison against single-core * caught at N=12 and N=16 and nothing else did (plan Phase 6.7). */ for (earlier = 0u; earlier < operand; earlier++) { if (operands[earlier].bank_base_address == operands[operand].bank_base_address) { ft_tt_log_error("dram shard: operands %u and %u share bank base 0x%08x — " "their members would silently overwrite each other", earlier, operand, operands[operand].bank_base_address); return FT_TT_EINVAL; } } } return FT_TT_OK; } /** * Fill in `distinct_endpoints`, `row_collisions` and the two balance figures. * * Reported rather than enforced, all four of them: a row collision above 12 streams is * structural on this part and a bank imbalance is a performance fact, so refusing either * would reject a plan that returns the right bytes. */ static void summarise(ft_tt_dram_shard_plan *plan, const ft_tt_dram_endpoint *doors, uint32_t door_count, uint32_t bank_count) { uint32_t streams[FT_TT_MAX_DRAM_BANKS] = {0u}; uint32_t index; uint32_t bank; plan->row_collisions = ft_tt_dram_stripe_shared_rows(doors, door_count, NULL, NULL); plan->distinct_endpoints = 0u; for (index = 0u; index < door_count; index++) { uint32_t earlier; bool seen = false; for (earlier = 0u; earlier < index; earlier++) { if (doors[earlier].logical_bank == doors[index].logical_bank && doors[earlier].port == doors[index].port) { seen = true; break; } } if (!seen) plan->distinct_endpoints++; if (doors[index].logical_bank < FT_TT_MAX_DRAM_BANKS) streams[doors[index].logical_bank]++; } plan->busiest_bank_streams = 0u; plan->quietest_bank_streams = 0xFFFFFFFFu; for (bank = 0u; bank < bank_count && bank < FT_TT_MAX_DRAM_BANKS; bank++) { if (streams[bank] > plan->busiest_bank_streams) plan->busiest_bank_streams = streams[bank]; if (streams[bank] < plan->quietest_bank_streams) plan->quietest_bank_streams = streams[bank]; } if (plan->quietest_bank_streams == 0xFFFFFFFFu) plan->quietest_bank_streams = 0u; } int ft_tt_dram_shard_plan_build(const ft_tt_grid_info *grid, const ft_tt_dram_shard_operand *operands, uint32_t operand_count, uint32_t member_count, ft_tt_coord_system system, ft_tt_dram_shard_plan *out) { /* One entry per (member, operand) concurrent stream, MEMBER-MAJOR. The order is part of * the contract: ft_tt_dram_stripe_choose_ports() is greedy in reader order, so a * different flattening is a different — still correct, differently fast — door map, and * a plan that is not reproducible cannot be the input to a byte comparison. */ uint32_t banks[FT_TT_DRAM_SHARD_MAX_MEMBERS * FT_TT_DRAM_SHARD_MAX_OPERANDS]; ft_tt_dram_endpoint doors[FT_TT_DRAM_SHARD_MAX_MEMBERS * FT_TT_DRAM_SHARD_MAX_OPERANDS]; /* How many pieces of each operand are already placed in each bank. More members than * banks is normal, and two of them at one base would have the second SILENTLY overwrite * the first — both would then read a valid piece of the same model from a live endpoint * and complete. */ uint32_t placed[FT_TT_DRAM_SHARD_MAX_OPERANDS][FT_TT_MAX_DRAM_BANKS]; uint64_t bank_bytes; uint32_t bank_count; uint32_t stream_count; uint32_t member; uint32_t operand; int rc = check_arguments(grid, operands, operand_count, member_count, out); if (rc != FT_TT_OK) return rc; bank_count = ft_tt_coord_dram_bank_count(grid); if (bank_count == 0u || bank_count > FT_TT_MAX_DRAM_BANKS) { ft_tt_log_error("dram shard: the grid reports %u DRAM banks", bank_count); return FT_TT_EUNAVAIL; } if (grid->arch == NULL) { ft_tt_log_error("dram shard: the grid has no architecture, so a bank has no size"); return FT_TT_EUNAVAIL; } bank_bytes = grid->arch->core.dram_bytes_per_bank; rc = check_extents(operands, operand_count, member_count, bank_count, bank_bytes); if (rc != FT_TT_OK) return rc; memset(placed, 0, sizeof(placed)); stream_count = 0u; for (member = 0u; member < member_count; member++) { for (operand = 0u; operand < operand_count; operand++) banks[stream_count++] = bank_of(bank_count, operand_count, operand, member); } rc = ft_tt_dram_stripe_choose_ports(grid, banks, stream_count, system, doors); if (rc != FT_TT_OK) return rc; memset(out, 0, sizeof(*out)); out->member_count = member_count; out->operand_count = operand_count; stream_count = 0u; for (member = 0u; member < member_count; member++) { for (operand = 0u; operand < operand_count; operand++) { ft_tt_dram_shard_placement *placement = &out->placement[operand][member]; const uint32_t bank = banks[stream_count]; const uint32_t slot = placed[operand][bank]++; const uint64_t address = (uint64_t)operands[operand].bank_base_address + (uint64_t)slot * operands[operand].member_bytes; placement->endpoint = doors[stream_count]; placement->address = (uint32_t)address; placement->bytes = operands[operand].member_bytes; placement->source_offset = operands[operand].spread == FT_TT_DRAM_SHARD_PARTITIONED ? member * operands[operand].member_bytes : 0u; stream_count++; } } summarise(out, doors, stream_count, bank_count); return FT_TT_OK; } const ft_tt_dram_shard_placement *ft_tt_dram_shard_placement_of( const ft_tt_dram_shard_plan *plan, uint32_t operand, uint32_t member) { if (plan == NULL || operand >= plan->operand_count || member >= plan->member_count) return NULL; return &plan->placement[operand][member]; }