/* SPDX-License-Identifier: AGPL-3.0-or-later */ /** * @file resident_place_writer.c * @brief Writer half of the RESIDENT MEMORY triad — EMBEDDING and KV_APPEND — and the processor * that PUBLISHES COMPLETION, per core and per group. * * IT IS NOT resident_vector_writer.c, AND THE DIFFERENCE IS THE WHOLE OP. That writer * drains `tile_count` whole tiles to consecutive destinations. This one writes a PARTIAL * destination: an append into a live KV cache must land `row_count` rows of each tile and * leave the other rows exactly as they were, because those rows are earlier positions of * the same sequence. So it emits FACE-ROW RUNS (resident_memory_protocol.h, * ftm_destination_runs) rather than tiles. * * THE RUNS COLLAPSE, WHICH IS WHY ONE CODE PATH SERVES DECODE AND PREFILL: * rows [r, r+n) of one face column-half are CONTIGUOUS, so a range is at most four * runs per tile; * rows [0, 32) is the whole tile, and a whole GROUP of tiles is contiguous, so a * 32-row append is ONE transaction of `group_tiles * TILE_BYTES`. * A one-token decode append is therefore `group_tiles * 2` writes of 32 bytes, and a * 32-token prefill append is one write of 8 KiB, out of the same loop. * * 32 BYTES IS A LEGAL DRAM WRITE and that is the load-bearing fact. The NoC's write * alignment is 16 B; only the read direction needs 64 (ft_tt_noc.c, * ft_tt_noc_alignment()). tests/test_phase6_memory.c measures it rather than trusting it, * and checks the NEIGHBOURS as well as the row — a partial write that quietly rounded up * to 64 bytes would pass a check that only looked at what it meant to write. * * WHY THE WRITER OWNS COMPLETION (resident_queue_protocol.h rule 1). An op is done when * the result has left L1. This processor owns `completed_count` and `completed_token`, * the reader owns `tail`, and nothing else writes either. It FETCHES on `head` so it * drains while the reader is still filling, and PUBLISHES only after `tail` has passed * its cursor — fetching on `tail` is a pipeline deadlock, not a precaution * (resident_matmul_writer.c records the run that cost). * * * C99 on Foundry's own `ft_tensix_dataflow.h` — no tt-metal include (plan Phase 6.6). * The wait mode, which used to be a `template ` argument, is now an * ordinary parameter carrying the same compile-time value; `-Os` folds it identically. * Compile-time args: * 0: CB_OUT_EMBEDDING index 1: tile size in bytes * 2: wait mode -- 0 upstream, 1 stoppable (ships), 2 bound-only * 3: CB_OUT_KV index -- KV_APPEND's own output CB, because A CB CARRIES ONE GROUP SIZE * (ft_tt_op_memory.h has the arithmetic and the incident). It must name the same two * buffers, in the same roles, as the reader's args 0 and 5: the two processors are * the producer and the consumer of ONE buffer and nothing checks that they agree. * Runtime args: the ten of resident_queue_protocol.h, identical on all five processors * * Build (NCRISC, NoC 1): * CTARGS=16,2048,1,17 NOC=1 KERNEL_TEXT_BASE=0x18000 bash \ * foundry/tt/build_tensix_dataflow_kernel.sh \ * foundry/tt/kernels/resident_place_writer.c ncrisc /tmp/ttk_memory * * @status GREEN ON SILICON 2026-08-12 — Tensix 30 and Tensix 32, 17 checks and 0 failures * each. The 32-byte DRAM write is MEASURED, neighbours included. 2656 B of text * and 24 B of bss on NCRISC, against a 12 KiB slab. RE-GREEN on Tensix 40, * 2026-08-13, with the output CB split per group size (ft_tt_op_memory.h @status * has the fix's two proofs). * @issues None * @todo None */ #include #include #include "resident_memory_protocol.h" #include "ft_tensix_dataflow.h" #include "resident_queue_protocol.h" #define CB_OUT_EMBEDDING ft_tensix_compile_time_arg(0) #define TILE_BYTES ft_tensix_compile_time_arg(1) #define WAIT_MODE ft_tensix_compile_time_arg(2) #define CB_OUT_KV ft_tensix_compile_time_arg(3) /** The output CB an opcode's group size belongs to — the consumer half of the reader's * output_circular_buffer(). ONE GROUP SIZE PER CB (ft_tt_op_memory.h). */ static uint32_t output_circular_buffer(uint32_t opcode) { return (opcode == kOpKvAppend) ? CB_OUT_KV : CB_OUT_EMBEDDING; } /** Put one group's assembled tiles on the destination. * * ONE barrier for the whole group, never one per run: a barrier drains this core's * outstanding transactions and is per core, not per transaction. */ /** @return false if any length came out zero — the caller refuses the descriptor rather * than issuing it (resident_memory_protocol.h, ftm_length_is_issuable). */ static bool write_one_group(uint32_t block, uint32_t destination_x, uint32_t destination_y, uint32_t destination_base, uint32_t group_tiles, uint32_t row0, uint32_t row_count, bool whole_tiles) { ftm_run runs[kMemMaxRuns]; uint32_t run_count; if (!ftm_length_is_issuable(group_tiles * TILE_BYTES)) return false; if (whole_tiles || (row0 == 0u && row_count == kMemTileRows)) { // The descriptor OWNS the destination tile row — either it said so, or it covers // every row of it. Both ends are contiguous, so it is ONE transaction. The rows it // did not place are the zeros the reader put in the circular-buffer page. ft_tensix_noc_write(block, ft_tensix_noc_address(destination_x, destination_y, destination_base), group_tiles * TILE_BYTES); ft_tensix_noc_write_barrier(); return true; } run_count = ftm_destination_runs(row0, row_count, TILE_BYTES, runs); if (run_count == 0u) return false; for (uint32_t index = 0; index < run_count; index++) { if (!ftm_length_is_issuable(runs[index].bytes)) return false; } for (uint32_t tile = 0; tile < group_tiles; tile++) { const uint32_t tile_offset = tile * TILE_BYTES; for (uint32_t index = 0; index < run_count; index++) { const uint32_t offset = tile_offset + runs[index].offset; ft_tensix_noc_write(block + offset, ft_tensix_noc_address(destination_x, destination_y, destination_base + offset), runs[index].bytes); } } ft_tensix_noc_write_barrier(); return true; } /** * Drain one descriptor: `group_count` groups of `group_tiles` tiles each. * * @return true if the whole descriptor was written; false if a wait was abandoned */ static bool write_one_memory_op(ftq_context *ctx, volatile FTQ_L1_PTR uint32_t *parameters, uint32_t cb_out) { const uint32_t flags = parameters[kMemFlags]; const uint32_t row_count = parameters[kMemRowCount]; const uint32_t group_count = parameters[kMemGroupCount]; const uint32_t group_tiles = parameters[kMemGroupTiles]; const uint32_t destination_x = parameters[kMemDstX]; const uint32_t destination_y = parameters[kMemDstY]; const uint32_t destination_base = parameters[kMemDstBase]; const uint32_t destination_group_stride = parameters[kMemDstGroupStride]; const uint32_t destination_row0 = parameters[kMemDstRow0]; const uint32_t row_in_tile = destination_row0 % kMemTileRows; const uint32_t tile_row = destination_row0 / kMemTileRows; uint32_t group_bytes; uint32_t tile_row_offset; if (row_count == 0u || row_count > kMemMaxRows || group_count == 0u || group_tiles == 0u || row_in_tile + row_count > kMemTileRows) { ftq_abort(ctx, kErrBadParameters); return false; } /* THE DESTINATION WAS UNBOUNDED. `destination_row0` is an absolute KV position, so * `tile_row * group_tiles * TILE_BYTES` is a raw byte offset with nothing above it — a * position past the cache capacity wrote into whatever DRAM follows, at full length, * with nothing faulting. The bound is kMemDstGroupStride, the bytes each group OWNS; * see ftm_destination_span_fits(). */ if (!ft_tensix_product_fits(group_tiles, TILE_BYTES, &group_bytes) || !ft_tensix_product_fits(tile_row, group_bytes, &tile_row_offset)) { ftq_abort(ctx, kErrBadParameters); return false; } /* And the CB CYCLE this file did not have: the output CB is POPPED `group_tiles` at a * time and the read pointer wraps by the same exact-hit rule as the write pointer, so a * descriptor that does not close the cycle leaves every later one off the grid. Kept * unweakened after the one-CB-per-group-size fix, for the reason the reader states. */ if (!ft_tensix_cb_pop_grid_is_safe(cb_out, group_tiles)) { ftq_abort(ctx, kErrBadParameters); return false; } for (uint32_t group = 0; group < group_count; group++) { const bool have_tiles = ftq_cb_wait_front_stoppable(ctx, (int32_t)cb_out, (int32_t)group_tiles, WAIT_MODE); if (have_tiles) { uint32_t group_offset; /* Inside the GROUP, not merely inside the region: the tighter statement, and the * one that also stops head h's append landing in head h+1. */ const bool bounded = ft_tensix_product_fits(group, destination_group_stride, &group_offset) && group_offset <= 0xFFFFFFFFu - tile_row_offset && ftm_destination_span_fits(destination_group_stride, tile_row_offset, group_bytes); const uint32_t base = destination_base + group_offset + tile_row_offset; if (!bounded || !write_one_group(ft_tensix_cb_read_pointer(cb_out), destination_x, destination_y, base, group_tiles, row_in_tile, row_count, (flags & kMemFlagDstWholeTiles) != 0u)) { ftq_abort(ctx, kErrBadParameters); ft_tensix_cb_pop_front(cb_out, group_tiles); return false; } } ft_tensix_cb_pop_front(cb_out, group_tiles); } return !ctx->aborting; } void kernel_main(void) { ftq_context ctx; uint32_t completed; bool running = true; ftq_context_init(&ctx); ftq_status_set(&ctx, kStateBoot); if (!ftq_check_control(&ctx)) { ftq_status_set(&ctx, kStateExit); return; } completed = ctx.control[kFieldCompletedCount]; ctx.cursor = completed; while (running) { // FETCH on `head`, so the writer drains the output CB while the reader is still // filling it. Publishing waits on `tail`, below. if (!ftq_wait_for_work(&ctx, kFieldHead)) break; ctx.aborting = false; ftq_status_set(&ctx, kStateExec); volatile FTQ_L1_PTR uint32_t *descriptor = ftq_descriptor(&ctx); const uint32_t opcode = descriptor[kDescOpcode]; const uint32_t token = descriptor[kDescToken]; const uint32_t group_sequence = descriptor[kDescA5]; bool done = true; ftq_status_opcode(&ctx, opcode); if (opcode == kOpEmbedding || opcode == kOpKvAppend) { ftq_status_set(&ctx, kStateCbWaitFront); done = write_one_memory_op(&ctx, ftq_parameters(descriptor), output_circular_buffer(opcode)); ftq_status_set(&ctx, kStateExec); } else if (opcode == kOpStop) { running = false; } if (!done) { running = false; continue; } // THE PRECONDITION, and the only place it belongs: the reader must have finished // this descriptor before its completion is published. if (!ftq_wait_until_reaches(&ctx, kFieldTail, ctx.cursor + 1u)) { running = false; continue; } ctx.cursor++; // Token before count: a host that sees the count is entitled to trust the token it // reads next. Both are plain stores from ONE in-order RISC-V. ctx.control[kFieldCompletedToken] = token; ctx.control[kFieldCompletedCount] = ++completed; ftq_group_post(&ctx, group_sequence, token); } // The writer does NOT touch kFieldError or kFieldState: the reader owns those, and one // writer per word is the whole ordering argument. ftq_status_set(&ctx, kStateExit); }