Foundry/tt/kernels/resident_place_writer.c

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/**
 * @file resident_place_writer.c
 * @brief Writer half of the RESIDENT MEMORY triad — EMBEDDING and KV_APPEND — and the processor
 * that PUBLISHES COMPLETION, per core and per group.
 *
 * IT IS NOT resident_vector_writer.c, AND THE DIFFERENCE IS THE WHOLE OP. That writer
 * drains `tile_count` whole tiles to consecutive destinations. This one writes a PARTIAL
 * destination: an append into a live KV cache must land `row_count` rows of each tile and
 * leave the other rows exactly as they were, because those rows are earlier positions of
 * the same sequence. So it emits FACE-ROW RUNS (resident_memory_protocol.h,
 * ftm_destination_runs) rather than tiles.
 *
 * THE RUNS COLLAPSE, WHICH IS WHY ONE CODE PATH SERVES DECODE AND PREFILL:
 *   rows [r, r+n) of one face column-half are CONTIGUOUS, so a range is at most four
 *   runs per tile;
 *   rows [0, 32) is the whole tile, and a whole GROUP of tiles is contiguous, so a
 *   32-row append is ONE transaction of `group_tiles * TILE_BYTES`.
 * A one-token decode append is therefore `group_tiles * 2` writes of 32 bytes, and a
 * 32-token prefill append is one write of 8 KiB, out of the same loop.
 *
 * 32 BYTES IS A LEGAL DRAM WRITE and that is the load-bearing fact. The NoC's write
 * alignment is 16 B; only the read direction needs 64 (ft_tt_noc.c,
 * ft_tt_noc_alignment()). tests/test_phase6_memory.c measures it rather than trusting it,
 * and checks the NEIGHBOURS as well as the row — a partial write that quietly rounded up
 * to 64 bytes would pass a check that only looked at what it meant to write.
 *
 * WHY THE WRITER OWNS COMPLETION (resident_queue_protocol.h rule 1). An op is done when
 * the result has left L1. This processor owns `completed_count` and `completed_token`,
 * the reader owns `tail`, and nothing else writes either. It FETCHES on `head` so it
 * drains while the reader is still filling, and PUBLISHES only after `tail` has passed
 * its cursor — fetching on `tail` is a pipeline deadlock, not a precaution
 * (resident_matmul_writer.c records the run that cost).
 *
 *
 * C99 on Foundry's own `ft_tensix_dataflow.h` — no tt-metal include (plan Phase 6.6).
 * The wait mode, which used to be a `template <uint32_t kMode>` argument, is now an
 * ordinary parameter carrying the same compile-time value; `-Os` folds it identically.
 * Compile-time args:
 *   0: CB_OUT_EMBEDDING index    1: tile size in bytes
 *   2: wait mode -- 0 upstream, 1 stoppable (ships), 2 bound-only
 *   3: CB_OUT_KV index -- KV_APPEND's own output CB, because A CB CARRIES ONE GROUP SIZE
 *      (ft_tt_op_memory.h has the arithmetic and the incident). It must name the same two
 *      buffers, in the same roles, as the reader's args 0 and 5: the two processors are
 *      the producer and the consumer of ONE buffer and nothing checks that they agree.
 * Runtime args: the ten of resident_queue_protocol.h, identical on all five processors
 *
 * Build (NCRISC, NoC 1):
 *   CTARGS=16,2048,1,17 NOC=1 KERNEL_TEXT_BASE=0x18000 bash \
 *       foundry/tt/build_tensix_dataflow_kernel.sh \
 *       foundry/tt/kernels/resident_place_writer.c ncrisc /tmp/ttk_memory
 *
 * @status GREEN ON SILICON 2026-08-12 — Tensix 30 and Tensix 32, 17 checks and 0 failures
 *         each. The 32-byte DRAM write is MEASURED, neighbours included. 2656 B of text
 *         and 24 B of bss on NCRISC, against a 12 KiB slab. RE-GREEN on Tensix 40,
 *         2026-08-13, with the output CB split per group size (ft_tt_op_memory.h @status
 *         has the fix's two proofs).
 * @issues None
 * @todo None
 */

#include <stdbool.h>
#include <stdint.h>

#include "resident_memory_protocol.h"
#include "ft_tensix_dataflow.h"
#include "resident_queue_protocol.h"

#define CB_OUT_EMBEDDING ft_tensix_compile_time_arg(0)
#define TILE_BYTES ft_tensix_compile_time_arg(1)
#define WAIT_MODE ft_tensix_compile_time_arg(2)
#define CB_OUT_KV ft_tensix_compile_time_arg(3)

/** The output CB an opcode's group size belongs to — the consumer half of the reader's
 *  output_circular_buffer(). ONE GROUP SIZE PER CB (ft_tt_op_memory.h). */
static uint32_t output_circular_buffer(uint32_t opcode)
{
    return (opcode == kOpKvAppend) ? CB_OUT_KV : CB_OUT_EMBEDDING;
}
/** Put one group's assembled tiles on the destination.
 *
 *  ONE barrier for the whole group, never one per run: a barrier drains this core's
 *  outstanding transactions and is per core, not per transaction. */
/** @return false if any length came out zero — the caller refuses the descriptor rather
 *          than issuing it (resident_memory_protocol.h, ftm_length_is_issuable). */
static bool write_one_group(uint32_t block, uint32_t destination_x, uint32_t destination_y,
                            uint32_t destination_base, uint32_t group_tiles, uint32_t row0,
                            uint32_t row_count, bool whole_tiles)
{
    ftm_run runs[kMemMaxRuns];
    uint32_t run_count;

    if (!ftm_length_is_issuable(group_tiles * TILE_BYTES))
        return false;
    if (whole_tiles || (row0 == 0u && row_count == kMemTileRows)) {
        // The descriptor OWNS the destination tile row — either it said so, or it covers
        // every row of it. Both ends are contiguous, so it is ONE transaction. The rows it
        // did not place are the zeros the reader put in the circular-buffer page.
        ft_tensix_noc_write(block, ft_tensix_noc_address(destination_x, destination_y,
                                                         destination_base),
                            group_tiles * TILE_BYTES);
        ft_tensix_noc_write_barrier();
        return true;
    }

    run_count = ftm_destination_runs(row0, row_count, TILE_BYTES, runs);
    if (run_count == 0u)
        return false;
    for (uint32_t index = 0; index < run_count; index++) {
        if (!ftm_length_is_issuable(runs[index].bytes))
            return false;
    }
    for (uint32_t tile = 0; tile < group_tiles; tile++) {
        const uint32_t tile_offset = tile * TILE_BYTES;

        for (uint32_t index = 0; index < run_count; index++) {
            const uint32_t offset = tile_offset + runs[index].offset;

            ft_tensix_noc_write(block + offset,
                                ft_tensix_noc_address(destination_x, destination_y,
                                                      destination_base + offset),
                                runs[index].bytes);
        }
    }
    ft_tensix_noc_write_barrier();
    return true;
}

/**
 * Drain one descriptor: `group_count` groups of `group_tiles` tiles each.
 *
 * @return true if the whole descriptor was written; false if a wait was abandoned
 */
static bool write_one_memory_op(ftq_context *ctx, volatile FTQ_L1_PTR uint32_t *parameters,
                                uint32_t cb_out)
{
    const uint32_t flags = parameters[kMemFlags];
    const uint32_t row_count = parameters[kMemRowCount];
    const uint32_t group_count = parameters[kMemGroupCount];
    const uint32_t group_tiles = parameters[kMemGroupTiles];
    const uint32_t destination_x = parameters[kMemDstX];
    const uint32_t destination_y = parameters[kMemDstY];
    const uint32_t destination_base = parameters[kMemDstBase];
    const uint32_t destination_group_stride = parameters[kMemDstGroupStride];
    const uint32_t destination_row0 = parameters[kMemDstRow0];
    const uint32_t row_in_tile = destination_row0 % kMemTileRows;
    const uint32_t tile_row = destination_row0 / kMemTileRows;
    uint32_t group_bytes;
    uint32_t tile_row_offset;

    if (row_count == 0u || row_count > kMemMaxRows || group_count == 0u || group_tiles == 0u ||
        row_in_tile + row_count > kMemTileRows) {
        ftq_abort(ctx, kErrBadParameters);
        return false;
    }
    /* THE DESTINATION WAS UNBOUNDED. `destination_row0` is an absolute KV position, so
     * `tile_row * group_tiles * TILE_BYTES` is a raw byte offset with nothing above it — a
     * position past the cache capacity wrote into whatever DRAM follows, at full length,
     * with nothing faulting. The bound is kMemDstGroupStride, the bytes each group OWNS;
     * see ftm_destination_span_fits(). */
    if (!ft_tensix_product_fits(group_tiles, TILE_BYTES, &group_bytes) ||
        !ft_tensix_product_fits(tile_row, group_bytes, &tile_row_offset)) {
        ftq_abort(ctx, kErrBadParameters);
        return false;
    }
    /* And the CB CYCLE this file did not have: the output CB is POPPED `group_tiles` at a
     * time and the read pointer wraps by the same exact-hit rule as the write pointer, so a
     * descriptor that does not close the cycle leaves every later one off the grid. Kept
     * unweakened after the one-CB-per-group-size fix, for the reason the reader states. */
    if (!ft_tensix_cb_pop_grid_is_safe(cb_out, group_tiles)) {
        ftq_abort(ctx, kErrBadParameters);
        return false;
    }

    for (uint32_t group = 0; group < group_count; group++) {
        const bool have_tiles =
            ftq_cb_wait_front_stoppable(ctx, (int32_t)cb_out, (int32_t)group_tiles, WAIT_MODE);

        if (have_tiles) {
            uint32_t group_offset;
            /* Inside the GROUP, not merely inside the region: the tighter statement, and the
             * one that also stops head h's append landing in head h+1. */
            const bool bounded =
                ft_tensix_product_fits(group, destination_group_stride, &group_offset) &&
                group_offset <= 0xFFFFFFFFu - tile_row_offset &&
                ftm_destination_span_fits(destination_group_stride, tile_row_offset,
                                          group_bytes);
            const uint32_t base = destination_base + group_offset + tile_row_offset;

            if (!bounded ||
                !write_one_group(ft_tensix_cb_read_pointer(cb_out), destination_x, destination_y,
                                 base, group_tiles, row_in_tile, row_count,
                                 (flags & kMemFlagDstWholeTiles) != 0u)) {
                ftq_abort(ctx, kErrBadParameters);
                ft_tensix_cb_pop_front(cb_out, group_tiles);
                return false;
            }
        }
        ft_tensix_cb_pop_front(cb_out, group_tiles);
    }
    return !ctx->aborting;
}

void kernel_main(void)
{
    ftq_context ctx;
    uint32_t completed;
    bool running = true;

    ftq_context_init(&ctx);
    ftq_status_set(&ctx, kStateBoot);

    if (!ftq_check_control(&ctx)) {
        ftq_status_set(&ctx, kStateExit);
        return;
    }

    completed = ctx.control[kFieldCompletedCount];
    ctx.cursor = completed;

    while (running) {
        // FETCH on `head`, so the writer drains the output CB while the reader is still
        // filling it. Publishing waits on `tail`, below.
        if (!ftq_wait_for_work(&ctx, kFieldHead))
            break;

        ctx.aborting = false;
        ftq_status_set(&ctx, kStateExec);

        volatile FTQ_L1_PTR uint32_t *descriptor = ftq_descriptor(&ctx);
        const uint32_t opcode = descriptor[kDescOpcode];
        const uint32_t token = descriptor[kDescToken];
        const uint32_t group_sequence = descriptor[kDescA5];
        bool done = true;

        ftq_status_opcode(&ctx, opcode);
        if (opcode == kOpEmbedding || opcode == kOpKvAppend) {
            ftq_status_set(&ctx, kStateCbWaitFront);
            done = write_one_memory_op(&ctx, ftq_parameters(descriptor),
                                       output_circular_buffer(opcode));
            ftq_status_set(&ctx, kStateExec);
        } else if (opcode == kOpStop) {
            running = false;
        }

        if (!done) {
            running = false;
            continue;
        }

        // THE PRECONDITION, and the only place it belongs: the reader must have finished
        // this descriptor before its completion is published.
        if (!ftq_wait_until_reaches(&ctx, kFieldTail, ctx.cursor + 1u)) {
            running = false;
            continue;
        }

        ctx.cursor++;
        // Token before count: a host that sees the count is entitled to trust the token it
        // reads next. Both are plain stores from ONE in-order RISC-V.
        ctx.control[kFieldCompletedToken] = token;
        ctx.control[kFieldCompletedCount] = ++completed;
        ftq_group_post(&ctx, group_sequence, token);
    }

    // The writer does NOT touch kFieldError or kFieldState: the reader owns those, and one
    // writer per word is the whole ordering argument.
    ftq_status_set(&ctx, kStateExit);
}