Foundry/cuda_kernels/tests/test_wkv7_cuda.cu

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
/*
 * test_wkv7_cuda.cu — standalone parity harness for the WKV7 fwd+bwd CUDA
 * kernel (sm_37/K80) vs the verified CPU oracle rx_wkv7_* (rwkvx/rx_pretrain.c).
 *
 * Loads the fixed-seed fixture (rwkvx/tests/wkv7_fixture.h — identical inputs
 * to the CPU gradient-check) for the small cases, plus a DETERMINISTICALLY
 * GENERATED N=64 case (RWKV-7 standard head_dim) that exercises the 48 KiB
 * sm_37 shared-mem fit of the backward.  Runs BOTH the CPU oracle and the GPU
 * kernel on GPU 0, reports per-tensor max relative error for y, dr,dw,dk,dv,
 * da,db,dS.  Pass criterion: max rel-err < 1e-3 (expect ~1e-4..1e-6; the GPU
 * dr reduction is algebraically identical to the CPU one, reordered, so the
 * gap is pure fp32 rounding).
 *
 * NOT wired into the fc_cuda_ptx.h embed pipeline — compiled directly by nvcc.
 */
#include <cuda_runtime.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <math.h>
#include <time.h>

#include "rwkvx/tests/wkv7_fixture.h"
#include "cuda_kernels/wkv7.cu"

/* CPU oracle (compiled from rwkvx/rx_pretrain.c as C, linked in) */
extern "C" int rx_wkv7_forward_seq(
    const float*,const float*,const float*,const float*,const float*,const float*,
    const float*,float*,float*,int,int,int,int);
extern "C" int rx_wkv7_backward(
    const float*,const float*,const float*,const float*,const float*,const float*,
    const float*,const float*,float*,float*,float*,float*,float*,float*,float*,
    int,int,int,int);

static double max_rel(const float *ga,const float *gb,int n){
    double m=0.0;
    for(int i=0;i<n;i++){
        double d=fabs((double)ga[i]-(double)gb[i]);
        double r=d/(fabs((double)gb[i])+1e-6);
        if(r>m)m=r;
    }
    return m;
}

static int g_fail=0;
/* fp32-parity metric: max-abs-error relative to the tensor's own scale
 * (inf-norm), i.e. max_i|gpu_i-cpu_i| / max_i|cpu_i|.  This is the canonical
 * way to compare two fp32 tensors and, unlike a per-element |b|+eps ratio, is
 * immune to entries that a signed reduction cancels to near-zero (at N=64 the
 * 64-wide reductions routinely produce such entries — even the UNCHANGED,
 * already-green forward tripped a per-element ratio there).  Bar held at 1e-3;
 * observed ~1e-7 (fp32 machine epsilon) since the GPU reductions are
 * algebraically identical to the CPU oracle, only reordered. */
static void chk(const char*tag,const char*tname,const float*gpu,const float*cpu,int n){
    double maxabs=0.0, scale=0.0;
    for(int i=0;i<n;i++){ double d=fabs((double)gpu[i]-(double)cpu[i]); if(d>maxabs)maxabs=d; double c=fabs((double)cpu[i]); if(c>scale)scale=c; }
    double rel = maxabs/(scale+1e-30);
    const char*st = (rel<1e-3)?"PASS":"FAIL";
    if(rel>=1e-3)g_fail=1;
    printf("    %-3s %-4s rel(max|d|/scale)=%.3e  maxabs=%.3e scale=%.3e  [%s]\n",tag,tname,rel,maxabs,scale,st);
}

static int run_case(const char*name,
    const float*r,const float*w,const float*k,const float*v,
    const float*a,const float*b,const float*dy,
    int B,int T,int H,int N)
{
    size_t nv=(size_t)B*T*H*N;
    size_t st=(size_t)B*H*N*N;
    float *y_c=(float*)malloc(nv*4);
    float *dr_c=(float*)malloc(nv*4),*dw_c=(float*)malloc(nv*4),*dk_c=(float*)malloc(nv*4);
    float *dv_c=(float*)malloc(nv*4),*da_c=(float*)malloc(nv*4),*db_c=(float*)malloc(nv*4);
    float *dS_c=(float*)malloc(st*4);
    rx_wkv7_forward_seq(r,w,k,v,a,b,NULL,y_c,NULL,B,T,H,N);
    rx_wkv7_backward(r,w,k,v,a,b,dy,NULL,dr_c,dw_c,dk_c,dv_c,da_c,db_c,dS_c,B,T,H,N);
    float *y_g=(float*)malloc(nv*4);
    float *dr_g=(float*)malloc(nv*4),*dw_g=(float*)malloc(nv*4),*dk_g=(float*)malloc(nv*4);
    float *dv_g=(float*)malloc(nv*4),*da_g=(float*)malloc(nv*4),*db_g=(float*)malloc(nv*4);
    float *dS_g=(float*)malloc(st*4);
    if(wkv7_forward_cuda(r,w,k,v,a,b,NULL,y_g,NULL,NULL,NULL,B,T,H,N)){printf("GPU fwd err\n");return 1;}
    if(wkv7_backward_cuda(r,w,k,v,a,b,dy,NULL,dr_g,dw_g,dk_g,dv_g,da_g,db_g,dS_g,B,T,H,N)){printf("GPU bwd err\n");return 1;}

    printf("  %s  (B=%d T=%d H=%d N=%d)\n",name,B,T,H,N);
    chk("fwd","y ",y_g,y_c,nv);
    chk("bwd","dr",dr_g,dr_c,nv);
    chk("bwd","dw",dw_g,dw_c,nv);
    chk("bwd","dk",dk_g,dk_c,nv);
    chk("bwd","dv",dv_g,dv_c,nv);
    chk("bwd","da",da_g,da_c,nv);
    chk("bwd","db",db_g,db_c,nv);
    chk("bwd","dS",dS_g,dS_c,st);

    free(y_c);free(dr_c);free(dw_c);free(dk_c);free(dv_c);free(da_c);free(db_c);free(dS_c);
    free(y_g);free(dr_g);free(dw_g);free(dk_g);free(dv_g);free(da_g);free(db_g);free(dS_g);
    return 0;
}

/* ---- deterministic generated case (splitmix64, no torch dependency) ---- */
static uint64_t sm_state;
static inline uint64_t sm_next(void){
    uint64_t z=(sm_state+=0x9E3779B97F4A7C15ULL);
    z=(z^(z>>30))*0xBF58476D1CE4E5B9ULL;
    z=(z^(z>>27))*0x94D049BB133111EBULL;
    return z^(z>>31);
}
static inline float sm_uf(void){ return (float)((sm_next()>>11)*(1.0/9007199254740992.0)); } /* [0,1) */
static inline float sm_sym(float s){ return (2.0f*sm_uf()-1.0f)*s; }

static int run_generated(const char*name,int B,int T,int H,int N,uint64_t seed){
    size_t nv=(size_t)B*T*H*N;
    float *r=(float*)malloc(nv*4),*w=(float*)malloc(nv*4),*k=(float*)malloc(nv*4),
          *v=(float*)malloc(nv*4),*a=(float*)malloc(nv*4),*b=(float*)malloc(nv*4),
          *dy=(float*)malloc(nv*4);
    sm_state=seed;
    for(size_t i=0;i<nv;i++){
        r[i]=sm_sym(0.5f); k[i]=sm_sym(0.5f); v[i]=sm_sym(0.6f);
        a[i]=sm_sym(0.5f); b[i]=sm_sym(0.4f); dy[i]=sm_sym(1.0f);
        w[i]=0.3f+0.6f*sm_uf();                 /* effective decay in (0.3,0.9) */
    }
    int rc=run_case(name,r,w,k,v,a,b,dy,B,T,H,N);
    free(r);free(w);free(k);free(v);free(a);free(b);free(dy);
    return rc;
}

static void bench(int B,int T,int H,int N,int iters){
    size_t nv=(size_t)B*T*H*N, st=(size_t)B*H*N*N, sasz=(size_t)T*B*H*N, scsz=(size_t)T*st;
    float *r=(float*)malloc(nv*4),*w=(float*)malloc(nv*4),*k=(float*)malloc(nv*4),
          *v=(float*)malloc(nv*4),*a=(float*)malloc(nv*4),*b=(float*)malloc(nv*4);
    for(size_t i=0;i<nv;i++){r[i]=0.01f;w[i]=0.5f;k[i]=0.02f;v[i]=0.03f;a[i]=-0.01f;b[i]=0.01f;}
    float *R,*W,*K,*V,*A,*Bb,*Y,*SC,*SA;
    cudaMalloc(&R,nv*4);cudaMalloc(&W,nv*4);cudaMalloc(&K,nv*4);cudaMalloc(&V,nv*4);
    cudaMalloc(&A,nv*4);cudaMalloc(&Bb,nv*4);cudaMalloc(&Y,nv*4);
    cudaMalloc(&SC,scsz*4);cudaMalloc(&SA,sasz*4);
    cudaMemcpy(R,r,nv*4,cudaMemcpyHostToDevice);cudaMemcpy(W,w,nv*4,cudaMemcpyHostToDevice);
    cudaMemcpy(K,k,nv*4,cudaMemcpyHostToDevice);cudaMemcpy(V,v,nv*4,cudaMemcpyHostToDevice);
    cudaMemcpy(A,a,nv*4,cudaMemcpyHostToDevice);cudaMemcpy(Bb,b,nv*4,cudaMemcpyHostToDevice);
    size_t shf=((size_t)N*N+7*N)*4;
    wkv7_fwd_kernel<<<B*H,N,shf>>>(R,W,K,V,A,Bb,NULL,Y,NULL,SC,SA,B,T,H,N);
    cudaDeviceSynchronize();
    struct timespec t0,t1; clock_gettime(CLOCK_MONOTONIC,&t0);
    for(int it=0;it<iters;it++)
        wkv7_fwd_kernel<<<B*H,N,shf>>>(R,W,K,V,A,Bb,NULL,Y,NULL,SC,SA,B,T,H,N);
    cudaDeviceSynchronize(); clock_gettime(CLOCK_MONOTONIC,&t1);
    double sec=(t1.tv_sec-t0.tv_sec)+(t1.tv_nsec-t0.tv_nsec)*1e-9;
    double toks=(double)B*T*iters;
    printf("  fwd bench B=%d T=%d H=%d N=%d : %d iters in %.4fs -> %.0f tokens/sec (one K80 die)\n",
        B,T,H,N,iters,sec,toks/sec);
    cudaFree(R);cudaFree(W);cudaFree(K);cudaFree(V);cudaFree(A);cudaFree(Bb);cudaFree(Y);cudaFree(SC);cudaFree(SA);
    free(r);free(w);free(k);free(v);free(a);free(b);
}

int main(void){
    int dev=0; cudaSetDevice(dev);
    cudaDeviceProp p; cudaGetDeviceProperties(&p,dev);
    printf("=== WKV7 fwd+bwd CUDA parity vs CPU oracle ===\n");
    printf("device %d: %s  cc %d.%d  shmem/block=%zuKiB\n\n",
        dev,p.name,p.major,p.minor,p.sharedMemPerBlock/1024);

    run_case("case0",c0_r,c0_w,c0_k,c0_v,c0_a,c0_b,c0_dy,C0_B,C0_T,C0_H,C0_N);
    run_case("case1",c1_r,c1_w,c1_k,c1_v,c1_a,c1_b,c1_dy,C1_B,C1_T,C1_H,C1_N);
    run_case("case2",c2_r,c2_w,c2_k,c2_v,c2_a,c2_b,c2_dy,C2_B,C2_T,C2_H,C2_N);
    /* N=64 = RWKV-7 standard head_dim: the real pretrainer target.  Backward
     * shared = 2*64*64 + 9*64 = 8768 floats = 34.25 KiB < 48 KiB sm_37 cap. */
    run_generated("caseN64",2,16,3,64,0x1234567ull);

    printf("\n=== throughput (forward, one die) ===\n");
    bench(8,512,12,64,50);

    cudaError_t e=cudaGetLastError();
    if(e!=cudaSuccess){printf("\nCUDA error state: %s\n",cudaGetErrorString(e));g_fail=1;}
    printf("\n%s\n", g_fail? "RESULT: FAIL":"RESULT: PASS (all tensors rel-err < 1e-3)");
    return g_fail;
}