David
/
Foundry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Foundry/tensor/tensor.h

857 lines
31 KiB
C
Raw Blame History

/* SPDX-License-Identifier: AGPL-3.0-or-later */
/* Copyright (C) 2026 David (Corey) Herrera */
#ifndef FOUNDRY_TENSOR_H
#define FOUNDRY_TENSOR_H
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Tensor operations with automatic differentiation.
* Replaces PyTorch core functionality.
*/
#define TM_MAX_DIMS 8
/* Data types */
typedef enum {
TM_F32, /* float32 - default */
TM_F64, /* float64 */
TM_F16, /* float16 */
TM_BF16, /* bfloat16 */
TM_I32, /* int32 */
TM_I64, /* int64 */
} tm_dtype;
/* Sparse formats */
typedef enum {
TM_SPARSE_COO, /* Coordinate format: (row, col, value) triples */
TM_SPARSE_CSR, /* Compressed Sparse Row: row_ptr, col_ind, values */
} tm_sparse_format;
/* Forward declarations */
typedef struct tm_tensor tm_tensor;
typedef struct tm_grad_fn tm_grad_fn;
/* Gradient function - computes gradients during backward pass */
struct tm_grad_fn {
void (*backward)(tm_tensor* out); /* Backward function */
tm_tensor* inputs[2]; /* Input tensors (for binary ops) */
int num_inputs;
void* ctx; /* Optional context */
};
/* Tensor structure */
struct tm_tensor {
void* data; /* Raw data pointer */
int64_t shape[TM_MAX_DIMS]; /* Shape of each dimension */
int64_t strides[TM_MAX_DIMS]; /* Strides in elements */
int ndim; /* Number of dimensions */
int64_t numel; /* Total number of elements */
tm_dtype dtype; /* Data type */
/* Autograd */
bool requires_grad; /* Track gradients? */
tm_tensor* grad; /* Gradient tensor */
tm_grad_fn* grad_fn; /* How to compute gradient */
/* Memory management */
int refcount; /* Reference count */
bool owns_data; /* Do we own the data? */
/* Sparse tensor support */
bool is_sparse; /* Is this a sparse tensor? */
tm_sparse_format sparse_format; /* COO or CSR */
int64_t nnz; /* Number of non-zero elements */
int64_t* indices; /* COO: [2, nnz] for 2D (row, col indices) */
int64_t* row_ptr; /* CSR: [rows + 1] row pointers */
int64_t* col_ind; /* CSR: [nnz] column indices */
};
/* ============================================================= */
/* Tensor Creation */
/* ============================================================= */
/* Create tensor with uninitialized data */
tm_tensor* tm_empty(int64_t* shape, int ndim, tm_dtype dtype);
/* Create tensor filled with zeros */
tm_tensor* tm_zeros(int64_t* shape, int ndim, tm_dtype dtype);
/* Create tensor filled with ones */
tm_tensor* tm_ones(int64_t* shape, int ndim, tm_dtype dtype);
/* Create tensor filled with value */
tm_tensor* tm_full(int64_t* shape, int ndim, tm_dtype dtype, double value);
/* Create tensor from existing data (does not copy) */
tm_tensor* tm_from_data(void* data, int64_t* shape, int ndim, tm_dtype dtype);
/* Create tensor with random normal values */
tm_tensor* tm_randn(int64_t* shape, int ndim, tm_dtype dtype);
/* Create tensor with random uniform values [0, 1) */
tm_tensor* tm_rand(int64_t* shape, int ndim, tm_dtype dtype);
/* Clone tensor (deep copy) */
tm_tensor* tm_clone(tm_tensor* t);
/* Create tensor with range of values [start, end) with step */
tm_tensor* tm_arange(double start, double end, double step, tm_dtype dtype);
/* Create tensor with random integers in [low, high) */
tm_tensor* tm_randint(int64_t low, int64_t high, int64_t* shape, int ndim);
/* Create identity matrix */
tm_tensor* tm_eye(int64_t n, tm_dtype dtype);
/* Create linearly spaced values */
tm_tensor* tm_linspace(double start, double end, int64_t steps, tm_dtype dtype);
/* Like functions - create tensor with same shape/dtype as input */
tm_tensor* tm_full_like(tm_tensor* t, double value);
tm_tensor* tm_empty_like(tm_tensor* t);
tm_tensor* tm_rand_like(tm_tensor* t);
tm_tensor* tm_randn_like(tm_tensor* t);
/* ============================================================= */
/* Memory Management */
/* ============================================================= */
/* Increment reference count */
tm_tensor* tm_retain(tm_tensor* t);
/* Decrement reference count, free if zero */
void tm_release(tm_tensor* t);
/* Convenience alias */
#define tm_free(t) tm_release(t)
/* ============================================================= */
/* Accessors */
/* ============================================================= */
/* Get element at index (1D) */
float tm_get_f32(tm_tensor* t, int64_t i);
double tm_get_f64(tm_tensor* t, int64_t i);
int32_t tm_get_i32(tm_tensor* t, int64_t i);
/* Set element at index (1D) */
void tm_set_f32(tm_tensor* t, int64_t i, float val);
void tm_set_f64(tm_tensor* t, int64_t i, double val);
void tm_set_i32(tm_tensor* t, int64_t i, int32_t val);
/* Get pointer to data */
float* tm_data_f32(tm_tensor* t);
double* tm_data_f64(tm_tensor* t);
/* Get size info */
int64_t tm_numel(tm_tensor* t);
int tm_ndim(tm_tensor* t);
int64_t tm_size(tm_tensor* t, int dim);
size_t tm_dtype_size(tm_dtype dtype);
/* ============================================================= */
/* Shape Operations */
/* ============================================================= */
/* Reshape (returns new view if possible, otherwise copies) */
tm_tensor* tm_reshape(tm_tensor* t, int64_t* shape, int ndim);
/* View (alias for reshape, for PyTorch compatibility) */
tm_tensor* tm_view(tm_tensor* t, int64_t* shape, int ndim);
/* Transpose (swap two dimensions) */
tm_tensor* tm_transpose(tm_tensor* t, int dim0, int dim1);
/* Permute dimensions (generalized transpose) */
tm_tensor* tm_permute(tm_tensor* t, int* dims, int ndim);
/* Squeeze (remove dimensions of size 1) */
tm_tensor* tm_squeeze(tm_tensor* t, int dim);
/* Unsqueeze (add dimension of size 1) */
tm_tensor* tm_unsqueeze(tm_tensor* t, int dim);
/* Flatten dimensions from start_dim to end_dim (inclusive) */
tm_tensor* tm_flatten(tm_tensor* t, int start_dim, int end_dim);
/* Select along dimension (reduces ndim by 1) */
tm_tensor* tm_select(tm_tensor* t, int dim, int64_t index);
/* Contiguous (ensure data is contiguous in memory) */
tm_tensor* tm_contiguous(tm_tensor* t);
/* Check if contiguous */
bool tm_is_contiguous(tm_tensor* t);
/* Concatenate tensors along dimension */
tm_tensor* tm_cat(tm_tensor** tensors, int num_tensors, int dim);
/* Stack tensors along new dimension */
tm_tensor* tm_stack(tm_tensor** tensors, int num_tensors, int dim);
/* Split tensor into chunks along dimension */
tm_tensor** tm_split(tm_tensor* t, int64_t split_size, int dim, int* num_chunks);
/* Split tensor into n equal chunks along dimension */
tm_tensor** tm_chunk(tm_tensor* t, int num_chunks, int dim, int* actual_chunks);
/* Narrow (slice) along dimension */
tm_tensor* tm_narrow(tm_tensor* t, int dim, int64_t start, int64_t length);
/* Expand tensor to new shape (broadcasting) */
tm_tensor* tm_expand(tm_tensor* t, int64_t* shape, int ndim);
/* Repeat tensor along dimensions */
tm_tensor* tm_repeat(tm_tensor* t, int64_t* repeats, int ndim);
/* Flip tensor along dimensions */
tm_tensor* tm_flip(tm_tensor* t, int* dims, int ndim);
/* Roll (circular shift) tensor */
tm_tensor* tm_roll(tm_tensor* t, int64_t shift, int dim);
/* 2D transpose alias */
tm_tensor* tm_t(tm_tensor* t);
/* ============================================================= */
/* Indexing Operations */
/* ============================================================= */
/* Gather elements along dimension using index tensor */
tm_tensor* tm_gather(tm_tensor* t, int dim, tm_tensor* index);
/* Scatter elements along dimension using index tensor */
tm_tensor* tm_scatter(tm_tensor* t, int dim, tm_tensor* index, tm_tensor* src);
/* Select elements along dimension using index tensor */
tm_tensor* tm_index_select(tm_tensor* t, int dim, tm_tensor* index);
/* Scatter with addition */
tm_tensor* tm_scatter_add(tm_tensor* t, int dim, tm_tensor* index, tm_tensor* src);
/* Select elements where mask is true */
tm_tensor* tm_masked_select(tm_tensor* t, tm_tensor* mask);
/* Get indices of non-zero elements */
tm_tensor* tm_nonzero(tm_tensor* t);
/* Fill at indices */
tm_tensor* tm_index_fill(tm_tensor* t, int dim, tm_tensor* index, float value);
/* Copy to indices */
tm_tensor* tm_index_copy(tm_tensor* t, int dim, tm_tensor* index, tm_tensor* src);
/* Add at indices */
tm_tensor* tm_index_add(tm_tensor* t, int dim, tm_tensor* index, tm_tensor* src);
/* Take elements by flat indices */
tm_tensor* tm_take(tm_tensor* t, tm_tensor* indices);
/* ============================================================= */
/* Element-wise Operations */
/* ============================================================= */
/* Binary ops */
tm_tensor* tm_add(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_sub(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_mul(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_div(tm_tensor* a, tm_tensor* b);
/* Scalar ops */
tm_tensor* tm_add_scalar(tm_tensor* a, double s);
tm_tensor* tm_sub_scalar(tm_tensor* a, double s);
tm_tensor* tm_mul_scalar(tm_tensor* a, double s);
tm_tensor* tm_div_scalar(tm_tensor* a, double s);
/* Unary ops */
tm_tensor* tm_neg(tm_tensor* a);
tm_tensor* tm_abs(tm_tensor* a);
tm_tensor* tm_exp(tm_tensor* a);
tm_tensor* tm_log(tm_tensor* a);
tm_tensor* tm_sqrt(tm_tensor* a);
tm_tensor* tm_rsqrt(tm_tensor* a);
tm_tensor* tm_pow(tm_tensor* a, double exponent);
tm_tensor* tm_sin(tm_tensor* a);
tm_tensor* tm_cos(tm_tensor* a);
tm_tensor* tm_tanh(tm_tensor* a);
tm_tensor* tm_relu(tm_tensor* a);
tm_tensor* tm_sigmoid(tm_tensor* a);
tm_tensor* tm_gelu(tm_tensor* a);
tm_tensor* tm_silu(tm_tensor* a);
/* Clamp values */
tm_tensor* tm_clamp(tm_tensor* a, double min_val, double max_val);
/* Conditional operations */
tm_tensor* tm_where(tm_tensor* condition, tm_tensor* a, tm_tensor* b);
tm_tensor* tm_masked_fill(tm_tensor* t, tm_tensor* mask, float value);
/* Element-wise max/min */
tm_tensor* tm_maximum(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_minimum(tm_tensor* a, tm_tensor* b);
/* Comparison operations (return boolean tensors as TM_I32) */
tm_tensor* tm_eq(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_ne(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_lt(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_le(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_gt(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_ge(tm_tensor* a, tm_tensor* b);
/* Check for special values */
tm_tensor* tm_isnan(tm_tensor* t);
tm_tensor* tm_isinf(tm_tensor* t);
/* Leaky ReLU activation */
tm_tensor* tm_leaky_relu(tm_tensor* a, float negative_slope);
/* ELU activation */
tm_tensor* tm_elu(tm_tensor* a, float alpha);
/* Rounding operations */
tm_tensor* tm_floor(tm_tensor* a);
tm_tensor* tm_ceil(tm_tensor* a);
tm_tensor* tm_round(tm_tensor* a);
tm_tensor* tm_trunc(tm_tensor* a);
/* Sign and reciprocal */
tm_tensor* tm_sign(tm_tensor* a);
tm_tensor* tm_reciprocal(tm_tensor* a);
/* Additional math functions */
tm_tensor* tm_log2(tm_tensor* a);
tm_tensor* tm_log10(tm_tensor* a);
tm_tensor* tm_exp2(tm_tensor* a);
tm_tensor* tm_expm1(tm_tensor* a);
tm_tensor* tm_log1p(tm_tensor* a);
tm_tensor* tm_tan(tm_tensor* a);
tm_tensor* tm_asin(tm_tensor* a);
tm_tensor* tm_acos(tm_tensor* a);
tm_tensor* tm_atan(tm_tensor* a);
tm_tensor* tm_sinh(tm_tensor* a);
tm_tensor* tm_cosh(tm_tensor* a);
/* Fused operations */
tm_tensor* tm_addcmul(tm_tensor* t, tm_tensor* a, tm_tensor* b, double value);
tm_tensor* tm_addcdiv(tm_tensor* t, tm_tensor* a, tm_tensor* b, double value);
tm_tensor* tm_lerp(tm_tensor* a, tm_tensor* b, double weight);
/* Logical operations */
tm_tensor* tm_logical_and(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_logical_or(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_logical_not(tm_tensor* a);
tm_tensor* tm_logical_xor(tm_tensor* a, tm_tensor* b);
/* Check functions */
tm_tensor* tm_isfinite(tm_tensor* t);
bool tm_equal(tm_tensor* a, tm_tensor* b);
bool tm_allclose(tm_tensor* a, tm_tensor* b, double rtol, double atol);
/* ============================================================= */
/* Matrix Operations */
/* ============================================================= */
/* Matrix multiplication */
tm_tensor* tm_matmul(tm_tensor* a, tm_tensor* b);
/* Batched matrix multiplication */
tm_tensor* tm_bmm(tm_tensor* a, tm_tensor* b);
/* Matrix multiplication aliases */
tm_tensor* tm_mm(tm_tensor* a, tm_tensor* b);
/* Fused t + a @ b */
tm_tensor* tm_addmm(tm_tensor* t, tm_tensor* a, tm_tensor* b, double beta, double alpha);
/* Vector operations */
tm_tensor* tm_dot(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_mv(tm_tensor* mat, tm_tensor* vec);
tm_tensor* tm_outer(tm_tensor* a, tm_tensor* b);
tm_tensor* tm_inner(tm_tensor* a, tm_tensor* b);
/* ============================================================= */
/* Reduction Operations */
/* ============================================================= */
/* Sum over dimension (dim=-1 means all) */
tm_tensor* tm_sum(tm_tensor* t, int dim, bool keepdim);
/* Mean over dimension */
tm_tensor* tm_mean(tm_tensor* t, int dim, bool keepdim);
/* Max over dimension */
tm_tensor* tm_max(tm_tensor* t, int dim, bool keepdim);
/* Min over dimension */
tm_tensor* tm_min(tm_tensor* t, int dim, bool keepdim);
/* Argmax - index of maximum value */
tm_tensor* tm_argmax(tm_tensor* t, int dim, bool keepdim);
/* Argmin - index of minimum value */
tm_tensor* tm_argmin(tm_tensor* t, int dim, bool keepdim);
/* Log-sum-exp (numerically stable) */
tm_tensor* tm_logsumexp(tm_tensor* t, int dim, bool keepdim);
/* Cumulative sum along dimension */
tm_tensor* tm_cumsum(tm_tensor* t, int dim);
/* Standard deviation and variance */
tm_tensor* tm_std(tm_tensor* t, int dim, bool keepdim, bool unbiased);
tm_tensor* tm_var(tm_tensor* t, int dim, bool keepdim, bool unbiased);
/* Product over dimension */
tm_tensor* tm_prod(tm_tensor* t, int dim, bool keepdim);
/* Cumulative product along dimension */
tm_tensor* tm_cumprod(tm_tensor* t, int dim);
/* Any/all along dimension */
tm_tensor* tm_any(tm_tensor* t, int dim, bool keepdim);
tm_tensor* tm_all(tm_tensor* t, int dim, bool keepdim);
/* Lp norm */
tm_tensor* tm_norm(tm_tensor* t, double p, int dim, bool keepdim);
/* Count non-zero elements */
tm_tensor* tm_count_nonzero(tm_tensor* t, int dim);
/* ============================================================= */
/* Type Conversion */
/* ============================================================= */
/* Convert tensor to different dtype */
tm_tensor* tm_to(tm_tensor* t, tm_dtype dtype);
/* Convenience type conversion functions */
tm_tensor* tm_float(tm_tensor* t); /* Convert to TM_F32 */
tm_tensor* tm_double(tm_tensor* t); /* Convert to TM_F64 */
tm_tensor* tm_half(tm_tensor* t); /* Convert to TM_F16 */
tm_tensor* tm_int(tm_tensor* t); /* Convert to TM_I32 */
tm_tensor* tm_long(tm_tensor* t); /* Convert to TM_I64 */
/* ============================================================= */
/* Neural Network Operations */
/* ============================================================= */
/* Softmax over dimension */
tm_tensor* tm_softmax(tm_tensor* t, int dim);
/* Log softmax */
tm_tensor* tm_log_softmax(tm_tensor* t, int dim);
/* Layer normalization */
tm_tensor* tm_layer_norm(tm_tensor* t, tm_tensor* weight, tm_tensor* bias, float eps);
/* Embedding lookup */
tm_tensor* tm_embedding(tm_tensor* weight, tm_tensor* indices);
/* Dropout (training mode) */
tm_tensor* tm_dropout(tm_tensor* t, float p, bool training);
/* RMS Normalization (used in Llama, modern LLMs) */
tm_tensor* tm_rms_norm(tm_tensor* t, tm_tensor* weight, float eps);
/* Linear layer: x @ weight.T + bias */
tm_tensor* tm_linear(tm_tensor* x, tm_tensor* weight, tm_tensor* bias);
/* Upper triangular matrix (for causal masks) */
tm_tensor* tm_triu(tm_tensor* t, int diagonal);
/* Lower triangular matrix */
tm_tensor* tm_tril(tm_tensor* t, int diagonal);
/* Hardswish activation: x * relu6(x + 3) / 6 */
tm_tensor* tm_hardswish(tm_tensor* t);
/* Hardtanh activation: clamp(x, min, max) */
tm_tensor* tm_hardtanh(tm_tensor* t, float min_val, float max_val);
/* GELU approximate (tanh approximation) */
tm_tensor* tm_gelu_approximate(tm_tensor* t);
/* Softplus: log(1 + exp(x)) */
tm_tensor* tm_softplus(tm_tensor* t, float beta, float threshold);
/* Mish activation: x * tanh(softplus(x)) */
tm_tensor* tm_mish(tm_tensor* t);
/* SELU activation: scale * (max(0,x) + min(0, alpha*(exp(x)-1))) */
tm_tensor* tm_selu(tm_tensor* t);
/* CELU activation: max(0,x) + min(0, alpha*(exp(x/alpha)-1)) */
tm_tensor* tm_celu(tm_tensor* t, float alpha);
/* ReLU6 activation: min(max(0,x), 6) */
tm_tensor* tm_relu6(tm_tensor* t);
/* GLU (Gated Linear Unit): a * sigmoid(b) where input split in half */
tm_tensor* tm_glu(tm_tensor* t, int dim);
/* Softsign activation: x / (1 + |x|) */
tm_tensor* tm_softsign(tm_tensor* t);
/* Tanhshrink activation: x - tanh(x) */
tm_tensor* tm_tanhshrink(tm_tensor* t);
/* Hardsigmoid: 0 if x<=-3, 1 if x>=3, else x/6 + 0.5 */
tm_tensor* tm_hardsigmoid(tm_tensor* t);
/* Threshold: x if x > threshold, else value */
tm_tensor* tm_threshold(tm_tensor* t, float threshold, float value);
/* Pad tensor (constant padding) */
tm_tensor* tm_pad(tm_tensor* t, int64_t* padding, int pad_len, float value);
/* Normalize (Lp normalization along dimension) */
tm_tensor* tm_normalize(tm_tensor* t, float p, int dim, float eps);
/* Interpolate/upsample (nearest neighbor) */
tm_tensor* tm_interpolate(tm_tensor* t, int64_t* size, int size_len, const char* mode);
/* Batch normalization: (x - mean) / sqrt(var + eps) * weight + bias */
tm_tensor* tm_batch_norm(tm_tensor* t, tm_tensor* weight, tm_tensor* bias,
tm_tensor* running_mean, tm_tensor* running_var,
bool training, float momentum, float eps);
/* Group normalization */
tm_tensor* tm_group_norm(tm_tensor* t, int num_groups, tm_tensor* weight,
tm_tensor* bias, float eps);
/* Instance normalization */
tm_tensor* tm_instance_norm(tm_tensor* t, tm_tensor* weight, tm_tensor* bias, float eps);
/* Scaled dot-product attention:
* Q: [batch, heads, seq_q, head_dim]
* K: [batch, heads, seq_k, head_dim]
* V: [batch, heads, seq_k, head_dim]
* attn_mask: optional [batch, heads, seq_q, seq_k] or NULL
* Returns: [batch, heads, seq_q, head_dim] */
tm_tensor* tm_scaled_dot_product_attention(tm_tensor* Q, tm_tensor* K, tm_tensor* V,
tm_tensor* attn_mask, float dropout_p,
bool is_causal);
/* Multi-head attention helper: split heads
* x: [batch, seq, embed_dim] -> [batch, heads, seq, head_dim] */
tm_tensor* tm_split_heads(tm_tensor* x, int num_heads);
/* Multi-head attention helper: merge heads
* x: [batch, heads, seq, head_dim] -> [batch, seq, embed_dim] */
tm_tensor* tm_merge_heads(tm_tensor* x);
/* 1D Convolution */
tm_tensor* tm_conv1d(tm_tensor* input, tm_tensor* weight, tm_tensor* bias,
int stride, int padding, int dilation, int groups);
/* 2D Convolution */
tm_tensor* tm_conv2d(tm_tensor* input, tm_tensor* weight, tm_tensor* bias,
int stride_h, int stride_w, int padding_h, int padding_w,
int dilation_h, int dilation_w, int groups);
/* 1D Average Pooling */
tm_tensor* tm_avg_pool1d(tm_tensor* input, int kernel_size, int stride, int padding);
/* 1D Max Pooling */
tm_tensor* tm_max_pool1d(tm_tensor* input, int kernel_size, int stride, int padding);
/* 2D Average Pooling */
tm_tensor* tm_avg_pool2d(tm_tensor* input, int kernel_h, int kernel_w,
int stride_h, int stride_w, int padding_h, int padding_w);
/* 2D Max Pooling */
tm_tensor* tm_max_pool2d(tm_tensor* input, int kernel_h, int kernel_w,
int stride_h, int stride_w, int padding_h, int padding_w);
/* Adaptive Average Pooling 1D: output has shape [batch, channels, output_size] */
tm_tensor* tm_adaptive_avg_pool1d(tm_tensor* input, int output_size);
/* Adaptive Average Pooling 2D: output has shape [batch, channels, output_h, output_w] */
tm_tensor* tm_adaptive_avg_pool2d(tm_tensor* input, int output_h, int output_w);
/* ============================================================= */
/* Loss Functions */
/* ============================================================= */
/* Cross-entropy loss: logits [batch, classes], targets [batch] (class indices)
* Returns scalar loss tensor */
tm_tensor* tm_cross_entropy(tm_tensor* logits, tm_tensor* targets);
/* Cross-entropy loss with soft targets: both [batch, classes]
* Returns scalar loss tensor */
tm_tensor* tm_cross_entropy_soft(tm_tensor* logits, tm_tensor* targets);
/* Mean squared error loss: pred [*], target [*] (same shape)
* Returns scalar loss tensor */
tm_tensor* tm_mse_loss(tm_tensor* pred, tm_tensor* target);
/* L1 (MAE) loss: sum(|pred - target|) / n */
tm_tensor* tm_l1_loss(tm_tensor* pred, tm_tensor* target);
/* Smooth L1 (Huber) loss: 0.5*x^2/beta if |x| < beta, else |x| - 0.5*beta */
tm_tensor* tm_smooth_l1_loss(tm_tensor* pred, tm_tensor* target, float beta);
/* Negative log likelihood loss: expects log probabilities and class indices */
tm_tensor* tm_nll_loss(tm_tensor* log_probs, tm_tensor* targets);
/* KL divergence: KL(target || input), input is log probs, target is probs */
tm_tensor* tm_kl_div(tm_tensor* input, tm_tensor* target);
/* Binary cross entropy: for binary classification */
tm_tensor* tm_binary_cross_entropy(tm_tensor* pred, tm_tensor* target);
/* Binary cross entropy with logits: applies sigmoid internally */
tm_tensor* tm_binary_cross_entropy_with_logits(tm_tensor* logits, tm_tensor* target);
/* Huber loss: smooth combination of L1 and L2 */
tm_tensor* tm_huber_loss(tm_tensor* pred, tm_tensor* target, float delta);
/* Cosine embedding loss: for similarity/dissimilarity learning */
tm_tensor* tm_cosine_embedding_loss(tm_tensor* x1, tm_tensor* x2, tm_tensor* y, float margin);
/* Triplet margin loss: for metric learning */
tm_tensor* tm_triplet_margin_loss(tm_tensor* anchor, tm_tensor* positive, tm_tensor* negative, float margin, float p);
/* Cross entropy with label smoothing */
tm_tensor* tm_cross_entropy_label_smooth(tm_tensor* logits, tm_tensor* targets, float smoothing);
/* ============================================================= */
/* Linear Algebra */
/* ============================================================= */
/* Matrix inverse (Gauss-Jordan elimination) */
tm_tensor* tm_inverse(tm_tensor* t);
/* Determinant (LU decomposition) */
tm_tensor* tm_det(tm_tensor* t);
/* Singular Value Decomposition: returns [U, S, V] where A = U * diag(S) * V^T */
tm_tensor** tm_svd(tm_tensor* t, int* num_outputs);
/* Eigenvalues (QR iteration, symmetric matrices) */
tm_tensor* tm_eig(tm_tensor* t);
/* ============================================================= */
/* Autograd */
/* ============================================================= */
/* Enable gradient tracking */
void tm_set_requires_grad(tm_tensor* t, bool requires);
/* Compute gradients (backpropagation) */
void tm_backward(tm_tensor* loss);
/* Zero all gradients */
void tm_zero_grad(tm_tensor** params, int n);
/* Detach from computation graph */
tm_tensor* tm_detach(tm_tensor* t);
/* No-grad context (returns previous state) */
bool tm_no_grad_begin(void);
void tm_no_grad_end(bool prev);
/* Retain gradient for non-leaf tensors */
void tm_retain_grad(tm_tensor* t);
/* Compute gradient of outputs w.r.t. inputs explicitly */
tm_tensor** tm_grad(tm_tensor** outputs, int num_outputs, tm_tensor** inputs, int num_inputs,
tm_tensor** grad_outputs);
/* Gradient checkpointing: recompute forward during backward to save memory
* Note: Full memory savings only on GPU; CPU version runs forward directly */
tm_tensor* tm_checkpoint(tm_tensor* (*fn)(tm_tensor**, int, void*),
tm_tensor** inputs, int num_inputs, void* ctx);
/* ============================================================= */
/* Higher-Order Gradients */
/* ============================================================= */
/* Compute Hessian matrix of loss w.r.t. parameters using finite differences
* loss_fn: function that computes scalar loss from parameters
* Returns n x n Hessian matrix where n = total number of parameters */
tm_tensor* tm_hessian(tm_tensor** params, int num_params,
tm_tensor* (*loss_fn)(tm_tensor**, int, void*), void* ctx);
/* Hessian-vector product: H @ v without computing full Hessian
* More memory efficient than tm_hessian for large models
* Returns 1D tensor of length = total parameters */
tm_tensor* tm_hvp(tm_tensor** params, int num_params, tm_tensor** vec, int num_vec,
tm_tensor* (*loss_fn)(tm_tensor**, int, void*), void* ctx);
/* Jacobian-vector product (forward-mode AD)
* Computes J @ tangents where J is Jacobian of fn w.r.t. inputs
* Returns array of output tensors (use num_outputs_out to get count) */
tm_tensor** tm_jvp(tm_tensor* (*fn)(tm_tensor**, int, void*), tm_tensor** inputs, int num_inputs,
tm_tensor** tangents, void* ctx, int* num_outputs_out);
/* ============================================================= */
/* Custom Autograd Functions */
/* ============================================================= */
/* Custom context for saving tensors during forward pass */
typedef struct tm_custom_ctx tm_custom_ctx;
/* Custom autograd function definition */
typedef struct tm_autograd_function tm_autograd_function;
/* Create a new custom context */
tm_custom_ctx* tm_custom_ctx_new(void);
/* Save tensor for backward pass */
void tm_save_for_backward(tm_custom_ctx* ctx, tm_tensor* t);
/* Get saved tensors (returns array, sets num_saved) */
tm_tensor** tm_get_saved_tensors(tm_custom_ctx* ctx, int* num_saved);
/* Set/get user data in context */
void tm_custom_ctx_set_data(tm_custom_ctx* ctx, void* user_data);
void* tm_custom_ctx_get_data(tm_custom_ctx* ctx);
/* Free custom context */
void tm_custom_ctx_free(tm_custom_ctx* ctx);
/* Register a custom autograd function
* forward: takes inputs and context, returns output (save tensors to ctx)
* backward: takes grad_output and context, returns array of input gradients
* Returns function ID (>=0) on success, -1 on failure */
int tm_register_autograd_function(const char* name,
tm_tensor* (*forward)(tm_tensor**, int, tm_custom_ctx*),
tm_tensor** (*backward)(tm_tensor*, tm_custom_ctx*, int*));
/* Apply a registered custom function by name */
tm_tensor* tm_apply_custom_function(const char* name, tm_tensor** inputs, int num_inputs);
/* Apply a registered custom function by ID (faster) */
tm_tensor* tm_apply_custom_function_id(int fn_id, tm_tensor** inputs, int num_inputs);
/* Clean up custom function registry */
void tm_cleanup_autograd_registry(void);
/* ============================================================= */
/* In-place Operations */
/* ============================================================= */
/* Create zeros tensor with same shape as input */
tm_tensor* tm_zeros_like(tm_tensor* t);
/* Create ones tensor with same shape as input */
tm_tensor* tm_ones_like(tm_tensor* t);
/* Fill tensor with value (in-place) */
void tm_fill_(tm_tensor* t, float value);
/* Add tensors in-place: a += b */
void tm_add_(tm_tensor* a, tm_tensor* b);
/* Scale tensor in-place: t *= s */
void tm_scale_(tm_tensor* t, float s);
/* Zero tensor in-place */
void tm_zero_(tm_tensor* t);
/* Copy data in-place: dst = src */
void tm_copy_(tm_tensor* dst, tm_tensor* src);
/* Clamp in-place */
void tm_clamp_(tm_tensor* t, float min_val, float max_val);
/* More in-place operations */
void tm_sub_(tm_tensor* a, tm_tensor* b);
void tm_mul_(tm_tensor* a, tm_tensor* b);
void tm_div_(tm_tensor* a, tm_tensor* b);
void tm_exp_(tm_tensor* t);
void tm_log_(tm_tensor* t);
void tm_sqrt_(tm_tensor* t);
void tm_abs_(tm_tensor* t);
void tm_neg_(tm_tensor* t);
void tm_relu_(tm_tensor* t);
void tm_sigmoid_(tm_tensor* t);
/* ============================================================= */
/* Sparse Tensor Operations */
/* ============================================================= */
/* Create sparse COO tensor from indices and values
* indices: [2, nnz] tensor of (row, col) pairs
* values: [nnz] tensor of non-zero values
* shape: dense shape [rows, cols] */
tm_tensor* tm_sparse_coo(tm_tensor* indices, tm_tensor* values, int64_t* shape, int ndim);
/* Create sparse CSR tensor from row_ptr, col_ind, and values
* row_ptr: [rows + 1] row pointers
* col_ind: [nnz] column indices
* values: [nnz] tensor of non-zero values
* shape: dense shape [rows, cols] */
tm_tensor* tm_sparse_csr(tm_tensor* row_ptr, tm_tensor* col_ind, tm_tensor* values,
int64_t* shape, int ndim);
/* Convert dense tensor to sparse COO format */
tm_tensor* tm_to_sparse(tm_tensor* t);
/* Convert dense tensor to sparse CSR format */
tm_tensor* tm_to_sparse_csr(tm_tensor* t);
/* Convert sparse tensor to dense */
tm_tensor* tm_to_dense(tm_tensor* t);
/* Check if tensor is sparse */
bool tm_is_sparse(tm_tensor* t);
/* Get number of non-zero elements */
int64_t tm_sparse_nnz(tm_tensor* t);
/* Get sparse format */
tm_sparse_format tm_get_sparse_format(tm_tensor* t);
/* Convert between sparse formats */
tm_tensor* tm_sparse_coo_to_csr(tm_tensor* t);
tm_tensor* tm_sparse_csr_to_coo(tm_tensor* t);
/* Sparse-dense matrix multiplication: sparse @ dense -> dense */
tm_tensor* tm_spmm(tm_tensor* sparse, tm_tensor* dense);
/* Sparse-sparse addition: sparse + sparse -> sparse */
tm_tensor* tm_sparse_add(tm_tensor* a, tm_tensor* b);
/* Sparse-scalar multiplication: sparse * scalar -> sparse */
tm_tensor* tm_sparse_mul_scalar(tm_tensor* t, double s);
/* Sparse matrix-vector multiplication: sparse @ vector -> vector */
tm_tensor* tm_spmv(tm_tensor* sparse, tm_tensor* vec);
/* Get values tensor from sparse tensor */
tm_tensor* tm_sparse_values(tm_tensor* t);
/* Get indices tensor from sparse COO tensor */
tm_tensor* tm_sparse_indices(tm_tensor* t);
/* Coalesce sparse tensor (combine duplicate indices) */
tm_tensor* tm_sparse_coalesce(tm_tensor* t);
/* ============================================================= */
/* Utility */
/* ============================================================= */
/* Print tensor info */
void tm_print(tm_tensor* t);
/* Print tensor with data */
void tm_print_data(tm_tensor* t);
/* Seed random number generator */
void tm_manual_seed(uint64_t seed);
#ifdef __cplusplus
}
#endif
#endif /* FOUNDRY_TENSOR_H */
Reference in New Issue View Git Blame Copy Permalink