3.5 KiB

Raw Permalink Blame History

optim

Optimizers for training neural networks. Python counterpart: PyTorch torch.optim.

Header: optim/optim.h Source: optim/optim.c Status: API Complete - Pending Verification

Overview

The optim module provides:

SGD with momentum and Nesterov
Adam with AMSGrad
AdamW (decoupled weight decay)
Learning rate schedulers
Gradient clipping

Quick Start

#include "optim/optim.h"
#include "tensor/tensor.h"

// Create parameters
tm_tensor* params[] = {weight, bias};
int num_params = 2;

// Create Adam optimizer
fo_adam_config cfg = fo_adam_default(1e-3);
fo_optimizer* opt = fo_adam_new(params, num_params, &cfg);

// Add warmup cosine scheduler
fo_scheduler_config sched = {
    .type = FO_SCHED_WARMUP_COSINE,
    .warmup_steps = 1000,
    .total_steps = 100000,
    .min_lr = 1e-6
};
fo_set_scheduler(opt, &sched);

// Training loop
for (int step = 0; step < 100000; step++) {
    fo_zero_grad(opt);

    // ... forward and backward pass ...

    fo_clip_grad_norm(opt, 1.0);
    fo_step(opt);
    fo_scheduler_step(opt);
}

fo_free(opt);

API Reference

Configuration

fo_sgd_config

typedef struct {
    float lr;           // Learning rate
    float momentum;     // Momentum factor (0 = no momentum)
    float weight_decay; // L2 regularization
    float dampening;    // Dampening for momentum
    bool nesterov;      // Use Nesterov momentum
} fo_sgd_config;

fo_adam_config

typedef struct {
    float lr;           // Learning rate
    float beta1;        // First moment decay (default: 0.9)
    float beta2;        // Second moment decay (default: 0.999)
    float eps;          // Numerical stability (default: 1e-8)
    float weight_decay; // Weight decay (0 for Adam, >0 for AdamW)
    bool amsgrad;       // Use AMSGrad variant
} fo_adam_config;

Default Configurations

fo_sgd_config fo_sgd_default(float lr);
fo_adam_config fo_adam_default(float lr);

Optimizer Creation

fo_optimizer* fo_sgd_new(tm_tensor** params, int num_params, fo_sgd_config* config);
fo_optimizer* fo_adam_new(tm_tensor** params, int num_params, fo_adam_config* config);
fo_optimizer* fo_adamw_new(tm_tensor** params, int num_params, fo_adam_config* config);
void fo_free(fo_optimizer* opt);

Optimizer Operations

void fo_step(fo_optimizer* opt);           // Update parameters
void fo_zero_grad(fo_optimizer* opt);      // Zero gradients
float fo_get_lr(fo_optimizer* opt);        // Get current LR
void fo_set_lr(fo_optimizer* opt, float lr); // Set LR
int fo_get_step(fo_optimizer* opt);        // Get step count

Learning Rate Schedulers

typedef enum {
    FO_SCHED_NONE,
    FO_SCHED_STEP,
    FO_SCHED_COSINE,
    FO_SCHED_LINEAR,
    FO_SCHED_WARMUP_COSINE,
} fo_scheduler_type;

void fo_set_scheduler(fo_optimizer* opt, fo_scheduler_config* config);
void fo_scheduler_step(fo_optimizer* opt);
float fo_get_scheduled_lr(fo_optimizer* opt, int step);

Gradient Clipping

float fo_clip_grad_norm(fo_optimizer* opt, float max_norm);  // Returns norm
void fo_clip_grad_value(fo_optimizer* opt, float clip_value);

Scheduler Types

Type	Description
`FO_SCHED_STEP`	Decay by gamma every step_size steps
`FO_SCHED_COSINE`	Cosine annealing to min_lr
`FO_SCHED_LINEAR`	Linear decay to min_lr
`FO_SCHED_WARMUP_COSINE`	Warmup then cosine decay

3.5 KiB Raw Permalink Blame History