primal-barracuda 0.4.0

barraCuda — BARrier-free Rust Abstracted Cross-platform Unified Dimensional Algebra. Vendor-agnostic GPU compute via WGSL.
Documentation
// SPDX-License-Identifier: AGPL-3.0-or-later
// AdamW Optimizer: Adam with Decoupled Weight Decay (f64 canonical)
// Key difference from Adam: Weight decay applied AFTER gradient update
// This makes weight decay independent of gradient-based learning rate adaptation
//
// Formula:
// m = beta1 * m + (1 - beta1) * grad
// v = beta2 * v + (1 - beta2) * grad²
// m_hat = m / (1 - beta1^t)
// v_hat = v / (1 - beta2^t)
// param = param - lr * m_hat / (sqrt(v_hat) + epsilon) - lr * wd * param
//
// Used in: BERT, GPT, modern transformers (superior to Adam for large models)

@group(0) @binding(0) var<storage, read> gradients: array<f64>;      // Input gradients
@group(0) @binding(1) var<storage, read_write> params: array<f64>;   // Model parameters to update
@group(0) @binding(2) var<storage, read_write> m: array<f64>;        // First moment (momentum)
@group(0) @binding(3) var<storage, read_write> v: array<f64>;        // Second moment (RMSprop)

struct AdamWParams {
    num_params: u32,
    learning_rate: f64,
    beta1: f64,          // Exponential decay rate for first moment (typically 0.9)
    beta2: f64,          // Exponential decay rate for second moment (typically 0.999)
    epsilon: f64,        // Small constant for numerical stability (typically 1e-8)
    weight_decay: f64,   // Decoupled weight decay (typically 0.01)
    step: u32,           // Current training step (for bias correction)
}
@group(0) @binding(4) var<uniform> adamw_params: AdamWParams;

@compute @workgroup_size(256)
fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
    let gid = global_id.x;

    if (gid >= adamw_params.num_params) {
        return;
    }

    // Get current values
    let grad = gradients[gid];
    let param = params[gid];
    var m_val = m[gid];
    var v_val = v[gid];

    // Update biased first moment estimate (momentum)
    m_val = adamw_params.beta1 * m_val + (1.0 - adamw_params.beta1) * grad;

    // Update biased second moment estimate (RMSprop)
    v_val = adamw_params.beta2 * v_val + (1.0 - adamw_params.beta2) * grad * grad;

    // Compute bias-corrected first moment estimate
    let m_hat = m_val / (1.0 - pow_f64(adamw_params.beta1, f64(adamw_params.step)));

    // Compute bias-corrected second moment estimate
    let v_hat = v_val / (1.0 - pow_f64(adamw_params.beta2, f64(adamw_params.step)));

    // Adam update (gradient-based)
    let adam_update = adamw_params.learning_rate * m_hat / (sqrt_f64(v_hat) + adamw_params.epsilon);

    // Decoupled weight decay (KEY DIFFERENCE from Adam!)
    let weight_decay_update = adamw_params.learning_rate * adamw_params.weight_decay * param;

    // Update parameters (Adam update + decoupled weight decay)
    params[gid] = param - adam_update - weight_decay_update;

    // Store updated moments
    m[gid] = m_val;
    v[gid] = v_val;
}