primal-barracuda 0.4.0

barraCuda — BARrier-free Rust Abstracted Cross-platform Unified Dimensional Algebra. Vendor-agnostic GPU compute via WGSL.
Documentation
// SPDX-License-Identifier: AGPL-3.0-or-later
// NAdam Optimizer (Nesterov-accelerated Adaptive Moment Estimation) (f64 canonical)
// Combines Adam with Nesterov momentum for faster convergence
//
// Update rule:
// m = beta1 * m + (1 - beta1) * gradient
// v = beta2 * v + (1 - beta2) * gradient²
// m_hat = m / (1 - beta1^t)
// v_hat = v / (1 - beta2^t)
// gradient_nesterov = (beta1 * m_hat + (1 - beta1) * gradient) / (1 - beta1^t)
// weight = weight - learning_rate * gradient_nesterov / (sqrt(v_hat) + epsilon)
//
// Used in: Training deep networks faster than Adam, modern optimizers

@group(0) @binding(0) var<storage, read> weights: array<f64>;
@group(0) @binding(1) var<storage, read> gradients: array<f64>;
@group(0) @binding(2) var<storage, read> m_in: array<f64>;
@group(0) @binding(3) var<storage, read> v_in: array<f64>;
@group(0) @binding(4) var<storage, read_write> weights_out: array<f64>;
@group(0) @binding(5) var<storage, read_write> m_out: array<f64>;
@group(0) @binding(6) var<storage, read_write> v_out: array<f64>;
@group(0) @binding(7) var<uniform> params: Params;

struct Params {
    learning_rate: f64,
    beta1: f64,           // First moment decay, typically 0.9
    beta2: f64,           // Second moment decay, typically 0.999
    epsilon: f64,         // Numerical stability, typically 1e-8
    weight_decay: f64,    // L2 regularization
    step: u32,            // Current step number
    _padding: vec2<u32>,
}

@compute @workgroup_size(256)
fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
    let idx = global_id.x;
    if idx >= arrayLength(&weights) {
        return;
    }

    let w = weights[idx];
    var g = gradients[idx];
    let m = m_in[idx];
    let v = v_in[idx];

    // Apply weight decay
    if params.weight_decay != 0.0 {
        g = g + params.weight_decay * w;
    }

    // Update biased first moment estimate
    let m_new = params.beta1 * m + (1.0 - params.beta1) * g;
    m_out[idx] = m_new;

    // Update biased second moment estimate
    let v_new = params.beta2 * v + (1.0 - params.beta2) * g * g;
    v_out[idx] = v_new;

    // Compute bias correction
    let step_f = f64(params.step);
    let beta1_t = pow_f64(params.beta1, step_f);
    let beta2_t = pow_f64(params.beta2, step_f);

    // Bias-corrected estimates
    let m_hat = m_new / (1.0 - beta1_t);
    let v_hat = v_new / (1.0 - beta2_t);

    // Nesterov momentum
    let gradient_nesterov = (params.beta1 * m_hat + (1.0 - params.beta1) * g) / (1.0 - beta1_t);

    // Update weights with Nesterov-accelerated gradient
    weights_out[idx] = w - params.learning_rate * gradient_nesterov / (sqrt_f64(v_hat) + params.epsilon);
}