primal-barracuda 0.4.0

barraCuda — BARrier-free Rust Abstracted Cross-platform Unified Dimensional Algebra. Vendor-agnostic GPU compute via WGSL.
Documentation
// SPDX-License-Identifier: AGPL-3.0-or-later
// Adam Optimizer: Adaptive moment estimation optimizer (f64 canonical)
// CUDA equivalent: Custom kernels or cuDNN optimizers
// Formula: Adaptive learning rate with momentum and RMSprop
// Use cases: Deep learning training, state-of-the-art optimization

@group(0) @binding(0) var<storage, read> gradients: array<f64>;      // Input gradients
@group(0) @binding(1) var<storage, read_write> params: array<f64>;   // Model parameters to update
@group(0) @binding(2) var<storage, read_write> m: array<f64>;        // First moment (momentum)
@group(0) @binding(3) var<storage, read_write> v: array<f64>;        // Second moment (RMSprop)

struct AdamParams {
    num_params: u32,
    learning_rate: f64,
    beta1: f64,          // Exponential decay rate for first moment (typically 0.9)
    beta2: f64,          // Exponential decay rate for second moment (typically 0.999)
    epsilon: f64,        // Small constant for numerical stability (typically 1e-8)
    weight_decay: f64,   // L2 regularization strength (typically 0 or 0.01)
    step: u32,           // Current training step (for bias correction)
}
@group(0) @binding(4) var<uniform> adam_params: AdamParams;

@compute @workgroup_size(256)
fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
    let gid = global_id.x;

    if (gid >= adam_params.num_params) {
        return;
    }

    // Get current values
    var grad = gradients[gid];
    let param = params[gid];
    var m_val = m[gid];
    var v_val = v[gid];

    // Apply weight decay (L2 regularization) if enabled
    if (adam_params.weight_decay > 0.0) {
        grad = grad + adam_params.weight_decay * param;
    }

    // Update biased first moment estimate (momentum)
    m_val = adam_params.beta1 * m_val + (1.0 - adam_params.beta1) * grad;

    // Update biased second moment estimate (RMSprop)
    v_val = adam_params.beta2 * v_val + (1.0 - adam_params.beta2) * grad * grad;

    // Compute bias-corrected first moment estimate
    let m_hat = m_val / (1.0 - pow_f64(adam_params.beta1, f64(adam_params.step)));

    // Compute bias-corrected second moment estimate
    let v_hat = v_val / (1.0 - pow_f64(adam_params.beta2, f64(adam_params.step)));

    // Update parameters
    let update = adam_params.learning_rate * m_hat / (sqrt_f64(v_hat) + adam_params.epsilon);
    params[gid] = param - update;

    // Store updated moments
    m[gid] = m_val;
    v[gid] = v_val;
}