primal-barracuda 0.4.0

barraCuda — BARrier-free Rust Abstracted Cross-platform Unified Dimensional Algebra. Vendor-agnostic GPU compute via WGSL.
Documentation
// SPDX-License-Identifier: AGPL-3.0-or-later
// AdaDelta Optimizer (f64 canonical)
// Extension of AdaGrad that seeks to reduce monotonically decreasing learning rate
//
// Update rule:
// E[g²] = rho * E[g²] + (1 - rho) * g²
// delta = sqrt((E[delta²] + epsilon) / (E[g²] + epsilon)) * g
// E[delta²] = rho * E[delta²] + (1 - rho) * delta²
// weight = weight - delta
//
// Benefits: No learning rate hyperparameter needed!
// Used in: When you want to avoid tuning learning rates

@group(0) @binding(0) var<storage, read> weights: array<f64>;
@group(0) @binding(1) var<storage, read> gradients: array<f64>;
@group(0) @binding(2) var<storage, read> acc_grad_in: array<f64>;    // E[g²]
@group(0) @binding(3) var<storage, read> acc_delta_in: array<f64>;   // E[delta²]
@group(0) @binding(4) var<storage, read_write> weights_out: array<f64>;
@group(0) @binding(5) var<storage, read_write> acc_grad_out: array<f64>;
@group(0) @binding(6) var<storage, read_write> acc_delta_out: array<f64>;
@group(0) @binding(7) var<uniform> params: Params;

struct Params {
    rho: f64,             // Decay rate, typically 0.95
    epsilon: f64,         // Numerical stability, typically 1e-6
    weight_decay: f64,    // L2 regularization
    _padding: u32,
}

@compute @workgroup_size(256)
fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
    let idx = global_id.x;
    if idx >= arrayLength(&weights) {
        return;
    }

    let w = weights[idx];
    var g = gradients[idx];
    let eg2 = acc_grad_in[idx];
    let ed2 = acc_delta_in[idx];

    // Apply weight decay
    if params.weight_decay != 0.0 {
        g = g + params.weight_decay * w;
    }

    // Accumulate gradient squared
    let eg2_new = params.rho * eg2 + (1.0 - params.rho) * g * g;
    acc_grad_out[idx] = eg2_new;

    // Compute update
    let rms_delta = sqrt_f64(ed2 + params.epsilon);
    let rms_grad = sqrt_f64(eg2_new + params.epsilon);
    let delta = (rms_delta / rms_grad) * g;

    // Accumulate delta squared
    let ed2_new = params.rho * ed2 + (1.0 - params.rho) * delta * delta;
    acc_delta_out[idx] = ed2_new;

    // Update weights
    weights_out[idx] = w - delta;
}