oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
//! Optional early detection of NaN and infinity in tensor operations.
//!
//! Disabled by default (near-zero overhead). Enable with [`set_detect_anomaly`],
//! the `--detect-anomaly` CLI flag, or `OXIDELM_DETECT_ANOMALY=1`.
//!
//! When enabled, each value-producing op checks its output and panics on the
//! first non-finite value, reporting the operation name, tensor shape, and the
//! training step set via [`set_anomaly_training_step`].

use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Mutex, MutexGuard};

static DETECT_ANOMALY: AtomicBool = AtomicBool::new(false);
static ANOMALY_TRAINING_STEP: AtomicUsize = AtomicUsize::new(0);
static ANOMALY_TEST_LOCK: Mutex<()> = Mutex::new(());

/// Exclusive lock for tests that toggle global anomaly state.
#[doc(hidden)]
pub fn test_lock() -> MutexGuard<'static, ()> {
    ANOMALY_TEST_LOCK
        .lock()
        .unwrap_or_else(|poisoned| poisoned.into_inner())
}

/// Enables or disables anomaly checks on tensor operation outputs.
pub fn set_detect_anomaly(enabled: bool) {
    DETECT_ANOMALY.store(enabled, Ordering::Relaxed);
}

/// Returns whether anomaly checks are currently enabled.
pub fn detect_anomaly_enabled() -> bool {
    DETECT_ANOMALY.load(Ordering::Relaxed)
}

/// Records the training step used in subsequent anomaly reports.
///
/// Trainers should call this at the start of each step (1-based step that will
/// appear in metrics). Outside training the value may be left at `0`.
pub fn set_anomaly_training_step(step: usize) {
    ANOMALY_TRAINING_STEP.store(step, Ordering::Relaxed);
}

/// Returns the training step last set for anomaly reports.
pub fn anomaly_training_step() -> usize {
    ANOMALY_TRAINING_STEP.load(Ordering::Relaxed)
}

/// Parses common truthy/falsey strings for CLI and environment use.
pub fn parse_detect_anomaly(value: &str) -> Result<bool, String> {
    match value.trim().to_ascii_lowercase().as_str() {
        "0" | "off" | "false" | "no" | "none" => Ok(false),
        "1" | "on" | "true" | "yes" => Ok(true),
        other => Err(format!(
            "unknown detect-anomaly value '{other}'; expected on or off"
        )),
    }
}

/// Apply `OXIDELM_DETECT_ANOMALY` if set. Invalid values are ignored.
pub fn init_from_env() {
    if let Ok(value) = std::env::var("OXIDELM_DETECT_ANOMALY") {
        if let Ok(enabled) = parse_detect_anomaly(&value) {
            set_detect_anomaly(enabled);
        }
    }
}

/// Formats the standard anomaly report message.
pub fn format_report(operation: &str, shape: &[usize]) -> String {
    format!(
        "anomaly: non-finite value in operation '{operation}' shape {shape:?} at training step {}",
        anomaly_training_step()
    )
}

/// Panics with an anomaly report when `values` contains NaN or infinity.
pub fn check_finite(operation: &str, shape: &[usize], values: &[f32]) {
    if !detect_anomaly_enabled() {
        return;
    }
    if values.iter().any(|value| !value.is_finite()) {
        panic!("{}", format_report(operation, shape));
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn parse_accepts_common_booleans() {
        assert!(parse_detect_anomaly("1").unwrap());
        assert!(parse_detect_anomaly("ON").unwrap());
        assert!(!parse_detect_anomaly("off").unwrap());
        assert!(parse_detect_anomaly("maybe").is_err());
    }

    #[test]
    fn report_includes_operation_shape_and_step() {
        set_anomaly_training_step(12);
        let report = format_report("div", &[2, 3]);
        assert!(report.contains("operation 'div'"));
        assert!(report.contains("shape [2, 3]"));
        assert!(report.contains("training step 12"));
        set_anomaly_training_step(0);
    }
}