oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
//! Calibration data loading for activation-aware PTQ (GPTQ / AWQ / SmoothQuant).
//!
//! Accepts a UTF-8 prompt corpus (`.txt` / `.jsonl`) or an OxideLM token-shard
//! directory (`manifest.json`). Sequence windows are sampled deterministically
//! from a fixed seed.

use crate::tokenizer::BpeTokenizer;
use crate::IndexTensor;
use rand::prelude::*;
use rand::rngs::StdRng;
use std::error::Error;
use std::fmt;
use std::fs;
use std::path::{Path, PathBuf};

/// Defaults used when CLI flags are omitted.
pub const DEFAULT_CALIBRATION_SEQUENCES: usize = 128;
pub const DEFAULT_CALIBRATION_SEED: u64 = 0;

#[derive(Debug)]
pub enum CalibrationError {
    Invalid(String),
    Io(std::io::Error),
    #[cfg(feature = "training")]
    DataLoader(crate::data_loader::DataLoaderError),
}

impl fmt::Display for CalibrationError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Invalid(message) => write!(formatter, "{message}"),
            Self::Io(error) => write!(formatter, "{error}"),
            #[cfg(feature = "training")]
            Self::DataLoader(error) => write!(formatter, "{error}"),
        }
    }
}

impl Error for CalibrationError {
    fn source(&self) -> Option<&(dyn Error + 'static)> {
        match self {
            Self::Io(error) => Some(error),
            #[cfg(feature = "training")]
            Self::DataLoader(error) => Some(error),
            _ => None,
        }
    }
}

impl From<std::io::Error> for CalibrationError {
    fn from(error: std::io::Error) -> Self {
        Self::Io(error)
    }
}

#[cfg(feature = "training")]
impl From<crate::data_loader::DataLoaderError> for CalibrationError {
    fn from(error: crate::data_loader::DataLoaderError) -> Self {
        Self::DataLoader(error)
    }
}

#[derive(Clone, Debug)]
pub struct CalibrationOptions {
    pub path: PathBuf,
    pub sequence_length: usize,
    pub max_sequences: usize,
    pub batch_size: usize,
    pub seed: u64,
}

impl CalibrationOptions {
    pub fn new(path: impl Into<PathBuf>, sequence_length: usize) -> Self {
        Self {
            path: path.into(),
            sequence_length,
            max_sequences: DEFAULT_CALIBRATION_SEQUENCES,
            batch_size: 1,
            seed: DEFAULT_CALIBRATION_SEED,
        }
    }
}

/// Packed calibration windows as token-id batches `[batch, sequence_length]`.
#[derive(Clone, Debug)]
pub struct CalibrationData {
    pub tokens: Vec<usize>,
    pub sequence_length: usize,
    pub sequences: Vec<Vec<usize>>,
    pub source: String,
    pub seed: u64,
}

impl CalibrationData {
    pub fn sequence_count(&self) -> usize {
        self.sequences.len()
    }

    /// Yields deterministic batches of shape `[B, T]`.
    pub fn batches(&self) -> impl Iterator<Item = IndexTensor> + '_ {
        let batch_size = self.batch_size_hint();
        let seq_len = self.sequence_length;
        (0..self.sequences.len())
            .step_by(batch_size)
            .map(move |start| {
                let end = (start + batch_size).min(self.sequences.len());
                let batch = end - start;
                let mut ids = Vec::with_capacity(batch * seq_len);
                for sequence in &self.sequences[start..end] {
                    ids.extend_from_slice(sequence);
                }
                IndexTensor::new(ids, vec![batch, seq_len])
            })
    }

    fn batch_size_hint(&self) -> usize {
        // Stored on options at load time via sequences packing; default 1.
        1
    }
}

/// Loads tokens from `--calibration` and packs deterministic windows.
pub fn load_calibration(
    options: &CalibrationOptions,
    tokenizer: &BpeTokenizer,
    vocab_size: usize,
) -> Result<CalibrationData, CalibrationError> {
    if options.sequence_length == 0 {
        return Err(CalibrationError::Invalid(
            "calibration sequence length must be positive".into(),
        ));
    }
    if options.max_sequences == 0 {
        return Err(CalibrationError::Invalid(
            "calibration max sequences must be positive".into(),
        ));
    }
    if options.batch_size == 0 {
        return Err(CalibrationError::Invalid(
            "calibration batch size must be positive".into(),
        ));
    }
    if tokenizer.vocab_size() != vocab_size {
        return Err(CalibrationError::Invalid(
            "tokenizer and model vocabulary sizes differ".into(),
        ));
    }

    let path = options.path.as_path();
    let (tokens, source) = load_calibration_tokens(path, tokenizer)?;
    if let Some(bad) = tokens.iter().copied().find(|token| *token >= vocab_size) {
        return Err(CalibrationError::Invalid(format!(
            "calibration token id {bad} is outside vocabulary {vocab_size}"
        )));
    }
    if tokens.len() < options.sequence_length {
        return Err(CalibrationError::Invalid(format!(
            "calibration corpus has {} tokens; need at least sequence length {}",
            tokens.len(),
            options.sequence_length
        )));
    }

    let sequences = sample_sequences(
        &tokens,
        options.sequence_length,
        options.max_sequences,
        options.seed,
    );
    if sequences.is_empty() {
        return Err(CalibrationError::Invalid(
            "no calibration sequences could be sampled".into(),
        ));
    }

    Ok(CalibrationData {
        tokens,
        sequence_length: options.sequence_length,
        sequences,
        source,
        seed: options.seed,
    })
}

fn load_calibration_tokens(
    path: &Path,
    tokenizer: &BpeTokenizer,
) -> Result<(Vec<usize>, String), CalibrationError> {
    if path.is_dir() {
        return load_token_shard_dir(path);
    }
    if !path.is_file() {
        return Err(CalibrationError::Invalid(format!(
            "calibration path {} does not exist",
            path.display()
        )));
    }
    let name = path
        .file_name()
        .and_then(|value| value.to_str())
        .unwrap_or("")
        .to_ascii_lowercase();
    if name == "manifest.json" {
        let dir = path.parent().unwrap_or_else(|| Path::new("."));
        return load_token_shard_dir(dir);
    }

    let text = fs::read_to_string(path)?;
    let tokens = if name.ends_with(".jsonl") {
        tokenize_jsonl(&text, tokenizer)?
    } else {
        // Plain text / prompt set: whole file, then also split blank-line prompts
        // and concatenate so short prompt files still yield enough tokens.
        let mut tokens = tokenizer.encode(&text);
        if tokens.len() < 64 {
            for prompt in text.split("\n\n") {
                let prompt = prompt.trim();
                if prompt.is_empty() {
                    continue;
                }
                tokens.extend(tokenizer.encode(prompt));
            }
        }
        tokens
    };
    Ok((tokens, format!("text:{}", path.display())))
}

fn tokenize_jsonl(text: &str, tokenizer: &BpeTokenizer) -> Result<Vec<usize>, CalibrationError> {
    let mut tokens = Vec::new();
    for (line_index, line) in text.lines().enumerate() {
        let line = line.trim();
        if line.is_empty() {
            continue;
        }
        let value: serde_json::Value = serde_json::from_str(line).map_err(|error| {
            CalibrationError::Invalid(format!(
                "invalid jsonl at line {}: {error}",
                line_index + 1
            ))
        })?;
        let prompt = value
            .get("text")
            .or_else(|| value.get("prompt"))
            .or_else(|| value.get("content"))
            .and_then(|value| value.as_str())
            .ok_or_else(|| {
                CalibrationError::Invalid(format!(
                    "jsonl line {} missing text/prompt/content string",
                    line_index + 1
                ))
            })?;
        tokens.extend(tokenizer.encode(prompt));
        tokens.push(tokenizer.special_tokens.eos);
    }
    if tokens.is_empty() {
        return Err(CalibrationError::Invalid(
            "jsonl calibration file produced no tokens".into(),
        ));
    }
    Ok(tokens)
}

fn load_token_shard_dir(path: &Path) -> Result<(Vec<usize>, String), CalibrationError> {
    #[cfg(feature = "training")]
    {
        let dataset = crate::data_loader::TokenDataset::open_shards(path)?;
        let mut tokens = Vec::with_capacity(dataset.token_count());
        dataset.read_range(0, dataset.token_count(), &mut tokens)?;
        return Ok((tokens, format!("shards:{}", path.display())));
    }
    #[cfg(not(feature = "training"))]
    {
        let _ = path;
        Err(CalibrationError::Invalid(
            "token-shard calibration requires the `training` feature".into(),
        ))
    }
}

/// Deterministic non-overlapping windows; if more windows exist than
/// `max_sequences`, subsample with a seeded RNG without replacement.
pub fn sample_sequences(
    tokens: &[usize],
    sequence_length: usize,
    max_sequences: usize,
    seed: u64,
) -> Vec<Vec<usize>> {
    if sequence_length == 0 || tokens.len() < sequence_length {
        return Vec::new();
    }
    let available = tokens.len() / sequence_length;
    if available == 0 {
        return Vec::new();
    }
    let mut starts: Vec<usize> = (0..available).map(|index| index * sequence_length).collect();
    if starts.len() > max_sequences {
        let mut rng = StdRng::seed_from_u64(seed);
        starts.shuffle(&mut rng);
        starts.truncate(max_sequences);
        starts.sort_unstable();
    }
    starts
        .into_iter()
        .map(|start| tokens[start..start + sequence_length].to_vec())
        .collect()
}

/// Runs `model.forward` over every calibration batch under `no_grad`.
pub fn run_calibration_forwards<F>(
    data: &CalibrationData,
    batch_size: usize,
    mut forward: F,
) -> Result<(), CalibrationError>
where
    F: FnMut(&IndexTensor) -> Result<(), String>,
{
    let batch_size = batch_size.max(1);
    let seq_len = data.sequence_length;
    let _guard = crate::no_grad();
    for start in (0..data.sequences.len()).step_by(batch_size) {
        let end = (start + batch_size).min(data.sequences.len());
        let batch = end - start;
        let mut ids = Vec::with_capacity(batch * seq_len);
        for sequence in &data.sequences[start..end] {
            ids.extend_from_slice(sequence);
        }
        let input = IndexTensor::new(ids, vec![batch, seq_len]);
        forward(&input).map_err(CalibrationError::Invalid)?;
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};

    fn tiny_tokenizer() -> BpeTokenizer {
        BpeTokenizer::train_from_bytes_with_min_frequency(
            b"hello world hello rust calibration prompts",
            FIRST_MERGE_ID + 16,
            1,
        )
        .unwrap()
    }

    #[test]
    fn sample_sequences_is_deterministic() {
        let tokens: Vec<usize> = (0..200).collect();
        let a = sample_sequences(&tokens, 8, 5, 42);
        let b = sample_sequences(&tokens, 8, 5, 42);
        assert_eq!(a, b);
        assert_eq!(a.len(), 5);
        assert!(a.windows(2).all(|pair| pair[0] < pair[1]));
    }

    #[test]
    fn load_text_calibration() {
        let directory = std::env::temp_dir().join(format!(
            "oxidelm-cal-{}",
            std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        ));
        fs::create_dir_all(&directory).unwrap();
        let path = directory.join("prompts.txt");
        let tokenizer = tiny_tokenizer();
        let mut text = String::new();
        for _ in 0..40 {
            text.push_str("hello world hello rust\n");
        }
        fs::write(&path, &text).unwrap();
        let options = CalibrationOptions {
            path: path.clone(),
            sequence_length: 8,
            max_sequences: 4,
            batch_size: 2,
            seed: 0,
        };
        let data = load_calibration(&options, &tokenizer, tokenizer.vocab_size()).unwrap();
        assert_eq!(data.sequence_count(), 4);
        assert_eq!(data.sequences[0].len(), 8);
        let _ = fs::remove_dir_all(&directory);
    }
}