use crate::tokenizer::BpeTokenizer;
use crate::IndexTensor;
use rand::prelude::*;
use rand::rngs::StdRng;
use std::error::Error;
use std::fmt;
use std::fs;
use std::path::{Path, PathBuf};
pub const DEFAULT_CALIBRATION_SEQUENCES: usize = 128;
pub const DEFAULT_CALIBRATION_SEED: u64 = 0;
#[derive(Debug)]
pub enum CalibrationError {
Invalid(String),
Io(std::io::Error),
#[cfg(feature = "training")]
DataLoader(crate::data_loader::DataLoaderError),
}
impl fmt::Display for CalibrationError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Invalid(message) => write!(formatter, "{message}"),
Self::Io(error) => write!(formatter, "{error}"),
#[cfg(feature = "training")]
Self::DataLoader(error) => write!(formatter, "{error}"),
}
}
}
impl Error for CalibrationError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Io(error) => Some(error),
#[cfg(feature = "training")]
Self::DataLoader(error) => Some(error),
_ => None,
}
}
}
impl From<std::io::Error> for CalibrationError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
#[cfg(feature = "training")]
impl From<crate::data_loader::DataLoaderError> for CalibrationError {
fn from(error: crate::data_loader::DataLoaderError) -> Self {
Self::DataLoader(error)
}
}
#[derive(Clone, Debug)]
pub struct CalibrationOptions {
pub path: PathBuf,
pub sequence_length: usize,
pub max_sequences: usize,
pub batch_size: usize,
pub seed: u64,
}
impl CalibrationOptions {
pub fn new(path: impl Into<PathBuf>, sequence_length: usize) -> Self {
Self {
path: path.into(),
sequence_length,
max_sequences: DEFAULT_CALIBRATION_SEQUENCES,
batch_size: 1,
seed: DEFAULT_CALIBRATION_SEED,
}
}
}
#[derive(Clone, Debug)]
pub struct CalibrationData {
pub tokens: Vec<usize>,
pub sequence_length: usize,
pub sequences: Vec<Vec<usize>>,
pub source: String,
pub seed: u64,
}
impl CalibrationData {
pub fn sequence_count(&self) -> usize {
self.sequences.len()
}
pub fn batches(&self) -> impl Iterator<Item = IndexTensor> + '_ {
let batch_size = self.batch_size_hint();
let seq_len = self.sequence_length;
(0..self.sequences.len())
.step_by(batch_size)
.map(move |start| {
let end = (start + batch_size).min(self.sequences.len());
let batch = end - start;
let mut ids = Vec::with_capacity(batch * seq_len);
for sequence in &self.sequences[start..end] {
ids.extend_from_slice(sequence);
}
IndexTensor::new(ids, vec![batch, seq_len])
})
}
fn batch_size_hint(&self) -> usize {
1
}
}
pub fn load_calibration(
options: &CalibrationOptions,
tokenizer: &BpeTokenizer,
vocab_size: usize,
) -> Result<CalibrationData, CalibrationError> {
if options.sequence_length == 0 {
return Err(CalibrationError::Invalid(
"calibration sequence length must be positive".into(),
));
}
if options.max_sequences == 0 {
return Err(CalibrationError::Invalid(
"calibration max sequences must be positive".into(),
));
}
if options.batch_size == 0 {
return Err(CalibrationError::Invalid(
"calibration batch size must be positive".into(),
));
}
if tokenizer.vocab_size() != vocab_size {
return Err(CalibrationError::Invalid(
"tokenizer and model vocabulary sizes differ".into(),
));
}
let path = options.path.as_path();
let (tokens, source) = load_calibration_tokens(path, tokenizer)?;
if let Some(bad) = tokens.iter().copied().find(|token| *token >= vocab_size) {
return Err(CalibrationError::Invalid(format!(
"calibration token id {bad} is outside vocabulary {vocab_size}"
)));
}
if tokens.len() < options.sequence_length {
return Err(CalibrationError::Invalid(format!(
"calibration corpus has {} tokens; need at least sequence length {}",
tokens.len(),
options.sequence_length
)));
}
let sequences = sample_sequences(
&tokens,
options.sequence_length,
options.max_sequences,
options.seed,
);
if sequences.is_empty() {
return Err(CalibrationError::Invalid(
"no calibration sequences could be sampled".into(),
));
}
Ok(CalibrationData {
tokens,
sequence_length: options.sequence_length,
sequences,
source,
seed: options.seed,
})
}
fn load_calibration_tokens(
path: &Path,
tokenizer: &BpeTokenizer,
) -> Result<(Vec<usize>, String), CalibrationError> {
if path.is_dir() {
return load_token_shard_dir(path);
}
if !path.is_file() {
return Err(CalibrationError::Invalid(format!(
"calibration path {} does not exist",
path.display()
)));
}
let name = path
.file_name()
.and_then(|value| value.to_str())
.unwrap_or("")
.to_ascii_lowercase();
if name == "manifest.json" {
let dir = path.parent().unwrap_or_else(|| Path::new("."));
return load_token_shard_dir(dir);
}
let text = fs::read_to_string(path)?;
let tokens = if name.ends_with(".jsonl") {
tokenize_jsonl(&text, tokenizer)?
} else {
let mut tokens = tokenizer.encode(&text);
if tokens.len() < 64 {
for prompt in text.split("\n\n") {
let prompt = prompt.trim();
if prompt.is_empty() {
continue;
}
tokens.extend(tokenizer.encode(prompt));
}
}
tokens
};
Ok((tokens, format!("text:{}", path.display())))
}
fn tokenize_jsonl(text: &str, tokenizer: &BpeTokenizer) -> Result<Vec<usize>, CalibrationError> {
let mut tokens = Vec::new();
for (line_index, line) in text.lines().enumerate() {
let line = line.trim();
if line.is_empty() {
continue;
}
let value: serde_json::Value = serde_json::from_str(line).map_err(|error| {
CalibrationError::Invalid(format!(
"invalid jsonl at line {}: {error}",
line_index + 1
))
})?;
let prompt = value
.get("text")
.or_else(|| value.get("prompt"))
.or_else(|| value.get("content"))
.and_then(|value| value.as_str())
.ok_or_else(|| {
CalibrationError::Invalid(format!(
"jsonl line {} missing text/prompt/content string",
line_index + 1
))
})?;
tokens.extend(tokenizer.encode(prompt));
tokens.push(tokenizer.special_tokens.eos);
}
if tokens.is_empty() {
return Err(CalibrationError::Invalid(
"jsonl calibration file produced no tokens".into(),
));
}
Ok(tokens)
}
fn load_token_shard_dir(path: &Path) -> Result<(Vec<usize>, String), CalibrationError> {
#[cfg(feature = "training")]
{
let dataset = crate::data_loader::TokenDataset::open_shards(path)?;
let mut tokens = Vec::with_capacity(dataset.token_count());
dataset.read_range(0, dataset.token_count(), &mut tokens)?;
return Ok((tokens, format!("shards:{}", path.display())));
}
#[cfg(not(feature = "training"))]
{
let _ = path;
Err(CalibrationError::Invalid(
"token-shard calibration requires the `training` feature".into(),
))
}
}
pub fn sample_sequences(
tokens: &[usize],
sequence_length: usize,
max_sequences: usize,
seed: u64,
) -> Vec<Vec<usize>> {
if sequence_length == 0 || tokens.len() < sequence_length {
return Vec::new();
}
let available = tokens.len() / sequence_length;
if available == 0 {
return Vec::new();
}
let mut starts: Vec<usize> = (0..available).map(|index| index * sequence_length).collect();
if starts.len() > max_sequences {
let mut rng = StdRng::seed_from_u64(seed);
starts.shuffle(&mut rng);
starts.truncate(max_sequences);
starts.sort_unstable();
}
starts
.into_iter()
.map(|start| tokens[start..start + sequence_length].to_vec())
.collect()
}
pub fn run_calibration_forwards<F>(
data: &CalibrationData,
batch_size: usize,
mut forward: F,
) -> Result<(), CalibrationError>
where
F: FnMut(&IndexTensor) -> Result<(), String>,
{
let batch_size = batch_size.max(1);
let seq_len = data.sequence_length;
let _guard = crate::no_grad();
for start in (0..data.sequences.len()).step_by(batch_size) {
let end = (start + batch_size).min(data.sequences.len());
let batch = end - start;
let mut ids = Vec::with_capacity(batch * seq_len);
for sequence in &data.sequences[start..end] {
ids.extend_from_slice(sequence);
}
let input = IndexTensor::new(ids, vec![batch, seq_len]);
forward(&input).map_err(CalibrationError::Invalid)?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
fn tiny_tokenizer() -> BpeTokenizer {
BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust calibration prompts",
FIRST_MERGE_ID + 16,
1,
)
.unwrap()
}
#[test]
fn sample_sequences_is_deterministic() {
let tokens: Vec<usize> = (0..200).collect();
let a = sample_sequences(&tokens, 8, 5, 42);
let b = sample_sequences(&tokens, 8, 5, 42);
assert_eq!(a, b);
assert_eq!(a.len(), 5);
assert!(a.windows(2).all(|pair| pair[0] < pair[1]));
}
#[test]
fn load_text_calibration() {
let directory = std::env::temp_dir().join(format!(
"oxidelm-cal-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
fs::create_dir_all(&directory).unwrap();
let path = directory.join("prompts.txt");
let tokenizer = tiny_tokenizer();
let mut text = String::new();
for _ in 0..40 {
text.push_str("hello world hello rust\n");
}
fs::write(&path, &text).unwrap();
let options = CalibrationOptions {
path: path.clone(),
sequence_length: 8,
max_sequences: 4,
batch_size: 2,
seed: 0,
};
let data = load_calibration(&options, &tokenizer, tokenizer.vocab_size()).unwrap();
assert_eq!(data.sequence_count(), 4);
assert_eq!(data.sequences[0].len(), 8);
let _ = fs::remove_dir_all(&directory);
}
}