use crate::IndexTensor;
use memmap2::{Mmap, MmapOptions};
use rand::prelude::*;
use serde::{Deserialize, Serialize};
use std::error::Error;
use std::fmt;
use std::fs::{create_dir_all, File};
use std::io::{BufReader, BufWriter, Write};
use std::path::{Component, Path, PathBuf};
use std::sync::mpsc::{sync_channel, Receiver};
use std::sync::Arc;
const SHARD_MAGIC: &[u8; 8] = b"OXTKSHD1";
const SHARD_HEADER_BYTES: usize = 16;
const MANIFEST_VERSION: u32 = 1;
#[derive(Debug)]
pub enum DataLoaderError {
InvalidData(String),
Io(std::io::Error),
Json(serde_json::Error),
WorkerStopped,
}
impl fmt::Display for DataLoaderError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidData(message) => write!(formatter, "token data error: {message}"),
Self::Io(error) => error.fmt(formatter),
Self::Json(error) => error.fmt(formatter),
Self::WorkerStopped => formatter.write_str("token prefetch worker stopped"),
}
}
}
impl Error for DataLoaderError {}
impl From<std::io::Error> for DataLoaderError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
impl From<serde_json::Error> for DataLoaderError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
#[derive(Clone)]
pub struct TokenDataset {
storage: Arc<TokenStorage>,
}
enum TokenStorage {
InMemory(Vec<usize>),
Mapped(MappedTokenShards),
}
struct MappedTokenShards {
shards: Vec<MappedTokenShard>,
token_count: usize,
}
struct MappedTokenShard {
mmap: Mmap,
global_start: usize,
token_count: usize,
}
impl TokenDataset {
pub fn from_tokens(tokens: Vec<usize>) -> Self {
Self {
storage: Arc::new(TokenStorage::InMemory(tokens)),
}
}
pub fn open_shards(directory: impl AsRef<Path>) -> Result<Self, DataLoaderError> {
let directory = directory.as_ref();
let manifest: TokenShardManifest =
serde_json::from_reader(BufReader::new(File::open(directory.join("manifest.json"))?))?;
if manifest.version != MANIFEST_VERSION {
return Err(DataLoaderError::InvalidData(format!(
"unsupported token shard manifest version {}",
manifest.version
)));
}
let mut shards = Vec::with_capacity(manifest.shards.len());
let mut global_start = 0usize;
for entry in manifest.shards {
let relative = Path::new(&entry.file);
if relative.components().count() != 1
|| !matches!(relative.components().next(), Some(Component::Normal(_)))
{
return Err(DataLoaderError::InvalidData(format!(
"invalid shard filename {}",
entry.file
)));
}
let file = File::open(directory.join(relative))?;
let file_length = usize::try_from(file.metadata()?.len()).map_err(|_| {
DataLoaderError::InvalidData("token shard is too large for this platform".into())
})?;
if file_length < SHARD_HEADER_BYTES {
return Err(DataLoaderError::InvalidData(format!(
"token shard {} is truncated",
entry.file
)));
}
let mmap = unsafe { MmapOptions::new().map(&file)? };
if &mmap[..8] != SHARD_MAGIC {
return Err(DataLoaderError::InvalidData(format!(
"token shard {} has invalid magic",
entry.file
)));
}
let header_count =
u64::from_le_bytes(mmap[8..16].try_into().expect("fixed token shard header"));
let token_count = usize::try_from(header_count).map_err(|_| {
DataLoaderError::InvalidData("token count is too large for this platform".into())
})?;
let expected_bytes = token_count
.checked_mul(size_of::<u32>())
.and_then(|bytes| bytes.checked_add(SHARD_HEADER_BYTES))
.ok_or_else(|| {
DataLoaderError::InvalidData("token shard byte size overflowed".into())
})?;
if token_count != entry.token_count || file_length != expected_bytes {
return Err(DataLoaderError::InvalidData(format!(
"token shard {} size does not match its manifest",
entry.file
)));
}
shards.push(MappedTokenShard {
mmap,
global_start,
token_count,
});
global_start = global_start.checked_add(token_count).ok_or_else(|| {
DataLoaderError::InvalidData("total token count overflowed".into())
})?;
}
if global_start != manifest.token_count {
return Err(DataLoaderError::InvalidData(
"manifest token count does not match its shards".into(),
));
}
Ok(Self {
storage: Arc::new(TokenStorage::Mapped(MappedTokenShards {
shards,
token_count: global_start,
})),
})
}
pub fn token_count(&self) -> usize {
match self.storage.as_ref() {
TokenStorage::InMemory(tokens) => tokens.len(),
TokenStorage::Mapped(shards) => shards.token_count,
}
}
pub fn read_range(
&self,
start: usize,
end: usize,
output: &mut Vec<usize>,
) -> Result<(), DataLoaderError> {
if start > end || end > self.token_count() {
return Err(DataLoaderError::InvalidData(format!(
"token range {start}..{end} exceeds dataset length {}",
self.token_count()
)));
}
output.reserve(end - start);
match self.storage.as_ref() {
TokenStorage::InMemory(tokens) => output.extend_from_slice(&tokens[start..end]),
TokenStorage::Mapped(shards) => {
let mut position = start;
while position < end {
let shard = shards
.shards
.iter()
.find(|shard| {
position >= shard.global_start
&& position < shard.global_start + shard.token_count
})
.ok_or_else(|| {
DataLoaderError::InvalidData(
"token range falls outside mapped shards".into(),
)
})?;
let local_start = position - shard.global_start;
let take = (end - position).min(shard.token_count - local_start);
let byte_start = SHARD_HEADER_BYTES + local_start * size_of::<u32>();
let byte_end = byte_start + take * size_of::<u32>();
for bytes in shard.mmap[byte_start..byte_end].chunks_exact(size_of::<u32>()) {
output.push(
u32::from_le_bytes(bytes.try_into().expect("four-byte token")) as usize,
);
}
position += take;
}
}
}
Ok(())
}
pub fn validate_token_ids(&self, vocab_size: usize) -> Result<(), DataLoaderError> {
const VALIDATION_CHUNK: usize = 65_536;
let mut start = 0usize;
let mut tokens = Vec::with_capacity(VALIDATION_CHUNK);
while start < self.token_count() {
tokens.clear();
let end = (start + VALIDATION_CHUNK).min(self.token_count());
self.read_range(start, end, &mut tokens)?;
if let Some(token) = tokens.iter().find(|token| **token >= vocab_size) {
return Err(DataLoaderError::InvalidData(format!(
"token {token} is outside vocabulary size {vocab_size}"
)));
}
start = end;
}
Ok(())
}
}
#[derive(Serialize, Deserialize)]
struct TokenShardManifest {
version: u32,
token_count: usize,
shards: Vec<TokenShardEntry>,
}
#[derive(Serialize, Deserialize)]
struct TokenShardEntry {
file: String,
token_count: usize,
}
pub struct TokenShardWriter {
directory: PathBuf,
shard_token_capacity: usize,
buffer: Vec<u32>,
entries: Vec<TokenShardEntry>,
token_count: usize,
}
impl TokenShardWriter {
pub fn create(
directory: impl AsRef<Path>,
shard_token_capacity: usize,
) -> Result<Self, DataLoaderError> {
if shard_token_capacity == 0 {
return Err(DataLoaderError::InvalidData(
"token shard capacity must be positive".into(),
));
}
let directory = directory.as_ref().to_path_buf();
create_dir_all(&directory)?;
Ok(Self {
directory,
shard_token_capacity,
buffer: Vec::with_capacity(shard_token_capacity),
entries: Vec::new(),
token_count: 0,
})
}
pub fn extend(&mut self, tokens: &[usize]) -> Result<(), DataLoaderError> {
for token in tokens {
let token = u32::try_from(*token).map_err(|_| {
DataLoaderError::InvalidData(format!("token ID {token} exceeds u32"))
})?;
self.buffer.push(token);
self.token_count += 1;
if self.buffer.len() == self.shard_token_capacity {
self.flush_shard()?;
}
}
Ok(())
}
pub fn finish(mut self) -> Result<TokenDataset, DataLoaderError> {
self.flush_shard()?;
if self.token_count == 0 {
return Err(DataLoaderError::InvalidData(
"cannot create an empty token dataset".into(),
));
}
let manifest = TokenShardManifest {
version: MANIFEST_VERSION,
token_count: self.token_count,
shards: self.entries,
};
serde_json::to_writer(
BufWriter::new(File::create(self.directory.join("manifest.json"))?),
&manifest,
)?;
TokenDataset::open_shards(self.directory)
}
fn flush_shard(&mut self) -> Result<(), DataLoaderError> {
if self.buffer.is_empty() {
return Ok(());
}
let filename = format!("shard-{:05}.bin", self.entries.len());
let mut writer = BufWriter::new(File::create(self.directory.join(&filename))?);
writer.write_all(SHARD_MAGIC)?;
writer.write_all(&(self.buffer.len() as u64).to_le_bytes())?;
for token in &self.buffer {
writer.write_all(&token.to_le_bytes())?;
}
writer.flush()?;
self.entries.push(TokenShardEntry {
file: filename,
token_count: self.buffer.len(),
});
self.buffer.clear();
Ok(())
}
}
pub struct DataBatch {
pub input_ids: IndexTensor,
pub target_ids: IndexTensor,
}
struct SyncLoader {
dataset: TokenDataset,
sampler: PackedSequenceSampler,
sequence_length: usize,
}
pub struct PrefetchDataLoader {
receiver: Option<Receiver<Result<PrefetchedBatch, DataLoaderError>>>,
sync: Option<SyncLoader>,
cursor: usize,
batch_size: usize,
}
impl PrefetchDataLoader {
#[allow(clippy::too_many_arguments)]
pub fn new(
dataset: TokenDataset,
sequence_length: usize,
batch_size: usize,
seed: u64,
cursor: usize,
prefetch_batches: usize,
) -> Result<Self, DataLoaderError> {
if sequence_length == 0 || batch_size == 0 {
return Err(DataLoaderError::InvalidData(
"sequence length and batch size must be positive".into(),
));
}
if dataset.token_count() <= sequence_length {
return Err(DataLoaderError::InvalidData(format!(
"dataset has {} tokens; need more than sequence length {sequence_length}",
dataset.token_count()
)));
}
if prefetch_batches == 0 {
let sampler =
PackedSequenceSampler::new(dataset.token_count(), sequence_length, seed, cursor);
return Ok(Self {
receiver: None,
sync: Some(SyncLoader {
dataset,
sampler,
sequence_length,
}),
cursor,
batch_size,
});
}
let mut sampler =
PackedSequenceSampler::new(dataset.token_count(), sequence_length, seed, cursor);
let (sender, receiver) = sync_channel(prefetch_batches);
std::thread::Builder::new()
.name("oxidelm-data-prefetch".into())
.spawn(move || loop {
let result = build_batch(&dataset, &mut sampler, sequence_length, batch_size);
let stop = result.is_err();
if sender.send(result).is_err() || stop {
break;
}
})?;
Ok(Self {
receiver: Some(receiver),
sync: None,
cursor,
batch_size,
})
}
pub fn next_batch(&mut self) -> Result<DataBatch, DataLoaderError> {
let batch = if let Some(receiver) = &self.receiver {
receiver
.recv()
.map_err(|_| DataLoaderError::WorkerStopped)??
} else if let Some(sync) = &mut self.sync {
build_batch(
&sync.dataset,
&mut sync.sampler,
sync.sequence_length,
self.batch_size,
)?
} else {
return Err(DataLoaderError::WorkerStopped);
};
self.cursor = self.cursor.saturating_add(self.batch_size);
let shape = vec![self.batch_size, batch.sequence_length];
Ok(DataBatch {
input_ids: IndexTensor::new(batch.inputs, shape.clone()),
target_ids: IndexTensor::new(batch.targets, shape),
})
}
pub fn cursor(&self) -> usize {
self.cursor
}
}
fn build_batch(
dataset: &TokenDataset,
sampler: &mut PackedSequenceSampler,
sequence_length: usize,
batch_size: usize,
) -> Result<PrefetchedBatch, DataLoaderError> {
let batch_elements = batch_size * sequence_length;
let mut inputs = Vec::with_capacity(batch_elements);
let mut targets = Vec::with_capacity(batch_elements);
let mut sequence = Vec::with_capacity(sequence_length + 1);
for _ in 0..batch_size {
sequence.clear();
let start = sampler.next_start();
dataset.read_range(start, start + sequence_length + 1, &mut sequence)?;
inputs.extend_from_slice(&sequence[..sequence_length]);
targets.extend_from_slice(&sequence[1..]);
}
Ok(PrefetchedBatch {
inputs,
targets,
sequence_length,
})
}
struct PrefetchedBatch {
inputs: Vec<usize>,
targets: Vec<usize>,
sequence_length: usize,
}
#[derive(Debug)]
pub(crate) struct PackedSequenceSampler {
sequence_length: usize,
sequence_count: usize,
seed: u64,
cursor: usize,
epoch: usize,
order: Vec<usize>,
}
impl PackedSequenceSampler {
pub(crate) fn new(
token_count: usize,
sequence_length: usize,
seed: u64,
cursor: usize,
) -> Self {
let sequence_count = (token_count - 1) / sequence_length;
let epoch = cursor / sequence_count;
let mut sampler = Self {
sequence_length,
sequence_count,
seed,
cursor,
epoch,
order: Vec::new(),
};
sampler.shuffle_epoch(epoch);
sampler
}
pub(crate) fn next_start(&mut self) -> usize {
let epoch = self.cursor / self.sequence_count;
if epoch != self.epoch {
self.shuffle_epoch(epoch);
}
let index_in_epoch = self.cursor % self.sequence_count;
let start = self.order[index_in_epoch] * self.sequence_length;
self.cursor += 1;
start
}
fn shuffle_epoch(&mut self, epoch: usize) {
self.epoch = epoch;
self.order = (0..self.sequence_count).collect();
let mut rng = StdRng::seed_from_u64(epoch_seed(self.seed, epoch));
self.order.shuffle(&mut rng);
}
}
fn epoch_seed(seed: u64, epoch: usize) -> u64 {
let mut value = seed ^ (epoch as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
value ^ (value >> 31)
}