use crate::network::NetworkError;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TokenBatch {
ids: Vec<u32>,
lengths: Vec<usize>,
seq_len: usize,
valid: Vec<bool>,
}
impl TokenBatch {
pub fn new<S: AsRef<[u32]>>(sequences: &[S]) -> Result<Self, NetworkError> {
if sequences.is_empty() {
return Err(NetworkError::EmptyDataset);
}
let lengths: Vec<usize> = sequences.iter().map(|s| s.as_ref().len()).collect();
if let Some(short) = lengths.iter().find(|&&length| length < 2) {
return Err(NetworkError::InvalidConfig(format!(
"a causal language-modelling loss needs at least two tokens per sequence, got {short}"
)));
}
let seq_len = lengths.iter().copied().max().unwrap_or(0);
let mut ids = vec![0u32; sequences.len() * seq_len];
let mut valid = vec![false; sequences.len() * seq_len];
for (index, sequence) in sequences.iter().enumerate() {
let sequence = sequence.as_ref();
let base = index * seq_len;
ids[base..base + sequence.len()].copy_from_slice(sequence);
valid[base..base + sequence.len()].fill(true);
}
Ok(Self {
ids,
lengths,
seq_len,
valid,
})
}
pub fn batch(&self) -> usize {
self.lengths.len()
}
pub fn seq_len(&self) -> usize {
self.seq_len
}
pub fn rows(&self) -> usize {
self.ids.len()
}
pub fn ids(&self) -> &[u32] {
&self.ids
}
pub fn lengths(&self) -> &[usize] {
&self.lengths
}
pub fn is_padded(&self) -> bool {
self.lengths.iter().any(|&length| length != self.seq_len)
}
pub fn predicted(&self) -> usize {
self.lengths.iter().map(|length| length - 1).sum()
}
pub fn layout(&self) -> Layout<'_> {
Layout {
seq_len: Some(self.seq_len),
valid: self.is_padded().then_some(&self.valid),
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Layout<'a> {
pub seq_len: Option<usize>,
pub valid: Option<&'a [bool]>,
}
impl<'a> Layout<'a> {
pub fn single(rows: usize) -> Self {
Self {
seq_len: Some(rows),
valid: None,
}
}
pub fn seq_len(&self, rows: usize) -> usize {
self.seq_len.unwrap_or(rows).max(1)
}
pub fn is_valid(&self, row: usize) -> bool {
self.valid.is_none_or(|mask| mask[row])
}
pub fn check(&self, rows: usize) -> Result<(), NetworkError> {
let seq_len = self.seq_len(rows);
if rows % seq_len != 0 {
return Err(NetworkError::InvalidConfig(format!(
"{rows} rows do not divide into sequences of {seq_len}"
)));
}
match self.valid {
Some(mask) if mask.len() != rows => Err(NetworkError::InvalidConfig(format!(
"padding mask covers {} rows, not {rows}",
mask.len()
))),
_ => Ok(()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn equal_length_sequences_need_no_padding() {
let batch = TokenBatch::new(&[[1u32, 2, 3], [4, 5, 6]]).unwrap();
assert_eq!(batch.rows(), 6);
assert_eq!(batch.seq_len(), 3);
assert!(!batch.is_padded());
assert_eq!(batch.layout().valid, None);
assert_eq!(batch.predicted(), 4);
}
#[test]
fn a_short_sequence_is_padded_on_the_right_and_masked() {
let batch = TokenBatch::new(&[vec![1u32, 2, 3, 4], vec![5, 6]]).unwrap();
assert_eq!(batch.ids(), &[1, 2, 3, 4, 5, 6, 0, 0]);
assert!(batch.is_padded());
let layout = batch.layout();
assert!(layout.is_valid(5));
assert!(!layout.is_valid(6));
assert_eq!(batch.predicted(), 3 + 1);
}
#[test]
fn a_one_token_sequence_is_rejected() {
assert!(TokenBatch::new(&[vec![1u32]]).is_err());
assert!(TokenBatch::new::<Vec<u32>>(&[]).is_err());
}
#[test]
fn the_default_layout_is_one_sequence() {
let layout = Layout::default();
assert_eq!(layout.seq_len(7), 7);
assert!(layout.is_valid(6));
assert!(layout.check(7).is_ok());
}
#[test]
fn a_layout_that_does_not_divide_the_rows_is_rejected() {
assert!(Layout::single(3).check(7).is_err());
let mask = [true, true];
let layout = Layout {
seq_len: Some(2),
valid: Some(&mask),
};
assert!(layout.check(4).is_err());
}
}