Struct Dataset 

pub struct Dataset { /* private fields */ }

Implementations

impl Dataset

pub fn new(tokens: Vec<u32>, seq_len: usize) -> Self

pub fn from_file<P: AsRef<Path>>( path: P, tokenizer: &Tokenizer, seq_len: usize, ) -> Result<Self>

Load dataset from a JSONL file where each line has a “text” field. Supports .gz and .zst/.zstd compressed files.

pub fn from_stdin(tokenizer: &Tokenizer, seq_len: usize) -> Result<Self>

Load dataset from stdin (JSONL format).

pub fn from_files<P: AsRef<Path>>( paths: &[P], tokenizer: &Tokenizer, seq_len: usize, ) -> Result<Self>

Load dataset from multiple JSONL files. Supports .gz and .zst/.zstd compressed files.

pub fn len(&self) -> usize

pub fn is_empty(&self) -> bool

pub fn get_batch( &self, indices: &[usize], device: &Device, ) -> Result<(Tensor, Tensor)>

pub fn tokens(&self) -> &[u32]