Struct TextDataset 

pub struct TextDataset { /* private fields */ }

Simple in-memory text dataset for LLM training

Implementations

impl TextDataset

pub fn new() -> Self

An empty dataset.

pub fn add_sample(&mut self, input_ids: Vec<u32>, target_ids: Vec<u32>)

Add a sample to the dataset

pub fn add_example(&mut self, example: TrainingExample)

Add a prepared example (with its loss mask).

pub fn from_sequences(sequences: Vec<Vec<u32>>) -> Self

Create from list of token sequences (next-token prediction)

pub fn from_jsonl_sft<P: AsRef<Path>>( path: P, tokenizer: &Tokenizer, use_chat_template: bool, max_len: usize, ) -> Result<Self, Error>

Create from a JSONL instruction dataset with {prompt,response} objects. Uses chat-template encoding and response-only loss masks when use_chat_template.

pub fn from_jsonl<P: AsRef<Path>>( path: P, tokenizer: &Tokenizer, ) -> Result<Self, Error>

Create from a JSONL instruction dataset with {prompt,response} objects.

pub fn from_text_file<P: AsRef<Path>>( path: P, tokenizer: &Tokenizer, ) -> Result<Self, Error>

Create from a plain text file, one training sample per non-empty line.

pub fn from_token_bin<P: AsRef<Path>>( path: P, sequence_length: usize, ) -> Result<Self, Error>

Create from a flat binary file of little-endian u32 token ids.

pub fn from_packed_tokens(tokens: Vec<u32>, sequence_length: usize) -> Self

Create fixed-length training windows from a packed token stream.

pub fn pack_sequences(sequences: Vec<Vec<u32>>, max_len: usize) -> Self

Greedily pack short sequences into longer windows up to max_len.

pub fn examples(&self) -> &[TrainingExample]

All examples, in insertion order.

Trait Implementations

impl Clone for TextDataset

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Dataset for TextDataset

fn len(&self) -> usize

Number of samples in the dataset

fn get(&self, idx: usize) -> (Vec<u32>, Vec<u32>)

Get a single sample by index Returns (input_ids, target_ids) as token vectors

fn get_loss_mask(&self, idx: usize) -> Option<Vec<u8>>

Optional per-token loss mask aligned with targets (1 = include, 0 = ignore)

fn is_empty(&self) -> bool

Check if dataset is empty

impl Default for TextDataset

fn default() -> Self

Returns the “default value” for a type.