Struct CitlDataLoader 

pub struct CitlDataLoader { /* private fields */ }

CITL DataLoader using alimentar for efficient data loading

Implementations

impl CitlDataLoader

pub fn new() -> Self

Create a new CITL DataLoader with default configuration

pub fn with_config(config: CitlLoaderConfig) -> Self

Create a new CITL DataLoader with custom configuration

pub fn batch_size(self, size: usize) -> Self

Set batch size

pub fn shuffle(self, shuffle: bool) -> Self

Enable/disable shuffling

pub fn min_confidence(self, confidence: f32) -> Self

Set minimum confidence threshold

pub fn merge_strategy(self, strategy: MergeStrategy) -> Self

Set merge strategy

pub fn load_parquet<P: AsRef<Path>>(&self, path: P) -> Result<CitlBatchIterator>

Load CITL corpus from Parquet file using alimentar

Returns an iterator over batches of TrainingExamples

pub fn load_jsonl<P: AsRef<Path>>( &self, path: P, ) -> Result<(Vec<TrainingExample>, ImportStats)>

Load CITL corpus from JSONL file (streaming)

pub fn config(&self) -> &CitlLoaderConfig

Get the configuration

Trait Implementations

impl Default for CitlDataLoader

fn default() -> Self

Returns the “default value” for a type.