Struct InstructionDataLoader 

pub struct InstructionDataLoader<C: CustomCollator<BatchItem = Tensor>> { /* private fields */ }

[Listing 7.6] Initializing the data loaders (InstructionDataLoader)

Implementations

impl<C: CustomCollator<BatchItem = Tensor> + Clone> InstructionDataLoader<C>

pub fn new( dataset: InstructionDataset, batch_size: usize, shuffle: bool, drop_last: bool, collator: C, ) -> Self

Creates a new InstructionDataLoader.

use candle_core::Device;
use llms_from_scratch_rs::listings::ch07::{
    AlpacaPromptFormatter, InstructionDataCollator, InstructionDataset,
    InstructionDataLoader, InstructionResponseExample
};
use tiktoken_rs::get_bpe_from_model;

let entry = InstructionResponseExample::new(
    "Some instruction",
    None,
    "Some output"
);
let data = vec![entry];
let tokenizer = get_bpe_from_model("gpt2").unwrap();
let prompt_formatter = AlpacaPromptFormatter;
let dataset = InstructionDataset::new(data, &tokenizer, &prompt_formatter);

// create InstructionDataLoader
let batch_size = 2_usize;
let shuffle = false;
let drop_last = false;
let collator = InstructionDataCollator::default();
let data_loader = InstructionDataLoader::new(
    dataset, batch_size, shuffle, drop_last, collator
);

pub fn len(&self) -> usize

pub fn is_empty(&self) -> bool

pub fn dataset(&self) -> &InstructionDataset

Trait Implementations

impl<C: CustomCollator<BatchItem = Tensor> + Clone> DataLoader for InstructionDataLoader<C>

fn batcher( &self, ) -> InstructionDataBatcher<C, IterResult1<InstructionDatasetIter>>

Returns a InstructionDataBatcher that itself provides batches over the associated dataset.

type Batcher = InstructionDataBatcher<C, IterResult1<InstructionDatasetIter>>