Struct GPTDataLoader 

pub struct GPTDataLoader { /* private fields */ }

A type for building a Batcher over a GPTDataset with specified params.

Implementations

impl GPTDataLoader

pub fn new( dataset: GPTDatasetV1, batch_size: usize, shuffle: bool, drop_last: bool, ) -> Self

Creates a new GPTDataLoader.

use llms_from_scratch_rs::listings::ch02::{GPTDatasetV1, GPTDataLoader};
use tiktoken_rs::get_bpe_from_model;

let txt = "In the heart of the city";
let tokenizer = tiktoken_rs::get_bpe_from_model("gpt2").unwrap();
let max_length = 3_usize;
let stride = 1_usize;
let dataset = GPTDatasetV1::new(txt, tokenizer, max_length, stride);

let batch_size = 2_usize;
let shuffle = false;
let drop_last = false;
let data_loader = GPTDataLoader::new(dataset, batch_size, shuffle, drop_last);

pub fn len(&self) -> usize

pub fn is_empty(&self) -> bool

Trait Implementations

impl DataLoader for GPTDataLoader

fn batcher(&self) -> GPTDataBatcher

Returns a GPTDataBatcher that itself provides batches over the associated dataset.

type Batcher = Batcher<IterResult2<GPTDatasetIter>>