minbpe 0.1.0

Port of Andrej Karpathy's minbpe to Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

use std::fs;
use std::path::PathBuf;

use minbpe::GPT4Tokenizer;
use minbpe::RegexTokenizerTrait;

fn main() -> std::io::Result<()> {
    let file_path = PathBuf::from("tests/taylorswift.txt");

    // Pre-initialize the tokenizer
    println!("Pre-initializing the tokenizer...");
    let start = std::time::Instant::now();
    GPT4Tokenizer::initialize();
    let duration = start.elapsed();
    println!(
        "GPT4Tokenizer static initialization completed in: {:?}",
        duration
    );

    // Get default instance of the tokenizer
    println!("Getting a default instance of GPT4Tokenizer...");
    let start = std::time::Instant::now();
    let tokenizer = GPT4Tokenizer::default();
    let duration = start.elapsed();
    println!(
        "GPT4Tokenizer default instance construction completed in: {:?}",
        duration
    );

    // Read the input file
    println!("Reading file: {:?}...", file_path);
    let start = std::time::Instant::now();
    let text = fs::read_to_string(file_path)?;
    let duration = start.elapsed();
    println!(
        "Reading {} characters completed in: {:?}",
        text.len(),
        duration
    );

    // Timing the encoding process, optional.
    let start = std::time::Instant::now();
    let tokens = tokenizer.encode(&text);
    let duration = start.elapsed();

    println!("Encoding completed in: {:?}", duration);
    println!("Produced {} encoded tokens", tokens.len());

    Ok(())
}