Skip to main content

bench_bpe_train/
bench_bpe_train.rs

1use std::io::Read;
2fn main() {
3    let mut args = std::env::args().skip(1);
4    let path = args.next().unwrap();
5    let megabytes: usize = args.next().unwrap().parse().unwrap();
6    let mut bytes = Vec::new();
7    std::fs::File::open(&path).unwrap().take((megabytes << 20) as u64).read_to_end(&mut bytes).unwrap();
8    let text = String::from_utf8_lossy(&bytes);
9    let lines: Vec<&str> = text.split_inclusive('\n').collect();
10    let start = std::time::Instant::now();
11    let json = rusting_brain::Bpe::train_json_split_digits(&lines, 32_000, &["<|endoftext|>"]).unwrap();
12    println!("{megabytes} MB, {} lines: {:.1} s, json {} bytes", lines.len(), start.elapsed().as_secs_f64(), json.len());
13}