bench_bpe_train/
bench_bpe_train.rs1use std::io::Read;
2fn main() {
3 let mut args = std::env::args().skip(1);
4 let path = args.next().unwrap();
5 let megabytes: usize = args.next().unwrap().parse().unwrap();
6 let mut bytes = Vec::new();
7 std::fs::File::open(&path).unwrap().take((megabytes << 20) as u64).read_to_end(&mut bytes).unwrap();
8 let text = String::from_utf8_lossy(&bytes);
9 let lines: Vec<&str> = text.split_inclusive('\n').collect();
10 let start = std::time::Instant::now();
11 let json = rusting_brain::Bpe::train_json_split_digits(&lines, 32_000, &["<|endoftext|>"]).unwrap();
12 println!("{megabytes} MB, {} lines: {:.1} s, json {} bytes", lines.len(), start.elapsed().as_secs_f64(), json.len());
13}