eld_llm 0.0.1

An LLM built from scratch in Rust
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

use std::collections::HashMap;

pub fn format_size(bytes: usize) -> String {
    const KB: usize = 1024;
    const MB: usize = KB * 1024;
    const GB: usize = MB * 1024;

    match bytes {
        b if b >= GB => format!("{:.2} GB", b as f64 / GB as f64),
        b if b >= MB => format!("{:.2} MB", b as f64 / MB as f64),
        b if b >= KB => format!("{:.2} KB", b as f64 / KB as f64),
        _ => format!("{} B", bytes),
    }
}

pub fn get_decoded_str(token: u64, vocab: &HashMap<u64, (u64, u64)>) -> String {
    if token < 256 {
        (token as u8 as char).to_string()
    } else {
        let (a, b) = vocab.get(&token).unwrap();
        format!("{}{}", get_decoded_str(*a, vocab), get_decoded_str(*b, vocab))
    }
}