eld_llm 0.0.1

An LLM built from scratch in Rust
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

// Metrics for the tokenizer's training process
pub struct TokenizerTrainingMetrics {
    pub training_time_sec: u64,
    
}

// Metrics for the tokenizer's vocabulary after training
pub struct TokenizerVocabMetrics {
    pub vocab_size: usize,
    pub avg_token_length: f64,
    pub max_token_length: usize, 
    pub min_token_length: usize,
} 


// Metrics for inference (encoding and decoding)
// Common metrics for inference, using a trained tokenizer on a test corpus 
pub struct TokenizerInferenceMetrics {
    pub avg_encode_time_ns: f64, 
    pub avg_decode_time_ns: f64,
    pub encode_throughput_tokens_per_sec: f64,
    pub decode_throughput_tokens_per_sec: f64,
    pub corpus_token_count: usize,
}


// Trait for development benchmarker results 
pub struct TokenizerDevelopmentResult {
    pub version: String,
    pub corpus_name: String,
    pub corpus_size_bytes: u64,
    pub training_metrics: TokenizerTrainingMetrics,
    pub vocab_metrics: TokenizerVocabMetrics,
    pub inference_metrics: TokenizerInferenceMetrics,
}