eld_llm 0.0.1

An LLM built from scratch in Rust
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

// cli parsing with clap 

use clap::{Parser, Subcommand}; 

#[derive(Parser)]
#[clap(name = "eld", version = "0.1", author = "Kjeti Indrehus", about = "A Rust library for tokenization and benchmarking")]
pub struct Cli {
    #[clap(subcommand)]
    pub command: Commands,
} 


// types of benchmarks we can run, e.g. tokenizer, embedding, etc. 
// For now we only have tokenizer benchmark, but we can add more later
#[derive(clap::ValueEnum, Clone)]
pub enum BenchType {
    Tokenizer,
}


// allowed commands 
#[derive(Subcommand)]
pub enum Commands {
    Bench {
        #[clap(long)]
        bench_type: BenchType,

        #[clap(long)]
        tokenizer: String, // e.g. "naive"


        #[clap(long)]
        data: String, // e.g. "data/shakespeare.txt"
    },
}