eld_llm 0.0.1

An LLM built from scratch in Rust
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

mod tokenizers;
mod utils;

use clap::Parser;
use utils::cli::Cli;
use utils::cli::commands::eld_commands::{BenchType, Commands};

fn main() { 
    // Load environment variables and initialize logging
    dotenvy::dotenv().ok();
    env_logger::init();
 
    // Parse command-line arguments
    let cli = Cli::parse();
    match cli.command {
        Commands::Bench { bench_type, tokenizer, data } => {
            log::info!("Command: bench");
            match bench_type {
                BenchType::Tokenizer => {
                    log::info!("Bench type: tokenizer");
                    tokenizers::benchmark::handler::run_bench(&tokenizer, &data);
                }
                _ => log::warn!("Unsupported bench type"),
            }
        }
        _ => log::warn!("Unsupported command"),
    }
}