llama-rust 0.1.0

LLM inference in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use anyhow::Result;
use clap::Parser;
use std::path::PathBuf;

///! How To Use
/// cargo build -- -m Qwen_Qwen2-0.5B-Instruct-GGUF_qwen2-0_5b-instruct-q2_k.gguf "hello, my name is"

#[derive(Parser)]
#[command(
    name = "llama-rust",
    about = "A simple CLI to run inference with a model",
    version = "0.1.0"
)]
pub struct Cli {
    #[arg(
        short = 'm',
        long = "model",
        value_name = "MODEL_PATH",
        help = "Path to the GGUF model file"
    )]
    pub model_path: PathBuf,

    #[arg(value_name = "PROMPT", help = "Input prompt for the model")]
    pub prompt: String,
}

pub fn run_inference(model_path: &PathBuf, prompt: &str) -> Result<String> {
    println!(
        "Loading model from: {}\nProcessing prompt: {}",
        model_path.display(),
        prompt
    );

    let simulated_output = format!(
        "Hello, my name is Grok! (Simulated response for '{}')",
        prompt
    );

    Ok(simulated_output)
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let cli = Cli::parse();

    if !cli.model_path.exists() {
        eprintln!(
            "Error: Model file '{}' does not exist",
            cli.model_path.display()
        );
        std::process::exit(1);
    }

    let output = run_inference(&cli.model_path, &cli.prompt)?;
    println!("Model output: {}", output);

    Ok(())
}