ferrum-cli 0.8.3

CLI for Ferrum — a Rust-native LLM inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

//! Ferrum CLI - Ollama-style command line interface for LLM inference
//!
//! Commands:
//! - serve: Start the inference server
//! - run: Run a model and start interactive chat
//! - stop: Stop the running server
//! - pull: Download a model
//! - list: List downloaded models
//! - doctor: Inspect a binary and model source without downloading weights

use clap::{Parser, Subcommand};
use colored::*;
use ferrum_cli::{commands::*, config::CliConfig, utils::setup_logging};
use std::process;

#[derive(Parser)]
#[command(name = "ferrum")]
#[command(about = "Ferrum - Fast LLM Inference Engine")]
#[command(version = env!("CARGO_PKG_VERSION"))]
#[command(
    long_about = "A high-performance LLM inference engine with Metal/CUDA acceleration.\n\nExamples:\n  ferrum doctor                                      # Inspect this binary\n  ferrum run qwen3.5:4b-q4_k_m                      # Metal chat\n  ferrum run qwen3.5:4b                             # CUDA chat\n  ferrum serve --model qwen3.5:4b --port 8000       # OpenAI-compatible server\n  ferrum list                                        # Show downloaded models"
)]
struct Cli {
    #[command(subcommand)]
    command: Commands,

    /// Verbose output
    #[arg(short, long, global = true)]
    verbose: bool,
}

#[derive(Subcommand)]
enum Commands {
    /// Run a model in interactive chat or with a one-shot prompt
    #[command(visible_alias = "r")]
    Run(run::RunCommand),

    /// Benchmark model throughput and latency
    #[command(hide = true)]
    Bench(bench::BenchCommand),

    /// HTTP serve-side bench with tokenizer-aware random prompts
    /// (apples-to-apples vs `vllm bench serve --dataset-name random`)
    #[command(hide = true)]
    BenchServe(bench_serve::BenchServeCommand),

    /// Validate and replay a request replay bundle without starting HTTP.
    #[command(hide = true)]
    ReplayBundle(replay_bundle::ReplayBundleCommand),

    /// Collect bitwise CUDA vNext evidence for the release model matrix.
    #[command(hide = true)]
    VnextDeterminism(vnext_determinism::VNextDeterminismCommand),

    /// Generate text embeddings using BERT models
    #[command(visible_alias = "e", hide = true)]
    Embed(embed::EmbedCommand),

    /// Transcribe audio files using Whisper models
    #[command(visible_alias = "t", hide = true)]
    Transcribe(transcribe::TranscribeCommand),

    /// Text-to-speech synthesis using Qwen3-TTS models
    #[command(hide = true)]
    Tts(tts::TtsCommand),

    /// Start the inference HTTP server
    Serve(serve::ServeCommand),

    /// Stop the running server
    Stop(stop::StopCommand),

    /// Download a model from HuggingFace Hub
    Pull(pull::PullCommand),

    /// List downloaded models
    #[command(visible_alias = "ls")]
    List(list::ListCommand),

    /// Inspect the binary, backend, cache, and a model source without downloading it
    Doctor(doctor::DoctorCommand),
}

#[tokio::main]
async fn main() {
    let cli = Cli::parse();

    // Setup logging
    let suppress_chat_template_warnings = matches!(cli.command, Commands::Run(_)) && !cli.verbose;
    if let Err(e) = setup_logging(cli.verbose, false, suppress_chat_template_warnings) {
        eprintln!("{} Failed to setup logging: {}", "Error:".red().bold(), e);
        process::exit(1);
    }

    // Load the optional local configuration without creating files.
    let config = match CliConfig::load("ferrum.toml").await {
        Ok(config) => config,
        Err(e) => {
            if cli.verbose {
                eprintln!("{} Config: {}", "⚠️".yellow(), e);
            }
            CliConfig::default()
        }
    };

    // Execute command
    let result = match cli.command {
        Commands::Run(cmd) => run::execute(cmd, config).await,
        Commands::Bench(cmd) => bench::execute(cmd, config).await,
        Commands::BenchServe(cmd) => bench_serve::execute(cmd, config).await,
        Commands::ReplayBundle(cmd) => replay_bundle::execute(cmd, config).await,
        Commands::VnextDeterminism(cmd) => vnext_determinism::execute(cmd).await,
        Commands::Embed(cmd) => embed::execute(cmd, config).await,
        Commands::Transcribe(cmd) => transcribe::execute(cmd, config).await,
        Commands::Tts(cmd) => tts::execute(cmd, config).await,
        Commands::Serve(cmd) => serve::execute(cmd, config).await,
        Commands::Stop(cmd) => stop::execute(cmd).await,
        Commands::Pull(cmd) => pull::execute(cmd, config).await,
        Commands::List(cmd) => list::execute(cmd, config).await,
        Commands::Doctor(cmd) => doctor::execute(cmd, config).await,
    };

    if let Err(e) = result {
        eprintln!("{} {}", "Error:".red().bold(), e);
        process::exit(1);
    }
}