llm-cascade 0.1.0

Resilient cascading LLM inference with automatic failover across multiple providers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use std::path::PathBuf;

use clap::Parser;

use llm_cascade::{config, db, run_cascade, Conversation};

#[derive(Parser, Debug)]
#[command(name = "llm-cascade", about = "Cascading LLM inference across multiple providers")]
struct Cli {
    /// Name of the cascade to use (defined in config)
    #[arg(short = 'C', long)]
    cascade: String,

    /// Text prompt to send
    #[arg(short, long, conflicts_with = "file")]
    prompt: Option<String>,

    /// Path to a JSON file containing the conversation
    #[arg(short, long, conflicts_with = "prompt")]
    file: Option<PathBuf>,

    /// Path to config file (default: ~/.config/llm-cascade/config.toml)
    #[arg(short, long)]
    config: Option<PathBuf>,
}

#[tokio::main]
async fn main() {
    tracing_subscriber::fmt()
        .with_env_filter(
            tracing_subscriber::EnvFilter::try_from_default_env()
                .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
        )
        .with_target(false)
        .init();

    let cli = Cli::parse();

    let config_path = cli.config.unwrap_or_else(config::default_config_path);
    let app_config = match config::load_config(&config_path) {
        Ok(c) => c,
        Err(e) => {
            eprintln!("Error loading config: {}", e);
            std::process::exit(1);
        }
    };

    let conn = match db::init_db(&app_config.database.path) {
        Ok(c) => c,
        Err(e) => {
            eprintln!("Error initializing database: {}", e);
            std::process::exit(1);
        }
    };

    let conversation = if let Some(file_path) = cli.file {
        match std::fs::read_to_string(&file_path) {
            Ok(content) => match serde_json::from_str::<Conversation>(&content) {
                Ok(conv) => conv,
                Err(e) => {
                    eprintln!("Error parsing conversation JSON from '{}': {}", file_path.display(), e);
                    std::process::exit(1);
                }
            },
            Err(e) => {
                eprintln!("Error reading file '{}': {}", file_path.display(), e);
                std::process::exit(1);
            }
        }
    } else if let Some(prompt) = cli.prompt {
        Conversation::single_user_prompt(prompt)
    } else {
        eprintln!("Error: either --prompt or --file must be provided");
        std::process::exit(1);
    };

    match run_cascade(&cli.cascade, &conversation, &app_config, &conn).await {
        Ok(response) => {
            let has_tool_calls = response.content.iter().any(|b| matches!(b, llm_cascade::ContentBlock::ToolCall { .. }));
            if has_tool_calls {
                let output = serde_json::to_string_pretty(&response).unwrap_or_default();
                println!("{}", output);
            } else {
                println!("{}", response.text_only());
            }
        }
        Err(e) => {
            eprintln!("{}", e);
            std::process::exit(1);
        }
    }
}