modelshelf 0.1.0

A shared local LLM model registry: discover, deduplicate, download, and update models across desktop apps.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

//! Show which curated models fit this machine, per task.
//!
//! ```console
//! $ cargo run -p modelshelf --example recommend            # chat (default)
//! $ cargo run -p modelshelf --example recommend -- stt     # any catalog task
//! ```
//!
//! To actually download the best pick, see
//! [`Shelf::provision_recommended`] — the commented block at the end.

use modelshelf::{Hardware, OpenOptions, RecommendOptions, Shelf};

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let task = std::env::args().nth(1).unwrap_or_else(|| "chat".to_owned());

    let shelf = Shelf::open(OpenOptions::default().app_id("com.example.recommend-demo"))?;
    let hw = Hardware::detect();
    let report = shelf.recommend_with(
        &hw,
        &RecommendOptions {
            task,
            ..Default::default()
        },
    )?;

    println!(
        "hardware: {:.1} GiB RAM, {} GPU(s){}",
        hw.ram_bytes as f64 / (1 << 30) as f64,
        hw.gpus.len(),
        if hw.unified_memory { " (unified)" } else { "" },
    );
    println!(
        "task {:?}: budget {:.1} GiB via {:?}, catalog v{} ({:?})\n",
        report.task,
        report.budget_bytes as f64 / (1 << 30) as f64,
        report.backend,
        report.catalog_version,
        report.catalog_origin,
    );

    for item in &report.items {
        let marker = if report.best.as_deref() == Some(item.entry.name.as_str()) {
            '*'
        } else {
            ' '
        };
        println!(
            "{} {:<24} {:>7.2} GiB download, {:>7.2} GiB needed  fits: {}",
            marker,
            item.entry.name,
            item.entry.total_bytes() as f64 / (1 << 30) as f64,
            item.required_bytes as f64 / (1 << 30) as f64,
            item.fits,
        );
    }

    match report.best_item() {
        Some(best) => println!("\nbest: {} — {}", best.entry.name, best.entry.notes),
        None => println!("\nno catalog model fits this machine for that task"),
    }
    if let Some(upgrade) = &report.upgrade {
        println!(
            "upgrade available: {} -> {}",
            upgrade.from_name, upgrade.to.name
        );
    }

    // Downloading the best pick (and marking it as this machine's
    // recommendation) is one call — uncomment to try; it may transfer
    // several GB:
    //
    // let outcome = shelf.provision_recommended(
    //     &hw,
    //     &RecommendOptions::default(),
    //     &modelshelf::PullOptions::default(),
    // )?;
    // println!("ready: {}", shelf.blob_path(&outcome.entry.id)?.display());

    Ok(())
}