smolvlm/
main.rs

1use anyhow::Result;
2use mistralrs::{IsqType, TextMessageRole, VisionMessages, VisionModelBuilder};
3
4const MODEL_ID: &str = "HuggingFaceTB/SmolVLM-Instruct";
5
6#[tokio::main]
7async fn main() -> Result<()> {
8    let model = VisionModelBuilder::new(MODEL_ID)
9        .with_isq(IsqType::Q8_0)
10        .with_logging()
11        .build()
12        .await?;
13
14    let bytes = match reqwest::blocking::get(
15        "https://cdn.britannica.com/45/5645-050-B9EC0205/head-treasure-flower-disk-flowers-inflorescence-ray.jpg",
16    ) {
17        Ok(http_resp) => http_resp.bytes()?.to_vec(),
18        Err(e) => anyhow::bail!(e),
19    };
20    let image = image::load_from_memory(&bytes)?;
21
22    let messages = VisionMessages::new().add_image_message(
23        TextMessageRole::User,
24        "What is this?",
25        vec![image],
26        &model,
27    )?;
28
29    let response = model.send_chat_request(messages).await?;
30
31    println!("{}", response.choices[0].message.content.as_ref().unwrap());
32    dbg!(
33        response.usage.avg_prompt_tok_per_sec,
34        response.usage.avg_compl_tok_per_sec
35    );
36
37    Ok(())
38}
smolvlm/main.rs

smolvlm/
main.rs
