uqff_vision/
main.rs

1use anyhow::Result;
2use mistralrs::{IsqType, TextMessageRole, UqffVisionModelBuilder, VisionMessages};
3
4#[tokio::main]
5async fn main() -> Result<()> {
6    let model = UqffVisionModelBuilder::new(
7        "EricB/Phi-3.5-vision-instruct-UQFF",
8        vec!["phi3.5-vision-instruct-q8_0.uqff".into()],
9    )
10    .into_inner()
11    .with_isq(IsqType::Q4K)
12    .with_logging()
13    .build()
14    .await?;
15
16    let bytes = match reqwest::blocking::get(
17        "https://cdn.britannica.com/45/5645-050-B9EC0205/head-treasure-flower-disk-flowers-inflorescence-ray.jpg",
18    ) {
19        Ok(http_resp) => http_resp.bytes()?.to_vec(),
20        Err(e) => anyhow::bail!(e),
21    };
22    let image = image::load_from_memory(&bytes)?;
23
24    let messages = VisionMessages::new().add_image_message(
25        TextMessageRole::User,
26        "What is depicted here? Please describe the scene in detail.",
27        vec![image],
28        &model,
29    )?;
30
31    let response = model.send_chat_request(messages).await?;
32
33    println!("{}", response.choices[0].message.content.as_ref().unwrap());
34    dbg!(
35        response.usage.avg_prompt_tok_per_sec,
36        response.usage.avg_compl_tok_per_sec
37    );
38
39    Ok(())
40}
uqff_vision/main.rs

uqff_vision/
main.rs
