isq/
main.rs

1use anyhow::Result;
2use mistralrs::{
3    IsqType, PagedAttentionMetaBuilder, TextMessageRole, TextMessages, TextModelBuilder,
4};
5
6#[tokio::main]
7async fn main() -> Result<()> {
8    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
9        .with_isq(IsqType::Q8_0)
10        .with_logging()
11        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
12        .build()
13        .await?;
14
15    let messages = TextMessages::new()
16        .add_message(
17            TextMessageRole::System,
18            "You are an AI agent with a specialty in programming.",
19        )
20        .add_message(
21            TextMessageRole::User,
22            "Hello! How are you? Please write generic binary search function in Rust.",
23        );
24
25    let response = model.send_chat_request(messages).await?;
26
27    println!("{}", response.choices[0].message.content.as_ref().unwrap());
28    dbg!(
29        response.usage.avg_prompt_tok_per_sec,
30        response.usage.avg_compl_tok_per_sec
31    );
32
33    // Next example: re-ISQ the model at runtime
34    model.re_isq_model(IsqType::HQQ4).await?;
35
36    let messages = TextMessages::new().add_message(TextMessageRole::User, "Why is the sky blue?");
37
38    let response = model.send_chat_request(messages).await?;
39
40    println!("{}", response.choices[0].message.content.as_ref().unwrap());
41    dbg!(
42        response.usage.avg_prompt_tok_per_sec,
43        response.usage.avg_compl_tok_per_sec
44    );
45
46    Ok(())
47}
isq/main.rs

isq/
main.rs
