gemma2/
main.rs

1use anyhow::Result;
2use mistralrs::{
3    IsqType, PagedAttentionMetaBuilder, RequestBuilder, TextMessageRole, TextModelBuilder,
4};
5
6#[tokio::main]
7async fn main() -> Result<()> {
8    let model = TextModelBuilder::new("google/gemma-2-9b-it")
9        .with_isq(IsqType::Q4K)
10        .with_logging()
11        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
12        .build()
13        .await?;
14
15    let request = RequestBuilder::new().add_message(
16        TextMessageRole::User,
17        "Please write a mathematical equation where a few numbers are added.",
18    );
19
20    let response = model.send_chat_request(request).await?;
21
22    println!("{}", response.choices[0].message.content.as_ref().unwrap());
23
24    Ok(())
25}
gemma2/main.rs

gemma2/
main.rs
