Skip to main content

grammar/
main.rs

1use anyhow::Result;
2use mistralrs::{
3    IsqType, PagedAttentionMetaBuilder, RequestBuilder, TextMessageRole, TextModelBuilder,
4};
5
6#[tokio::main]
7async fn main() -> Result<()> {
8    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
9        .with_isq(IsqType::Q4K)
10        .with_logging()
11        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
12        .build()
13        .await?;
14
15    // Bullet list regex
16    let request = RequestBuilder::new()
17        .set_constraint(mistralrs::Constraint::Regex(
18            "(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
19        ))
20        .add_message(TextMessageRole::User, "Please write a few jokes.");
21
22    let response = model.send_chat_request(request).await?;
23
24    println!("{}", response.choices[0].message.content.as_ref().unwrap());
25
26    Ok(())
27}