llguidance/
main.rs

1use anyhow::Result;
2use mistralrs::{
3    llguidance::api::GrammarWithLexer, IsqType, LlguidanceGrammar, PagedAttentionMetaBuilder,
4    RequestBuilder, TextMessageRole, TextModelBuilder,
5};
6use serde_json::json;
7
8#[tokio::main]
9async fn main() -> Result<()> {
10    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
11        .with_isq(IsqType::Q4K)
12        .with_logging()
13        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
14        .build()
15        .await?;
16
17    let top =
18        GrammarWithLexer::from_lark(r#"start: "Reasoning: " /.+/ "\nJSON: " @myobj"#.to_string());
19    let schema = GrammarWithLexer {
20        name: Some("myobj".to_string()),
21        json_schema: Some(json!({
22            "type": "object",
23            "properties": {
24                "answer": {"type": "string", "enum": ["Yes", "No"]},
25            },
26            "required": ["answer"],
27            "additionalProperties": false,
28        })),
29        ..Default::default()
30    };
31
32    let request = RequestBuilder::new()
33        .set_constraint(mistralrs::Constraint::Llguidance(LlguidanceGrammar {
34            grammars: vec![top, schema],
35            max_tokens: None,
36        }))
37        .set_sampler_max_len(100)
38        .add_message(
39            TextMessageRole::User,
40            "If all dogs are mammals, and all mammals are animals, are dogs animals?",
41        );
42
43    let response = model.send_chat_request(request).await?;
44
45    println!("{}", response.choices[0].message.content.as_ref().unwrap());
46
47    Ok(())
48}
llguidance/main.rs

llguidance/
main.rs
