json_schema/
main.rs

1use anyhow::Result;
2use mistralrs::{
3    IsqType, PagedAttentionMetaBuilder, RequestBuilder, TextMessageRole, TextModelBuilder,
4};
5use serde_json::json;
6
7#[tokio::main]
8async fn main() -> Result<()> {
9    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
10        .with_isq(IsqType::Q4K)
11        .with_logging()
12        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
13        .build()
14        .await?;
15
16    let request = RequestBuilder::new()
17        .set_constraint(mistralrs::Constraint::JsonSchema(json!(
18            {
19                "type": "object",
20                "properties": {
21                    "street": {"type": "string"},
22                    "city": {"type": "string"},
23                    "state": {"type": "string", "pattern": "^[A-Z]{2}$"},
24                    "zip": {"type": "integer", "minimum": 10000, "maximum": 99999},
25                },
26                "required": ["street", "city", "state", "zip"],
27                "additionalProperties": false,
28            }
29        )))
30        .set_sampler_max_len(100)
31        .add_message(TextMessageRole::User, "A sample address please.");
32
33    let response = model.send_chat_request(request).await?;
34
35    println!("{}", response.choices[0].message.content.as_ref().unwrap());
36
37    Ok(())
38}
json_schema/main.rs

json_schema/
main.rs
