Struct RequestBuilder 

pub struct RequestBuilder { /* private fields */ }

A way to add messages with finer control given.

This includes control over:

• Logits processors
• Constraints
• Logprobs
• Tools
• Sampling
• Enable thinking for models that support the configuration

Implementations

impl RequestBuilder

pub fn new() -> Self

Examples found in repository

examples/gemma2/main.rs (line 15)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("google/gemma-2-9b-it")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new().add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/grammar/main.rs (line 16)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    // Bullet list regex
    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Regex(
            "(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
        ))
        .add_message(TextMessageRole::User, "Please write a few jokes.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/custom_logits_processor/main.rs (line 35)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let mut rng = rand::rng();
    let random_value: f64 = rng.random_range(0.0..=1.0);
    let threshold: f64 = rng.random_range(0.0..=0.5);

    let request = RequestBuilder::new()
        .add_logits_processor(Arc::new(move |logits: &Tensor, _context: &[u32]| {
            logits * random_value
        }))
        .add_logits_processor(Arc::new(ThresholdLogitsProcessor { threshold }))
        .add_message(
            TextMessageRole::User,
            "Please write a mathematical equation where a few numbers are added.",
        );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/json_schema/main.rs (line 16)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::JsonSchema(json!(
            {
                "type": "object",
                "properties": {
                    "street": {"type": "string"},
                    "city": {"type": "string"},
                    "state": {"type": "string", "pattern": "^[A-Z]{2}$"},
                    "zip": {"type": "integer", "minimum": 10000, "maximum": 99999},
                },
                "required": ["street", "city", "state", "zip"],
                "additionalProperties": false,
            }
        )))
        .set_sampler_max_len(100)
        .add_message(TextMessageRole::User, "A sample address please.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/llguidance/main.rs (line 32)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let top =
        GrammarWithLexer::from_lark(r#"start: "Reasoning: " /.+/ "\nJSON: " @myobj"#.to_string());
    let schema = GrammarWithLexer {
        name: Some("myobj".to_string()),
        json_schema: Some(json!({
            "type": "object",
            "properties": {
                "answer": {"type": "string", "enum": ["Yes", "No"]},
            },
            "required": ["answer"],
            "additionalProperties": false,
        })),
        ..Default::default()
    };

    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Llguidance(LlguidanceGrammar {
            grammars: vec![top, schema],
            max_tokens: None,
        }))
        .set_sampler_max_len(100)
        .add_message(
            TextMessageRole::User,
            "If all dogs are mammals, and all mammals are animals, are dogs animals?",
        );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/async/main.rs (line 34)

async fn run() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q8_0)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    // Next example: Return some logprobs with the `RequestBuilder`, which enables higher configurability.
    let request = RequestBuilder::new().return_logprobs(true).add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!(
        "Logprobs: {:?}",
        &response.choices[0]
            .logprobs
            .as_ref()
            .unwrap()
            .content
            .as_ref()
            .unwrap()[0..3]
    );
    Ok(())
}

Additional examples can be found in:

• examples/simple/main.rs
• examples/uqff/main.rs
• examples/gguf_locally/main.rs
• examples/speculative/main.rs
• examples/text_auto_device_map/main.rs
• examples/simple_stream/main.rs
• examples/tools/main.rs

pub fn with_web_search_options( self, web_search_options: WebSearchOptions, ) -> Self

Examples found in repository

examples/web_search/main.rs (line 21)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_search(SearchEmbeddingModel::default())
        .build()
        .await?;

    let messages = TextMessages::new().add_message(
        TextMessageRole::User,
        "What is the weather forecast for Boston?",
    );
    let messages =
        RequestBuilder::from(messages).with_web_search_options(WebSearchOptions::default());

    let response = model.send_chat_request(messages).await?;

    println!("What is the weather forecast for Boston?\n\n");
    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/custom_search/main.rs (line 66)

async fn main() -> Result<()> {
    // Build the model enabling web-search support.  We supply a custom search callback that is
    // used **instead** of the default remote web search when we set `web_search_options` later
    // in the request.

    // The EmbeddingGemma reranker is **not** required even when using a custom callback – it is
    // used inside the retrieval pipeline to cluster / rank the results that our callback returns.
    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_search_callback(Arc::new(|params: &mistralrs::SearchFunctionParameters| {
            // In a real application there could be network or database calls here – but for the
            // sake of demonstration we simply perform a local filesystem search.
            local_search(&params.query)
        }))
        .build()
        .await?;

    let messages =
        TextMessages::new().add_message(TextMessageRole::User, "Where is Cargo.toml in this repo?");

    // Enable searching for this request.  Because we provided a custom search callback above, the
    // model will call **our** function instead of performing an online web search.
    let messages =
        RequestBuilder::from(messages).with_web_search_options(WebSearchOptions::default());

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

pub fn add_message(self, role: TextMessageRole, text: impl ToString) -> Self

Add a message to the request.

For messages with tool calls, use Self::add_message_with_tool_call. For messages with tool outputs, use Self::add_tool_message.

Examples found in repository

examples/gemma2/main.rs (lines 15-18)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("google/gemma-2-9b-it")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new().add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/grammar/main.rs (line 20)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    // Bullet list regex
    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Regex(
            "(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
        ))
        .add_message(TextMessageRole::User, "Please write a few jokes.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/custom_logits_processor/main.rs (lines 40-43)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let mut rng = rand::rng();
    let random_value: f64 = rng.random_range(0.0..=1.0);
    let threshold: f64 = rng.random_range(0.0..=0.5);

    let request = RequestBuilder::new()
        .add_logits_processor(Arc::new(move |logits: &Tensor, _context: &[u32]| {
            logits * random_value
        }))
        .add_logits_processor(Arc::new(ThresholdLogitsProcessor { threshold }))
        .add_message(
            TextMessageRole::User,
            "Please write a mathematical equation where a few numbers are added.",
        );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/json_schema/main.rs (line 31)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::JsonSchema(json!(
            {
                "type": "object",
                "properties": {
                    "street": {"type": "string"},
                    "city": {"type": "string"},
                    "state": {"type": "string", "pattern": "^[A-Z]{2}$"},
                    "zip": {"type": "integer", "minimum": 10000, "maximum": 99999},
                },
                "required": ["street", "city", "state", "zip"],
                "additionalProperties": false,
            }
        )))
        .set_sampler_max_len(100)
        .add_message(TextMessageRole::User, "A sample address please.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/llguidance/main.rs (lines 38-41)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let top =
        GrammarWithLexer::from_lark(r#"start: "Reasoning: " /.+/ "\nJSON: " @myobj"#.to_string());
    let schema = GrammarWithLexer {
        name: Some("myobj".to_string()),
        json_schema: Some(json!({
            "type": "object",
            "properties": {
                "answer": {"type": "string", "enum": ["Yes", "No"]},
            },
            "required": ["answer"],
            "additionalProperties": false,
        })),
        ..Default::default()
    };

    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Llguidance(LlguidanceGrammar {
            grammars: vec![top, schema],
            max_tokens: None,
        }))
        .set_sampler_max_len(100)
        .add_message(
            TextMessageRole::User,
            "If all dogs are mammals, and all mammals are animals, are dogs animals?",
        );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/async/main.rs (lines 34-37)

async fn run() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q8_0)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    // Next example: Return some logprobs with the `RequestBuilder`, which enables higher configurability.
    let request = RequestBuilder::new().return_logprobs(true).add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!(
        "Logprobs: {:?}",
        &response.choices[0]
            .logprobs
            .as_ref()
            .unwrap()
            .content
            .as_ref()
            .unwrap()[0..3]
    );
    Ok(())
}

Additional examples can be found in:

• examples/simple/main.rs
• examples/uqff/main.rs
• examples/gguf_locally/main.rs
• examples/speculative/main.rs
• examples/text_auto_device_map/main.rs
• examples/simple_stream/main.rs
• examples/tools/main.rs

pub fn add_tool_message( self, tool_content: impl ToString, tool_id: impl ToString, ) -> Self

Add a message with the output of a tool call.

Examples found in repository

examples/tools/main.rs (line 74)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("meta-llama/Meta-Llama-3.1-8B-Instruct")
        .with_logging()
        .with_isq(IsqType::Q8_0)
        .build()
        .await?;

    let parameters: HashMap<String, Value> = serde_json::from_value(json!({
        "type": "object",
        "properties": {
            "place": {
                "type": "string",
                "description": "The place to get the weather for.",
            },
        },
        "required": ["place"],
    }))?;

    let tools = vec![Tool {
        tp: ToolType::Function,
        function: Function {
            description: Some("Get the weather for a certain city.".to_string()),
            name: "get_weather".to_string(),
            parameters: Some(parameters),
        },
    }];

    // We will keep all the messages here
    let mut messages = RequestBuilder::new()
        .add_message(TextMessageRole::User, "What is the weather in Boston?")
        .set_tools(tools)
        .set_tool_choice(ToolChoice::Auto);

    let response = model.send_chat_request(messages.clone()).await?;

    let message = &response.choices[0].message;

    if let Some(tool_calls) = &message.tool_calls {
        let called = &tool_calls[0];
        if called.function.name == "get_weather" {
            let input: GetWeatherInput = serde_json::from_str(&called.function.arguments)?;
            println!("Called tool `get_weather` with arguments {input:?}");

            let result = get_weather(input);
            println!("Output of tool call: {result}");

            // Add tool call message from assistant so it knows what it called
            // Then, add message from the tool
            messages = messages
                .add_message_with_tool_call(
                    TextMessageRole::Assistant,
                    String::new(),
                    vec![called.clone()],
                )
                .add_tool_message(result, called.id.clone())
                .set_tool_choice(ToolChoice::None);

            let response = model.send_chat_request(messages.clone()).await?;

            let message = &response.choices[0].message;
            println!("Output of model: {:?}", message.content);
        }
    }

    Ok(())
}

pub fn add_message_with_tool_call( self, role: TextMessageRole, text: impl ToString, tool_calls: Vec<ToolCallResponse>, ) -> Self

Examples found in repository

examples/tools/main.rs (lines 69-73)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("meta-llama/Meta-Llama-3.1-8B-Instruct")
        .with_logging()
        .with_isq(IsqType::Q8_0)
        .build()
        .await?;

    let parameters: HashMap<String, Value> = serde_json::from_value(json!({
        "type": "object",
        "properties": {
            "place": {
                "type": "string",
                "description": "The place to get the weather for.",
            },
        },
        "required": ["place"],
    }))?;

    let tools = vec![Tool {
        tp: ToolType::Function,
        function: Function {
            description: Some("Get the weather for a certain city.".to_string()),
            name: "get_weather".to_string(),
            parameters: Some(parameters),
        },
    }];

    // We will keep all the messages here
    let mut messages = RequestBuilder::new()
        .add_message(TextMessageRole::User, "What is the weather in Boston?")
        .set_tools(tools)
        .set_tool_choice(ToolChoice::Auto);

    let response = model.send_chat_request(messages.clone()).await?;

    let message = &response.choices[0].message;

    if let Some(tool_calls) = &message.tool_calls {
        let called = &tool_calls[0];
        if called.function.name == "get_weather" {
            let input: GetWeatherInput = serde_json::from_str(&called.function.arguments)?;
            println!("Called tool `get_weather` with arguments {input:?}");

            let result = get_weather(input);
            println!("Output of tool call: {result}");

            // Add tool call message from assistant so it knows what it called
            // Then, add message from the tool
            messages = messages
                .add_message_with_tool_call(
                    TextMessageRole::Assistant,
                    String::new(),
                    vec![called.clone()],
                )
                .add_tool_message(result, called.id.clone())
                .set_tool_choice(ToolChoice::None);

            let response = model.send_chat_request(messages.clone()).await?;

            let message = &response.choices[0].message;
            println!("Output of model: {:?}", message.content);
        }
    }

    Ok(())
}

pub fn add_image_message( self, role: TextMessageRole, text: impl ToString, images: Vec<DynamicImage>, model: &Model, ) -> Result<Self>

pub fn add_audio_message( self, role: TextMessageRole, text: impl ToString, audios: Vec<AudioInput>, model: &Model, ) -> Result<Self>

pub fn add_multimodal_message( self, role: TextMessageRole, text: impl ToString, images: Vec<DynamicImage>, audios: Vec<AudioInput>, model: &Model, ) -> Result<Self>

By convention, all images are added before all audios.

pub fn add_logits_processor( self, processor: Arc<dyn CustomLogitsProcessor>, ) -> Self

Examples found in repository

examples/custom_logits_processor/main.rs (lines 36-38)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let mut rng = rand::rng();
    let random_value: f64 = rng.random_range(0.0..=1.0);
    let threshold: f64 = rng.random_range(0.0..=0.5);

    let request = RequestBuilder::new()
        .add_logits_processor(Arc::new(move |logits: &Tensor, _context: &[u32]| {
            logits * random_value
        }))
        .add_logits_processor(Arc::new(ThresholdLogitsProcessor { threshold }))
        .add_message(
            TextMessageRole::User,
            "Please write a mathematical equation where a few numbers are added.",
        );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

pub fn set_adapters(self, adapters: Vec<String>) -> Self

pub fn set_tools(self, tools: Vec<Tool>) -> Self

The default tool choice is auto.

Examples found in repository

examples/custom_tool_call/main.rs (line 68)

async fn main() -> Result<()> {
    // Build the model and register the *tool callback*.
    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_tool_callback(
            "local_search",
            Arc::new(|f: &CalledFunction| {
                let args: serde_json::Value = serde_json::from_str(&f.arguments)?;
                let query = args["query"].as_str().unwrap_or("");
                Ok(serde_json::to_string(&local_search(query)?)?)
            }),
        )
        .build()
        .await?;

    // Define the JSON schema for the tool the model can call.
    let parameters = std::collections::HashMap::from([(
        "query".to_string(),
        serde_json::json!({"type": "string", "description": "Query"}),
    )]);
    let tool = Tool {
        tp: ToolType::Function,
        function: mistralrs::Function {
            description: Some("Local filesystem search".to_string()),
            name: "local_search".to_string(),
            parameters: Some(parameters),
        },
    };

    // Ask the user question and allow the model to call the tool automatically.
    let messages =
        TextMessages::new().add_message(TextMessageRole::User, "Where is Cargo.toml in this repo?");
    let messages = RequestBuilder::from(messages)
        .set_tools(vec![tool])
        .set_tool_choice(ToolChoice::Auto);

    let response = model.send_chat_request(messages).await?;
    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    Ok(())
}

examples/tools/main.rs (line 50)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("meta-llama/Meta-Llama-3.1-8B-Instruct")
        .with_logging()
        .with_isq(IsqType::Q8_0)
        .build()
        .await?;

    let parameters: HashMap<String, Value> = serde_json::from_value(json!({
        "type": "object",
        "properties": {
            "place": {
                "type": "string",
                "description": "The place to get the weather for.",
            },
        },
        "required": ["place"],
    }))?;

    let tools = vec![Tool {
        tp: ToolType::Function,
        function: Function {
            description: Some("Get the weather for a certain city.".to_string()),
            name: "get_weather".to_string(),
            parameters: Some(parameters),
        },
    }];

    // We will keep all the messages here
    let mut messages = RequestBuilder::new()
        .add_message(TextMessageRole::User, "What is the weather in Boston?")
        .set_tools(tools)
        .set_tool_choice(ToolChoice::Auto);

    let response = model.send_chat_request(messages.clone()).await?;

    let message = &response.choices[0].message;

    if let Some(tool_calls) = &message.tool_calls {
        let called = &tool_calls[0];
        if called.function.name == "get_weather" {
            let input: GetWeatherInput = serde_json::from_str(&called.function.arguments)?;
            println!("Called tool `get_weather` with arguments {input:?}");

            let result = get_weather(input);
            println!("Output of tool call: {result}");

            // Add tool call message from assistant so it knows what it called
            // Then, add message from the tool
            messages = messages
                .add_message_with_tool_call(
                    TextMessageRole::Assistant,
                    String::new(),
                    vec![called.clone()],
                )
                .add_tool_message(result, called.id.clone())
                .set_tool_choice(ToolChoice::None);

            let response = model.send_chat_request(messages.clone()).await?;

            let message = &response.choices[0].message;
            println!("Output of model: {:?}", message.content);
        }
    }

    Ok(())
}

pub fn set_tool_choice(self, tool_choice: ToolChoice) -> Self

Examples found in repository

examples/custom_tool_call/main.rs (line 69)

async fn main() -> Result<()> {
    // Build the model and register the *tool callback*.
    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_tool_callback(
            "local_search",
            Arc::new(|f: &CalledFunction| {
                let args: serde_json::Value = serde_json::from_str(&f.arguments)?;
                let query = args["query"].as_str().unwrap_or("");
                Ok(serde_json::to_string(&local_search(query)?)?)
            }),
        )
        .build()
        .await?;

    // Define the JSON schema for the tool the model can call.
    let parameters = std::collections::HashMap::from([(
        "query".to_string(),
        serde_json::json!({"type": "string", "description": "Query"}),
    )]);
    let tool = Tool {
        tp: ToolType::Function,
        function: mistralrs::Function {
            description: Some("Local filesystem search".to_string()),
            name: "local_search".to_string(),
            parameters: Some(parameters),
        },
    };

    // Ask the user question and allow the model to call the tool automatically.
    let messages =
        TextMessages::new().add_message(TextMessageRole::User, "Where is Cargo.toml in this repo?");
    let messages = RequestBuilder::from(messages)
        .set_tools(vec![tool])
        .set_tool_choice(ToolChoice::Auto);

    let response = model.send_chat_request(messages).await?;
    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    Ok(())
}

examples/tools/main.rs (line 51)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("meta-llama/Meta-Llama-3.1-8B-Instruct")
        .with_logging()
        .with_isq(IsqType::Q8_0)
        .build()
        .await?;

    let parameters: HashMap<String, Value> = serde_json::from_value(json!({
        "type": "object",
        "properties": {
            "place": {
                "type": "string",
                "description": "The place to get the weather for.",
            },
        },
        "required": ["place"],
    }))?;

    let tools = vec![Tool {
        tp: ToolType::Function,
        function: Function {
            description: Some("Get the weather for a certain city.".to_string()),
            name: "get_weather".to_string(),
            parameters: Some(parameters),
        },
    }];

    // We will keep all the messages here
    let mut messages = RequestBuilder::new()
        .add_message(TextMessageRole::User, "What is the weather in Boston?")
        .set_tools(tools)
        .set_tool_choice(ToolChoice::Auto);

    let response = model.send_chat_request(messages.clone()).await?;

    let message = &response.choices[0].message;

    if let Some(tool_calls) = &message.tool_calls {
        let called = &tool_calls[0];
        if called.function.name == "get_weather" {
            let input: GetWeatherInput = serde_json::from_str(&called.function.arguments)?;
            println!("Called tool `get_weather` with arguments {input:?}");

            let result = get_weather(input);
            println!("Output of tool call: {result}");

            // Add tool call message from assistant so it knows what it called
            // Then, add message from the tool
            messages = messages
                .add_message_with_tool_call(
                    TextMessageRole::Assistant,
                    String::new(),
                    vec![called.clone()],
                )
                .add_tool_message(result, called.id.clone())
                .set_tool_choice(ToolChoice::None);

            let response = model.send_chat_request(messages.clone()).await?;

            let message = &response.choices[0].message;
            println!("Output of model: {:?}", message.content);
        }
    }

    Ok(())
}

pub fn return_logprobs(self, return_logprobs: bool) -> Self

Examples found in repository

examples/async/main.rs (line 34)

async fn run() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q8_0)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    // Next example: Return some logprobs with the `RequestBuilder`, which enables higher configurability.
    let request = RequestBuilder::new().return_logprobs(true).add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!(
        "Logprobs: {:?}",
        &response.choices[0]
            .logprobs
            .as_ref()
            .unwrap()
            .content
            .as_ref()
            .unwrap()[0..3]
    );
    Ok(())
}

examples/simple/main.rs (line 35)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q8_0)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    // Next example: Return some logprobs with the `RequestBuilder`, which enables higher configurability.
    let request = RequestBuilder::new().return_logprobs(true).add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!(
        "Logprobs: {:?}",
        &response.choices[0]
            .logprobs
            .as_ref()
            .unwrap()
            .content
            .as_ref()
            .unwrap()[0..3]
    );

    Ok(())
}

examples/uqff/main.rs (line 37)

async fn main() -> Result<()> {
    let model = UqffTextModelBuilder::new(
        "EricB/Phi-3.5-mini-instruct-UQFF",
        vec!["phi3.5-mini-instruct-q8_0.uqff".into()],
    )
    .into_inner()
    .with_logging()
    .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
    .build()
    .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    // Next example: Return some logprobs with the `RequestBuilder`, which enables higher configurability.
    let request = RequestBuilder::new().return_logprobs(true).add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!(
        "Logprobs: {:?}",
        &response.choices[0]
            .logprobs
            .as_ref()
            .unwrap()
            .content
            .as_ref()
            .unwrap()[0..3]
    );

    Ok(())
}

examples/gguf_locally/main.rs (line 35)

async fn main() -> Result<()> {
    // We do not use any files from remote servers here, and instead load the
    // chat template from the specified file, and the tokenizer and model from a
    // local GGUF file at the path specified.
    let model = GgufModelBuilder::new(
        "gguf_models/mistral_v0.1/",
        vec!["mistral-7b-instruct-v0.1.Q4_K_M.gguf"],
    )
    .with_chat_template("chat_templates/mistral.json")
    .with_logging()
    .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
    .build()
    .await?;

    let messages = TextMessages::new().add_message(
        TextMessageRole::User,
        "Hello! How are you? Please write generic binary search function in Rust.",
    );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    // Next example: Return some logprobs with the `RequestBuilder`, which enables higher configurability.
    let request = RequestBuilder::new().return_logprobs(true).add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!(
        "Logprobs: {:?}",
        &response.choices[0]
            .logprobs
            .as_ref()
            .unwrap()
            .content
            .as_ref()
            .unwrap()[0..3]
    );

    Ok(())
}

examples/speculative/main.rs (line 37)

async fn main() -> Result<()> {
    let target = TextModelBuilder::new("meta-llama/Llama-3.1-8B-Instruct").with_logging();
    let draft = TextModelBuilder::new("meta-llama/Llama-3.2-1B-Instruct")
        .with_logging()
        .with_isq(IsqType::Q8_0);
    let spec_cfg = SpeculativeConfig { gamma: 16 };
    let model = TextSpeculativeBuilder::new(target, draft, spec_cfg)?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    // Next example: Return some logprobs with the `RequestBuilder`, which enables higher configurability.
    let request = RequestBuilder::new().return_logprobs(true).add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!(
        "Logprobs: {:?}",
        &response.choices[0]
            .logprobs
            .as_ref()
            .unwrap()
            .content
            .as_ref()
            .unwrap()[0..3]
    );

    Ok(())
}

examples/text_auto_device_map/main.rs (line 40)

async fn main() -> Result<()> {
    let auto_map_params = AutoDeviceMapParams::Text {
        max_seq_len: 4096,
        max_batch_size: 2,
    };
    let model = TextModelBuilder::new("meta-llama/Llama-3.3-70B-Instruct")
        .with_isq(IsqType::Q8_0)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .with_device_mapping(DeviceMapSetting::Auto(auto_map_params))
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    // Next example: Return some logprobs with the `RequestBuilder`, which enables higher configurability.
    let request = RequestBuilder::new().return_logprobs(true).add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!(
        "Logprobs: {:?}",
        &response.choices[0]
            .logprobs
            .as_ref()
            .unwrap()
            .content
            .as_ref()
            .unwrap()[0..3]
    );

    Ok(())
}

Additional examples can be found in:

• examples/simple_stream/main.rs

pub fn set_constraint(self, constraint: Constraint) -> Self

Examples found in repository

examples/grammar/main.rs (lines 17-19)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    // Bullet list regex
    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Regex(
            "(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
        ))
        .add_message(TextMessageRole::User, "Please write a few jokes.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/json_schema/main.rs (lines 17-29)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::JsonSchema(json!(
            {
                "type": "object",
                "properties": {
                    "street": {"type": "string"},
                    "city": {"type": "string"},
                    "state": {"type": "string", "pattern": "^[A-Z]{2}$"},
                    "zip": {"type": "integer", "minimum": 10000, "maximum": 99999},
                },
                "required": ["street", "city", "state", "zip"],
                "additionalProperties": false,
            }
        )))
        .set_sampler_max_len(100)
        .add_message(TextMessageRole::User, "A sample address please.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/llguidance/main.rs (lines 33-36)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let top =
        GrammarWithLexer::from_lark(r#"start: "Reasoning: " /.+/ "\nJSON: " @myobj"#.to_string());
    let schema = GrammarWithLexer {
        name: Some("myobj".to_string()),
        json_schema: Some(json!({
            "type": "object",
            "properties": {
                "answer": {"type": "string", "enum": ["Yes", "No"]},
            },
            "required": ["answer"],
            "additionalProperties": false,
        })),
        ..Default::default()
    };

    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Llguidance(LlguidanceGrammar {
            grammars: vec![top, schema],
            max_tokens: None,
        }))
        .set_sampler_max_len(100)
        .add_message(
            TextMessageRole::User,
            "If all dogs are mammals, and all mammals are animals, are dogs animals?",
        );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

pub fn set_sampling(self, params: SamplingParams) -> Self

Set the sampling parameters as given.

pub fn set_deterministic_sampler(self) -> Self

Set the sampling parameters for deterministic generation. This sets up the parameters so that there is:

• No temperature, topk, topp, minp
• No penalties, stop tokens, or logit bias
• No maximum length

pub fn set_sampler_temperature(self, temperature: f64) -> Self

pub fn set_sampler_topk(self, topk: usize) -> Self

pub fn set_sampler_topp(self, topp: f64) -> Self

pub fn set_sampler_minp(self, minp: f64) -> Self

pub fn set_sampler_topn_logprobs(self, top_n_logprobs: usize) -> Self

pub fn set_sampler_frequency_penalty(self, frequency_penalty: f32) -> Self

pub fn set_sampler_presence_penalty(self, presence_penalty: f32) -> Self

pub fn set_sampler_stop_toks(self, stop_toks: StopTokens) -> Self

pub fn set_sampler_max_len(self, max_len: usize) -> Self

Examples found in repository

examples/json_schema/main.rs (line 30)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::JsonSchema(json!(
            {
                "type": "object",
                "properties": {
                    "street": {"type": "string"},
                    "city": {"type": "string"},
                    "state": {"type": "string", "pattern": "^[A-Z]{2}$"},
                    "zip": {"type": "integer", "minimum": 10000, "maximum": 99999},
                },
                "required": ["street", "city", "state", "zip"],
                "additionalProperties": false,
            }
        )))
        .set_sampler_max_len(100)
        .add_message(TextMessageRole::User, "A sample address please.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/llguidance/main.rs (line 37)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let top =
        GrammarWithLexer::from_lark(r#"start: "Reasoning: " /.+/ "\nJSON: " @myobj"#.to_string());
    let schema = GrammarWithLexer {
        name: Some("myobj".to_string()),
        json_schema: Some(json!({
            "type": "object",
            "properties": {
                "answer": {"type": "string", "enum": ["Yes", "No"]},
            },
            "required": ["answer"],
            "additionalProperties": false,
        })),
        ..Default::default()
    };

    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Llguidance(LlguidanceGrammar {
            grammars: vec![top, schema],
            max_tokens: None,
        }))
        .set_sampler_max_len(100)
        .add_message(
            TextMessageRole::User,
            "If all dogs are mammals, and all mammals are animals, are dogs animals?",
        );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/llama_vision_multiturn/main.rs (line 17)

async fn main() -> Result<()> {
    let model = VisionModelBuilder::new(MODEL_ID)
        .with_logging()
        .with_isq(mistralrs::IsqType::Q8_0)
        .build()
        .await?;

    let mut messages = VisionMessages::new().add_message(TextMessageRole::User, "Hello!");

    let resp = model
        .send_chat_request(RequestBuilder::from(messages.clone()).set_sampler_max_len(100))
        .await?
        .choices[0]
        .message
        .content
        .clone()
        .unwrap();
    println!("\n\n{resp}");
    messages = messages.add_message(TextMessageRole::Assistant, resp);

    let bytes = match reqwest::blocking::get(
        // "https://s3.amazonaws.com/cdn.tulips.com/images/large/Timeless-Tulip.jpg",
        "https://niche-museums.imgix.net/pioneer-history.jpeg",
    ) {
        Ok(http_resp) => http_resp.bytes()?.to_vec(),
        Err(e) => anyhow::bail!(e),
    };
    let image = image::load_from_memory(&bytes)?;

    messages = messages.add_image_message(
        TextMessageRole::User,
        "What is depicted here? Please describe the scene in detail.",
        vec![image],
        &model,
    )?;
    let resp = model
        .send_chat_request(RequestBuilder::from(messages.clone()).set_sampler_max_len(100))
        .await?
        .choices[0]
        .message
        .content
        .clone()
        .unwrap();
    println!("\n\n{resp}");
    messages = messages.add_message(TextMessageRole::Assistant, resp);

    let bytes = match reqwest::blocking::get(
            "https://www.nhmagazine.com/content/uploads/2019/05/mtwashingtonFranconia-2-19-18-108-Edit-Edit.jpg"
        ) {
            Ok(http_resp) => http_resp.bytes()?.to_vec(),
            Err(e) => anyhow::bail!(e),
        };
    let image = image::load_from_memory(&bytes)?;

    messages =
        messages.add_image_message(TextMessageRole::User, "What is this?", vec![image], &model)?;
    let resp = model
        .send_chat_request(RequestBuilder::from(messages.clone()).set_sampler_max_len(100))
        .await?
        .choices[0]
        .message
        .content
        .clone()
        .unwrap();
    println!("\n\n{resp}");
    messages = messages.add_message(TextMessageRole::Assistant, resp);

    let bytes =
        match reqwest::blocking::get("https://cdn.britannica.com/79/4679-050-BC127236/Titanic.jpg")
        {
            Ok(http_resp) => http_resp.bytes()?.to_vec(),
            Err(e) => anyhow::bail!(e),
        };
    let image = image::load_from_memory(&bytes)?;

    messages =
        messages.add_image_message(TextMessageRole::User, "What is this?", vec![image], &model)?;
    let resp = model
        .send_chat_request(RequestBuilder::from(messages.clone()).set_sampler_max_len(100))
        .await?
        .choices[0]
        .message
        .content
        .clone()
        .unwrap();
    println!("\n\nModel response*: {resp}");
    messages = messages.add_message(TextMessageRole::Assistant, resp);

    println!("Final chat history: {messages:?}");

    Ok(())
}

pub fn set_sampler_logits_bias(self, logits_bias: HashMap<u32, f32>) -> Self

pub fn set_sampler_n_choices(self, n_choices: usize) -> Self

pub fn set_sampler_dry_params(self, dry_params: DrySamplingParams) -> Self

pub fn enable_thinking(self, enable_thinking: bool) -> Self

pub fn with_truncate_sequence(self, truncate_sequence: bool) -> Self

Truncate prompts that exceed the model’s maximum context length.

Trait Implementations

impl Clone for RequestBuilder

fn clone(&self) -> RequestBuilder

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Default for RequestBuilder

fn default() -> Self

Returns the “default value” for a type.

impl From<TextMessages> for RequestBuilder

fn from(value: TextMessages) -> Self

Converts to this type from the input type.

impl From<VisionMessages> for RequestBuilder

fn from(value: VisionMessages) -> Self

Converts to this type from the input type.

impl RequestLike for RequestBuilder

fn messages_ref(&self) -> &[IndexMap<String, MessageContent>]

fn images_ref(&self) -> &[DynamicImage]

fn take_messages(&mut self) -> RequestMessage

fn enable_search(&self) -> Option<bool>

fn take_logits_processors( &mut self, ) -> Option<Vec<Arc<dyn CustomLogitsProcessor>>>

fn take_adapters(&mut self) -> Option<Vec<String>>

fn return_logprobs(&self) -> bool

fn take_constraint(&mut self) -> Constraint

fn take_tools(&mut self) -> Option<(Vec<Tool>, ToolChoice)>

fn take_sampling_params(&mut self) -> SamplingParams

fn take_web_search_options(&mut self) -> Option<WebSearchOptions>

fn truncate_sequence(&self) -> bool