web_search/
main.rs

1use anyhow::Result;
2use mistralrs::{
3    IsqType, RequestBuilder, SearchEmbeddingModel, TextMessageRole, TextMessages, TextModelBuilder,
4    WebSearchOptions,
5};
6
7#[tokio::main]
8async fn main() -> Result<()> {
9    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
10        .with_isq(IsqType::Q4K)
11        .with_logging()
12        .with_search(SearchEmbeddingModel::default())
13        .build()
14        .await?;
15
16    let messages = TextMessages::new().add_message(
17        TextMessageRole::User,
18        "What is the weather forecast for Boston?",
19    );
20    let messages =
21        RequestBuilder::from(messages).with_web_search_options(WebSearchOptions::default());
22
23    let response = model.send_chat_request(messages).await?;
24
25    println!("What is the weather forecast for Boston?\n\n");
26    println!("{}", response.choices[0].message.content.as_ref().unwrap());
27    dbg!(
28        response.usage.avg_prompt_tok_per_sec,
29        response.usage.avg_compl_tok_per_sec
30    );
31
32    Ok(())
33}
web_search/main.rs

web_search/
main.rs
