mistralrs 0.8.1

Fast, flexible LLM inference.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

//! Audio input processing with a multimodal model.
//!
//! Run with: `cargo run --release --example audio -p mistralrs`

use anyhow::Result;
use mistralrs::{AudioInput, MultimodalMessages, MultimodalModelBuilder, TextMessageRole};

#[tokio::main]
async fn main() -> Result<()> {
    let model = MultimodalModelBuilder::new("../hf_models/gemma3n_e4b")
        .with_logging()
        .build()
        .await?;

    let audio_bytes = std::fs::read("sample_speech.wav")?;
    let audio = AudioInput::from_bytes(&audio_bytes)?;

    let messages = MultimodalMessages::new().add_audio_message(
        TextMessageRole::User,
        "What is being said?",
        vec![audio],
    );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    Ok(())
}