Struct TextModelBuilder 

pub struct TextModelBuilder { /* private fields */ }

Configure a text model with the various parameters for loading, running, and other inference behaviors.

Implementations

impl TextModelBuilder

pub fn new(model_id: impl ToString) -> Self

A few defaults are applied here:

• MoQE ISQ organization
• Token source is from the cache (.cache/huggingface/token)
• Maximum number of sequences running is 32
• Number of sequences to hold in prefix cache is 16.
• Automatic device mapping with model defaults according to AutoDeviceMapParams
• By default, web searching compatible with the OpenAI web_search_options setting is disabled.

Examples found in repository

examples/gemma2/main.rs (line 8)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("google/gemma-2-9b-it")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new().add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/grammar/main.rs (line 8)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    // Bullet list regex
    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Regex(
            "(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
        ))
        .add_message(TextMessageRole::User, "Please write a few jokes.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/lora/main.rs (line 7)

async fn main() -> Result<()> {
    let model = LoraModelBuilder::from_text_model_builder(
        TextModelBuilder::new("meta-llama/Llama-3.2-1B-Instruct").with_logging(),
        vec!["danielhanchen/llama-3.2-lora".to_string()],
    )
    .build()
    .await?;

    let messages = TextMessages::new().add_message(
        TextMessageRole::User,
        "Hello! How are you? Please write generic binary search function in Rust.",
    );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/gpt_oss/main.rs (line 6)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("openai/gpt-oss-20b")
        .with_logging()
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/glm4_moe_lite/main.rs (line 6)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("zai-org/GLM-4.7-Flash")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/xlora/main.rs (line 10)

async fn main() -> Result<()> {
    let model =
        XLoraModelBuilder::from_text_model_builder(
            TextModelBuilder::new("HuggingFaceH4/zephyr-7b-beta").with_logging(),
            "lamm-mit/x-lora",
            serde_json::from_reader(File::open("my-ordering-file.json").unwrap_or_else(|_| {
                panic!("Could not load ordering file at my-ordering-file.json")
            }))?,
        )
        .build()
        .await?;

    let messages =
        TextMessages::new().add_message(TextMessageRole::User, "Hello! What is graphene.");

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

Additional examples can be found in:

• examples/web_search/main.rs
• examples/deepseekr1/main.rs
• examples/deepseekv2/main.rs
• examples/phi3_5_moe/main.rs
• examples/granite/main.rs
• examples/smollm3/main.rs
• examples/mixture_of_quant_experts/main.rs
• examples/imatrix/main.rs
• examples/custom_logits_processor/main.rs
• examples/paged_attn/main.rs
• examples/json_schema/main.rs
• examples/llguidance/main.rs
• examples/isq/main.rs
• examples/async/main.rs
• examples/simple/main.rs
• examples/anymoe_lora/main.rs
• examples/custom_search/main.rs
• examples/custom_tool_call/main.rs
• examples/speculative/main.rs
• examples/batching/main.rs
• examples/text_auto_device_map/main.rs
• examples/anymoe/main.rs
• examples/topology/main.rs
• examples/simple_stream/main.rs
• examples/qwen3/main.rs
• examples/tools/main.rs
• examples/perplexity/main.rs
• examples/agent/main.rs
• examples/multi_model/main.rs
• examples/agent_streaming/main.rs
• examples/mcp_client/main.rs

pub fn with_search(self, search_embedding_model: SearchEmbeddingModel) -> Self

Enable searching compatible with the OpenAI web_search_options setting. This loads the selected search embedding reranker (EmbeddingGemma by default).

Examples found in repository

examples/web_search/main.rs (line 12)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_search(SearchEmbeddingModel::default())
        .build()
        .await?;

    let messages = TextMessages::new().add_message(
        TextMessageRole::User,
        "What is the weather forecast for Boston?",
    );
    let messages =
        RequestBuilder::from(messages).with_web_search_options(WebSearchOptions::default());

    let response = model.send_chat_request(messages).await?;

    println!("What is the weather forecast for Boston?\n\n");
    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

pub fn with_search_callback(self, callback: Arc<SearchCallback>) -> Self

Override the search function used when web_search_options is enabled.

Examples found in repository

examples/custom_search/main.rs (lines 52-56)

async fn main() -> Result<()> {
    // Build the model enabling web-search support.  We supply a custom search callback that is
    // used **instead** of the default remote web search when we set `web_search_options` later
    // in the request.

    // The EmbeddingGemma reranker is **not** required even when using a custom callback – it is
    // used inside the retrieval pipeline to cluster / rank the results that our callback returns.
    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_search_callback(Arc::new(|params: &mistralrs::SearchFunctionParameters| {
            // In a real application there could be network or database calls here – but for the
            // sake of demonstration we simply perform a local filesystem search.
            local_search(&params.query)
        }))
        .build()
        .await?;

    let messages =
        TextMessages::new().add_message(TextMessageRole::User, "Where is Cargo.toml in this repo?");

    // Enable searching for this request.  Because we provided a custom search callback above, the
    // model will call **our** function instead of performing an online web search.
    let messages =
        RequestBuilder::from(messages).with_web_search_options(WebSearchOptions::default());

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

pub fn with_tool_callback( self, name: impl Into<String>, callback: Arc<ToolCallback>, ) -> Self

Register a callback for a specific tool name.

Examples found in repository

examples/custom_tool_call/main.rs (lines 39-46)

async fn main() -> Result<()> {
    // Build the model and register the *tool callback*.
    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_tool_callback(
            "local_search",
            Arc::new(|f: &CalledFunction| {
                let args: serde_json::Value = serde_json::from_str(&f.arguments)?;
                let query = args["query"].as_str().unwrap_or("");
                Ok(serde_json::to_string(&local_search(query)?)?)
            }),
        )
        .build()
        .await?;

    // Define the JSON schema for the tool the model can call.
    let parameters = std::collections::HashMap::from([(
        "query".to_string(),
        serde_json::json!({"type": "string", "description": "Query"}),
    )]);
    let tool = Tool {
        tp: ToolType::Function,
        function: mistralrs::Function {
            description: Some("Local filesystem search".to_string()),
            name: "local_search".to_string(),
            parameters: Some(parameters),
        },
    };

    // Ask the user question and allow the model to call the tool automatically.
    let messages =
        TextMessages::new().add_message(TextMessageRole::User, "Where is Cargo.toml in this repo?");
    let messages = RequestBuilder::from(messages)
        .set_tools(vec![tool])
        .set_tool_choice(ToolChoice::Auto);

    let response = model.send_chat_request(messages).await?;
    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    Ok(())
}

pub fn with_tool_callback_and_tool( self, name: impl Into<String>, callback: Arc<ToolCallback>, tool: Tool, ) -> Self

Register a callback with an associated Tool definition that will be automatically added to requests when tool callbacks are active.

pub fn with_mcp_client(self, config: McpClientConfig) -> Self

Configure MCP client to connect to external MCP servers and automatically register their tools for use in automatic tool calling.

Examples found in repository

examples/mcp_client/main.rs (line 130)

async fn main() -> Result<()> {
    // Simple MCP client configuration using defaults
    // Most fields use sensible defaults (enabled=true, UUID for id/prefix, no timeouts)
    let mcp_config_simple = McpClientConfig {
        servers: vec![McpServerConfig {
            name: "Filesystem Tools".to_string(),
            source: McpServerSource::Process {
                command: "npx".to_string(),
                args: vec![
                    "@modelcontextprotocol/server-filesystem".to_string(),
                    ".".to_string(),
                ],
                work_dir: None,
                env: None,
            },
            ..Default::default()
        }],
        ..Default::default()
    };

    // Alternative: Full configuration with multiple transport types
    let _mcp_config_full = McpClientConfig {
        servers: vec![
            // Example: Process-based MCP server (enabled by default)
            McpServerConfig {
                name: "Filesystem Tools".to_string(),
                source: McpServerSource::Process {
                    command: "npx".to_string(),
                    args: vec![
                        "@modelcontextprotocol/server-filesystem".to_string(),
                        ".".to_string(),
                    ],
                    work_dir: None,
                    env: None,
                },
                tool_prefix: Some("fs".to_string()),
                ..Default::default()
            },
            // Example: HTTP-based MCP server with Bearer token authentication (disabled by default)
            McpServerConfig {
                id: "hf_server".to_string(),
                name: "Hugging Face MCP".to_string(),
                source: McpServerSource::Http {
                    url: "https://hf.co/mcp".to_string(),
                    timeout_secs: Some(30),
                    headers: None, // Additional headers can be specified here if needed
                },
                enabled: false,                      // Disabled by default
                tool_prefix: Some("hf".to_string()), // Prefixes tool names to avoid conflicts
                resources: None,
                bearer_token: Some("hf_xxx".to_string()), // Replace with your actual Hugging Face token
            },
            //
            // // Example with both Bearer token and additional headers (uncomment HashMap import above)
            // McpServerConfig {
            //     id: "authenticated_server".to_string(),
            //     name: "Authenticated MCP Server".to_string(),
            //     source: McpServerSource::Http {
            //         url: "https://api.example.com/mcp".to_string(),
            //         timeout_secs: Some(60),
            //         headers: Some({
            //             let mut headers = HashMap::new();
            //             headers.insert("X-API-Version".to_string(), "v1".to_string());
            //             headers.insert("X-Client-ID".to_string(), "mistral-rs".to_string());
            //             headers
            //         }),
            //     },
            //     enabled: false,
            //     tool_prefix: Some("auth".to_string()),
            //     resources: None,
            //     bearer_token: Some("your-bearer-token".to_string()), // Will be added as Authorization: Bearer <token>
            // },
            // Example WebSocket-based MCP server (disabled by default)
            McpServerConfig {
                id: "websocket_server".to_string(),
                name: "WebSocket Example".to_string(),
                source: McpServerSource::WebSocket {
                    url: "wss://api.example.com/mcp".to_string(),
                    timeout_secs: Some(30),
                    headers: None,
                },
                enabled: false, // Disabled by default
                tool_prefix: Some("ws".to_string()),
                resources: None,
                bearer_token: Some("your-websocket-token".to_string()), // WebSocket Bearer token support
            },
        ],
        // Automatically discover and register tools from connected MCP servers
        auto_register_tools: true,
        // Timeout for individual tool calls (30 seconds)
        tool_timeout_secs: Some(30),
        // Maximum concurrent tool calls across all servers
        max_concurrent_calls: Some(5),
    };

    // Use the simple configuration for this example
    let mcp_config = mcp_config_simple;

    println!("Building model with MCP client support...");

    // Build the model with MCP client configuration
    // The MCP client will automatically connect to configured servers and discover available tools
    let model = TextModelBuilder::new("Qwen/Qwen3-4B".to_string())
        .with_isq(IsqType::Q8_0) // Use 8-bit quantization for efficiency
        .with_logging()
        .with_paged_attn(|| {
            PagedAttentionMetaBuilder::default()
                .with_gpu_memory(MemoryGpuConfig::ContextSize(8192))
                .build()
        })?
        .with_mcp_client(mcp_config) // This automatically connects to MCP servers and registers tools
        .build()
        .await?;

    println!("Model built successfully! MCP servers connected and tools registered.");
    println!("MCP tools are now available for automatic tool calling during conversations.");
    println!(
        "Note: Install filesystem server with: npx @modelcontextprotocol/server-filesystem . -y"
    );

    // Create a conversation that demonstrates MCP tool usage
    // The system message informs the model about available external tools
    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI assistant with access to external tools via MCP servers. \
             You can access filesystem operations and other external services \
             provided by connected MCP servers. Use these tools when appropriate to \
             help answer user questions. Tools are automatically available and you \
             can call them as needed.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! Can you list the files in the current directory and create a test.txt file?",
        );

    println!("\nSending chat request...");
    println!("The model will automatically use MCP tools if needed to answer the question.");
    let response = model.send_chat_request(messages).await?;

    println!("\nResponse:");
    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    // Display performance metrics
    println!("\nPerformance metrics:");
    println!(
        "Prompt tokens/sec: {:.2}",
        response.usage.avg_prompt_tok_per_sec
    );
    println!(
        "Completion tokens/sec: {:.2}",
        response.usage.avg_compl_tok_per_sec
    );

    // Display any MCP tool calls that were made during the conversation
    if let Some(tool_calls) = &response.choices[0].message.tool_calls {
        println!("\nMCP tool calls made:");
        for tool_call in tool_calls {
            println!(
                "- Tool: {} | Arguments: {}",
                tool_call.function.name, tool_call.function.arguments
            );
        }
    } else {
        println!("\nNo tool calls were made for this request.");
    }

    Ok(())
}

pub fn with_throughput_logging(self) -> Self

Enable runner throughput logging.

pub fn with_jinja_explicit(self, jinja_explicit: String) -> Self

Explicit JINJA chat template file (.jinja) to be used. If specified, this overrides all other chat templates.

pub fn with_topology(self, topology: Topology) -> Self

Set the model topology for use during loading. If there is an overlap, the topology type is used over the ISQ type.

Examples found in repository

examples/topology/main.rs (lines 11-41)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q8_0)
        .with_topology(
            Topology::empty()
                .with_range(
                    0..8,
                    LayerTopology {
                        isq: Some(IsqType::Q3K),
                        device: None,
                    },
                )
                .with_range(
                    8..16,
                    LayerTopology {
                        isq: Some(IsqType::Q4K),
                        device: None,
                    },
                )
                .with_range(
                    16..24,
                    LayerTopology {
                        isq: Some(IsqType::Q6K),
                        device: None,
                    },
                )
                .with_range(
                    24..32,
                    LayerTopology {
                        isq: Some(IsqType::Q8_0),
                        device: None,
                    },
                ),
        )
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

pub fn with_topology_from_path<P: AsRef<Path>>(self, path: P) -> Result<Self>

Set the model topology from a path. This preserves the path for unload/reload support. If there is an overlap, the topology type is used over the ISQ type.

pub fn with_mixture_qexperts_isq(self) -> Self

Organize ISQ to enable MoQE (Mixture of Quantized Experts, https://arxiv.org/abs/2310.02410)

Examples found in repository

examples/mixture_of_quant_experts/main.rs (line 10)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-MoE-instruct")
        .with_isq(IsqType::Q4K)
        .with_mixture_qexperts_isq()
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

pub fn with_chat_template(self, chat_template: impl ToString) -> Self

Literal Jinja chat template OR Path (ending in .json) to one.

pub fn with_tokenizer_json(self, tokenizer_json: impl ToString) -> Self

Path to a discrete tokenizer.json file.

pub fn with_loader_type(self, loader_type: NormalLoaderType) -> Self

Manually set the model loader type. Otherwise, it will attempt to automatically determine the loader type.

pub fn with_dtype(self, dtype: ModelDType) -> Self

Load the model in a certain dtype.

pub fn with_force_cpu(self) -> Self

Force usage of the CPU device. Do not use PagedAttention with this.

pub fn with_token_source(self, token_source: TokenSource) -> Self

Source of the Hugging Face token.

pub fn with_hf_revision(self, revision: impl ToString) -> Self

Set the revision to use for a Hugging Face remote model.

pub fn with_isq(self, isq: IsqType) -> Self

Use ISQ of a certain type. If there is an overlap, the topology type is used over the ISQ type.

Examples found in repository

examples/gemma2/main.rs (line 9)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("google/gemma-2-9b-it")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new().add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/grammar/main.rs (line 9)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    // Bullet list regex
    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Regex(
            "(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
        ))
        .add_message(TextMessageRole::User, "Please write a few jokes.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/glm4_moe_lite/main.rs (line 7)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("zai-org/GLM-4.7-Flash")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/web_search/main.rs (line 10)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_search(SearchEmbeddingModel::default())
        .build()
        .await?;

    let messages = TextMessages::new().add_message(
        TextMessageRole::User,
        "What is the weather forecast for Boston?",
    );
    let messages =
        RequestBuilder::from(messages).with_web_search_options(WebSearchOptions::default());

    let response = model.send_chat_request(messages).await?;

    println!("What is the weather forecast for Boston?\n\n");
    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/deepseekr1/main.rs (line 9)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("deepseek-ai/DeepSeek-R1")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/deepseekv2/main.rs (line 9)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("deepseek-ai/DeepSeek-V2-Lite")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

Additional examples can be found in:

• examples/phi3_5_moe/main.rs
• examples/granite/main.rs
• examples/smollm3/main.rs
• examples/mixture_of_quant_experts/main.rs
• examples/imatrix/main.rs
• examples/custom_logits_processor/main.rs
• examples/paged_attn/main.rs
• examples/json_schema/main.rs
• examples/llguidance/main.rs
• examples/isq/main.rs
• examples/async/main.rs
• examples/simple/main.rs
• examples/anymoe_lora/main.rs
• examples/custom_search/main.rs
• examples/custom_tool_call/main.rs
• examples/speculative/main.rs
• examples/batching/main.rs
• examples/text_auto_device_map/main.rs
• examples/anymoe/main.rs
• examples/topology/main.rs
• examples/simple_stream/main.rs
• examples/qwen3/main.rs
• examples/tools/main.rs
• examples/perplexity/main.rs
• examples/agent/main.rs
• examples/multi_model/main.rs
• examples/agent_streaming/main.rs
• examples/mcp_client/main.rs

pub fn with_imatrix(self, path: PathBuf) -> Self

Utilise this imatrix file during ISQ. Incompatible with specifying a calibration file.

pub fn with_calibration_file(self, path: PathBuf) -> Self

Utilise this calibration file to collcet an imatrix. Incompatible with specifying a calibration file.

Examples found in repository

examples/imatrix/main.rs (line 10)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("meta-llama/Llama-3.2-3B-Instruct")
        .with_isq(IsqType::Q4K)
        .with_calibration_file("calibration_data/calibration_datav3_small.txt".into())
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/perplexity/main.rs (line 90)

async fn main() -> Result<()> {
    let args = Args::parse();

    let quant = if let Some(isq) = &args.isq {
        Some(parse_isq_value(isq, None).map_err(anyhow::Error::msg)?)
    } else {
        None
    };

    let prompt_chunksize = 1024;
    let mut model_builder = TextModelBuilder::new(&args.model_id).with_logging();
    if let Some(quant) = quant {
        model_builder = model_builder.with_isq(quant);
    }
    if let Some(calibration_file) = &args.calibration_file {
        model_builder = model_builder.with_calibration_file(calibration_file.clone());
    }

    let model = model_builder.build().await?;

    let text = read_to_string(&args.file)?;
    let tokens = model
        .tokenize(Either::Right(text), None, false, false, None)
        .await?;
    let bos_token = model
        .tokenize(Either::Right(" ".to_string()), None, true, false, None)
        .await?[0];
    let inner = model.inner();

    println!("Using bos token id `{bos_token}`.");

    let n_chunks = tokens.len().div_ceil(prompt_chunksize);
    let mut ppl_measurements = Vec::new();
    for (i, chunk) in tokens.chunks(prompt_chunksize).enumerate() {
        let start = Instant::now();
        let (logits, tokens) = {
            let chunk = [vec![bos_token], chunk.to_vec()].concat();
            process_chunk(inner, chunk).await?
        };

        // Upcast to float if we need to compute the loss to avoid potential precision issues
        let logits = logits.to_device(&Device::Cpu)?.to_dtype(DType::F32)?;
        // Shift so that tokens < n predict n
        let shift_logits = logits.narrow(0, 0, logits.dim(0)? - 1)?.contiguous()?;
        let shift_labels = Tensor::from_slice(&tokens[1..], (tokens.len() - 1,), &Device::Cpu)?;

        let loss_fct = cross_entropy_loss(&shift_logits, &shift_labels)?;
        let perplexity = loss_fct.exp()?.to_scalar::<f32>()?;
        let end = Instant::now();

        ppl_measurements.push(perplexity);
        println!(
            "Chunk {i}/{n_chunks} ({} tokens): Perplexity for `{}`, ISQ `{:?}`, {}s: {perplexity}",
            tokens.len(),
            args.file,
            quant,
            end.duration_since(start).as_secs_f32(),
        );
    }

    let mean = ppl_measurements.iter().sum::<f32>() / ppl_measurements.len() as f32;
    let variance = ppl_measurements
        .iter()
        .map(|e| (mean - e).powf(2.))
        .sum::<f32>()
        / ppl_measurements.len() as f32;
    let std_dev = variance.sqrt();
    println!();
    println!(
        "Final perplexity for `{}`, ISQ `{:?}`: {}±{} ppl",
        args.file, quant, mean, std_dev
    );

    Ok(())
}

pub fn with_paged_attn( self, paged_attn_cfg: impl FnOnce() -> Result<PagedAttentionConfig>, ) -> Result<Self>

Enable PagedAttention. Configure PagedAttention with a PagedAttentionConfig object, which can be created with sensible values with a PagedAttentionMetaBuilder.

If PagedAttention is not supported (query with paged_attn_supported), this will do nothing.

Examples found in repository

examples/gemma2/main.rs (line 11)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("google/gemma-2-9b-it")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new().add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/grammar/main.rs (line 11)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    // Bullet list regex
    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Regex(
            "(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
        ))
        .add_message(TextMessageRole::User, "Please write a few jokes.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/deepseekr1/main.rs (line 11)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("deepseek-ai/DeepSeek-R1")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/deepseekv2/main.rs (line 11)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("deepseek-ai/DeepSeek-V2-Lite")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/phi3_5_moe/main.rs (line 11)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-MoE-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/granite/main.rs (line 11)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("ibm-granite/granite-4.0-tiny-preview")
        .with_isq(IsqType::Q8_0)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

Additional examples can be found in:

• examples/smollm3/main.rs
• examples/mixture_of_quant_experts/main.rs
• examples/imatrix/main.rs
• examples/custom_logits_processor/main.rs
• examples/paged_attn/main.rs
• examples/json_schema/main.rs
• examples/llguidance/main.rs
• examples/isq/main.rs
• examples/async/main.rs
• examples/simple/main.rs
• examples/uqff/main.rs
• examples/anymoe_lora/main.rs
• examples/batching/main.rs
• examples/text_auto_device_map/main.rs
• examples/anymoe/main.rs
• examples/topology/main.rs
• examples/simple_stream/main.rs
• examples/agent/main.rs
• examples/agent_streaming/main.rs
• examples/mcp_client/main.rs

pub fn with_max_num_seqs(self, max_num_seqs: usize) -> Self

Set the maximum number of sequences which can be run at once.

pub fn with_no_kv_cache(self) -> Self

Disable KV cache. Trade performance for memory usage.

pub fn with_prefix_cache_n(self, n_seqs: Option<usize>) -> Self

Set the number of sequences to hold in the prefix cache. Set to None to disable the prefix cacher.

pub fn with_logging(self) -> Self

Enable logging.

Examples found in repository

examples/gemma2/main.rs (line 10)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("google/gemma-2-9b-it")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new().add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/grammar/main.rs (line 10)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    // Bullet list regex
    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Regex(
            "(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
        ))
        .add_message(TextMessageRole::User, "Please write a few jokes.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/lora/main.rs (line 7)

async fn main() -> Result<()> {
    let model = LoraModelBuilder::from_text_model_builder(
        TextModelBuilder::new("meta-llama/Llama-3.2-1B-Instruct").with_logging(),
        vec!["danielhanchen/llama-3.2-lora".to_string()],
    )
    .build()
    .await?;

    let messages = TextMessages::new().add_message(
        TextMessageRole::User,
        "Hello! How are you? Please write generic binary search function in Rust.",
    );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/gpt_oss/main.rs (line 7)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("openai/gpt-oss-20b")
        .with_logging()
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/glm4_moe_lite/main.rs (line 8)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("zai-org/GLM-4.7-Flash")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/xlora/main.rs (line 10)

async fn main() -> Result<()> {
    let model =
        XLoraModelBuilder::from_text_model_builder(
            TextModelBuilder::new("HuggingFaceH4/zephyr-7b-beta").with_logging(),
            "lamm-mit/x-lora",
            serde_json::from_reader(File::open("my-ordering-file.json").unwrap_or_else(|_| {
                panic!("Could not load ordering file at my-ordering-file.json")
            }))?,
        )
        .build()
        .await?;

    let messages =
        TextMessages::new().add_message(TextMessageRole::User, "Hello! What is graphene.");

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

Additional examples can be found in:

• examples/web_search/main.rs
• examples/deepseekr1/main.rs
• examples/deepseekv2/main.rs
• examples/phi3_5_moe/main.rs
• examples/granite/main.rs
• examples/smollm3/main.rs
• examples/mixture_of_quant_experts/main.rs
• examples/imatrix/main.rs
• examples/custom_logits_processor/main.rs
• examples/paged_attn/main.rs
• examples/json_schema/main.rs
• examples/llguidance/main.rs
• examples/isq/main.rs
• examples/async/main.rs
• examples/simple/main.rs
• examples/uqff/main.rs
• examples/anymoe_lora/main.rs
• examples/custom_search/main.rs
• examples/custom_tool_call/main.rs
• examples/speculative/main.rs
• examples/batching/main.rs
• examples/text_auto_device_map/main.rs
• examples/anymoe/main.rs
• examples/topology/main.rs
• examples/simple_stream/main.rs
• examples/qwen3/main.rs
• examples/tools/main.rs
• examples/perplexity/main.rs
• examples/agent/main.rs
• examples/agent_streaming/main.rs
• examples/mcp_client/main.rs

pub fn with_device_mapping(self, device_mapping: DeviceMapSetting) -> Self

Provide metadata to initialize the device mapper.

Examples found in repository

examples/text_auto_device_map/main.rs (line 17)

async fn main() -> Result<()> {
    let auto_map_params = AutoDeviceMapParams::Text {
        max_seq_len: 4096,
        max_batch_size: 2,
    };
    let model = TextModelBuilder::new("meta-llama/Llama-3.3-70B-Instruct")
        .with_isq(IsqType::Q8_0)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .with_device_mapping(DeviceMapSetting::Auto(auto_map_params))
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    // Next example: Return some logprobs with the `RequestBuilder`, which enables higher configurability.
    let request = RequestBuilder::new().return_logprobs(true).add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!(
        "Logprobs: {:?}",
        &response.choices[0]
            .logprobs
            .as_ref()
            .unwrap()
            .content
            .as_ref()
            .unwrap()[0..3]
    );

    Ok(())
}

pub fn from_uqff(self, path: Vec<PathBuf>) -> Self

👎Deprecated: Use UqffTextModelBuilder to load a UQFF model instead of the generic from_uqff

Path to read a .uqff file from. Other necessary configuration files must be present at this location.

For example, these include:

• residual.safetensors
• tokenizer.json
• config.json
• More depending on the model

pub fn write_uqff(self, path: PathBuf) -> Self

Path to write a .uqff file to and serialize the other necessary files.

The parent (part of the path excluding the filename) will determine where any other files serialized are written to.

For example, these include:

• residual.safetensors
• tokenizer.json
• config.json
• More depending on the model

pub fn from_hf_cache_pathf(self, hf_cache_path: PathBuf) -> Self

Cache path for Hugging Face models downloaded locally

pub fn with_device(self, device: Device) -> Self

Set the main device to load this model onto. Automatic device mapping will be performed starting with this device.

pub fn with_matformer_config_path(self, path: PathBuf) -> Self

Path to a Matryoshka Transformer configuration CSV file.

pub fn with_matformer_slice_name(self, name: String) -> Self

Name of the slice to use from the Matryoshka Transformer configuration.

pub async fn build(self) -> Result<Model>

Examples found in repository

examples/gemma2/main.rs (line 12)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("google/gemma-2-9b-it")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let request = RequestBuilder::new().add_message(
        TextMessageRole::User,
        "Please write a mathematical equation where a few numbers are added.",
    );

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/grammar/main.rs (line 12)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    // Bullet list regex
    let request = RequestBuilder::new()
        .set_constraint(mistralrs::Constraint::Regex(
            "(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
        ))
        .add_message(TextMessageRole::User, "Please write a few jokes.");

    let response = model.send_chat_request(request).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

examples/gpt_oss/main.rs (line 8)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("openai/gpt-oss-20b")
        .with_logging()
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/glm4_moe_lite/main.rs (line 9)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("zai-org/GLM-4.7-Flash")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/web_search/main.rs (line 13)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_search(SearchEmbeddingModel::default())
        .build()
        .await?;

    let messages = TextMessages::new().add_message(
        TextMessageRole::User,
        "What is the weather forecast for Boston?",
    );
    let messages =
        RequestBuilder::from(messages).with_web_search_options(WebSearchOptions::default());

    let response = model.send_chat_request(messages).await?;

    println!("What is the weather forecast for Boston?\n\n");
    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

examples/deepseekr1/main.rs (line 12)

async fn main() -> Result<()> {
    let model = TextModelBuilder::new("deepseek-ai/DeepSeek-R1")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(
            TextMessageRole::System,
            "You are an AI agent with a specialty in programming.",
        )
        .add_message(
            TextMessageRole::User,
            "Hello! How are you? Please write generic binary search function in Rust.",
        );

    let response = model.send_chat_request(messages).await?;

    println!("{}", response.choices[0].message.content.as_ref().unwrap());
    dbg!(
        response.usage.avg_prompt_tok_per_sec,
        response.usage.avg_compl_tok_per_sec
    );

    Ok(())
}

Additional examples can be found in:

• examples/deepseekv2/main.rs
• examples/phi3_5_moe/main.rs
• examples/granite/main.rs
• examples/smollm3/main.rs
• examples/mixture_of_quant_experts/main.rs
• examples/imatrix/main.rs
• examples/custom_logits_processor/main.rs
• examples/paged_attn/main.rs
• examples/json_schema/main.rs
• examples/llguidance/main.rs
• examples/isq/main.rs
• examples/async/main.rs
• examples/simple/main.rs
• examples/uqff/main.rs
• examples/custom_search/main.rs
• examples/custom_tool_call/main.rs
• examples/batching/main.rs
• examples/text_auto_device_map/main.rs
• examples/topology/main.rs
• examples/simple_stream/main.rs
• examples/qwen3/main.rs
• examples/tools/main.rs
• examples/perplexity/main.rs
• examples/agent/main.rs
• examples/agent_streaming/main.rs
• examples/mcp_client/main.rs

Trait Implementations

impl Clone for TextModelBuilder

fn clone(&self) -> TextModelBuilder

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl From<TextModelBuilder> for AnyModelBuilder

fn from(b: TextModelBuilder) -> Self

Converts to this type from the input type.

impl From<UqffTextModelBuilder> for TextModelBuilder

fn from(value: UqffTextModelBuilder) -> Self

Converts to this type from the input type.