mistralrs_core/
request.rs

1use either::Either;
2use indexmap::IndexMap;
3use mistralrs_audio::AudioInput;
4use mistralrs_quant::IsqType;
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7
8use crate::VideoInput;
9
10use crate::{
11    response::Response, sampler::SamplingParams, tools::ToolChoice, CustomLogitsProcessor,
12    DiffusionGenerationParams, Tool,
13};
14use std::{fmt::Debug, path::PathBuf, sync::Arc};
15use tokio::sync::mpsc::Sender;
16
17pub type LlguidanceGrammar = llguidance::api::TopLevelGrammar;
18
19#[derive(Clone, Serialize, Deserialize)]
20/// Control the constraint with llguidance.
21pub enum Constraint {
22    Regex(String),
23    Lark(String),
24    JsonSchema(serde_json::Value),
25    Llguidance(LlguidanceGrammar),
26    None,
27}
28
29#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
30#[cfg_attr(feature = "pyo3_macros", pyo3::pyclass(eq, eq_int))]
31#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
32/// Image generation response format
33pub enum ImageGenerationResponseFormat {
34    Url,
35    B64Json,
36}
37
38pub type MessageContent = Either<String, Vec<IndexMap<String, Value>>>;
39
40/// Reasoning effort level for models that support it (e.g., GPT-OSS with Harmony format).
41/// Controls the depth of reasoning/analysis in the model's response.
42#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
43#[cfg_attr(feature = "pyo3_macros", pyo3::pyclass(eq, eq_int))]
44#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
45#[serde(rename_all = "lowercase")]
46pub enum ReasoningEffort {
47    /// Minimal reasoning, faster responses
48    Low,
49    /// Balanced reasoning depth
50    #[default]
51    Medium,
52    /// Deep reasoning, more thorough analysis
53    High,
54}
55
56impl ReasoningEffort {
57    /// Convert to string representation for chat template
58    pub fn as_str(&self) -> &'static str {
59        match self {
60            Self::Low => "low",
61            Self::Medium => "medium",
62            Self::High => "high",
63        }
64    }
65}
66
67#[derive(Clone, Debug, Serialize, Deserialize)]
68/// Message or messages for a [`Request`].
69pub enum RequestMessage {
70    Chat {
71        messages: Vec<IndexMap<String, MessageContent>>,
72        enable_thinking: Option<bool>,
73        /// Reasoning effort level for Harmony-format models
74        reasoning_effort: Option<ReasoningEffort>,
75    },
76    Completion {
77        text: String,
78        echo_prompt: bool,
79        best_of: Option<usize>,
80    },
81    CompletionTokens(Vec<u32>),
82    MultimodalChat {
83        #[serde(skip)] // TODO
84        images: Vec<image::DynamicImage>,
85        #[serde(skip)] // TODO
86        audios: Vec<AudioInput>,
87        #[serde(skip)]
88        videos: Vec<VideoInput>,
89        messages: Vec<IndexMap<String, MessageContent>>,
90        enable_thinking: Option<bool>,
91        /// Reasoning effort level for Harmony-format models
92        reasoning_effort: Option<ReasoningEffort>,
93    },
94    ImageGeneration {
95        prompt: String,
96        format: ImageGenerationResponseFormat,
97        generation_params: DiffusionGenerationParams,
98        save_file: Option<PathBuf>,
99    },
100    SpeechGeneration {
101        prompt: String,
102    },
103    Embedding {
104        prompt: String,
105    },
106    EmbeddingTokens {
107        prompt: Vec<u32>,
108    },
109}
110
111fn default_responder<T>() -> Sender<T> {
112    let (sender, _) = tokio::sync::mpsc::channel(1);
113    sender
114}
115
116#[cfg_attr(feature = "pyo3_macros", pyo3::pyclass(eq, eq_int))]
117#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
118#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Default)]
119pub enum SearchContextSize {
120    #[serde(rename = "low")]
121    Low,
122    #[default]
123    #[serde(rename = "medium")]
124    Medium,
125    #[serde(rename = "high")]
126    High,
127}
128
129#[cfg_attr(feature = "pyo3_macros", pyo3::pyclass(eq))]
130#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
131#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
132pub struct ApproximateUserLocation {
133    pub city: String,
134    pub country: String,
135    pub region: String,
136    pub timezone: String,
137}
138
139#[cfg_attr(feature = "pyo3_macros", pyo3::pyclass(eq))]
140#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
141#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
142#[serde(tag = "type")]
143pub enum WebSearchUserLocation {
144    #[serde(rename = "approximate")]
145    Approximate {
146        approximate: ApproximateUserLocation,
147    },
148}
149
150#[cfg_attr(feature = "pyo3_macros", pyo3::pyclass(eq))]
151#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
152#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)]
153pub struct WebSearchOptions {
154    pub search_context_size: Option<SearchContextSize>,
155    pub user_location: Option<WebSearchUserLocation>,
156    /// Override the description for the search tool.
157    pub search_description: Option<String>,
158    /// Override the description for the extraction tool.
159    pub extract_description: Option<String>,
160}
161
162#[derive(Clone, Serialize, Deserialize)]
163/// A normal request request to the `MistralRs`.
164/// - `messages`: Messages for the request
165/// - `sampling_params`: Sampling parameters for generation
166/// - `response`: Object to send the result through
167/// - `return_logprobs`: Whether to return logprobs
168/// - `is_streaming`: Control whether the request is streaming, if so chunk responses will be sent
169/// - `id`: Request ID
170/// - `constraint`: Constraint to use during generation
171/// - `suffix`: Suffix to add
172/// - `tools`: Tools available in this request
173/// - `tool_choice`: Choice of tools
174/// - `logits_processors`: Custom logits processors. Order of application:
175///     1) Apply penalties from `sampling_params`
176///     2) Apply these custom logits processors sequentially
177///     3) Apply temperature and softmax
178///     4) Sample the next token (topk, topp, minp, etc)
179/// - `return_raw_logits`: Return raw logits.
180/// - `truncate_sequence`: Whether to truncate the prompt if it exceeds the model's maximum context length.
181pub struct NormalRequest {
182    pub messages: RequestMessage,
183    pub sampling_params: SamplingParams,
184    #[serde(default = "default_responder")]
185    #[serde(skip)]
186    pub response: Sender<Response>,
187    pub return_logprobs: bool,
188    pub is_streaming: bool,
189    pub id: usize,
190    pub constraint: Constraint,
191    pub suffix: Option<String>,
192    pub tools: Option<Vec<Tool>>,
193    pub tool_choice: Option<ToolChoice>,
194    #[serde(skip)]
195    pub logits_processors: Option<Vec<Arc<dyn CustomLogitsProcessor>>>,
196    pub return_raw_logits: bool,
197    pub web_search_options: Option<WebSearchOptions>,
198    pub model_id: Option<String>,
199    #[serde(default)]
200    pub truncate_sequence: bool,
201}
202
203impl NormalRequest {
204    pub fn new_simple(
205        messages: RequestMessage,
206        sampling_params: SamplingParams,
207        response: Sender<Response>,
208        id: usize,
209        tools: Option<Vec<Tool>>,
210        tool_choice: Option<ToolChoice>,
211    ) -> Self {
212        Self {
213            messages,
214            sampling_params,
215            response,
216            id,
217            tools,
218            tool_choice,
219            return_logprobs: false,
220            is_streaming: false,
221            constraint: Constraint::None,
222            suffix: None,
223            logits_processors: None,
224            return_raw_logits: false,
225            web_search_options: None,
226            model_id: None,
227            truncate_sequence: false,
228        }
229    }
230}
231
232#[derive(Clone, Serialize, Deserialize)]
233/// Request to tokenize some messages or some text.
234/// - `add_generation_prompt` is only applicable if chat messages are provided and not a raw string.
235pub struct TokenizationRequest {
236    pub text: Either<Vec<IndexMap<String, MessageContent>>, String>,
237    pub tools: Option<Vec<Tool>>,
238    pub add_generation_prompt: bool,
239    pub add_special_tokens: bool,
240    pub enable_thinking: Option<bool>,
241    pub reasoning_effort: Option<ReasoningEffort>,
242    #[serde(default = "default_responder")]
243    #[serde(skip)]
244    pub response: Sender<anyhow::Result<Vec<u32>>>,
245}
246
247#[derive(Clone, Serialize, Deserialize)]
248/// Request to detokenize some text.
249pub struct DetokenizationRequest {
250    pub tokens: Vec<u32>,
251    pub skip_special_tokens: bool,
252    #[serde(default = "default_responder")]
253    #[serde(skip)]
254    pub response: Sender<anyhow::Result<String>>,
255}
256
257#[derive(Clone, Serialize, Deserialize)]
258/// A request to the Engine, encapsulating the various parameters as well as
259/// the `mpsc` response `Sender` used to return the [`Response`].
260pub enum Request {
261    Normal(Box<NormalRequest>),
262    ReIsq(IsqType),
263    Tokenize(TokenizationRequest),
264    Detokenize(DetokenizationRequest),
265    // Sending a terminate request causes the `run` function to return to the thread created in `MistralRs::new`,
266    // and then Engine will be dropped.
267    Terminate,
268    TerminateAllSeqsNextStep,
269}
270
271impl Debug for Request {
272    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
273        match self {
274            Request::Normal(boxed_req) => {
275                let NormalRequest {
276                    messages,
277                    sampling_params,
278                    is_streaming,
279                    id,
280                    ..
281                } = &**boxed_req;
282                write!(
283                    f,
284                    "Request {id} {{ messages: `{messages:?}`, sampling_params: {sampling_params:?}, is_streaming: {is_streaming}}}",
285                )
286            }
287            Request::ReIsq(tp) => {
288                write!(f, "Re ISQ Request {tp:?}",)
289            }
290            Request::Tokenize(req) => {
291                write!(f, "Tokenization Request {:?}", req.text)
292            }
293            Request::Detokenize(req) => {
294                write!(f, "Tokenization Request {:?}", req.tokens)
295            }
296            Request::Terminate => write!(f, "Termination Request"),
297            Request::TerminateAllSeqsNextStep => write!(f, "Terminate All Seqs Next Step"),
298        }
299    }
300}
mistralrs_core/request.rs

mistralrs_core/
request.rs
