mistralrs_core/pipeline/
mod.rs

1mod amoe;
2mod auto;
3pub mod chat_template;
4mod diffusion;
5mod embedding;
6mod ggml;
7mod gguf;
8pub(crate) mod hf;
9mod inputs_processor;
10mod isq;
11pub(crate) mod llg;
12mod loaders;
13mod macros;
14mod multimodal;
15mod normal;
16mod paths;
17mod processing;
18mod response;
19mod sampling;
20mod speculative;
21mod speech;
22
23pub use super::diffusion_models::DiffusionGenerationParams;
24use crate::amoe::{AnyMoeConfig, AnyMoeExpertType, AnyMoeTrainingInputs, AnyMoeTrainingResult};
25use crate::device_map::DeviceMapper;
26use crate::paged_attention::{CacheConfig, CacheEngine, ModelConfigLike};
27use crate::prefix_cacher::PrefixCacheManagerV2;
28use crate::PagedAttentionConfig;
29pub use amoe::{AnyMoeLoader, AnyMoePipeline};
30pub use auto::{AutoLoader, AutoLoaderBuilder};
31use chat_template::ChatTemplate;
32pub use diffusion::{DiffusionLoader, DiffusionLoaderBuilder};
33pub use embedding::{EmbeddingLoader, EmbeddingLoaderBuilder, EmbeddingSpecificConfig};
34pub use ggml::{GGMLLoader, GGMLLoaderBuilder, GGMLSpecificConfig};
35pub use gguf::{GGUFLoader, GGUFLoaderBuilder, GGUFSpecificConfig};
36use image::DynamicImage;
37pub use inputs_processor::InputProcessorOutput;
38pub(crate) use isq::IsqModelLoader;
39pub use isq::{
40    expand_isq_value, parse_isq_value, IsqModel, IsqOrganization, UQFF_MULTI_FILE_DELIMITER,
41};
42use llguidance::toktrie::TokEnv;
43pub use loaders::{
44    AdapterKind, AutoDeviceMapParams, AutoEmbeddingLoader, AutoMultimodalLoader, AutoNormalLoader,
45    DeepSeekV2Loader, DeepSeekV3Loader, DeviceMappedModelLoader, DiffusionLoaderType,
46    DiffusionModel, DiffusionModelLoader, EmbeddingGemmaLoader, EmbeddingLoaderType,
47    EmbeddingModel, EmbeddingModelLoader, EmbeddingModelPaths, EmbeddingModule,
48    EmbeddingModulePaths, EmbeddingModuleType, FluxLoader, GLM4Loader, GLM4MoeLiteLoader,
49    GLM4MoeLoader, Gemma2Loader, Gemma3Loader, Gemma3nLoader, Gemma4Loader, GemmaLoader,
50    GptOssLoader, GraniteMoeHybridLoader, Idefics2Loader, Idefics3Loader, LLaVALoader,
51    LLaVANextLoader, LlamaLoader, Loader, LocalModelPaths, MiniCpmOLoader, Mistral3Loader,
52    MistralLoader, MixtralLoader, ModelKind, ModelPaths, MultimodalLoaderType, MultimodalModel,
53    MultimodalModelLoader, NormalLoaderType, NormalLoadingMetadata, NormalModel, NormalModelLoader,
54    Phi2Loader, Phi3Loader, Phi3VLoader, Phi3_5MoELoader, Phi4MMLoader, PrettyName,
55    QuantizationKind, Qwen2Loader, Qwen2VLLoader, Qwen2_5VLLoader, Qwen3EmbeddingLoader,
56    Qwen3Loader, Qwen3MoELoader, Qwen3NextLoader, Qwen3VLLoader, Qwen3VLMoELoader, Qwen3_5Loader,
57    Qwen3_5MoeLoader, SmolLm3Loader, Starcoder2Loader, TokenSource, VLlama4Loader, VLlamaLoader,
58    VoxtralLoader,
59};
60#[allow(clippy::too_many_arguments)]
61pub(crate) fn get_device_layers_for_loader(
62    loader: &dyn loaders::DeviceMappedModelLoader,
63    config: &str,
64    num_layers: usize,
65    layer_sizes_in_bytes: Vec<usize>,
66    non_mapped_size_in_bytes: usize,
67    total_model_size_in_bytes: usize,
68    devices: &[Device],
69    dtype: DType,
70    params: &loaders::AutoDeviceMapParams,
71    paged_attn_config: Option<&PagedAttentionConfig>,
72) -> Result<crate::device_map::DeviceMapMetadata> {
73    loaders::auto_device_map::get_device_layers(
74        loader,
75        config,
76        num_layers,
77        layer_sizes_in_bytes,
78        non_mapped_size_in_bytes,
79        total_model_size_in_bytes,
80        devices,
81        dtype,
82        params,
83        paged_attn_config,
84    )
85}
86use mistralrs_quant::IsqType;
87pub use multimodal::{MultimodalLoader, MultimodalLoaderBuilder, MultimodalSpecificConfig};
88pub use normal::{NormalLoader, NormalLoaderBuilder, NormalSpecificConfig};
89pub(crate) use paths::{get_chat_template, get_model_paths, get_xlora_paths};
90pub use paths::{AdapterPaths, LoraAdapterPaths};
91pub(crate) use processing::{
92    apply_chat_template, BasicProcessor, MessagesAction, Processor, ProcessorCreator,
93};
94use rand_isaac::Isaac64Rng;
95pub use speculative::{SpeculativeConfig, SpeculativeLoader, SpeculativePipeline};
96pub use speech::{SpeechLoader, SpeechPipeline};
97use std::any::Any;
98use std::fmt::Debug;
99use std::sync::atomic::AtomicUsize;
100use std::sync::Arc;
101use std::time::{Duration, Instant};
102use tokenizers::Tokenizer;
103
104use anyhow::Result;
105use candle_core::{DType, Device, IndexOp, Tensor, Var};
106
107use crate::sequence::Sequence;
108
109pub use self::inputs_processor::{
110    text_models_inputs_processor, InputsProcessor, InputsProcessorType,
111};
112use self::text_models_inputs_processor::PagedAttentionMeta;
113pub use crate::kv_cache::{
114    Cache, CacheManager, EitherCache, HybridLayerCache, KvCache, LayerCaches, NormalCache,
115    NormalCacheType,
116};
117
118#[derive(Clone, PartialEq, Eq)]
119pub enum SupportedModality {
120    Text,
121    Audio,
122    Vision,
123    Video,
124    Embedding,
125}
126
127impl Debug for SupportedModality {
128    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
129        match self {
130            Self::Text => write!(f, "📝 Text"),
131            Self::Audio => write!(f, "🔊 Audio"),
132            Self::Vision => write!(f, "🖼️ Vision"),
133            Self::Video => write!(f, "🎬 Video"),
134            Self::Embedding => write!(f, "🔢 Embedding"),
135        }
136    }
137}
138
139#[derive(Debug, Clone)]
140pub struct Modalities {
141    pub input: Vec<SupportedModality>,
142    pub output: Vec<SupportedModality>,
143}
144
145pub struct GeneralMetadata {
146    pub max_seq_len: usize,
147    /// Only None if it doesn't make sense for the model
148    pub llg_factory: Option<Arc<llguidance::ParserFactory>>,
149    pub no_kv_cache: bool,
150    pub no_prefix_cache: bool,
151    pub num_hidden_layers: usize,
152    pub eos_tok: Vec<u32>,
153    pub kind: ModelKind,
154    // TODO: Replace is_xlora queries to check via kind instead:
155    pub is_xlora: bool,
156    pub activation_dtype: DType,
157    pub sliding_window: Option<usize>,
158    // PagedAttention stuff
159    pub cache_config: Option<CacheConfig>,
160    pub cache_engine: Option<CacheEngine>,
161    pub model_metadata: Option<Arc<dyn ModelConfigLike + Send + Sync>>,
162    pub modalities: Modalities,
163}
164
165impl GeneralMetadata {
166    pub fn tok_env(&self) -> Option<TokEnv> {
167        self.llg_factory.as_ref().map(|f| f.tok_env().clone())
168    }
169}
170
171#[derive(Clone, Copy)]
172pub enum CacheInstruction {
173    In,
174    Out,
175    /// load_preallocated_cache means to load the preallocated cache, if applicable.
176    Reset {
177        load_preallocated_cache: bool,
178        reset_non_granular: bool,
179    },
180    Nothing,
181}
182
183pub trait PreProcessingMixin: MetadataMixin {
184    fn get_processor(&self) -> Arc<dyn Processor> {
185        Arc::new(BasicProcessor)
186    }
187    /// Only None if it doesnt make sense for the model
188    fn get_chat_template(&self) -> Option<Arc<ChatTemplate>>;
189    fn get_input_processor_config(&self) -> Option<Arc<dyn Any>>;
190}
191
192pub trait IsqPipelineMixin {
193    fn re_isq_model(&mut self, dtype: IsqType) -> Result<()>;
194}
195
196pub trait CacheManagerMixin {
197    /// Clone the cache FROM the sequences' cache TO the model cache. Only called for completion seqs.
198    /// It is not a guarantee that this will be called for each completion step.
199    fn clone_in_cache(&self, seqs: &mut [&mut Sequence]);
200    /// Clone the cache FROM the model cache TO the sequences. Called for prompt and completion seqs.
201    /// It is not a guarantee that this will be called for each step.
202    fn clone_out_cache(&self, seqs: &mut [&mut Sequence]);
203    /// Set the model cache to all None. Only called for prompt seqs.
204    /// It is not a guarantee that this will be called for each prompt step.
205    /// This may also reset the non granular state if applicable.
206    fn set_none_cache(
207        &self,
208        seqs: &mut [&mut Sequence],
209        reset_non_granular: bool,
210        modify_draft_cache: bool,
211        load_preallocated_cache: bool,
212    );
213    fn cache(&self) -> &EitherCache;
214}
215
216pub trait MetadataMixin {
217    fn device(&self) -> Device;
218    /// Only None if it doesnt make sense for the model
219    fn tokenizer(&self) -> Option<Arc<Tokenizer>>;
220    fn name(&self) -> String;
221    fn reset_non_granular_state(&self);
222    fn get_metadata(&self) -> Arc<GeneralMetadata>;
223    fn generation_defaults(&self) -> Option<crate::ModelGenerationDefaults> {
224        None
225    }
226    fn device_mapper(&self) -> Option<&dyn DeviceMapper>;
227}
228
229/// Implemented by the base model of an AnyMoe.
230pub trait AnyMoePipelineMixin {
231    /// Get vars for each gating layer
232    fn amoe_layer_vars(&self) -> Vec<Vec<Var>> {
233        unreachable!()
234    }
235    fn amoe_finish_training(&mut self, _gate_model_id: Option<String>) -> candle_core::Result<()> {
236        unreachable!()
237    }
238    fn amoe_base_model_trainable_params(&self) -> usize {
239        unreachable!()
240    }
241    fn amoe_supported(&self) -> bool {
242        false
243    }
244    /// Per-layer cached outputs.
245    fn amoe_take_cached_gating_outputs(&mut self) -> Vec<Tensor> {
246        unreachable!()
247    }
248    /// Inject the MoE layers
249    #[allow(clippy::too_many_arguments)]
250    fn amoe_create_layers(
251        &mut self,
252        _model_ids: Vec<String>,
253        _token: &TokenSource,
254        _revision: Option<String>,
255        _match_regex: &str,
256        _config: AnyMoeConfig,
257        _dtype: DType,
258        _dev: &Device,
259        (_prefix, _mlp): (String, String),
260        _layers: Vec<usize>,
261        _expert_type: AnyMoeExpertType,
262        _silent: bool,
263        _gate_model_id: Option<String>,
264    ) -> candle_core::Result<()> {
265        unreachable!()
266    }
267    /// Pre-train the gating layers
268    #[allow(clippy::too_many_arguments)]
269    fn amoe_pre_train(
270        &self,
271        _inputs: AnyMoeTrainingInputs,
272        (_prefix, _mlp): (String, String),
273        _model_ids: Vec<String>,
274        _token: TokenSource,
275        _revision: Option<String>,
276        _layers: Vec<usize>,
277        _silent: bool,
278    ) -> Result<Option<AnyMoeTrainingResult>, candle_core::Error> {
279        unreachable!()
280    }
281}
282
283/// Category of the model. This can also be used to extract model-category specific tools,
284/// such as the multimodal model prompt prefixer.
285#[derive(Clone)]
286pub enum ModelCategory {
287    Text,
288    Multimodal {
289        prefixer: Arc<dyn MultimodalPromptPrefixer>,
290    },
291    Diffusion,
292    Audio,
293    Speech,
294    Embedding,
295}
296
297impl std::fmt::Debug for ModelCategory {
298    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
299        match self {
300            ModelCategory::Text => write!(f, "ModelCategory::Text"),
301            ModelCategory::Multimodal { .. } => {
302                write!(f, "ModelCategory::Multimodal {{ prefixer: .. }}")
303            }
304            ModelCategory::Diffusion => write!(f, "ModelCategory::Diffusion"),
305            ModelCategory::Audio => write!(f, "ModelCategory::Audio"),
306            ModelCategory::Speech => write!(f, "ModelCategory::Speech"),
307            ModelCategory::Embedding => write!(f, "ModelCategory::Embedding"),
308        }
309    }
310}
311
312impl PartialEq for ModelCategory {
313    fn eq(&self, other: &Self) -> bool {
314        match (self, other) {
315            (Self::Text, Self::Text) => true,
316            (Self::Multimodal { .. }, Self::Multimodal { .. }) => true,
317            (Self::Audio, Self::Audio) => true,
318            (Self::Speech, Self::Speech) => true,
319            (Self::Diffusion, Self::Diffusion) => true,
320            (Self::Embedding, Self::Embedding) => true,
321            (
322                Self::Text
323                | Self::Multimodal { .. }
324                | Self::Diffusion
325                | Self::Audio
326                | Self::Speech
327                | Self::Embedding,
328                _,
329            ) => false,
330        }
331    }
332}
333
334/// Prepend a vision tag appropriate for the model to the prompt. Image indexing is assumed that start at 0.
335pub trait MultimodalPromptPrefixer: Send + Sync {
336    /// Prefix for inclusion in messages (may do nothing if the chat template handles it).
337    fn prefix_image(&self, _image_indices: Vec<usize>, prompt: &str) -> String {
338        prompt.to_string()
339    }
340    /// Prefix for inclusion in messages (may do nothing if the chat template handles it).
341    fn prefix_audio(&self, _audio_indexes: Vec<usize>, prompt: &str) -> String {
342        prompt.to_string()
343    }
344    /// Prefix for inclusion in messages (may do nothing if the chat template handles it).
345    fn prefix_video(&self, _video_indexes: Vec<usize>, prompt: &str) -> String {
346        prompt.to_string()
347    }
348}
349
350#[derive(Clone)]
351pub enum CacheBackendMetadata {
352    DefaultInstructions {
353        pre_op: CacheInstruction,
354        post_op: CacheInstruction,
355    },
356    PagedAttention {
357        metadata: PagedAttentionMeta,
358    },
359}
360
361#[derive(Clone, Debug)]
362pub enum ForwardInputsResult {
363    RawLogits {
364        logits: Tensor,
365    },
366    Embeddings {
367        embeddings: Tensor,
368    },
369    CausalGeneration {
370        logits: Tensor,
371    },
372    Image {
373        images: Vec<DynamicImage>,
374    },
375    Speech {
376        pcms: Vec<Arc<Vec<f32>>>,
377        rates: Vec<usize>,
378        channels: Vec<usize>,
379    },
380}
381
382impl ForwardInputsResult {
383    fn index_bs(&self, bs_idx: usize) -> candle_core::Result<Self> {
384        match self {
385            Self::CausalGeneration { logits } => Ok(Self::CausalGeneration {
386                logits: logits.i(bs_idx)?,
387            }),
388            Self::Embeddings { embeddings } => Ok(Self::Embeddings {
389                embeddings: embeddings.i(bs_idx)?,
390            }),
391            Self::RawLogits { logits } => Ok(Self::RawLogits {
392                logits: logits.i(bs_idx)?,
393            }),
394            Self::Image { images } => Ok(Self::Image {
395                images: vec![images[bs_idx].clone()],
396            }),
397            Self::Speech {
398                pcms,
399                rates,
400                channels,
401            } => Ok(Self::Speech {
402                pcms: vec![pcms[bs_idx].clone()],
403                rates: vec![rates[bs_idx]],
404                channels: vec![channels[bs_idx]],
405            }),
406        }
407    }
408
409    fn to_device(&self, device: &Device) -> candle_core::Result<Self> {
410        match self {
411            Self::CausalGeneration { logits } => Ok(Self::CausalGeneration {
412                logits: logits.to_device(device)?,
413            }),
414            Self::RawLogits { logits } => Ok(Self::RawLogits {
415                logits: logits.to_device(device)?,
416            }),
417            Self::Embeddings { embeddings } => Ok(Self::Embeddings {
418                embeddings: embeddings.to_device(device)?,
419            }),
420            Self::Image { .. } => Ok(self.clone()),
421            Self::Speech { .. } => Ok(self.clone()),
422        }
423    }
424}
425
426#[derive(serde::Serialize, serde::Deserialize)]
427pub(crate) struct FileListCache {
428    files: Vec<String>,
429}
430
431#[async_trait::async_trait]
432pub trait Pipeline:
433    Send
434    + Sync
435    + PreProcessingMixin
436    + IsqPipelineMixin
437    + CacheManagerMixin
438    + MetadataMixin
439    + AnyMoePipelineMixin
440{
441    fn forward_inputs(
442        &mut self,
443        inputs: Box<dyn Any>,
444        return_raw_logits: bool,
445    ) -> Result<ForwardInputsResult, candle_core::Error>;
446
447    /// Returns the total of model execution time.
448    #[allow(clippy::too_many_arguments)]
449    async fn step(
450        &mut self,
451        input_seqs: &mut [&mut Sequence],
452        is_prompt: bool,
453        return_raw_logits: bool,
454        prefix_cacher: &mut PrefixCacheManagerV2,
455        disable_eos_stop: bool,
456        rng: Arc<std::sync::Mutex<Isaac64Rng>>,
457        backend_metadata: CacheBackendMetadata,
458    ) -> Result<Duration, candle_core::Error> {
459        match backend_metadata {
460            CacheBackendMetadata::DefaultInstructions { pre_op, post_op } => {
461                let inputs_iter =
462                    std::iter::once(self.get_processor().inputs_processor().process_inputs(
463                        self.tokenizer(),
464                        input_seqs,
465                        is_prompt,
466                        self.get_metadata().is_xlora,
467                        &self.device(),
468                        self.get_metadata().no_kv_cache,
469                        None,
470                        return_raw_logits,
471                        self.get_metadata().sliding_window,
472                        self.get_input_processor_config(),
473                        None,
474                        self.device_mapper(),
475                    ));
476
477                let mut logits = vec![None; input_seqs.len()];
478                let len_inputs = 1;
479                let mut raw_out_logits = vec![vec![None; len_inputs]; input_seqs.len()];
480                let mut embedding_logits = vec![None; input_seqs.len()];
481
482                let mut exec_duration = Duration::ZERO;
483                for (i, inputs) in inputs_iter.into_iter().enumerate() {
484                    let InputProcessorOutput {
485                        inputs,
486                        seq_indices,
487                    } = inputs.map_err(candle_core::Error::msg)?;
488                    if i == 0 {
489                        match pre_op {
490                            CacheInstruction::In => self.clone_in_cache(input_seqs),
491                            CacheInstruction::Nothing => (),
492                            CacheInstruction::Reset {
493                                load_preallocated_cache,
494                                reset_non_granular,
495                            } => self.set_none_cache(
496                                input_seqs,
497                                reset_non_granular,
498                                false,
499                                load_preallocated_cache,
500                            ),
501                            _ => unreachable!("Unreachable PRE cache op."),
502                        }
503                    }
504
505                    let start = Instant::now();
506                    let raw_logits = self.forward_inputs(inputs, return_raw_logits)?;
507                    let end = Instant::now();
508                    exec_duration += end.duration_since(start);
509
510                    for (logit_idx, seq_idx) in seq_indices.into_iter().enumerate() {
511                        if let ForwardInputsResult::RawLogits { logits } = &raw_logits {
512                            raw_out_logits[seq_idx][i] =
513                                Some(logits.i(logit_idx)?.to_device(&Device::Cpu)?);
514                        } else if let ForwardInputsResult::Embeddings { embeddings } = &raw_logits {
515                            embedding_logits[seq_idx] =
516                                Some(embeddings.i(logit_idx)?.to_device(&Device::Cpu)?);
517                        } else {
518                            logits[seq_idx] = Some(raw_logits.index_bs(logit_idx)?);
519                        }
520                    }
521                }
522
523                match post_op {
524                    CacheInstruction::Out => self.clone_out_cache(input_seqs),
525                    CacheInstruction::Nothing => (),
526                    CacheInstruction::Reset {
527                        load_preallocated_cache,
528                        reset_non_granular,
529                    } => self.set_none_cache(
530                        input_seqs,
531                        reset_non_granular,
532                        false,
533                        load_preallocated_cache,
534                    ),
535                    _ => unreachable!("Unreachable POST cache op."),
536                }
537
538                if raw_out_logits[0][0].is_some() {
539                    let start = Instant::now();
540                    response::send_raw_responses(
541                        input_seqs,
542                        raw_out_logits
543                            .into_iter()
544                            .map(|raw| raw.into_iter().flatten().collect::<Vec<_>>())
545                            .collect(),
546                    )
547                    .await?;
548                    let end = Instant::now();
549                    exec_duration += end.duration_since(start);
550
551                    return Ok(exec_duration);
552                }
553                if embedding_logits[0].is_some() {
554                    let start = Instant::now();
555                    response::send_embedding_responses(
556                        input_seqs,
557                        embedding_logits
558                            .into_iter()
559                            .map(|raw| {
560                                raw.unwrap()
561                                    .to_dtype(DType::F32)
562                                    .unwrap()
563                                    .to_vec1::<f32>()
564                                    .unwrap()
565                            })
566                            .collect(),
567                    )
568                    .await?;
569                    let end = Instant::now();
570                    exec_duration += end.duration_since(start);
571
572                    return Ok(exec_duration);
573                }
574
575                let start = Instant::now();
576                let logits_on_cpu = logits.len() > 1;
577                let logits = logits
578                    .into_iter()
579                    .map(|l| {
580                        let l = l.expect("Did not get any inputs. This is shocking.");
581                        if logits_on_cpu {
582                            l.to_device(&Device::Cpu)
583                        } else {
584                            Ok(l)
585                        }
586                    })
587                    .collect::<candle_core::Result<Vec<_>>>()?;
588
589                match &logits[0] {
590                    ForwardInputsResult::RawLogits { .. }
591                    | ForwardInputsResult::Embeddings { .. } => unreachable!(),
592                    ForwardInputsResult::CausalGeneration { .. } => {
593                        self.sample_causal_gen(
594                            input_seqs,
595                            logits
596                                .into_iter()
597                                .map(|r| {
598                                    #[allow(irrefutable_let_patterns)]
599                                    let ForwardInputsResult::CausalGeneration { logits } = r
600                                    else {
601                                        unreachable!(
602                                            "All results must have same type, `CausalGeneration`"
603                                        )
604                                    };
605                                    logits
606                                })
607                                .collect::<Vec<_>>(),
608                            prefix_cacher,
609                            disable_eos_stop,
610                            rng,
611                        )
612                        .await?;
613                    }
614                    ForwardInputsResult::Image { .. } => {
615                        response::send_image_responses(
616                            input_seqs,
617                            logits
618                                .into_iter()
619                                .map(|r| {
620                                    #[allow(irrefutable_let_patterns)]
621                                    let ForwardInputsResult::Image { images } = r
622                                    else {
623                                        unreachable!("All results must have same type, `Image`")
624                                    };
625                                    images
626                                        .into_iter()
627                                        .next()
628                                        .expect("Must have at least 1 element.")
629                                })
630                                .collect::<Vec<_>>(),
631                        )
632                        .await?;
633                    }
634                    ForwardInputsResult::Speech { .. } => {
635                        let rates = logits
636                            .iter()
637                            .map(|r| {
638                                #[allow(irrefutable_let_patterns)]
639                                let ForwardInputsResult::Speech { rates, .. } = r
640                                else {
641                                    unreachable!("All results must have same type, `Speech`")
642                                };
643                                assert_eq!(rates.len(), 1, "Each sequence must have 1 PCM output.");
644                                *rates.first().unwrap()
645                            })
646                            .collect::<Vec<_>>();
647                        let channels = logits
648                            .iter()
649                            .map(|r| {
650                                #[allow(irrefutable_let_patterns)]
651                                let ForwardInputsResult::Speech { channels, .. } = r
652                                else {
653                                    unreachable!("All results must have same type, `Speech`")
654                                };
655                                assert_eq!(
656                                    channels.len(),
657                                    1,
658                                    "Each sequence must have 1 PCM output."
659                                );
660                                *channels.first().unwrap()
661                            })
662                            .collect::<Vec<_>>();
663                        let pcms = logits
664                            .into_iter()
665                            .map(|r| {
666                                #[allow(irrefutable_let_patterns)]
667                                let ForwardInputsResult::Speech { pcms, .. } = r
668                                else {
669                                    unreachable!("All results must have same type, `Speech`")
670                                };
671                                assert_eq!(pcms.len(), 1, "Each sequence must have 1 PCM output.");
672                                pcms.into_iter().nth(0).unwrap()
673                            })
674                            .collect::<Vec<_>>();
675                        response::send_speech_responses(input_seqs, &pcms, &rates, &channels)
676                            .await?;
677                    }
678                }
679                let end = Instant::now();
680                exec_duration += end.duration_since(start);
681
682                Ok(exec_duration)
683            }
684            CacheBackendMetadata::PagedAttention { metadata } => {
685                // For hybrid models, build state_indices tensor from sequences'
686                // recurrent_state_idx so recurrent layers are active during forward.
687                // Paged attention manages KV caches separately, but recurrent state
688                // pool access still needs the indices tensor to be set.
689                if self.cache().is_hybrid() {
690                    let mut hybrid_cache = self.cache().hybrid();
691                    let recurrent_device = hybrid_cache.caches.iter().find_map(|c| {
692                        if let HybridLayerCache::Recurrent(pool) = c {
693                            Some(pool.device().clone())
694                        } else {
695                            None
696                        }
697                    });
698                    if let Some(device) = recurrent_device {
699                        #[allow(clippy::cast_possible_truncation)]
700                        let indices: Vec<u32> = input_seqs
701                            .iter()
702                            .filter_map(|seq| seq.recurrent_state_idx().map(|idx| idx as u32))
703                            .collect();
704                        if indices.len() == input_seqs.len() {
705                            if let Ok(si) = Tensor::from_vec(indices, (input_seqs.len(),), &device)
706                            {
707                                hybrid_cache.set_state_indices(Some(si));
708                            }
709                        }
710                    }
711                }
712
713                let inputs_iter =
714                    std::iter::once(self.get_processor().inputs_processor().process_inputs(
715                        self.tokenizer(),
716                        input_seqs,
717                        is_prompt,
718                        self.get_metadata().is_xlora,
719                        &self.device(),
720                        self.get_metadata().no_kv_cache,
721                        None,
722                        return_raw_logits,
723                        self.get_metadata().sliding_window,
724                        self.get_input_processor_config(),
725                        Some(metadata),
726                        self.device_mapper(),
727                    ));
728
729                let mut logits = vec![None; input_seqs.len()];
730                let len_inputs = 1;
731                let mut raw_out_logits = vec![vec![None; len_inputs]; input_seqs.len()];
732                let mut embedding_logits = vec![None; input_seqs.len()];
733
734                let mut exec_duration = Duration::ZERO;
735                for (i, inputs) in inputs_iter.into_iter().enumerate() {
736                    let InputProcessorOutput {
737                        inputs,
738                        seq_indices,
739                    } = inputs.map_err(candle_core::Error::msg)?;
740
741                    let start = Instant::now();
742                    let raw_logits = self.forward_inputs(inputs, return_raw_logits)?;
743                    let end = Instant::now();
744                    exec_duration += end.duration_since(start);
745
746                    for (logit_idx, seq_idx) in seq_indices.into_iter().enumerate() {
747                        if let ForwardInputsResult::RawLogits { logits } = &raw_logits {
748                            raw_out_logits[seq_idx][i] =
749                                Some(logits.i(logit_idx)?.to_device(&Device::Cpu)?);
750                        } else if let ForwardInputsResult::Embeddings { embeddings } = &raw_logits {
751                            embedding_logits[seq_idx] =
752                                Some(embeddings.i(logit_idx)?.to_device(&Device::Cpu)?);
753                        } else {
754                            logits[seq_idx] = Some(raw_logits.index_bs(logit_idx)?);
755                        }
756                    }
757                }
758
759                if raw_out_logits[0][0].is_some() {
760                    let start = Instant::now();
761                    response::send_raw_responses(
762                        input_seqs,
763                        raw_out_logits
764                            .into_iter()
765                            .map(|raw| raw.into_iter().flatten().collect::<Vec<_>>())
766                            .collect(),
767                    )
768                    .await?;
769                    let end = Instant::now();
770                    exec_duration += end.duration_since(start);
771
772                    return Ok(exec_duration);
773                }
774                if embedding_logits[0].is_some() {
775                    let start = Instant::now();
776                    response::send_embedding_responses(
777                        input_seqs,
778                        embedding_logits
779                            .into_iter()
780                            .map(|raw| {
781                                raw.unwrap()
782                                    .to_dtype(DType::F32)
783                                    .unwrap()
784                                    .to_vec1::<f32>()
785                                    .unwrap()
786                            })
787                            .collect(),
788                    )
789                    .await?;
790                    let end = Instant::now();
791                    exec_duration += end.duration_since(start);
792
793                    return Ok(exec_duration);
794                }
795
796                let start = Instant::now();
797                let logits_on_cpu = logits.len() > 1;
798                let logits = logits
799                    .into_iter()
800                    .map(|l| {
801                        let l = l.expect("Did not get any inputs. This is shocking.");
802                        if logits_on_cpu {
803                            l.to_device(&Device::Cpu)
804                        } else {
805                            Ok(l)
806                        }
807                    })
808                    .collect::<candle_core::Result<Vec<_>>>()?;
809
810                match &logits[0] {
811                    ForwardInputsResult::RawLogits { .. }
812                    | ForwardInputsResult::Embeddings { .. } => unreachable!(),
813                    ForwardInputsResult::CausalGeneration { .. } => {
814                        self.sample_causal_gen(
815                            input_seqs,
816                            logits
817                                .into_iter()
818                                .map(|r| {
819                                    #[allow(irrefutable_let_patterns)]
820                                    let ForwardInputsResult::CausalGeneration { logits } = r
821                                    else {
822                                        unreachable!("All results must have same type")
823                                    };
824                                    logits
825                                })
826                                .collect::<Vec<_>>(),
827                            prefix_cacher,
828                            disable_eos_stop,
829                            rng,
830                        )
831                        .await?;
832                    }
833                    ForwardInputsResult::Image { .. } => {
834                        response::send_image_responses(
835                            input_seqs,
836                            logits
837                                .into_iter()
838                                .map(|r| {
839                                    #[allow(irrefutable_let_patterns)]
840                                    let ForwardInputsResult::Image { images } = r
841                                    else {
842                                        unreachable!("All results must have same type, `Image`")
843                                    };
844                                    images
845                                        .into_iter()
846                                        .next()
847                                        .expect("Must have at least 1 element.")
848                                })
849                                .collect::<Vec<_>>(),
850                        )
851                        .await?;
852                    }
853                    ForwardInputsResult::Speech { .. } => {
854                        let rates = logits
855                            .iter()
856                            .map(|r| {
857                                #[allow(irrefutable_let_patterns)]
858                                let ForwardInputsResult::Speech { rates, .. } = r
859                                else {
860                                    unreachable!("All results must have same type, `Speech`")
861                                };
862                                assert_eq!(rates.len(), 1, "Each sequence must have 1 PCM output.");
863                                *rates.first().unwrap()
864                            })
865                            .collect::<Vec<_>>();
866                        let channels = logits
867                            .iter()
868                            .map(|r| {
869                                #[allow(irrefutable_let_patterns)]
870                                let ForwardInputsResult::Speech { channels, .. } = r
871                                else {
872                                    unreachable!("All results must have same type, `Speech`")
873                                };
874                                assert_eq!(
875                                    channels.len(),
876                                    1,
877                                    "Each sequence must have 1 PCM output."
878                                );
879                                *channels.first().unwrap()
880                            })
881                            .collect::<Vec<_>>();
882                        let pcms = logits
883                            .into_iter()
884                            .map(|r| {
885                                #[allow(irrefutable_let_patterns)]
886                                let ForwardInputsResult::Speech { pcms, .. } = r
887                                else {
888                                    unreachable!("All results must have same type, `Speech`")
889                                };
890                                assert_eq!(pcms.len(), 1, "Each sequence must have 1 PCM output.");
891                                pcms.into_iter().nth(0).unwrap()
892                            })
893                            .collect::<Vec<_>>();
894                        response::send_speech_responses(input_seqs, &pcms, &rates, &channels)
895                            .await?;
896                    }
897                }
898                let end = Instant::now();
899                exec_duration += end.duration_since(start);
900
901                Ok(exec_duration)
902            }
903        }
904    }
905
906    async fn sample_causal_gen(
907        &self,
908        seqs: &mut [&mut Sequence],
909        logits: Vec<Tensor>,
910        prefix_cacher: &mut PrefixCacheManagerV2,
911        disable_eos_stop: bool,
912        rng: Arc<std::sync::Mutex<Isaac64Rng>>,
913    ) -> Result<(), candle_core::Error>;
914
915    fn category(&self) -> ModelCategory;
916
917    /// Return encoder cache hit/miss counters (hits, misses) if this pipeline has an encoder cache.
918    fn encoder_cache_counters(&self) -> Option<(Arc<AtomicUsize>, Arc<AtomicUsize>)> {
919        None
920    }
921}
922
923pub(crate) fn extract_logits(
924    logits: &Tensor,
925    context_lens: Vec<(usize, usize)>,
926) -> candle_core::Result<Tensor> {
927    let mut toks = Vec::new();
928    for (dim, (start, len)) in logits.chunk(logits.dims()[0], 0)?.iter().zip(context_lens) {
929        toks.push(dim.narrow(1, start, len)?);
930    }
931    Tensor::cat(&toks, 0)
932}
933
934#[cfg(test)]
935mod tests {
936    use crate::MessageContent;
937    use either::Either;
938    use indexmap::IndexMap;
939    use serde_json::Value;
940
941    macro_rules! hashmap {
942        (@single $($x:tt)*) => (());
943        (@count $($rest:expr),*) => (<[()]>::len(&[$(hashmap!(@single $rest)),*]));
944
945        ($($key:expr => $value:expr,)+) => { hashmap!($($key => $value),+) };
946        ($($key:expr => $value:expr),*) => {
947            {
948                let _cap = hashmap!(@count $($key),*);
949                let mut _map = ::indexmap::IndexMap::with_capacity(_cap);
950                $(
951                    let _ = _map.insert($key, Value::String($value));
952                )*
953                _map
954            }
955        };
956    }
957
958    #[cfg(test)]
959    #[track_caller]
960    fn test_with_inputs(
961        templates: &[(bool, &str, &str, &str, &str)],
962        expected_outputs: &[&str],
963        inputs: Vec<IndexMap<String, MessageContent>>,
964    ) {
965        use crate::pipeline::chat_template::ChatTemplateValue;
966
967        use super::chat_template::apply_chat_template_to;
968        let mut failed = Vec::new();
969        let n_templates = templates.len();
970        for ((has_system, bos, eos, unk, template), expected) in
971            templates.iter().zip(expected_outputs)
972        {
973            let output = match apply_chat_template_to(
974                if !has_system {
975                    inputs[1..].to_vec()
976                } else {
977                    inputs.clone()
978                },
979                true,
980                None,
981                None, // reasoning_effort
982                &ChatTemplateValue(Either::Left(template.to_string())),
983                Some(bos.to_string()),
984                Some(eos.to_string()),
985                Some(unk.to_string()),
986                Vec::new(),
987            ) {
988                Ok(v) => v,
989                Err(e) => {
990                    failed.push(format!("Failed with {e}."));
991                    continue;
992                }
993            };
994            if output != *expected {
995                failed.push(format!(
996                    "Expected: `{}` \n\nGot:      `{}`",
997                    expected.replace('\n', "\\n"),
998                    output.replace('\n', "\\n")
999                ));
1000            }
1001        }
1002        if !failed.is_empty() {
1003            for (i, line) in failed.iter().enumerate() {
1004                println!("------------ Template {i} ------------");
1005                println!("{line}");
1006            }
1007            println!("------------------------");
1008            panic!("{}/{n_templates} chat templates failed.", failed.len());
1009        }
1010    }
1011
1012    #[test]
1013    /// Generating these cases:
1014    /// ```py
1015    /// >>> t=transformers.AutoTokenizer.from_pretrained(...)
1016    /// # If non-system prompt model
1017    /// >>> t.apply_chat_template([{"role":"user","content":"Hello"},{"role":"assistant","content":"Hi there"},{"role":"user","content":"Who are you"},{"role":"assistant","content":"   I am an assistant   "},{"role":"user","content":"Another question"}], add_generation_prompt=True, tokenize=False)
1018    /// # If system prompt model
1019    /// >>> t.apply_chat_template([{"role":"system","content":"You are a helpful assistant"},{"role":"user","content":"Hello"},{"role":"assistant","content":"Hi there"},{"role":"user","content":"Who are you"},{"role":"assistant","content":"   I am an assistant   "},{"role":"user","content":"Another question"}], add_generation_prompt=True, tokenize=False)
1020    /// ```
1021    fn test_chat_templates() {
1022        let templates = [
1023            // ChatML: https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B
1024            (true, "<s>", "</s>", "<unk>", "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"),
1025            // mistralai/Mistral-7B-Instruct-v0.1
1026            (false, "<s>", "</s>", "<unk>", "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}"),
1027            // meta-llama/Llama-2-13b-chat-hf
1028            (true, "<s>", "</s>", "<unk>", "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}"),
1029            // mistralai/Mixtral-8x7B-Instruct-v0.1
1030            (false, "<s>", "</s>", "<unk>", "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}"),
1031            // google/gemma-7b-it
1032            (false, "<bos>", "<eos>", "<unk>", "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}"),
1033            // HuggingFaceM4/idefics2-8b-chatty
1034            (true, "<s>", "</s>", "<unk>", "{% for message in messages %}{{message['role'].capitalize()}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}"),
1035        ];
1036        let expected_outputs = [
1037            // ChatML: https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B
1038            "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nWho are you<|im_end|>\n<|im_start|>assistant\n   I am an assistant   <|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant\n",
1039            // mistralai/Mistral-7B-Instruct-v0.1
1040            "<s>[INST] Hello [/INST]Hi there</s> [INST] Who are you [/INST]   I am an assistant   </s> [INST] Another question [/INST]",
1041            // meta-llama/Llama-2-13b-chat-hf
1042            "<s>[INST] <<SYS>>\nYou are a helpful assistant\n<</SYS>>\n\nHello [/INST] Hi there </s><s>[INST] Who are you [/INST] I am an assistant </s><s>[INST] Another question [/INST]",
1043            // mistralai/Mixtral-8x7B-Instruct-v0.1
1044            "<s>[INST] Hello [/INST]Hi there</s>[INST] Who are you [/INST]   I am an assistant   </s>[INST] Another question [/INST]",
1045            // google/gemma-7b-it
1046            "<bos><start_of_turn>user\nHello<end_of_turn>\n<start_of_turn>model\nHi there<end_of_turn>\n<start_of_turn>user\nWho are you<end_of_turn>\n<start_of_turn>model\nI am an assistant<end_of_turn>\n<start_of_turn>user\nAnother question<end_of_turn>\n<start_of_turn>model\n",
1047        ];
1048        let messages = [
1049            ["system", "You are a helpful assistant"],
1050            ["user", "Hello"],
1051            ["assistant", "Hi there"],
1052            ["user", "Who are you"],
1053            ["assistant", "   I am an assistant   "],
1054            ["user", "Another question"],
1055        ];
1056        let mut inputs = Vec::new();
1057        for [role, content] in messages {
1058            let mut message: IndexMap<String, Either<String, Vec<IndexMap<String, Value>>>> =
1059                IndexMap::new();
1060            message.insert("role".to_string(), Either::Left(role.to_string()));
1061            message.insert("content".to_string(), Either::Left(content.to_string()));
1062            inputs.push(message);
1063        }
1064        test_with_inputs(&templates, &expected_outputs, inputs);
1065    }
1066
1067    #[test]
1068    /// Generating these cases:
1069    /// ```py
1070    /// >>> processor=transformers.AutoProcessor.from_pretrained(...)
1071    /// >>> processor.apply_chat_template([
1072    ///         {"role":"system","content":[{"type":"text", "text": "You are a helpful assistant"}]},
1073    ///         {"role":"user","content":[{"type":"image"}, {"type":"text", "text": "Hello, please describe the above."}]},
1074    ///         {"role":"assistant","content":[{"type":"text", "text": "Hi there"}]},
1075    ///         {"role":"user","content":[{"type":"text", "text": "Who are you"}]},
1076    ///         {"role":"assistant","content":[{"type":"text", "text": "   I am an assistant   "}]},
1077    ///         {"role":"user","content":[{"type":"text", "text": "Another question"}]}
1078    ///     ], add_generation_prompt=True, tokenize=False)
1079    /// ```
1080    fn test_image_chat_templates() {
1081        let templates = [
1082            // HuggingFaceM4/idefics2-8b-chatty
1083            (true, "<s>", "</s>", "<unk>", "{% for message in messages %}{{message['role'].capitalize()}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}"),
1084        ];
1085        let expected_outputs = [
1086            // HuggingFaceM4/idefics2-8b-chatty
1087            "System: You are a helpful assistant<end_of_utterance>\nUser:<image>Hello, please describe the above.<end_of_utterance>\nAssistant: Hi there<end_of_utterance>\nUser:<image>This is me, who are you<end_of_utterance>\nAssistant:    I am an assistant   <end_of_utterance>\nUser:<image>Another question, what is this?<end_of_utterance>\nAssistant:",
1088        ];
1089
1090        let mut inputs = Vec::new();
1091
1092        let mut message: IndexMap<String, Either<String, Vec<IndexMap<String, Value>>>> =
1093            IndexMap::new();
1094        message.insert("role".to_string(), Either::Left("system".to_string()));
1095        message.insert(
1096            "content".to_string(),
1097            Either::Right(vec![hashmap! {
1098                "type".to_string() => "text".to_string(),
1099                "text".to_string() => "You are a helpful assistant".to_string()
1100            }]),
1101        );
1102        inputs.push(message);
1103
1104        let mut message: IndexMap<String, Either<String, Vec<IndexMap<String, Value>>>> =
1105            IndexMap::new();
1106        message.insert("role".to_string(), Either::Left("user".to_string()));
1107        message.insert(
1108            "content".to_string(),
1109            Either::Right(vec![
1110                hashmap! {
1111                    "type".to_string() => "image".to_string()
1112                },
1113                hashmap! {
1114                    "type".to_string() => "text".to_string(),
1115                    "text".to_string() => "Hello, please describe the above.".to_string()
1116                },
1117            ]),
1118        );
1119        inputs.push(message);
1120
1121        let mut message: IndexMap<String, Either<String, Vec<IndexMap<String, Value>>>> =
1122            IndexMap::new();
1123        message.insert("role".to_string(), Either::Left("assistant".to_string()));
1124        message.insert(
1125            "content".to_string(),
1126            Either::Right(vec![hashmap! {
1127                "type".to_string() => "text".to_string(),
1128                "text".to_string() => "Hi there".to_string()
1129            }]),
1130        );
1131        inputs.push(message);
1132
1133        let mut message: IndexMap<String, Either<String, Vec<IndexMap<String, Value>>>> =
1134            IndexMap::new();
1135        message.insert("role".to_string(), Either::Left("user".to_string()));
1136        message.insert(
1137            "content".to_string(),
1138            Either::Right(vec![
1139                hashmap! {
1140                    "type".to_string() => "image".to_string()
1141                },
1142                hashmap! {
1143                    "type".to_string() => "text".to_string(),
1144                    "text".to_string() => "This is me, who are you".to_string()
1145                },
1146            ]),
1147        );
1148        inputs.push(message);
1149
1150        let mut message: IndexMap<String, Either<String, Vec<IndexMap<String, Value>>>> =
1151            IndexMap::new();
1152        message.insert("role".to_string(), Either::Left("assistant".to_string()));
1153        message.insert(
1154            "content".to_string(),
1155            Either::Right(vec![hashmap! {
1156                "type".to_string() => "text".to_string(),
1157                "text".to_string() => "   I am an assistant   ".to_string()
1158            }]),
1159        );
1160        inputs.push(message);
1161
1162        let mut message: IndexMap<String, Either<String, Vec<IndexMap<String, Value>>>> =
1163            IndexMap::new();
1164        message.insert("role".to_string(), Either::Left("user".to_string()));
1165        message.insert(
1166            "content".to_string(),
1167            Either::Right(vec![
1168                hashmap! {
1169                    "type".to_string() => "image".to_string()
1170                },
1171                hashmap! {
1172                    "type".to_string() => "text".to_string(),
1173                    "text".to_string() => "Another question, what is this?".to_string()
1174                },
1175            ]),
1176        );
1177        inputs.push(message);
1178
1179        test_with_inputs(&templates, &expected_outputs, inputs);
1180    }
1181}
mistralrs_core/pipeline/mod.rs

mistralrs_core/pipeline/
mod.rs
