Struct ModelManager 

pub struct ModelManager { /* private fields */ }

Implementations

impl ModelManager

pub fn new() -> Self

pub fn get_model_entries(&self) -> Vec<ModelEntry>

pub fn has_model_any(&self, model: &str) -> bool

pub fn model_display_names(&self) -> HashSet<String>

pub fn list_chat_completions_models(&self) -> Vec<String>

pub fn list_completions_models(&self) -> Vec<String>

pub fn list_embeddings_models(&self) -> Vec<String>

pub fn list_tensor_models(&self) -> Vec<String>

pub fn add_completions_model( &self, model: &str, engine: OpenAICompletionsStreamingEngine, ) -> Result<(), ModelManagerError>

pub fn add_chat_completions_model( &self, model: &str, engine: OpenAIChatCompletionsStreamingEngine, ) -> Result<(), ModelManagerError>

pub fn add_embeddings_model( &self, model: &str, engine: OpenAIEmbeddingsStreamingEngine, ) -> Result<(), ModelManagerError>

pub fn add_tensor_model( &self, model: &str, engine: TensorStreamingEngine, ) -> Result<(), ModelManagerError>

pub fn remove_completions_model( &self, model: &str, ) -> Result<(), ModelManagerError>

pub fn remove_chat_completions_model( &self, model: &str, ) -> Result<(), ModelManagerError>

pub fn remove_embeddings_model( &self, model: &str, ) -> Result<(), ModelManagerError>

pub fn remove_tensor_model(&self, model: &str) -> Result<(), ModelManagerError>

pub fn get_embeddings_engine( &self, model: &str, ) -> Result<OpenAIEmbeddingsStreamingEngine, ModelManagerError>

pub fn get_completions_engine( &self, model: &str, ) -> Result<OpenAICompletionsStreamingEngine, ModelManagerError>

pub fn get_chat_completions_engine( &self, model: &str, ) -> Result<OpenAIChatCompletionsStreamingEngine, ModelManagerError>

pub fn get_tensor_engine( &self, model: &str, ) -> Result<TensorStreamingEngine, ModelManagerError>

pub fn save_model_entry(&self, key: &str, entry: ModelEntry)

Save a ModelEntry under an instance’s etcd models/ key so we can fetch it later when the key is deleted from etcd.

pub fn remove_model_entry(&self, key: &str) -> Option<ModelEntry>

Remove and return model entry for this instance’s etcd key. We do this when the instance stops.

pub async fn kv_chooser_for( &self, model_name: &str, component: &Component, kv_cache_block_size: u32, kv_router_config: Option<KvRouterConfig>, ) -> Result<Arc<KvRouter>>

pub fn get_model_tool_call_parser(&self, model: &str) -> Option<String>

pub fn get_parsing_options(&self, model: &str) -> ParsingOptions

Creates parsing options with tool call parser and reasoning parser for the specified model. Currently reasoning parser is not implemented (returns None).

Trait Implementations

impl Default for ModelManager

fn default() -> Self

Returns the “default value” for a type.