Enum LLMBackend 

pub enum LLMBackend {
    OpenAI(OpenAIBackend),
    Anthropic(AnthropicBackend),
    GenericApi(GenericApiBackend),
}

Variants

OpenAI(OpenAIBackend)

Anthropic(AnthropicBackend)

GenericApi(GenericApiBackend)

Implementations

impl LLMBackend

pub async fn clear_cache( self: &Arc<LLMBackend>, ) -> Result<CompletionResponse, CompletionError>

pub async fn set_cache( self: &Arc<LLMBackend>, prompt: &LLMPrompt, ) -> Result<CompletionResponse, CompletionError>

pub fn new_prompt(&self) -> LLMPrompt

pub fn get_total_prompt_tokens(&self, prompt: &LLMPrompt) -> Result<u64, Error>

pub fn model_id(&self) -> &str

pub fn model_ctx_size(&self) -> u64

pub fn inference_ctx_size(&self) -> u64

pub fn tokenizer(&self) -> &Arc<Tokenizer>

pub fn build_logit_bias( &self, logit_bias: &mut Option<LogitBias>, ) -> Result<(), Error>

pub fn openai(&self) -> Result<&OpenAIBackend, Error>

pub fn anthropic(&self) -> Result<&AnthropicBackend, Error>

pub fn generic_api(&self) -> Result<&GenericApiBackend, Error>

pub fn shutdown(&self)