Enum LLMBackend

pub enum LLMBackend {
    OpenAI(OpenAIBackend),
    Anthropic(AnthropicBackend),
    GenericApi(GenericApiBackend),
}

Variants

OpenAI(OpenAIBackend)

Anthropic(AnthropicBackend)

GenericApi(GenericApiBackend)

Implementations

impl LLMBackend

pub async fn clear_cache( self: &Arc<Self>, ) -> Result<CompletionResponse, CompletionError>

pub async fn set_cache( self: &Arc<Self>, prompt: &LLMPrompt, ) -> Result<CompletionResponse, CompletionError>

pub fn new_prompt(&self) -> LLMPrompt

pub fn get_total_prompt_tokens(&self, prompt: &LLMPrompt) -> Result<u64>

pub fn model_id(&self) -> &str

pub fn model_ctx_size(&self) -> u64

pub fn inference_ctx_size(&self) -> u64

pub fn tokenizer(&self) -> &Arc<Tokenizer>

pub fn build_logit_bias(&self, logit_bias: &mut Option<LogitBias>) -> Result<()>

pub fn openai(&self) -> Result<&OpenAIBackend>

pub fn anthropic(&self) -> Result<&AnthropicBackend>

pub fn generic_api(&self) -> Result<&GenericApiBackend>

pub fn shutdown(&self)