llm-cascade 0.1.0

Resilient cascading LLM inference with automatic failover across multiple providers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

use crate::error::ProviderError;
use crate::models::{ContentBlock, Conversation, LlmResponse};

#[async_trait::async_trait]
pub trait LlmProvider: Send + Sync {
    async fn complete(&self, conversation: &Conversation) -> Result<LlmResponse, ProviderError>;

    fn provider_name(&self) -> &str;

    fn model_name(&self) -> &str;

    fn entry_key(&self) -> String {
        format!("{}/{}", self.provider_name(), self.model_name())
    }
}