llm 1.3.8

A Rust library unifying multiple LLM backends.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

use llm::models::ModelListRequest;
use llm::LLMProvider;
use std::sync::Arc;

use crate::config::ModelConfig;

use super::entry::ModelInfo;

#[derive(Debug, Clone)]
pub struct ModelCatalog {
    models: Vec<ModelInfo>,
}

impl ModelCatalog {
    pub fn from_config(models: &[ModelConfig]) -> Self {
        let entries = models
            .iter()
            .map(|cfg| ModelInfo {
                id: cfg.id.clone(),
                context_window: cfg.context_window,
                max_output_tokens: cfg.max_output_tokens,
                pricing: cfg.pricing.clone(),
                supports_tools: cfg.supports_tools,
                supports_vision: cfg.supports_vision,
                supports_streaming: cfg.supports_streaming,
            })
            .collect();
        Self { models: entries }
    }

    pub async fn from_provider(provider: Arc<dyn LLMProvider>) -> Self {
        let response = provider
            .list_models(Some(&ModelListRequest::default()))
            .await;
        let entries = response
            .map(|resp| {
                resp.get_models()
                    .into_iter()
                    .map(ModelInfo::new)
                    .collect::<Vec<_>>()
            })
            .unwrap_or_default();
        Self { models: entries }
    }

    pub fn is_empty(&self) -> bool {
        self.models.is_empty()
    }

    pub fn list(&self) -> &[ModelInfo] {
        &self.models
    }
}