llm 1.3.8

A Rust library unifying multiple LLM backends.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

use std::pin::Pin;

use async_trait::async_trait;
use futures::stream::Stream;

use crate::{
    chat::{ChatMessage, ChatProvider, ChatResponse, StreamChunk, StreamResponse, Tool},
    error::LLMError,
};

use super::wrapper::ResilientLLM;

#[async_trait]
impl ChatProvider for ResilientLLM {
    async fn chat_with_tools(
        &self,
        messages: &[ChatMessage],
        tools: Option<&[Tool]>,
    ) -> Result<Box<dyn ChatResponse>, LLMError> {
        self.retry(|| self.inner.chat_with_tools(messages, tools))
            .await
    }

    async fn chat_stream(
        &self,
        messages: &[ChatMessage],
    ) -> Result<Pin<Box<dyn Stream<Item = Result<String, LLMError>> + Send>>, LLMError> {
        self.retry(|| self.inner.chat_stream(messages)).await
    }

    async fn chat_stream_struct(
        &self,
        messages: &[ChatMessage],
    ) -> Result<Pin<Box<dyn Stream<Item = Result<StreamResponse, LLMError>> + Send>>, LLMError>
    {
        self.retry(|| self.inner.chat_stream_struct(messages)).await
    }

    async fn chat_stream_with_tools(
        &self,
        messages: &[ChatMessage],
        tools: Option<&[Tool]>,
    ) -> Result<Pin<Box<dyn Stream<Item = Result<StreamChunk, LLMError>> + Send>>, LLMError> {
        self.retry(|| self.inner.chat_stream_with_tools(messages, tools))
            .await
    }
}