llm 1.3.8

A Rust library unifying multiple LLM backends.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

use std::time::Instant;

use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;

use llm::error::LLMError;

use super::manager::StreamRequest;
use crate::runtime::{AppEvent, StreamEvent};

use super::stream_struct::stream_struct;
use super::stream_text::stream_text;
use super::stream_tools::stream_with_tools;

pub async fn run_stream(
    request: StreamRequest,
    sender: mpsc::Sender<AppEvent>,
    cancel: CancellationToken,
) {
    let _ = sender
        .send(AppEvent::Stream(StreamEvent::Started {
            conversation_id: request.conversation_id,
        }))
        .await;
    let start_time = Instant::now();

    if let Err(err) = stream_with_fallback(&request, &sender, &cancel).await {
        let event = StreamEvent::Error {
            conversation_id: request.conversation_id,
            message_id: request.message_id,
            error: err.to_string(),
        };
        let _ = sender.send(AppEvent::Stream(event)).await;
    }

    let _ = sender
        .send(AppEvent::Stream(StreamEvent::Done {
            conversation_id: request.conversation_id,
        }))
        .await;
    log::debug!("stream finished in {:?}", start_time.elapsed());
}

async fn stream_with_fallback(
    request: &StreamRequest,
    sender: &mpsc::Sender<AppEvent>,
    cancel: &CancellationToken,
) -> Result<(), LLMError> {
    if request.capabilities.tool_streaming
        && stream_with_tools(request, sender, cancel).await.is_ok()
    {
        return Ok(());
    }
    if stream_struct(request, sender, cancel).await.is_ok() {
        return Ok(());
    }
    stream_text(request, sender, cancel).await
}