llm 1.3.8

A Rust library unifying multiple LLM backends.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

use tokio::sync::mpsc;

use super::manager::StreamRequest;
use crate::runtime::{AppEvent, StreamEvent};

pub const TOKEN_BATCH_SIZE: usize = 32;

pub async fn flush_text_if_needed(
    size: usize,
    buffer: &mut String,
    request: &StreamRequest,
    sender: &mpsc::Sender<AppEvent>,
) {
    if size >= TOKEN_BATCH_SIZE {
        flush_text(buffer, request, sender).await;
    }
}

pub async fn flush_text(
    buffer: &mut String,
    request: &StreamRequest,
    sender: &mpsc::Sender<AppEvent>,
) {
    if buffer.is_empty() {
        return;
    }
    let delta = std::mem::take(buffer);
    let event = StreamEvent::TextDelta {
        conversation_id: request.conversation_id,
        message_id: request.message_id,
        delta,
    };
    let _ = sender.send(AppEvent::Stream(event)).await;
}