ai-lib 0.4.0

A unified AI SDK for Rust providing a single interface for multiple AI providers with hybrid architecture
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

//! Batch processing module.
//!
//! This module handles concurrent execution of multiple chat completion requests.
//! It supports:
//! - Fixed concurrency limits
//! - Smart batching (auto-tuning concurrency)

use super::AiClient;
use crate::types::{AiLibError, ChatCompletionRequest, ChatCompletionResponse};

pub async fn chat_completion_batch(
    client: &AiClient,
    requests: Vec<ChatCompletionRequest>,
    concurrency_limit: Option<usize>,
) -> Result<Vec<Result<ChatCompletionResponse, AiLibError>>, AiLibError> {
    client
        .chat_provider
        .batch(requests, concurrency_limit)
        .await
}

pub async fn chat_completion_batch_smart(
    client: &AiClient,
    requests: Vec<ChatCompletionRequest>,
) -> Result<Vec<Result<ChatCompletionResponse, AiLibError>>, AiLibError> {
    // Use sequential processing for small batches, concurrent processing for large batches
    let concurrency_limit = if requests.len() <= 3 { None } else { Some(10) };
    chat_completion_batch(client, requests, concurrency_limit).await
}