af-llm 0.5.0

Unified async LLM client with timeout and circuit breaking for OpenAI-compatible endpoints.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

use std::io::{Read, Write};
use std::net::TcpListener;
use std::time::Duration;

use af_llm::{ChatMessage, CompletionRequest, LlmClient, LlmConfig, LlmError};

#[tokio::test]
async fn premature_http_stream_end_without_done_fails_closed() {
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let address = listener.local_addr().unwrap();
    let server = std::thread::spawn(move || {
        let (mut socket, _) = listener.accept().unwrap();
        let mut request = [0; 4096];
        let _ = socket.read(&mut request).unwrap();
        let body = b"data: {\"id\":\"x\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"partial\"},\"finish_reason\":\"stop\"}]}\n\n";
        write!(
            socket,
            "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
            body.len()
        )
        .unwrap();
        socket.write_all(body).unwrap();
    });

    let mut config = LlmConfig::new(format!("http://{address}/v1"), "");
    config.timeout = Duration::from_secs(2);
    let client = LlmClient::new(config).unwrap();
    let request = CompletionRequest::new("model", vec![ChatMessage::user("hello")]).stream(true);
    let error = client
        .complete_stream_single_attempt(&request, |_, _| {})
        .await
        .unwrap_err();

    server.join().unwrap();
    assert!(matches!(error, LlmError::StreamProtocol(message) if message.contains("[DONE]")));
}