af-llm 0.14.0

Unified async LLM client with timeout and circuit breaking for OpenAI-compatible endpoints.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

//! Error type for the LLM client.

use thiserror::Error;

/// Everything that can go wrong on the way to a completion.
///
/// Mirrors the failure modes of the Python `llm_completion` wrapper, but where
/// that returned `None` and swallowed the cause, we keep a typed error so the
/// caller decides how to degrade.
#[derive(Debug, Error)]
pub enum LlmError {
    /// Invalid, unsupported, or unauthorized input rejected before model dispatch.
    /// Adapters must not use this variant after sending a provider request.
    #[error("model input rejected: {0}")]
    InvalidInput(String),
    /// Caller cancelled the provider attempt.
    #[error("llm request cancelled")]
    Cancelled,

    /// Caller deadline elapsed before the provider attempt settled.
    #[error("llm request deadline exceeded")]
    DeadlineExceeded,
    /// Circuit breaker is open — we fail fast without hitting the provider.
    #[error("llm circuit breaker is open; failing fast")]
    CircuitOpen,

    /// Transport-level failure (connection, TLS, timeout).
    #[error("llm transport error: {0}")]
    Transport(#[from] reqwest::Error),

    /// Provider returned a non-2xx status.
    #[error("llm api error: status {status}, body: {body}")]
    Api {
        /// HTTP status returned.
        status: u16,
        /// Response body, truncated for logs.
        body: String,
    },

    /// Response body could not be decoded into the expected shape.
    #[error("llm response decode error: {0}")]
    Decode(#[from] serde_json::Error),

    /// A streaming response violated the SSE completion contract.
    #[error("llm stream protocol error: {0}")]
    StreamProtocol(String),

    /// A streaming response contained bytes that were not valid UTF-8.
    #[error("llm stream utf-8 error: {0}")]
    InvalidUtf8(#[from] std::string::FromUtf8Error),
}

impl LlmError {
    /// Provider explicitly rejected the input context; generic payload errors do not qualify.
    pub fn is_context_overflow(&self) -> bool {
        let Self::Api {
            status: 400 | 413,
            body,
        } = self
        else {
            return false;
        };
        let Ok(value) = serde_json::from_str::<serde_json::Value>(body) else {
            return false;
        };
        value
            .pointer("/error/code")
            .and_then(serde_json::Value::as_str)
            == Some("context_length_exceeded")
    }

    /// Whether retrying this error class could plausibly succeed.
    ///
    /// A 4xx (except 408/429) is the caller's fault and will fail identically on
    /// retry; transport errors and 5xx/408/429 are worth another attempt.
    pub fn is_retryable(&self) -> bool {
        match self {
            LlmError::Transport(_) | LlmError::StreamProtocol(_) | LlmError::InvalidUtf8(_) => true,
            LlmError::Api { status, .. } => {
                *status == 408 || *status == 429 || (500..600).contains(status)
            }
            // Circuit-open and decode failures are not improved by retry.
            LlmError::InvalidInput(_)
            | LlmError::CircuitOpen
            | LlmError::Decode(_)
            | LlmError::Cancelled
            | LlmError::DeadlineExceeded => false,
        }
    }
}

/// Result alias for this crate's error.
pub type Result<T> = std::result::Result<T, LlmError>;