svir 0.1.6

A small, composable SDK for talking to large language models
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

//! Telling a context overflow from any other failure the server reports.
//!
//! Servers agree on no single sign of it. Some set an error code, some an error type, and some
//! only say it in words, so all three are read (D30).

use serde_json::Value;

use super::wire::{CODE, MESSAGE, TYPE};

/// Values of `error.code` or `error.type` that mean the request did not fit.
const NAMES: [&str; 3] = [
    "context_length_exceeded",
    "context_window_exceeded",
    "exceed_context_size_error",
];

/// What a server that names nothing says about it, in lowercase. Every server observed speaks of
/// the context's length, size, window, or budget, or of compacted context.
const PHRASES: [&str; 5] = [
    "context length",
    "context size",
    "context window",
    "context budget",
    "compacted context",
];

/// Whether an error the server reported is a context overflow. `error` is its error object, or
/// its message alone.
pub(crate) fn is_overflow(error: &Value) -> bool {
    let field = |key: &str| error.get(key).and_then(Value::as_str);
    let named = [field(CODE), field(TYPE)]
        .into_iter()
        .flatten()
        .any(|name| NAMES.contains(&name));

    named
        || field(MESSAGE)
            .or_else(|| error.as_str())
            .is_some_and(says_overflow)
}

/// Whether a message speaks of the context being exceeded, in any letter case.
pub(crate) fn says_overflow(message: &str) -> bool {
    let message = message.to_ascii_lowercase();

    PHRASES.iter().any(|phrase| message.contains(phrase))
}

#[cfg(test)]
mod tests {
    use serde_json::json;

    use super::*;

    #[test]
    fn a_code_or_a_type_names_an_overflow() {
        assert!(is_overflow(&json!({"code": "context_length_exceeded"})));
        assert!(is_overflow(&json!({"code": "context_window_exceeded"})));
        assert!(is_overflow(
            &json!({"code": 400, "type": "exceed_context_size_error"})
        ));
        assert!(!is_overflow(&json!({"code": "unsupported_parameter"})));
        assert!(!is_overflow(
            &json!({"code": 400, "type": "invalid_request_error"})
        ));
    }

    #[test]
    fn a_message_says_it_in_the_words_servers_use() {
        for message in [
            "This model's maximum context length is 8192 tokens.",
            "The number of tokens to keep from the initial prompt is greater than the context length.",
            "the request exceeds the available context size, try increasing it",
            "Input exceeds the Context Window of this model",
            "Protected conversation exceeds the available context budget.",
            "Output reservation leaves no room for compacted context.",
        ] {
            assert!(is_overflow(&json!({"message": message})), "{message}");
            assert!(is_overflow(&json!(message)), "{message}");
        }

        for message in ["engine died", "max_tokens is too large", ""] {
            assert!(!is_overflow(&json!({"message": message})), "{message}");
        }
        assert!(!is_overflow(&json!({})));
        assert!(!is_overflow(&json!(null)));
    }
}