limit-llm 0.0.46

Multi-provider LLM client for Rust with streaming support. Supports Anthropic Claude, OpenAI, and z.ai.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

//! Helper functions for applying cache control to messages.

use crate::config::CacheSettings;
use crate::types::{CacheControl, Message, Role};

/// Apply cache control to strategic message positions.
///
/// Strategy (Anthropic/OpenAI best practices):
/// 1. System message (if present) - first message if role is System
/// 2. Last user message - enables caching of conversation context
///
/// Returns a new Vec with cache_control applied where appropriate.
pub fn apply_cache_control(messages: &[Message], settings: &CacheSettings) -> Vec<Message> {
    if !settings.is_enabled() {
        return messages.to_vec();
    }

    let cache_control = if settings.is_long_retention() {
        CacheControl::ephemeral_long()
    } else {
        CacheControl::ephemeral()
    };

    let mut result = messages.to_vec();
    let len = result.len();

    // Apply to system message (first message if role is System)
    if len > 0 && result[0].role == Role::System {
        result[0].cache_control = Some(cache_control.clone());
    }

    // Apply to last user message
    for i in (0..len).rev() {
        if result[i].role == Role::User {
            result[i].cache_control = Some(cache_control.clone());
            break;
        }
    }

    result
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_no_caching_when_disabled() {
        let settings = CacheSettings {
            retention: "none".to_string(),
        };
        let messages = vec![Message {
            role: Role::User,
            content: Some(crate::MessageContent::text("Hello")),
            tool_calls: None,
            tool_call_id: None,
            cache_control: None,
        }];

        let result = apply_cache_control(&messages, &settings);
        assert!(result[0].cache_control.is_none());
    }

    #[test]
    fn test_cache_applied_to_user_message() {
        let settings = CacheSettings {
            retention: "short".to_string(),
        };
        let messages = vec![Message {
            role: Role::User,
            content: Some(crate::MessageContent::text("Hello")),
            tool_calls: None,
            tool_call_id: None,
            cache_control: None,
        }];

        let result = apply_cache_control(&messages, &settings);
        assert!(result[0].cache_control.is_some());
    }

    #[test]
    fn test_cache_applied_to_system_and_last_user() {
        let settings = CacheSettings {
            retention: "short".to_string(),
        };
        let messages = vec![
            Message {
                role: Role::System,
                content: Some(crate::MessageContent::text("System prompt")),
                tool_calls: None,
                tool_call_id: None,
                cache_control: None,
            },
            Message {
                role: Role::User,
                content: Some(crate::MessageContent::text("Hello")),
                tool_calls: None,
                tool_call_id: None,
                cache_control: None,
            },
            Message {
                role: Role::Assistant,
                content: Some(crate::MessageContent::text("Hi!")),
                tool_calls: None,
                tool_call_id: None,
                cache_control: None,
            },
            Message {
                role: Role::User,
                content: Some(crate::MessageContent::text("How are you?")),
                tool_calls: None,
                tool_call_id: None,
                cache_control: None,
            },
        ];

        let result = apply_cache_control(&messages, &settings);

        // System message should have cache
        assert!(result[0].cache_control.is_some());
        // First user message should NOT have cache (only last)
        assert!(result[1].cache_control.is_none());
        // Assistant should NOT have cache
        assert!(result[2].cache_control.is_none());
        // Last user message should have cache
        assert!(result[3].cache_control.is_some());
    }

    #[test]
    fn test_long_retention_ttl() {
        let settings = CacheSettings {
            retention: "long".to_string(),
        };
        let messages = vec![Message {
            role: Role::User,
            content: Some(crate::MessageContent::text("Hello")),
            tool_calls: None,
            tool_call_id: None,
            cache_control: None,
        }];

        let result = apply_cache_control(&messages, &settings);
        let cc = result[0].cache_control.as_ref().unwrap();
        assert_eq!(cc.ttl, Some("1h".to_string()));
    }
}