flyllm 0.3.1

A Rust library for unifying LLM backends as an abstraction layer with load balancing.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153

use log::debug;
use rand::Rng;

use crate::load_balancer::tracker::InstanceTracker;

/// Trait defining the interface for load balancing strategies
///
/// Implementations of this trait determine how to select which LLM instance
/// will handle a particular request based on instance metrics.
pub trait LoadBalancingStrategy {
    /// Select an instance from available candidates
    ///
    /// # Parameters
    /// * `trackers` - Array of (id, tracker) tuples for available instances
    ///
    /// # Returns
    /// * Index into the trackers array of the selected instance
    fn select_instance(&mut self, trackers: &[(usize, &InstanceTracker)]) -> usize;
}

/// Strategy that selects the instance that was used least recently
/// 
/// This strategy helps distribute load by prioritizing instances
/// that haven't been used in the longest time.
pub struct LeastRecentlyUsedStrategy;

impl LeastRecentlyUsedStrategy {
    /// Creates a new LeastRecentlyUsedStrategy
    pub fn new() -> Self {
        Self {}
    }
}

impl LoadBalancingStrategy for LeastRecentlyUsedStrategy {
    /// Select the instance with the oldest last_used timestamp
    ///
    /// # Parameters
    /// * `trackers` - Array of (id, tracker) tuples for available instances
    ///
    /// # Returns
    /// * Index into the trackers array of the least recently used instance
    ///
    /// # Panics
    /// Panics if `trackers` is empty
    fn select_instance(&mut self, trackers: &[(usize, &InstanceTracker)]) -> usize {
        if trackers.is_empty() {
            panic!("LoadBalancingStrategy::select_instance called with empty trackers slice");
        }
        
        let mut oldest_index = 0;
        let mut oldest_time = trackers[0].1.last_used;
        
        for (i, (_id, tracker)) in trackers.iter().enumerate().skip(1) {
            if tracker.last_used < oldest_time {
                oldest_index = i;
                oldest_time = tracker.last_used;
            }
        }
        
        debug!(
            "LeastRecentlyUsedStrategy: Selected index {} (ID: {}) from {} eligible trackers with last_used: {:?}",
            oldest_index, trackers[oldest_index].0, trackers.len(), oldest_time
        );
        
        oldest_index
    }
}


/// Strategy that selects the instance with the lowest average response time.
#[derive(Debug, Default)]
pub struct LowestLatencyStrategy;

impl LowestLatencyStrategy {
     /// Creates a new LowestLatencyStrategy
     pub fn new() -> Self {
         Self {} 
     }
}

impl LoadBalancingStrategy for LowestLatencyStrategy {
    /// Select the instance with the minimum `avg_response_time`.
    ///
    /// # Parameters
    /// * `trackers` - Array of (id, tracker) tuples for available instances.
    ///
    /// # Returns
    /// * Index into the trackers array of the fastest instance.
    ///
    /// # Panics
    /// * Panics if `trackers` is empty.
    fn select_instance(&mut self, trackers: &[(usize, &InstanceTracker)]) -> usize {
        if trackers.is_empty() {
            panic!("LowestLatencyStrategy::select_instance called with empty trackers slice");
        }

        let mut best_index = 0;
        let mut lowest_time = trackers[0].1.avg_response_time();

        for (i, (_id, tracker)) in trackers.iter().enumerate().skip(1) {
            let avg_time = tracker.avg_response_time();
            if avg_time < lowest_time {
                best_index = i;
                lowest_time = avg_time;
            }
        }

        debug!(
            "LowestLatencyStrategy: Selected index {} (ID: {}) from {} eligible trackers with avg_response_time: {:?}",
            best_index, trackers[best_index].0, trackers.len(), lowest_time
        );

        best_index
    }
}

/// Strategy that selects a random instance from the available pool.
#[derive(Debug, Default)]
pub struct RandomStrategy;

impl RandomStrategy {
     /// Creates a new RandomStrategy
     pub fn new() -> Self {
         Self {}
     }
}

impl LoadBalancingStrategy for RandomStrategy {
    /// Select a random instance.
    ///
    /// # Parameters
    /// * `trackers` - Array of (id, tracker) tuples for available instances.
    ///
    /// # Returns
    /// * Index into the trackers array of a randomly chosen instance.
    ///
    /// # Panics
    /// * Panics if `trackers` is empty.
    fn select_instance(&mut self, trackers: &[(usize, &InstanceTracker)]) -> usize {
        if trackers.is_empty() {
            panic!("RandomStrategy::select_instance called with empty trackers slice");
        }

        let index = rand::rng().random_range(0..trackers.len());

        debug!(
            "RandomStrategy: Selected random index {} (ID: {}) from {} eligible trackers",
            index, trackers[index].0, trackers.len()
        );

        index
    }
}