llmux 2.4.0

Hook-driven LLM model multiplexer with pluggable switch policy
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

use super::{PolicyContext, PolicyDecision, SwitchContext, SwitchPolicy};
use async_trait::async_trait;
use std::time::Duration;

/// FIFO policy — switch immediately on first request for a non-active model.
///
/// No background scheduler needed: every request spawns its own switch attempt
/// via `maybe_trigger_switch`, and the switch lock serializes them.
pub struct FifoPolicy {
    request_timeout: Option<Duration>,
    drain_before_switch: bool,
    min_active_duration: Duration,
}

impl FifoPolicy {
    pub fn new(
        request_timeout: Option<Duration>,
        drain_before_switch: bool,
        min_active_duration: Duration,
    ) -> Self {
        Self {
            request_timeout,
            drain_before_switch,
            min_active_duration,
        }
    }
}

impl Default for FifoPolicy {
    fn default() -> Self {
        Self::new(None, true, Duration::ZERO)
    }
}

#[async_trait]
impl SwitchPolicy for FifoPolicy {
    async fn on_pending_request(&self, _ctx: &PolicyContext) -> PolicyDecision {
        PolicyDecision::SwitchNow
    }

    async fn prepare_switch(&self, ctx: &mut SwitchContext) {
        if self.drain_before_switch {
            ctx.wait_for_in_flight().await;
        }
    }

    fn request_timeout(&self) -> Option<Duration> {
        self.request_timeout
    }

    fn min_active_duration(&self) -> Duration {
        self.min_active_duration
    }
}