sipp-rs 0.1.0

Unified Rust library for extensible Sipp inference
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

use crate::core::CapabilitySupport;
use crate::engine::ModelCapabilities;

/////////////////////////////////////////////////////////////////////////////////
/// TESTS
/////////////////////////////////////////////////////////////////////////////////

#[cfg(test)]
#[path = "../tests/client/endpoint_tests.rs"]
mod endpoint_tests;

/////////////////////////////////////////////////////////////////////////////////
/// SRC
/////////////////////////////////////////////////////////////////////////////////

/// Addressable inference destination.
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub enum EndpointRef {
    /// Local engine endpoint registered in the client.
    Local {
        /// Client-scoped endpoint id.
        id: String,
    },
    /// HTTP gateway endpoint registered in the client.
    Gateway {
        /// Client-scoped endpoint id.
        id: String,
    },
    /// Direct provider endpoint registered in the client.
    Provider {
        /// Client-scoped endpoint id.
        id: String,
    },
}

impl EndpointRef {
    /// Return the stable client-scoped endpoint id.
    pub fn id(&self) -> &str {
        match self {
            Self::Local { id } | Self::Gateway { id } | Self::Provider { id } => id,
        }
    }

    /// Return the stable endpoint kind label.
    pub const fn kind(&self) -> &'static str {
        match self {
            Self::Local { .. } => "local",
            Self::Gateway { .. } => "gateway",
            Self::Provider { .. } => "provider",
        }
    }

    pub(crate) fn is_local(&self) -> bool {
        matches!(self, Self::Local { .. })
    }

    /// Create a reference for a gateway endpoint.
    pub fn gateway(id: impl Into<String>) -> Self {
        Self::Gateway { id: id.into() }
    }
}

/// Cached support for the three public inference verbs.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EndpointCapabilities {
    /// Raw-prompt text generation support.
    pub query: CapabilitySupport,
    /// Chat generation support.
    pub chat: CapabilitySupport,
    /// Embedding support.
    pub embed: CapabilitySupport,
}

impl EndpointCapabilities {
    pub(crate) fn from_local(capabilities: &ModelCapabilities) -> Self {
        Self {
            query: support(capabilities.supports_text_generation),
            chat: support(capabilities.supports_text_generation && capabilities.has_chat_template),
            embed: support(capabilities.supports_embeddings),
        }
    }

    /// Return capabilities that will be determined by the endpoint at runtime.
    pub const fn unknown() -> Self {
        Self {
            query: CapabilitySupport::Unknown,
            chat: CapabilitySupport::Unknown,
            embed: CapabilitySupport::Unknown,
        }
    }

    /// Return support for one canonical inference operation.
    pub fn for_operation(&self, operation: &'static str) -> CapabilitySupport {
        match operation {
            "query" => self.query,
            "chat" => self.chat,
            "embed" => self.embed,
            _ => CapabilitySupport::Unsupported,
        }
    }
}

const fn support(enabled: bool) -> CapabilitySupport {
    if enabled {
        CapabilitySupport::Supported
    } else {
        CapabilitySupport::Unsupported
    }
}