llmshim 0.12.0

Blazing fast LLM API translation layer in pure Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

use crate::error::Result;
use serde_json::Value;
use std::{future::Future, pin::Pin};

pub struct ProviderRequest {
    pub url: String,
    pub headers: Vec<(String, String)>,
    pub body: Value,
}

#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RequestAdmissionPolicy {
    native_namespace: Option<String>,
    protected_native_fields: &'static [&'static str],
    native_prompt_fields: &'static [&'static str],
    native_output_limit_fields: &'static [&'static str],
}

impl RequestAdmissionPolicy {
    pub fn namespaced(
        native_namespace: impl Into<String>,
        protected_native_fields: &'static [&'static str],
        native_prompt_fields: &'static [&'static str],
        native_output_limit_fields: &'static [&'static str],
    ) -> Self {
        Self {
            native_namespace: Some(native_namespace.into()),
            protected_native_fields,
            native_prompt_fields,
            native_output_limit_fields,
        }
    }

    pub fn native_namespace(&self) -> Option<&str> {
        self.native_namespace.as_deref()
    }

    pub fn protected_native_fields(&self) -> &'static [&'static str] {
        self.protected_native_fields
    }

    pub fn native_prompt_fields(&self) -> &'static [&'static str] {
        self.native_prompt_fields
    }

    pub fn native_output_limit_fields(&self) -> &'static [&'static str] {
        self.native_output_limit_fields
    }
}

impl ProviderRequest {
    /// Check endpoint, credentials, settings and prefix for a continuation.
    /// Dispatch still sends a full stateless request.
    pub fn can_continue_from(&self, previous: &Self) -> bool {
        self.url == previous.url
            && self.headers == previous.headers
            && crate::cache::continuation_matches(&previous.body, &self.body)
    }
}

/// Core trait every provider implements.
/// Takes OpenAI-format JSON in, emits provider-native JSON out, and back again.
pub trait Provider: Send + Sync {
    fn name(&self) -> &str;

    /// Describe native request fields that affect proxy admission. Providers
    /// without a merged native namespace keep the empty default.
    fn request_admission_policy(&self) -> RequestAdmissionPolicy {
        RequestAdmissionPolicy::default()
    }

    /// Native wire and issuer used by shared reasoning replay. Custom
    /// Chat-Completions adapters inherit a conservative unbound account.
    fn replay_target(&self, model: &str) -> crate::reasoning::ReplayTarget {
        crate::reasoning::ReplayTarget::new(
            self.name(),
            model,
            crate::reasoning::WireFormat::OpenAiChat,
        )
    }

    /// Capture the identity of the request actually sent, including native
    /// model overrides and refreshed OAuth account headers.
    fn request_replay_target(
        &self,
        model: &str,
        request: &ProviderRequest,
    ) -> crate::reasoning::ReplayTarget {
        self.replay_target(request.body["model"].as_str().unwrap_or(model))
    }

    /// Transform an OpenAI-format request into the provider's native format.
    /// `model` is the raw model string (after prefix stripping).
    fn transform_request(&self, model: &str, request: &Value) -> Result<ProviderRequest>;

    /// Prepare credentials asynchronously before dispatch. API-key providers
    /// use the synchronous transform; OAuth providers can refresh here.
    fn prepare_request<'a>(
        &'a self,
        model: &'a str,
        request: &'a Value,
    ) -> Pin<Box<dyn Future<Output = Result<ProviderRequest>> + Send + 'a>> {
        Box::pin(async move { self.transform_request(model, request) })
    }

    /// Transform the provider's native response back into OpenAI format.
    fn transform_response(&self, model: &str, response: Value) -> Result<Value>;

    /// Create per-response streaming state. Use this when manually consuming
    /// SSE; tool ids, JSON arguments and trailing signatures require state.
    fn stream_normalizer(&self, model: &str) -> crate::streaming::StreamNormalizer {
        crate::streaming::StreamNormalizer::new(self.replay_target(model))
    }

    /// Low-level stateless content/reasoning parser. Tool events are consumed by
    /// `stream_normalizer`, not emitted as incomplete callable tool records.
    fn transform_stream_chunk(&self, model: &str, chunk: &str) -> Result<Option<String>>;
}