synapse-gateway 0.5.21

OpenAI-compatible LLM router and gateway with streaming, tool calling, multi-provider fallback, native Vertex AI, and per-tenant cost accounting
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

//! Pure lane detection: standard vs native-Vertex.

use crate::routing::request::ChatRequest;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Lane {
    Standard,
    NativeVertex,
}

/// Classify by inspecting the request for native-Vertex triggers.
/// Pure and allocation-free; safe to call on the hot path.
pub fn classify(req: &ChatRequest) -> Lane {
    let v = match &req.vertex {
        Some(v) => v,
        None => return Lane::Standard,
    };
    let triggers_native = v.cached_content.is_some()
        || v.response_schema.is_some()
        || v.media_uris
            .as_ref()
            .is_some_and(|uris| uris.iter().any(|u| u.starts_with("gs://")));
    if triggers_native {
        Lane::NativeVertex
    } else {
        Lane::Standard
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::routing::request::{ChatRequest, VertexExt};

    fn base() -> ChatRequest {
        serde_json::from_value(serde_json::json!({
            "model": "m", "messages": [{"role": "user", "content": "hi"}]
        }))
        .unwrap()
    }

    #[test]
    fn no_vertex_block_is_standard() {
        assert_eq!(classify(&base()), Lane::Standard);
    }

    #[test]
    fn cached_content_is_native() {
        let req = ChatRequest {
            vertex: Some(VertexExt {
                cached_content: Some("cachedContents/x".into()),
                ..Default::default()
            }),
            ..base()
        };
        assert_eq!(classify(&req), Lane::NativeVertex);
    }

    #[test]
    fn response_schema_is_native() {
        let req = ChatRequest {
            vertex: Some(VertexExt {
                response_schema: Some(serde_json::json!({"type": "object"})),
                ..Default::default()
            }),
            ..base()
        };
        assert_eq!(classify(&req), Lane::NativeVertex);
    }

    #[test]
    fn gs_media_uri_is_native_but_https_is_not() {
        let gs = ChatRequest {
            vertex: Some(VertexExt {
                media_uris: Some(vec!["gs://b/v.mp4".into()]),
                ..Default::default()
            }),
            ..base()
        };
        assert_eq!(classify(&gs), Lane::NativeVertex);
        let https = ChatRequest {
            vertex: Some(VertexExt {
                media_uris: Some(vec!["https://x/v.mp4".into()]),
                ..Default::default()
            }),
            ..base()
        };
        assert_eq!(classify(&https), Lane::Standard);
    }
}