synapse-gateway 0.5.9

OpenAI-compatible LLM router and gateway with streaming, tool calling, multi-provider fallback, native Vertex AI, and per-tenant cost accounting
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

//! OpenAI-compatible chat request body + native extension block.

use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};

#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ChatRequest {
    pub model: String,
    pub messages: Vec<Message>,
    #[serde(default)]
    pub temperature: Option<f32>,
    #[serde(default)]
    pub max_tokens: Option<u32>,
    #[serde(default)]
    pub stream: Option<bool>,
    #[serde(default)]
    pub response_format: Option<ResponseFormat>,
    #[serde(default)]
    pub routing_strategy: Option<String>,
    #[serde(default)]
    pub vertex: Option<VertexExt>,
    #[serde(default)]
    pub tools: Option<Vec<Value>>,
    #[serde(default)]
    pub tool_choice: Option<Value>,
    #[serde(flatten, default)]
    pub passthrough: Map<String, Value>,
}

#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Message {
    pub role: String,
    #[serde(default, skip_serializing_if = "Value::is_null")]
    pub content: Value, // string, array of parts, or null on a tool-call turn
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub tool_calls: Option<Vec<Value>>, // assistant turn: [{id,type,function:{name,arguments}}]
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub tool_call_id: Option<String>, // role:"tool" result correlation
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub name: Option<String>, // optional name (tool result or system message)
}

#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ResponseFormat {
    #[serde(rename = "type")]
    pub kind: String, // "text" | "json_object" | "json_schema"
    #[serde(default)]
    pub json_schema: Option<Value>,
}

#[derive(Debug, Clone, Default, Deserialize, Serialize)]
pub struct VertexExt {
    #[serde(default)]
    pub cached_content: Option<String>,
    #[serde(default)]
    pub media_uris: Option<Vec<String>>,
    #[serde(default)]
    pub response_schema: Option<Value>,
    /// Raw Vertex `generationConfig.thinkingConfig` passthrough (e.g.
    /// `{"thinkingLevel":"low"}` for Gemini 3, or `{"thinkingBudget":N}` for
    /// Gemini 2.5). Threaded verbatim so the gateway stays model-agnostic.
    #[serde(default)]
    pub thinking_config: Option<Value>,
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn parses_plain_openai_body() {
        let body = serde_json::json!({
            "model": "gemini-pro",
            "messages": [{"role": "user", "content": "hi"}],
            "temperature": 0.2
        });
        let req: ChatRequest = serde_json::from_value(body).unwrap();
        assert_eq!(req.model, "gemini-pro");
        assert_eq!(req.messages.len(), 1);
        assert!(req.vertex.is_none());
        assert!(req.passthrough.is_empty());
    }

    #[test]
    fn captures_vertex_extension_and_passthrough() {
        let body = serde_json::json!({
            "model": "gemini-pro",
            "messages": [{"role": "user", "content": "hi"}],
            "top_k": 40,
            "vertex": { "cached_content": "cachedContents/abc" }
        });
        let req: ChatRequest = serde_json::from_value(body).unwrap();
        assert_eq!(
            req.vertex.unwrap().cached_content.as_deref(),
            Some("cachedContents/abc")
        );
        assert_eq!(req.passthrough.get("top_k"), Some(&serde_json::json!(40)));
    }

    #[test]
    fn captures_vertex_thinking_config() {
        let body = serde_json::json!({
            "model": "gemini-3-pro",
            "messages": [{"role": "user", "content": "hi"}],
            "vertex": { "thinking_config": { "thinkingLevel": "low" } }
        });
        let req: ChatRequest = serde_json::from_value(body).unwrap();
        assert_eq!(
            req.vertex.unwrap().thinking_config,
            Some(serde_json::json!({ "thinkingLevel": "low" }))
        );
    }

    #[test]
    fn parses_tools_and_tool_messages() {
        let body = serde_json::json!({
            "model": "gemini-pro",
            "messages": [
                {"role": "user", "content": "weather in SF?"},
                {"role": "assistant", "content": null,
                 "tool_calls": [{"id": "call_0", "type": "function",
                    "function": {"name": "get_weather", "arguments": "{\"city\":\"SF\"}"}}]},
                {"role": "tool", "tool_call_id": "call_0", "content": "21C"}
            ],
            "tools": [{"type": "function", "function": {"name": "get_weather",
                "description": "Lookup", "parameters": {"type": "object"}}}],
            "tool_choice": "auto"
        });
        let req: ChatRequest = serde_json::from_value(body).unwrap();
        assert_eq!(req.tools.as_ref().unwrap().len(), 1);
        assert_eq!(req.tool_choice, Some(serde_json::json!("auto")));
        let asst = &req.messages[1];
        assert!(asst.tool_calls.is_some());
        let tool = &req.messages[2];
        assert_eq!(tool.tool_call_id.as_deref(), Some("call_0"));
    }
}