atomr-infer-runtime-gemini 0.8.0

Google Gemini provider for atomr-infer — implements ModelRunner against Vertex AI and AI Studio endpoints, with cost table, error classification, and SSE streaming.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

use serde::{Deserialize, Serialize};
use url::Url;

use atomr_infer_core::deployment::{RateLimits, RetryPolicy, Timeouts};
use atomr_infer_core::runtime::CircuitBreakerConfig;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GeminiConfig {
    #[serde(flatten)]
    pub variant: GeminiVariant,
    /// Auth credential. AI Studio uses an API key (`StaticApiKey`);
    /// Vertex uses an OAuth2 access token via the operator-supplied
    /// credential provider.
    pub credential: SecretRef,
    #[serde(default)]
    pub safety: Vec<SafetySetting>,
    #[serde(default)]
    pub rate_limits: RateLimits,
    #[serde(default)]
    pub retry: RetryPolicy,
    #[serde(default)]
    pub circuit_breaker: CircuitBreakerConfig,
    #[serde(default)]
    pub timeouts: Timeouts,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "variant", rename_all = "snake_case")]
pub enum GeminiVariant {
    AiStudio {
        #[serde(default = "default_aistudio_endpoint")]
        endpoint: Url,
    },
    Vertex {
        project: String,
        region: String,
    },
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SafetySetting {
    pub category: String,
    pub threshold: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "from", rename_all = "snake_case")]
pub enum SecretRef {
    Env {
        name: String,
    },
    File {
        path: std::path::PathBuf,
    },
    Inline {
        value: String,
    },
    /// Vertex uses application default credentials; resolved by the
    /// operator-supplied `CredentialProvider`.
    Adc,
}

fn default_aistudio_endpoint() -> Url {
    Url::parse("https://generativelanguage.googleapis.com/v1beta/").expect("static url")
}

impl GeminiConfig {
    pub fn generate_content_url(&self, model: &str, stream: bool) -> Result<Url, url::ParseError> {
        let suffix = if stream {
            ":streamGenerateContent?alt=sse"
        } else {
            ":generateContent"
        };
        match &self.variant {
            GeminiVariant::AiStudio { endpoint } => {
                endpoint.join(&format!("models/{model}{suffix}"))
            }
            GeminiVariant::Vertex { project, region } => Url::parse(&format!(
                "https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{region}/publishers/google/models/{model}{suffix}"
            )),
        }
    }
}