car_inference/

remote.rs

//! Remote inference backend — HTTP client for cloud API models.
//!
//! Supports three protocols:
//! - **OpenAI-compatible** — `/v1/chat/completions` (OpenAI, Ollama, vLLM, etc.)
//! - **Anthropic** — `/v1/messages`
//! - **Google** — Gemini API
//!
//! The backend reads API keys from environment variables (never stored in config).
//! All calls are async and return the same types as local inference.

use reqwest::Client;
use serde::{Deserialize, Serialize};
use tracing::debug;

use crate::schema::{ApiProtocol, ModelSchema, ModelSource};
use crate::InferenceError;

/// Remote inference client. Reuses a single HTTP client for connection pooling.
pub struct RemoteBackend {
    client: Client,
}

impl RemoteBackend {
    pub fn new() -> Self {
        let client = Client::builder()
            .timeout(std::time::Duration::from_secs(120))
            .build()
            .unwrap_or_default();
        Self { client }
    }

    /// Generate text via a remote API.
    pub async fn generate(
        &self,
        schema: &ModelSchema,
        prompt: &str,
        context: Option<&str>,
        temperature: f64,
        max_tokens: usize,
    ) -> Result<String, InferenceError> {
        let (endpoint, api_key, protocol) = extract_remote_config(schema)?;

        match protocol {
            ApiProtocol::OpenAiCompat => {
                self.generate_openai(&endpoint, &api_key, &schema.name, prompt, context, temperature, max_tokens).await
            }
            ApiProtocol::Anthropic => {
                self.generate_anthropic(&endpoint, &api_key, &schema.name, prompt, context, temperature, max_tokens).await
            }
            ApiProtocol::Google => {
                self.generate_google(&endpoint, &api_key, &schema.name, prompt, context, temperature, max_tokens).await
            }
        }
    }

    /// Generate embeddings via a remote API (OpenAI-compatible only for now).
    pub async fn embed(
        &self,
        schema: &ModelSchema,
        texts: &[String],
    ) -> Result<Vec<Vec<f32>>, InferenceError> {
        let (endpoint, api_key, protocol) = extract_remote_config(schema)?;

        match protocol {
            ApiProtocol::OpenAiCompat => {
                self.embed_openai(&endpoint, &api_key, &schema.name, texts).await
            }
            _ => Err(InferenceError::InferenceFailed(format!(
                "embedding not supported for {:?} protocol", protocol
            ))),
        }
    }

    // --- OpenAI-compatible ---

    async fn generate_openai(
        &self,
        endpoint: &str,
        api_key: &str,
        model: &str,
        prompt: &str,
        context: Option<&str>,
        temperature: f64,
        max_tokens: usize,
    ) -> Result<String, InferenceError> {
        let url = format_endpoint(endpoint, "/v1/chat/completions");

        let mut messages = Vec::new();
        if let Some(ctx) = context {
            messages.push(serde_json::json!({
                "role": "system",
                "content": ctx,
            }));
        }
        messages.push(serde_json::json!({
            "role": "user",
            "content": prompt,
        }));

        let body = serde_json::json!({
            "model": model,
            "messages": messages,
            "temperature": temperature,
            "max_tokens": max_tokens,
        });

        debug!(url = %url, model = %model, "openai-compat generate request");

        let resp = self.client
            .post(&url)
            .header("Authorization", format!("Bearer {api_key}"))
            .header("Content-Type", "application/json")
            .json(&body)
            .send()
            .await
            .map_err(|e| InferenceError::InferenceFailed(format!("HTTP error: {e}")))?;

        let status = resp.status();
        let text = resp.text().await
            .map_err(|e| InferenceError::InferenceFailed(format!("read body: {e}")))?;

        if !status.is_success() {
            return Err(InferenceError::InferenceFailed(format!(
                "API returned {status}: {text}"
            )));
        }

        let parsed: OpenAiResponse = serde_json::from_str(&text)
            .map_err(|e| InferenceError::InferenceFailed(format!("parse response: {e}")))?;

        parsed.choices.first()
            .and_then(|c| c.message.content.clone())
            .ok_or_else(|| InferenceError::InferenceFailed("empty response".into()))
    }

    async fn embed_openai(
        &self,
        endpoint: &str,
        api_key: &str,
        model: &str,
        texts: &[String],
    ) -> Result<Vec<Vec<f32>>, InferenceError> {
        let url = format_endpoint(endpoint, "/v1/embeddings");

        let body = serde_json::json!({
            "model": model,
            "input": texts,
        });

        let resp = self.client
            .post(&url)
            .header("Authorization", format!("Bearer {api_key}"))
            .header("Content-Type", "application/json")
            .json(&body)
            .send()
            .await
            .map_err(|e| InferenceError::InferenceFailed(format!("HTTP error: {e}")))?;

        let status = resp.status();
        let text = resp.text().await
            .map_err(|e| InferenceError::InferenceFailed(format!("read body: {e}")))?;

        if !status.is_success() {
            return Err(InferenceError::InferenceFailed(format!(
                "API returned {status}: {text}"
            )));
        }

        let parsed: OpenAiEmbedResponse = serde_json::from_str(&text)
            .map_err(|e| InferenceError::InferenceFailed(format!("parse response: {e}")))?;

        Ok(parsed.data.into_iter().map(|d| d.embedding).collect())
    }

    // --- Anthropic ---

    async fn generate_anthropic(
        &self,
        endpoint: &str,
        api_key: &str,
        model: &str,
        prompt: &str,
        context: Option<&str>,
        temperature: f64,
        max_tokens: usize,
    ) -> Result<String, InferenceError> {
        let url = format_endpoint(endpoint, "/v1/messages");

        let mut body = serde_json::json!({
            "model": model,
            "max_tokens": max_tokens,
            "temperature": temperature,
            "messages": [{
                "role": "user",
                "content": prompt,
            }],
        });

        if let Some(ctx) = context {
            body["system"] = serde_json::Value::String(ctx.to_string());
        }

        debug!(url = %url, model = %model, "anthropic generate request");

        let resp = self.client
            .post(&url)
            .header("x-api-key", api_key)
            .header("anthropic-version", "2023-06-01")
            .header("Content-Type", "application/json")
            .json(&body)
            .send()
            .await
            .map_err(|e| InferenceError::InferenceFailed(format!("HTTP error: {e}")))?;

        let status = resp.status();
        let text = resp.text().await
            .map_err(|e| InferenceError::InferenceFailed(format!("read body: {e}")))?;

        if !status.is_success() {
            return Err(InferenceError::InferenceFailed(format!(
                "API returned {status}: {text}"
            )));
        }

        let parsed: AnthropicResponse = serde_json::from_str(&text)
            .map_err(|e| InferenceError::InferenceFailed(format!("parse response: {e}")))?;

        parsed.content.into_iter()
            .find(|c| c.content_type == "text")
            .map(|c| c.text)
            .ok_or_else(|| InferenceError::InferenceFailed("no text in response".into()))
    }

    // --- Google (Gemini) ---

    async fn generate_google(
        &self,
        endpoint: &str,
        api_key: &str,
        model: &str,
        prompt: &str,
        context: Option<&str>,
        _temperature: f64,
        _max_tokens: usize,
    ) -> Result<String, InferenceError> {
        // Gemini uses ?key= query param, not header auth
        let url = format!(
            "{}/v1beta/models/{}:generateContent?key={}",
            endpoint.trim_end_matches('/'),
            model,
            api_key,
        );

        let mut parts = vec![serde_json::json!({"text": prompt})];
        if let Some(ctx) = context {
            parts.insert(0, serde_json::json!({"text": ctx}));
        }

        let body = serde_json::json!({
            "contents": [{
                "parts": parts,
            }],
        });

        debug!(model = %model, "google generate request");

        let resp = self.client
            .post(&url)
            .header("Content-Type", "application/json")
            .json(&body)
            .send()
            .await
            .map_err(|e| InferenceError::InferenceFailed(format!("HTTP error: {e}")))?;

        let status = resp.status();
        let text = resp.text().await
            .map_err(|e| InferenceError::InferenceFailed(format!("read body: {e}")))?;

        if !status.is_success() {
            return Err(InferenceError::InferenceFailed(format!(
                "API returned {status}: {text}"
            )));
        }

        let parsed: GoogleResponse = serde_json::from_str(&text)
            .map_err(|e| InferenceError::InferenceFailed(format!("parse response: {e}")))?;

        parsed.candidates.into_iter()
            .next()
            .and_then(|c| c.content.parts.into_iter().next())
            .map(|p| p.text)
            .ok_or_else(|| InferenceError::InferenceFailed("no text in response".into()))
    }
}

impl Default for RemoteBackend {
    fn default() -> Self {
        Self::new()
    }
}

// --- Helpers ---

/// Extract endpoint, API key, and protocol from a model schema.
fn extract_remote_config(schema: &ModelSchema) -> Result<(String, String, ApiProtocol), InferenceError> {
    match &schema.source {
        ModelSource::RemoteApi { endpoint, api_key_env, protocol, .. } => {
            let api_key = std::env::var(api_key_env).map_err(|_| {
                InferenceError::InferenceFailed(format!(
                    "API key env var {} not set for model {}",
                    api_key_env, schema.id
                ))
            })?;
            Ok((endpoint.clone(), api_key, *protocol))
        }
        ModelSource::Ollama { model_tag, host } => {
            // Ollama uses OpenAI-compatible API
            Ok((host.clone(), String::new(), ApiProtocol::OpenAiCompat))
        }
        _ => Err(InferenceError::InferenceFailed(format!(
            "model {} is not remote", schema.id
        ))),
    }
}

/// Normalize endpoint URL for a given path.
fn format_endpoint(base: &str, path: &str) -> String {
    let base = base.trim_end_matches('/');
    // If the base already ends with the path, use it as-is
    if base.ends_with(path.trim_start_matches('/')) {
        base.to_string()
    } else {
        format!("{}{}", base, path)
    }
}

// --- Response types ---

#[derive(Debug, Deserialize)]
struct OpenAiResponse {
    choices: Vec<OpenAiChoice>,
}

#[derive(Debug, Deserialize)]
struct OpenAiChoice {
    message: OpenAiMessage,
}

#[derive(Debug, Deserialize)]
struct OpenAiMessage {
    content: Option<String>,
}

#[derive(Debug, Deserialize)]
struct OpenAiEmbedResponse {
    data: Vec<OpenAiEmbedData>,
}

#[derive(Debug, Deserialize)]
struct OpenAiEmbedData {
    embedding: Vec<f32>,
}

#[derive(Debug, Deserialize)]
struct AnthropicResponse {
    content: Vec<AnthropicContent>,
}

#[derive(Debug, Deserialize)]
struct AnthropicContent {
    #[serde(rename = "type")]
    content_type: String,
    #[serde(default)]
    text: String,
}

#[derive(Debug, Deserialize)]
struct GoogleResponse {
    candidates: Vec<GoogleCandidate>,
}

#[derive(Debug, Deserialize)]
struct GoogleCandidate {
    content: GoogleContent,
}

#[derive(Debug, Deserialize)]
struct GoogleContent {
    parts: Vec<GooglePart>,
}

#[derive(Debug, Deserialize)]
struct GooglePart {
    text: String,
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn format_endpoint_no_dup() {
        assert_eq!(
            format_endpoint("https://api.openai.com", "/v1/chat/completions"),
            "https://api.openai.com/v1/chat/completions"
        );
        assert_eq!(
            format_endpoint("https://api.openai.com/v1/chat/completions", "/v1/chat/completions"),
            "https://api.openai.com/v1/chat/completions"
        );
        assert_eq!(
            format_endpoint("https://api.openai.com/", "/v1/chat/completions"),
            "https://api.openai.com/v1/chat/completions"
        );
    }

    #[test]
    fn extract_config_missing_env() {
        let schema = ModelSchema {
            id: "test/model:v1".into(),
            name: "Test".into(),
            provider: "test".into(),
            family: "test".into(),
            version: "1".into(),
            capabilities: vec![],
            context_length: 4096,
            param_count: String::new(),
            quantization: None,
            performance: Default::default(),
            cost: Default::default(),
            source: ModelSource::RemoteApi {
                endpoint: "https://api.test.com".into(),
                api_key_env: "NONEXISTENT_TEST_KEY_12345".into(),
                api_version: None,
                protocol: ApiProtocol::OpenAiCompat,
            },
            tags: vec![],
            available: false,
        };
        let result = extract_remote_config(&schema);
        assert!(result.is_err());
    }

    #[test]
    fn parse_openai_response() {
        let json = r#"{"choices":[{"message":{"content":"Hello world"}}]}"#;
        let resp: OpenAiResponse = serde_json::from_str(json).unwrap();
        assert_eq!(resp.choices[0].message.content.as_deref(), Some("Hello world"));
    }

    #[test]
    fn parse_anthropic_response() {
        let json = r#"{"content":[{"type":"text","text":"Hello world"}]}"#;
        let resp: AnthropicResponse = serde_json::from_str(json).unwrap();
        assert_eq!(resp.content[0].text, "Hello world");
    }

    #[test]
    fn parse_google_response() {
        let json = r#"{"candidates":[{"content":{"parts":[{"text":"Hello world"}]}}]}"#;
        let resp: GoogleResponse = serde_json::from_str(json).unwrap();
        assert_eq!(resp.candidates[0].content.parts[0].text, "Hello world");
    }
}