car-inference-types 0.32.0

//! Pure-serde conversation/message wire types for CAR inference.
//!
//! These types — [`Message`], [`ToolCall`], [`ContentBlock`] — define the
//! multi-turn conversation wire form that car-inference's protocol handlers
//! and local chat templates consume. They were extracted out of
//! `car-inference` (where they still live as re-exports) so a **dependency-
//! light** consumer can build and pattern-match the REAL types without pulling
//! the Candle/MLX inference stack:
//!
//! - `car-inference` re-exports every type here (`pub use
//!   car_inference_types::{Message, ToolCall, ContentBlock}`), so its own
//!   callers are unchanged.
//! - `car-sync`'s transcript-resume projection (multi-device sync B2) builds
//!   `Vec<Message>` directly from folded conversation ops. Because it depends
//!   on this crate — not on a hand-copied mirror — a change to `Message`'s
//!   shape is a **compile error** at the resume site, not a runtime
//!   `from_value::<Message>` break in the daemon (the drift trap the kernel
//!   review flagged).
//!
//! The crate is deliberately serde-only. Anything that needs Candle, a
//! tokenizer, or a model backend belongs in `car-inference`, not here.

use serde::{Deserialize, Serialize};

/// A tool call returned by the model.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ToolCall {
    /// Provider-assigned tool call ID (e.g. OpenAI `call_abc123`, Anthropic `toolu_abc123`).
    /// When present, protocol handlers use this for round-trip correlation instead of
    /// synthesizing positional IDs like `call_0`.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub id: Option<String>,
    /// Tool/function name.
    pub name: String,
    /// Arguments as key-value pairs.
    pub arguments: std::collections::HashMap<String, serde_json::Value>,
}

/// A content block in a multimodal message.
///
/// The image variants (`ImageBase64`, `ImageUrl`) are fully wired on
/// the native Qwen2.5-VL backend. The video variants
/// (`VideoPath`, `VideoUrl`, `VideoBase64`) are defined on the public
/// request surface so higher-level tooling can express Qwen2.5-VL
/// video-understanding payloads, but the native backend returns
/// `UnsupportedMode` for them until the video-tokenization path lands.
/// Remote multimodal providers (Anthropic, Google Vertex) accept them
/// through the protocol handlers today.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ContentBlock {
    /// Plain text content.
    Text { text: String },
    /// Base64-encoded image.
    ImageBase64 {
        /// Base64-encoded image data.
        data: String,
        /// MIME type (e.g., "image/png", "image/jpeg").
        media_type: String,
    },
    /// Image from URL.
    ImageUrl {
        /// URL of the image.
        url: String,
        /// Detail level for image processing ("auto", "low", "high").
        #[serde(default = "default_detail")]
        detail: String,
    },
    /// Video loaded from a local filesystem path. Qwen2.5-VL samples
    /// the clip at `fps` frames/sec (default: backend-chosen) and
    /// caps at `max_frames` to respect context budgets.
    VideoPath {
        path: String,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        fps: Option<f32>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max_frames: Option<u32>,
    },
    /// Video accessible over HTTP(S). Semantics as [`ContentBlock::VideoPath`].
    VideoUrl {
        url: String,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        fps: Option<f32>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max_frames: Option<u32>,
    },
    /// Base64-encoded video bytes. Prefer `VideoPath` when possible;
    /// inline base64 is expensive to round-trip.
    VideoBase64 {
        data: String,
        media_type: String,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        fps: Option<f32>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max_frames: Option<u32>,
    },
    /// Audio loaded from a local filesystem path. Used for
    /// audio-understanding models (Gemma 4 small variants, Gemini).
    AudioPath {
        path: String,
        /// Optional explicit sample-rate hint. Most backends will
        /// resample internally; this is a best-effort declaration.
        #[serde(default, skip_serializing_if = "Option::is_none")]
        sample_rate: Option<u32>,
    },
    /// Audio accessible over HTTP(S).
    AudioUrl {
        url: String,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        sample_rate: Option<u32>,
    },
    /// Base64-encoded audio bytes.
    AudioBase64 {
        data: String,
        media_type: String,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        sample_rate: Option<u32>,
    },
}

impl ContentBlock {
    /// Return true if this block carries video data (any encoding).
    /// Used by backends that need to refuse video inputs until the
    /// tokenization path is wired.
    pub fn is_video(&self) -> bool {
        matches!(
            self,
            ContentBlock::VideoPath { .. }
                | ContentBlock::VideoUrl { .. }
                | ContentBlock::VideoBase64 { .. }
        )
    }

    /// Return true if this block carries audio data (any encoding).
    /// Used by backends that need to refuse audio inputs until the
    /// tokenization path is wired. Gemma 4 small variants and Gemini
    /// accept audio; everything else in CAR rejects with
    /// `UnsupportedMode`.
    pub fn is_audio(&self) -> bool {
        matches!(
            self,
            ContentBlock::AudioPath { .. }
                | ContentBlock::AudioUrl { .. }
                | ContentBlock::AudioBase64 { .. }
        )
    }
}

fn default_detail() -> String {
    "auto".to_string()
}

/// A message in a multi-turn conversation.
///
/// The `System` variant exists so callers can express a first-class
/// system prompt inside `messages: Vec<Message>` without threading it
/// through the legacy `context: Option<String>` field on the request.
/// Protocol handlers and local chat templates that have a native
/// system-role slot (OpenAI, Anthropic, Gemini, Gemma 4, Qwen) emit it
/// in the right place; ones that don't can fold it into the first user
/// turn.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "role", rename_all = "snake_case")]
pub enum Message {
    /// A system prompt. Appears once, at the start of the conversation.
    System { content: String },
    /// A user message (text only).
    User { content: String },
    /// A user message with multimodal content (text + images + video + audio).
    UserMultimodal { content: Vec<ContentBlock> },
    /// An assistant response, possibly with tool calls.
    Assistant {
        #[serde(default)]
        content: String,
        #[serde(default)]
        tool_calls: Vec<ToolCall>,
    },
    /// The result of executing a tool call.
    ToolResult {
        tool_use_id: String,
        content: String,
    },
    /// Provider-specific output items that need to round-trip
    /// verbatim across turns. The OpenAI Responses API returns
    /// reasoning blobs, encrypted_content, web-search results, etc.
    /// as opaque structured items; the next request must include
    /// them in the same form to preserve provider-side state.
    ///
    /// `protocol` identifies the provider format that produced the
    /// items (currently `"openai-responses"`). Builder paths that
    /// don't recognize the protocol drop the variant — there is no
    /// portable rendering across providers.
    ProviderOutputItems {
        protocol: String,
        items: Vec<serde_json::Value>,
    },
}