car_inference_types/lib.rs
1//! Pure-serde conversation/message wire types for CAR inference.
2//!
3//! These types — [`Message`], [`ToolCall`], [`ContentBlock`] — define the
4//! multi-turn conversation wire form that car-inference's protocol handlers
5//! and local chat templates consume. They were extracted out of
6//! `car-inference` (where they still live as re-exports) so a **dependency-
7//! light** consumer can build and pattern-match the REAL types without pulling
8//! the Candle/MLX inference stack:
9//!
10//! - `car-inference` re-exports every type here (`pub use
11//! car_inference_types::{Message, ToolCall, ContentBlock}`), so its own
12//! callers are unchanged.
13//! - `car-sync`'s transcript-resume projection (multi-device sync B2) builds
14//! `Vec<Message>` directly from folded conversation ops. Because it depends
15//! on this crate — not on a hand-copied mirror — a change to `Message`'s
16//! shape is a **compile error** at the resume site, not a runtime
17//! `from_value::<Message>` break in the daemon (the drift trap the kernel
18//! review flagged).
19//!
20//! The crate is deliberately serde-only. Anything that needs Candle, a
21//! tokenizer, or a model backend belongs in `car-inference`, not here.
22
23use serde::{Deserialize, Serialize};
24
25/// A tool call returned by the model.
26#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
27pub struct ToolCall {
28 /// Provider-assigned tool call ID (e.g. OpenAI `call_abc123`, Anthropic `toolu_abc123`).
29 /// When present, protocol handlers use this for round-trip correlation instead of
30 /// synthesizing positional IDs like `call_0`.
31 #[serde(default, skip_serializing_if = "Option::is_none")]
32 pub id: Option<String>,
33 /// Tool/function name.
34 pub name: String,
35 /// Arguments as key-value pairs.
36 pub arguments: std::collections::HashMap<String, serde_json::Value>,
37}
38
39/// A content block in a multimodal message.
40///
41/// The image variants (`ImageBase64`, `ImageUrl`) are fully wired on
42/// the native Qwen2.5-VL backend. The video variants
43/// (`VideoPath`, `VideoUrl`, `VideoBase64`) are defined on the public
44/// request surface so higher-level tooling can express Qwen2.5-VL
45/// video-understanding payloads, but the native backend returns
46/// `UnsupportedMode` for them until the video-tokenization path lands.
47/// Remote multimodal providers (Anthropic, Google Vertex) accept them
48/// through the protocol handlers today.
49#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
50#[serde(tag = "type", rename_all = "snake_case")]
51pub enum ContentBlock {
52 /// Plain text content.
53 Text { text: String },
54 /// Base64-encoded image.
55 ImageBase64 {
56 /// Base64-encoded image data.
57 data: String,
58 /// MIME type (e.g., "image/png", "image/jpeg").
59 media_type: String,
60 },
61 /// Image from URL.
62 ImageUrl {
63 /// URL of the image.
64 url: String,
65 /// Detail level for image processing ("auto", "low", "high").
66 #[serde(default = "default_detail")]
67 detail: String,
68 },
69 /// Video loaded from a local filesystem path. Qwen2.5-VL samples
70 /// the clip at `fps` frames/sec (default: backend-chosen) and
71 /// caps at `max_frames` to respect context budgets.
72 VideoPath {
73 path: String,
74 #[serde(default, skip_serializing_if = "Option::is_none")]
75 fps: Option<f32>,
76 #[serde(default, skip_serializing_if = "Option::is_none")]
77 max_frames: Option<u32>,
78 },
79 /// Video accessible over HTTP(S). Semantics as [`ContentBlock::VideoPath`].
80 VideoUrl {
81 url: String,
82 #[serde(default, skip_serializing_if = "Option::is_none")]
83 fps: Option<f32>,
84 #[serde(default, skip_serializing_if = "Option::is_none")]
85 max_frames: Option<u32>,
86 },
87 /// Base64-encoded video bytes. Prefer `VideoPath` when possible;
88 /// inline base64 is expensive to round-trip.
89 VideoBase64 {
90 data: String,
91 media_type: String,
92 #[serde(default, skip_serializing_if = "Option::is_none")]
93 fps: Option<f32>,
94 #[serde(default, skip_serializing_if = "Option::is_none")]
95 max_frames: Option<u32>,
96 },
97 /// Audio loaded from a local filesystem path. Used for
98 /// audio-understanding models (Gemma 4 small variants, Gemini).
99 AudioPath {
100 path: String,
101 /// Optional explicit sample-rate hint. Most backends will
102 /// resample internally; this is a best-effort declaration.
103 #[serde(default, skip_serializing_if = "Option::is_none")]
104 sample_rate: Option<u32>,
105 },
106 /// Audio accessible over HTTP(S).
107 AudioUrl {
108 url: String,
109 #[serde(default, skip_serializing_if = "Option::is_none")]
110 sample_rate: Option<u32>,
111 },
112 /// Base64-encoded audio bytes.
113 AudioBase64 {
114 data: String,
115 media_type: String,
116 #[serde(default, skip_serializing_if = "Option::is_none")]
117 sample_rate: Option<u32>,
118 },
119}
120
121impl ContentBlock {
122 /// Return true if this block carries video data (any encoding).
123 /// Used by backends that need to refuse video inputs until the
124 /// tokenization path is wired.
125 pub fn is_video(&self) -> bool {
126 matches!(
127 self,
128 ContentBlock::VideoPath { .. }
129 | ContentBlock::VideoUrl { .. }
130 | ContentBlock::VideoBase64 { .. }
131 )
132 }
133
134 /// Return true if this block carries audio data (any encoding).
135 /// Used by backends that need to refuse audio inputs until the
136 /// tokenization path is wired. Gemma 4 small variants and Gemini
137 /// accept audio; everything else in CAR rejects with
138 /// `UnsupportedMode`.
139 pub fn is_audio(&self) -> bool {
140 matches!(
141 self,
142 ContentBlock::AudioPath { .. }
143 | ContentBlock::AudioUrl { .. }
144 | ContentBlock::AudioBase64 { .. }
145 )
146 }
147}
148
149fn default_detail() -> String {
150 "auto".to_string()
151}
152
153/// A message in a multi-turn conversation.
154///
155/// The `System` variant exists so callers can express a first-class
156/// system prompt inside `messages: Vec<Message>` without threading it
157/// through the legacy `context: Option<String>` field on the request.
158/// Protocol handlers and local chat templates that have a native
159/// system-role slot (OpenAI, Anthropic, Gemini, Gemma 4, Qwen) emit it
160/// in the right place; ones that don't can fold it into the first user
161/// turn.
162#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
163#[serde(tag = "role", rename_all = "snake_case")]
164pub enum Message {
165 /// A system prompt. Appears once, at the start of the conversation.
166 System { content: String },
167 /// A user message (text only).
168 User { content: String },
169 /// A user message with multimodal content (text + images + video + audio).
170 UserMultimodal { content: Vec<ContentBlock> },
171 /// An assistant response, possibly with tool calls.
172 Assistant {
173 #[serde(default)]
174 content: String,
175 #[serde(default)]
176 tool_calls: Vec<ToolCall>,
177 },
178 /// The result of executing a tool call.
179 ToolResult {
180 tool_use_id: String,
181 content: String,
182 },
183 /// Provider-specific output items that need to round-trip
184 /// verbatim across turns. The OpenAI Responses API returns
185 /// reasoning blobs, encrypted_content, web-search results, etc.
186 /// as opaque structured items; the next request must include
187 /// them in the same form to preserve provider-side state.
188 ///
189 /// `protocol` identifies the provider format that produced the
190 /// items (currently `"openai-responses"`). Builder paths that
191 /// don't recognize the protocol drop the variant — there is no
192 /// portable rendering across providers.
193 ProviderOutputItems {
194 protocol: String,
195 items: Vec<serde_json::Value>,
196 },
197}