serdes_ai_core/messages/
content.rs

1//! Multi-modal content types for user prompts.
2//!
3//! This module defines the content types that can be included in user messages,
4//! supporting text, images, audio, video, documents, and generic files.
5
6use serde::{Deserialize, Serialize};
7
8use super::media::{AudioMediaType, DocumentMediaType, ImageMediaType, VideoMediaType};
9
10/// User message content.
11#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
12#[serde(untagged)]
13pub enum UserContent {
14    /// Plain text content.
15    Text(String),
16    /// Multi-part content.
17    Parts(Vec<UserContentPart>),
18}
19
20impl UserContent {
21    /// Create text content.
22    #[must_use]
23    pub fn text(s: impl Into<String>) -> Self {
24        Self::Text(s.into())
25    }
26
27    /// Create multi-part content.
28    #[must_use]
29    pub fn parts(parts: Vec<UserContentPart>) -> Self {
30        Self::Parts(parts)
31    }
32
33    /// Check if this is text content.
34    #[must_use]
35    pub fn is_text(&self) -> bool {
36        matches!(self, Self::Text(_))
37    }
38
39    /// Get as text if this is text content.
40    #[must_use]
41    pub fn as_text(&self) -> Option<&str> {
42        match self {
43            Self::Text(s) => Some(s),
44            _ => None,
45        }
46    }
47
48    /// Get all parts (wraps text in a single-element vec if needed).
49    #[must_use]
50    pub fn to_parts(&self) -> Vec<UserContentPart> {
51        match self {
52            Self::Text(s) => vec![UserContentPart::Text { text: s.clone() }],
53            Self::Parts(parts) => parts.clone(),
54        }
55    }
56}
57
58impl Default for UserContent {
59    fn default() -> Self {
60        Self::Text(String::new())
61    }
62}
63
64impl From<String> for UserContent {
65    fn from(s: String) -> Self {
66        Self::Text(s)
67    }
68}
69
70impl From<&str> for UserContent {
71    fn from(s: &str) -> Self {
72        Self::Text(s.to_string())
73    }
74}
75
76impl From<Vec<UserContentPart>> for UserContent {
77    fn from(parts: Vec<UserContentPart>) -> Self {
78        Self::Parts(parts)
79    }
80}
81
82/// Individual content part in a multi-part message.
83#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
84#[serde(tag = "type", rename_all = "snake_case")]
85pub enum UserContentPart {
86    /// Text content.
87    Text {
88        /// The text.
89        text: String,
90    },
91    /// Image content.
92    Image {
93        /// The image.
94        #[serde(flatten)]
95        image: ImageContent,
96    },
97    /// Audio content.
98    Audio {
99        /// The audio.
100        #[serde(flatten)]
101        audio: AudioContent,
102    },
103    /// Video content.
104    Video {
105        /// The video.
106        #[serde(flatten)]
107        video: VideoContent,
108    },
109    /// Document content.
110    Document {
111        /// The document.
112        #[serde(flatten)]
113        document: DocumentContent,
114    },
115    /// Generic file content.
116    File {
117        /// The file.
118        #[serde(flatten)]
119        file: FileContent,
120    },
121}
122
123impl UserContentPart {
124    /// Create text content.
125    #[must_use]
126    pub fn text(s: impl Into<String>) -> Self {
127        Self::Text { text: s.into() }
128    }
129
130    /// Create image content from URL.
131    #[must_use]
132    pub fn image_url(url: impl Into<String>) -> Self {
133        Self::Image {
134            image: ImageContent::url(url),
135        }
136    }
137
138    /// Create image content from binary data.
139    #[must_use]
140    pub fn image_binary(data: Vec<u8>, media_type: ImageMediaType) -> Self {
141        Self::Image {
142            image: ImageContent::binary(data, media_type),
143        }
144    }
145}
146
147/// Image content.
148#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
149#[serde(untagged)]
150pub enum ImageContent {
151    /// Image from URL.
152    Url(ImageUrl),
153    /// Binary image data.
154    Binary(BinaryImage),
155}
156
157impl ImageContent {
158    /// Create from URL.
159    #[must_use]
160    pub fn url(url: impl Into<String>) -> Self {
161        Self::Url(ImageUrl::new(url))
162    }
163
164    /// Create from binary data.
165    #[must_use]
166    pub fn binary(data: Vec<u8>, media_type: ImageMediaType) -> Self {
167        Self::Binary(BinaryImage::new(data, media_type))
168    }
169
170    /// Get the media type if known.
171    #[must_use]
172    pub fn media_type(&self) -> Option<ImageMediaType> {
173        match self {
174            Self::Url(u) => u.media_type,
175            Self::Binary(b) => Some(b.media_type),
176        }
177    }
178}
179
180/// Image from URL.
181#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
182pub struct ImageUrl {
183    /// The image URL.
184    pub url: String,
185    /// Media type hint.
186    #[serde(skip_serializing_if = "Option::is_none")]
187    pub media_type: Option<ImageMediaType>,
188    /// Force download instead of using URL directly.
189    #[serde(default)]
190    pub force_download: bool,
191    /// Vendor-specific metadata.
192    #[serde(skip_serializing_if = "Option::is_none")]
193    pub vendor_metadata: Option<serde_json::Value>,
194}
195
196impl ImageUrl {
197    /// Create a new image URL.
198    #[must_use]
199    pub fn new(url: impl Into<String>) -> Self {
200        Self {
201            url: url.into(),
202            media_type: None,
203            force_download: false,
204            vendor_metadata: None,
205        }
206    }
207
208    /// Set the media type.
209    #[must_use]
210    pub fn with_media_type(mut self, media_type: ImageMediaType) -> Self {
211        self.media_type = Some(media_type);
212        self
213    }
214
215    /// Set force download.
216    #[must_use]
217    pub fn with_force_download(mut self, force: bool) -> Self {
218        self.force_download = force;
219        self
220    }
221
222    /// Set vendor metadata.
223    #[must_use]
224    pub fn with_vendor_metadata(mut self, metadata: serde_json::Value) -> Self {
225        self.vendor_metadata = Some(metadata);
226        self
227    }
228}
229
230/// Binary image data.
231#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
232pub struct BinaryImage {
233    /// The raw image data.
234    #[serde(with = "base64_serde")]
235    pub data: Vec<u8>,
236    /// The media type.
237    pub media_type: ImageMediaType,
238}
239
240impl BinaryImage {
241    /// Create new binary image.
242    #[must_use]
243    pub fn new(data: Vec<u8>, media_type: ImageMediaType) -> Self {
244        Self { data, media_type }
245    }
246
247    /// Get data as base64 string.
248    #[must_use]
249    pub fn to_base64(&self) -> String {
250        base64::Engine::encode(&base64::engine::general_purpose::STANDARD, &self.data)
251    }
252
253    /// Get as data URL.
254    #[must_use]
255    pub fn to_data_url(&self) -> String {
256        format!(
257            "data:{};base64,{}",
258            self.media_type.mime_type(),
259            self.to_base64()
260        )
261    }
262}
263
264/// Audio content.
265#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
266#[serde(untagged)]
267pub enum AudioContent {
268    /// Audio from URL.
269    Url(AudioUrl),
270    /// Binary audio data.
271    Binary(BinaryAudio),
272}
273
274impl AudioContent {
275    /// Create from URL.
276    #[must_use]
277    pub fn url(url: impl Into<String>) -> Self {
278        Self::Url(AudioUrl::new(url))
279    }
280
281    /// Create from binary data.
282    #[must_use]
283    pub fn binary(data: Vec<u8>, media_type: AudioMediaType) -> Self {
284        Self::Binary(BinaryAudio::new(data, media_type))
285    }
286}
287
288/// Audio from URL.
289#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
290pub struct AudioUrl {
291    /// The audio URL.
292    pub url: String,
293    /// Media type hint.
294    #[serde(skip_serializing_if = "Option::is_none")]
295    pub media_type: Option<AudioMediaType>,
296    /// Force download instead of using URL directly.
297    #[serde(default)]
298    pub force_download: bool,
299    /// Vendor-specific metadata.
300    #[serde(skip_serializing_if = "Option::is_none")]
301    pub vendor_metadata: Option<serde_json::Value>,
302}
303
304impl AudioUrl {
305    /// Create a new audio URL.
306    #[must_use]
307    pub fn new(url: impl Into<String>) -> Self {
308        Self {
309            url: url.into(),
310            media_type: None,
311            force_download: false,
312            vendor_metadata: None,
313        }
314    }
315
316    /// Set the media type.
317    #[must_use]
318    pub fn with_media_type(mut self, media_type: AudioMediaType) -> Self {
319        self.media_type = Some(media_type);
320        self
321    }
322}
323
324/// Binary audio data.
325#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
326pub struct BinaryAudio {
327    /// The raw audio data.
328    #[serde(with = "base64_serde")]
329    pub data: Vec<u8>,
330    /// The media type.
331    pub media_type: AudioMediaType,
332}
333
334impl BinaryAudio {
335    /// Create new binary audio.
336    #[must_use]
337    pub fn new(data: Vec<u8>, media_type: AudioMediaType) -> Self {
338        Self { data, media_type }
339    }
340}
341
342/// Video content.
343#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
344#[serde(untagged)]
345pub enum VideoContent {
346    /// Video from URL.
347    Url(VideoUrl),
348    /// Binary video data.
349    Binary(BinaryVideo),
350}
351
352impl VideoContent {
353    /// Create from URL.
354    #[must_use]
355    pub fn url(url: impl Into<String>) -> Self {
356        Self::Url(VideoUrl::new(url))
357    }
358
359    /// Create from binary data.
360    #[must_use]
361    pub fn binary(data: Vec<u8>, media_type: VideoMediaType) -> Self {
362        Self::Binary(BinaryVideo::new(data, media_type))
363    }
364}
365
366/// Video from URL.
367#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
368pub struct VideoUrl {
369    /// The video URL.
370    pub url: String,
371    /// Media type hint.
372    #[serde(skip_serializing_if = "Option::is_none")]
373    pub media_type: Option<VideoMediaType>,
374    /// Force download instead of using URL directly.
375    #[serde(default)]
376    pub force_download: bool,
377    /// Vendor-specific metadata.
378    #[serde(skip_serializing_if = "Option::is_none")]
379    pub vendor_metadata: Option<serde_json::Value>,
380}
381
382impl VideoUrl {
383    /// Create a new video URL.
384    #[must_use]
385    pub fn new(url: impl Into<String>) -> Self {
386        Self {
387            url: url.into(),
388            media_type: None,
389            force_download: false,
390            vendor_metadata: None,
391        }
392    }
393
394    /// Set the media type.
395    #[must_use]
396    pub fn with_media_type(mut self, media_type: VideoMediaType) -> Self {
397        self.media_type = Some(media_type);
398        self
399    }
400}
401
402/// Binary video data.
403#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
404pub struct BinaryVideo {
405    /// The raw video data.
406    #[serde(with = "base64_serde")]
407    pub data: Vec<u8>,
408    /// The media type.
409    pub media_type: VideoMediaType,
410}
411
412impl BinaryVideo {
413    /// Create new binary video.
414    #[must_use]
415    pub fn new(data: Vec<u8>, media_type: VideoMediaType) -> Self {
416        Self { data, media_type }
417    }
418}
419
420/// Document content.
421#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
422#[serde(untagged)]
423pub enum DocumentContent {
424    /// Document from URL.
425    Url(DocumentUrl),
426    /// Binary document data.
427    Binary(BinaryDocument),
428}
429
430impl DocumentContent {
431    /// Create from URL.
432    #[must_use]
433    pub fn url(url: impl Into<String>) -> Self {
434        Self::Url(DocumentUrl::new(url))
435    }
436
437    /// Create from binary data.
438    #[must_use]
439    pub fn binary(data: Vec<u8>, media_type: DocumentMediaType) -> Self {
440        Self::Binary(BinaryDocument::new(data, media_type))
441    }
442}
443
444/// Document from URL.
445#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
446pub struct DocumentUrl {
447    /// The document URL.
448    pub url: String,
449    /// Media type hint.
450    #[serde(skip_serializing_if = "Option::is_none")]
451    pub media_type: Option<DocumentMediaType>,
452    /// Force download instead of using URL directly.
453    #[serde(default)]
454    pub force_download: bool,
455    /// Vendor-specific metadata.
456    #[serde(skip_serializing_if = "Option::is_none")]
457    pub vendor_metadata: Option<serde_json::Value>,
458}
459
460impl DocumentUrl {
461    /// Create a new document URL.
462    #[must_use]
463    pub fn new(url: impl Into<String>) -> Self {
464        Self {
465            url: url.into(),
466            media_type: None,
467            force_download: false,
468            vendor_metadata: None,
469        }
470    }
471
472    /// Set the media type.
473    #[must_use]
474    pub fn with_media_type(mut self, media_type: DocumentMediaType) -> Self {
475        self.media_type = Some(media_type);
476        self
477    }
478}
479
480/// Binary document data.
481#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
482pub struct BinaryDocument {
483    /// The raw document data.
484    #[serde(with = "base64_serde")]
485    pub data: Vec<u8>,
486    /// The media type.
487    pub media_type: DocumentMediaType,
488    /// Optional filename.
489    #[serde(skip_serializing_if = "Option::is_none")]
490    pub filename: Option<String>,
491}
492
493impl BinaryDocument {
494    /// Create new binary document.
495    #[must_use]
496    pub fn new(data: Vec<u8>, media_type: DocumentMediaType) -> Self {
497        Self {
498            data,
499            media_type,
500            filename: None,
501        }
502    }
503
504    /// Set the filename.
505    #[must_use]
506    pub fn with_filename(mut self, filename: impl Into<String>) -> Self {
507        self.filename = Some(filename.into());
508        self
509    }
510}
511
512/// Generic file content.
513#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
514#[serde(untagged)]
515pub enum FileContent {
516    /// File from URL.
517    Url(FileUrl),
518    /// Binary file data.
519    Binary(BinaryFile),
520}
521
522impl FileContent {
523    /// Create from URL.
524    #[must_use]
525    pub fn url(url: impl Into<String>) -> Self {
526        Self::Url(FileUrl::new(url))
527    }
528
529    /// Create from binary data.
530    #[must_use]
531    pub fn binary(data: Vec<u8>, mime_type: impl Into<String>) -> Self {
532        Self::Binary(BinaryFile::new(data, mime_type))
533    }
534}
535
536/// File from URL.
537#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
538pub struct FileUrl {
539    /// The file URL.
540    pub url: String,
541    /// MIME type hint.
542    #[serde(skip_serializing_if = "Option::is_none")]
543    pub mime_type: Option<String>,
544    /// Force download instead of using URL directly.
545    #[serde(default)]
546    pub force_download: bool,
547    /// Vendor-specific metadata.
548    #[serde(skip_serializing_if = "Option::is_none")]
549    pub vendor_metadata: Option<serde_json::Value>,
550}
551
552impl FileUrl {
553    /// Create a new file URL.
554    #[must_use]
555    pub fn new(url: impl Into<String>) -> Self {
556        Self {
557            url: url.into(),
558            mime_type: None,
559            force_download: false,
560            vendor_metadata: None,
561        }
562    }
563
564    /// Set the MIME type.
565    #[must_use]
566    pub fn with_mime_type(mut self, mime_type: impl Into<String>) -> Self {
567        self.mime_type = Some(mime_type.into());
568        self
569    }
570}
571
572/// Binary file data.
573#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
574pub struct BinaryFile {
575    /// The raw file data.
576    #[serde(with = "base64_serde")]
577    pub data: Vec<u8>,
578    /// The MIME type.
579    pub mime_type: String,
580    /// Optional filename.
581    #[serde(skip_serializing_if = "Option::is_none")]
582    pub filename: Option<String>,
583}
584
585impl BinaryFile {
586    /// Create new binary file.
587    #[must_use]
588    pub fn new(data: Vec<u8>, mime_type: impl Into<String>) -> Self {
589        Self {
590            data,
591            mime_type: mime_type.into(),
592            filename: None,
593        }
594    }
595
596    /// Set the filename.
597    #[must_use]
598    pub fn with_filename(mut self, filename: impl Into<String>) -> Self {
599        self.filename = Some(filename.into());
600        self
601    }
602}
603
604/// Serde helper for base64 encoding.
605mod base64_serde {
606    use base64::{engine::general_purpose::STANDARD, Engine};
607    use serde::{Deserialize, Deserializer, Serializer};
608
609    pub fn serialize<S>(data: &[u8], serializer: S) -> Result<S::Ok, S::Error>
610    where
611        S: Serializer,
612    {
613        serializer.serialize_str(&STANDARD.encode(data))
614    }
615
616    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
617    where
618        D: Deserializer<'de>,
619    {
620        let s = String::deserialize(deserializer)?;
621        STANDARD.decode(s).map_err(serde::de::Error::custom)
622    }
623}
624
625#[cfg(test)]
626mod tests {
627    use super::*;
628
629    #[test]
630    fn test_user_content_text() {
631        let content = UserContent::text("Hello, world!");
632        assert!(content.is_text());
633        assert_eq!(content.as_text(), Some("Hello, world!"));
634    }
635
636    #[test]
637    fn test_user_content_from_string() {
638        let content: UserContent = "Hello".into();
639        assert!(content.is_text());
640    }
641
642    #[test]
643    fn test_image_url() {
644        let img =
645            ImageUrl::new("https://example.com/image.png").with_media_type(ImageMediaType::Png);
646        assert_eq!(img.url, "https://example.com/image.png");
647        assert_eq!(img.media_type, Some(ImageMediaType::Png));
648    }
649
650    #[test]
651    fn test_binary_image_to_data_url() {
652        let img = BinaryImage::new(vec![1, 2, 3, 4], ImageMediaType::Png);
653        let data_url = img.to_data_url();
654        assert!(data_url.starts_with("data:image/png;base64,"));
655    }
656
657    #[test]
658    fn test_serde_roundtrip() {
659        let content = UserContent::parts(vec![
660            UserContentPart::text("Hello"),
661            UserContentPart::image_url("https://example.com/img.jpg"),
662        ]);
663        let json = serde_json::to_string(&content).unwrap();
664        let parsed: UserContent = serde_json::from_str(&json).unwrap();
665        assert_eq!(content, parsed);
666    }
667}
serdes_ai_core/messages/content.rs

serdes_ai_core/messages/
content.rs
