lc_schema/messages/
message.rs

1//! Message data structures for chat models.
2
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::collections::HashMap;
6
7use lc_shared::tools::ToolCall;
8
9use super::audio::AudioContent;
10use super::file::FileContent;
11use super::image::ImageContent;
12use super::video::VideoContent;
13
14/// Modality of a media attachment on a multimodal message.
15///
16/// B7 (v0.22.4): the common vocabulary used by the provider-neutral
17/// [`MediaPart`] view; each chat backend maps these onto its own wire blocks
18/// (`image_url` / `input_audio` / `input_video` / `file`, Anthropic
19/// `image`/`document`, Gemini `inline_data`/`file_data`).
20#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
21#[serde(rename_all = "lowercase")]
22pub enum Modality {
23    /// Still image (vision input).
24    Image,
25    /// Audio clip (e.g. voice input / transcription inside a chat turn).
26    Audio,
27    /// Video clip.
28    Video,
29    /// Attached file/document (e.g. PDF).
30    File,
31}
32
33/// One media attachment of a [`Message`], exposed by [`Message::media_parts`].
34///
35/// This is the provider-neutral unified view (B7, v0.22.4) over the parallel
36/// `images` / `audio` / `videos` / `files` vectors: provider request builders
37/// iterate [`Message::media_parts`] once instead of each growing their own
38/// modality-specific special cases.
39#[derive(Debug, Clone, PartialEq)]
40pub enum MediaPart<'a> {
41    /// An image attachment.
42    Image(&'a ImageContent),
43    /// An audio attachment.
44    Audio(&'a AudioContent),
45    /// A video attachment.
46    Video(&'a VideoContent),
47    /// A file/document attachment.
48    File(&'a FileContent),
49}
50
51impl<'a> MediaPart<'a> {
52    /// Returns the attachment's modality.
53    pub fn modality(&self) -> Modality {
54        match self {
55            MediaPart::Image(_) => Modality::Image,
56            MediaPart::Audio(_) => Modality::Audio,
57            MediaPart::Video(_) => Modality::Video,
58            MediaPart::File(_) => Modality::File,
59        }
60    }
61
62    /// Returns the raw value (https URL, `gs://` URI, or `data:` URI).
63    pub fn url(&self) -> &str {
64        match self {
65            MediaPart::Image(m) => &m.url,
66            MediaPart::Audio(m) => &m.url,
67            MediaPart::Video(m) => &m.url,
68            MediaPart::File(m) => &m.url,
69        }
70    }
71
72    /// Returns the explicit MIME type if the attachment carries one.
73    ///
74    /// [`FileContent`] can store an out-of-band MIME type; for data URIs the
75    /// type embedded in the `data:` prefix is parsed instead.
76    pub fn mime_type(&self) -> Option<&str> {
77        match self {
78            MediaPart::File(f) => f.mime_type.as_deref().or_else(|| data_uri_mime(&f.url)),
79            MediaPart::Image(i) => data_uri_mime(&i.url),
80            MediaPart::Audio(a) => data_uri_mime(&a.url),
81            MediaPart::Video(v) => data_uri_mime(&v.url),
82        }
83    }
84
85    /// Returns the optional filename (file attachments only).
86    pub fn name(&self) -> Option<&str> {
87        match self {
88            MediaPart::File(f) => f.name.as_deref(),
89            _ => None,
90        }
91    }
92
93    /// Returns the raw base64 payload if the value is a base64 data URI.
94    pub fn base64_data(&self) -> Option<&str> {
95        let url = self.url();
96        url.split_once(',')
97            .filter(|(prefix, _)| prefix.contains("base64"))
98            .map(|(_, data)| data)
99    }
100
101    /// Returns whether the value is a `data:` URI.
102    pub fn is_data_uri(&self) -> bool {
103        self.url().starts_with("data:")
104    }
105}
106
107/// Extracts the MIME segment from a data URI (`data:<mime>;base64,...`).
108pub(crate) fn data_uri_mime(url: &str) -> Option<&str> {
109    url.strip_prefix("data:")?
110        .split([';', ','])
111        .next()
112        .filter(|mime| !mime.is_empty())
113}
114
115/// Message type classification.
116#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
117#[serde(rename_all = "lowercase")]
118pub enum MessageType {
119    /// System message
120    System,
121    /// Human (user) message
122    Human,
123    /// AI (assistant) message
124    AI,
125    /// Tool result message, carrying the matching tool_call_id
126    Tool {
127        /// Associated tool call ID
128        tool_call_id: String,
129    },
130}
131
132/// Complete message structure for chat interactions.
133#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
134pub struct Message {
135    /// Message text content
136    pub content: String,
137
138    /// Image content (multimodal vision)
139    #[serde(default)]
140    pub images: Vec<ImageContent>,
141
142    /// Audio content (multimodal audio)
143    #[serde(default)]
144    pub audio: Vec<AudioContent>,
145
146    /// Video content (multimodal video) — B7 (v0.22.4)
147    #[serde(default)]
148    pub videos: Vec<VideoContent>,
149
150    /// File content (multimodal document)
151    #[serde(default)]
152    pub files: Vec<FileContent>,
153
154    /// Message type
155    #[serde(rename = "type")]
156    pub message_type: MessageType,
157
158    /// Message name (optional)
159    #[serde(skip_serializing_if = "Option::is_none")]
160    pub name: Option<String>,
161
162    /// Additional keyword arguments
163    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
164    pub additional_kwargs: HashMap<String, Value>,
165
166    /// Message ID (optional)
167    #[serde(skip_serializing_if = "Option::is_none")]
168    pub id: Option<String>,
169
170    /// Tool call list (optional)
171    #[serde(skip_serializing_if = "Option::is_none")]
172    pub tool_calls: Option<Vec<ToolCall>>,
173}
174
175impl Message {
176    /// Creates a system message.
177    pub fn system(content: impl Into<String>) -> Self {
178        Self {
179            content: content.into(),
180            images: Vec::new(),
181            audio: Vec::new(),
182            videos: Vec::new(),
183            files: Vec::new(),
184            message_type: MessageType::System,
185            name: None,
186            additional_kwargs: HashMap::new(),
187            id: None,
188            tool_calls: None,
189        }
190    }
191
192    /// Creates a human (user) message.
193    pub fn human(content: impl Into<String>) -> Self {
194        Self {
195            content: content.into(),
196            images: Vec::new(),
197            audio: Vec::new(),
198            videos: Vec::new(),
199            files: Vec::new(),
200            message_type: MessageType::Human,
201            name: None,
202            additional_kwargs: HashMap::new(),
203            id: None,
204            tool_calls: None,
205        }
206    }
207
208    /// Creates a human message with an image (vision).
209    pub fn human_with_image(content: impl Into<String>, image_url: impl Into<String>) -> Self {
210        Self {
211            content: content.into(),
212            images: vec![ImageContent::from_url(image_url)],
213            audio: Vec::new(),
214            videos: Vec::new(),
215            files: Vec::new(),
216            message_type: MessageType::Human,
217            name: None,
218            additional_kwargs: HashMap::new(),
219            id: None,
220            tool_calls: None,
221        }
222    }
223
224    /// Creates a human message with multiple images.
225    pub fn human_with_images(content: impl Into<String>, images: Vec<ImageContent>) -> Self {
226        Self {
227            content: content.into(),
228            images,
229            audio: Vec::new(),
230            videos: Vec::new(),
231            files: Vec::new(),
232            message_type: MessageType::Human,
233            name: None,
234            additional_kwargs: HashMap::new(),
235            id: None,
236            tool_calls: None,
237        }
238    }
239
240    /// Creates a human message with audio content.
241    pub fn human_with_audio(content: impl Into<String>, audio: AudioContent) -> Self {
242        Self {
243            content: content.into(),
244            images: Vec::new(),
245            audio: vec![audio],
246            videos: Vec::new(),
247            files: Vec::new(),
248            message_type: MessageType::Human,
249            name: None,
250            additional_kwargs: HashMap::new(),
251            id: None,
252            tool_calls: None,
253        }
254    }
255
256    /// Creates a human message with video content.
257    pub fn human_with_video(content: impl Into<String>, video: VideoContent) -> Self {
258        Self {
259            content: content.into(),
260            images: Vec::new(),
261            audio: Vec::new(),
262            videos: vec![video],
263            files: Vec::new(),
264            message_type: MessageType::Human,
265            name: None,
266            additional_kwargs: HashMap::new(),
267            id: None,
268            tool_calls: None,
269        }
270    }
271
272    /// Creates a human message with file content.
273    pub fn human_with_file(content: impl Into<String>, file: FileContent) -> Self {
274        Self {
275            content: content.into(),
276            images: Vec::new(),
277            audio: Vec::new(),
278            videos: Vec::new(),
279            files: vec![file],
280            message_type: MessageType::Human,
281            name: None,
282            additional_kwargs: HashMap::new(),
283            id: None,
284            tool_calls: None,
285        }
286    }
287
288    /// Creates an AI (assistant) message.
289    pub fn ai(content: impl Into<String>) -> Self {
290        Self {
291            content: content.into(),
292            images: Vec::new(),
293            audio: Vec::new(),
294            videos: Vec::new(),
295            files: Vec::new(),
296            message_type: MessageType::AI,
297            name: None,
298            additional_kwargs: HashMap::new(),
299            id: None,
300            tool_calls: None,
301        }
302    }
303
304    /// Creates an AI message with tool calls.
305    pub fn ai_with_tool_calls(content: impl Into<String>, tool_calls: Vec<ToolCall>) -> Self {
306        Self {
307            content: content.into(),
308            images: Vec::new(),
309            audio: Vec::new(),
310            videos: Vec::new(),
311            files: Vec::new(),
312            message_type: MessageType::AI,
313            name: None,
314            additional_kwargs: HashMap::new(),
315            id: None,
316            tool_calls: Some(tool_calls),
317        }
318    }
319
320    /// Creates a tool result message.
321    pub fn tool(tool_call_id: impl Into<String>, content: impl Into<String>) -> Self {
322        Self {
323            content: content.into(),
324            images: Vec::new(),
325            audio: Vec::new(),
326            videos: Vec::new(),
327            files: Vec::new(),
328            message_type: MessageType::Tool {
329                tool_call_id: tool_call_id.into(),
330            },
331            name: None,
332            additional_kwargs: HashMap::new(),
333            id: None,
334            tool_calls: None,
335        }
336    }
337
338    /// Sets the message name.
339    pub fn with_name(mut self, name: impl Into<String>) -> Self {
340        self.name = Some(name.into());
341        self
342    }
343
344    /// Sets the message ID.
345    pub fn with_id(mut self, id: impl Into<String>) -> Self {
346        self.id = Some(id.into());
347        self
348    }
349
350    /// Adds an additional keyword argument.
351    pub fn with_additional_kwarg(mut self, key: impl Into<String>, value: Value) -> Self {
352        self.additional_kwargs.insert(key.into(), value);
353        self
354    }
355
356    /// Adds an image to the message (vision).
357    pub fn with_image(mut self, image: ImageContent) -> Self {
358        self.images.push(image);
359        self
360    }
361
362    /// Adds audio content to the message.
363    pub fn with_audio(mut self, audio: AudioContent) -> Self {
364        self.audio.push(audio);
365        self
366    }
367
368    /// Adds video content to the message.
369    pub fn with_video(mut self, video: VideoContent) -> Self {
370        self.videos.push(video);
371        self
372    }
373
374    /// Adds file content to the message.
375    pub fn with_file(mut self, file: FileContent) -> Self {
376        self.files.push(file);
377        self
378    }
379
380    /// Returns whether the message has images.
381    pub fn has_images(&self) -> bool {
382        !self.images.is_empty()
383    }
384
385    /// Returns whether the message has audio content.
386    pub fn has_audio(&self) -> bool {
387        !self.audio.is_empty()
388    }
389
390    /// Returns whether the message has video content.
391    pub fn has_videos(&self) -> bool {
392        !self.videos.is_empty()
393    }
394
395    /// Returns whether the message has file content.
396    pub fn has_files(&self) -> bool {
397        !self.files.is_empty()
398    }
399
400    /// Returns whether the message has any multimodal content (images, audio,
401    /// video, or files).
402    pub fn is_multimodal(&self) -> bool {
403        self.has_images() || self.has_audio() || self.has_videos() || self.has_files()
404    }
405
406    /// Provider-neutral view over every attachment, in canonical order
407    /// (images → audio → video → files, each in insertion order).
408    ///
409    /// B7 (v0.22.4): chat backends map this single stream onto their wire
410    /// blocks instead of special-casing the parallel content vectors.
411    pub fn media_parts(&self) -> Vec<MediaPart<'_>> {
412        let mut parts: Vec<MediaPart<'_>> = Vec::with_capacity(
413            self.images.len() + self.audio.len() + self.videos.len() + self.files.len(),
414        );
415        parts.extend(self.images.iter().map(MediaPart::Image));
416        parts.extend(self.audio.iter().map(MediaPart::Audio));
417        parts.extend(self.videos.iter().map(MediaPart::Video));
418        parts.extend(self.files.iter().map(MediaPart::File));
419        parts
420    }
421
422    /// Returns the message type as a string.
423    ///
424    /// Tool messages include their `tool_call_id` (e.g. `"tool:call_123"`) so
425    /// the type string is unambiguous about which tool result the message holds.
426    pub fn type_str(&self) -> String {
427        match &self.message_type {
428            MessageType::System => "system".to_string(),
429            MessageType::Human => "human".to_string(),
430            MessageType::AI => "ai".to_string(),
431            MessageType::Tool { tool_call_id } => format!("tool:{tool_call_id}"),
432        }
433    }
434
435    /// Returns whether the message has tool calls.
436    pub fn has_tool_calls(&self) -> bool {
437        self.tool_calls.as_deref().is_some_and(|t| !t.is_empty())
438    }
439
440    /// Returns the tool calls if present.
441    pub fn get_tool_calls(&self) -> Option<&[ToolCall]> {
442        self.tool_calls.as_deref()
443    }
444}
445
446#[cfg(test)]
447mod tests {
448    use super::*;
449
450    #[test]
451    fn test_human_with_image() {
452        let msg = Message::human_with_image("描述这张图", "https://example.com/img.jpg");
453        assert_eq!(msg.content, "描述这张图");
454        assert_eq!(msg.images.len(), 1);
455        assert_eq!(msg.images[0].url, "https://example.com/img.jpg");
456        assert!(msg.has_images());
457    }
458
459    #[test]
460    fn test_human_no_images_by_default() {
461        let msg = Message::human("纯文本");
462        assert!(msg.images.is_empty());
463        assert!(!msg.has_images());
464    }
465
466    #[test]
467    fn test_with_image_builder() {
468        let msg = Message::human("看图")
469            .with_image(ImageContent::from_url("https://example.com/a.png"))
470            .with_image(ImageContent::from_base64("abc"));
471        assert_eq!(msg.images.len(), 2);
472    }
473
474    #[test]
475    fn test_message_deserialize_without_images_field() {
476        // The old format (no images field) must still deserialize (#[serde(default)])
477        let json = r#"{"content":"hi","type":"human"}"#;
478        let msg: Message = serde_json::from_str(json).unwrap();
479        assert_eq!(msg.content, "hi");
480        assert!(msg.images.is_empty());
481    }
482
483    #[test]
484    fn test_human_with_images_multiple() {
485        let msg = Message::human_with_images(
486            "多图",
487            vec![
488                ImageContent::from_url("https://example.com/1.jpg"),
489                ImageContent::from_url("https://example.com/2.jpg"),
490            ],
491        );
492        assert_eq!(msg.images.len(), 2);
493    }
494
495    #[test]
496    fn test_system_ai_no_images() {
497        assert!(Message::system("s").images.is_empty());
498        assert!(Message::ai("a").images.is_empty());
499        assert!(Message::tool("id", "c").images.is_empty());
500    }
501
502    #[test]
503    fn test_type_str_includes_tool_call_id() {
504        assert_eq!(Message::system("s").type_str(), "system");
505        assert_eq!(Message::human("h").type_str(), "human");
506        assert_eq!(Message::ai("a").type_str(), "ai");
507        assert_eq!(
508            Message::tool("call_123", "result").type_str(),
509            "tool:call_123"
510        );
511    }
512
513    #[test]
514    fn test_has_tool_calls_empty_and_present() {
515        let with_calls = Message::ai_with_tool_calls(
516            "call tool",
517            vec![ToolCall::builder("call_1")
518                .name("weather")
519                .arguments(r#"{"city":"beijing"}"#)
520                .build()],
521        );
522        assert!(with_calls.has_tool_calls());
523        assert_eq!(with_calls.get_tool_calls().unwrap().len(), 1);
524
525        // No panic on None or on an empty vec
526        assert!(!Message::ai("plain").has_tool_calls());
527        let empty = Message::ai_with_tool_calls("no calls", vec![]);
528        assert!(!empty.has_tool_calls());
529    }
530
531    // --- B7: video + unified MediaPart view ---
532
533    #[test]
534    fn test_human_with_video() {
535        let msg = Message::human_with_video(
536            "看视频",
537            VideoContent::from_url("https://example.com/clip.mp4"),
538        );
539        assert!(msg.has_videos());
540        assert!(msg.is_multimodal());
541        assert_eq!(msg.videos.len(), 1);
542        assert_eq!(msg.videos[0].url, "https://example.com/clip.mp4");
543    }
544
545    #[test]
546    fn test_with_video_builder() {
547        let msg = Message::human("视频")
548            .with_video(VideoContent::from_url("https://example.com/a.mp4"))
549            .with_video(VideoContent::from_base64("abc"));
550        assert_eq!(msg.videos.len(), 2);
551    }
552
553    #[test]
554    fn test_video_field_defaults_on_old_json() {
555        // Messages serialized before the videos field existed must still deserialize.
556        let json = r#"{"content":"hi","type":"human","images":[],"audio":[],"files":[]}"#;
557        let msg: Message = serde_json::from_str(json).unwrap();
558        assert!(msg.videos.is_empty());
559        assert!(!msg.has_videos());
560    }
561
562    #[test]
563    fn test_media_parts_canonical_order_and_modality() {
564        let msg = Message::human("mixed")
565            .with_image(ImageContent::from_url("https://e.com/a.png"))
566            .with_audio(AudioContent::from_url("https://e.com/a.mp3"))
567            .with_video(VideoContent::from_url("https://e.com/a.mp4"))
568            .with_file(FileContent::from_base64("abc", "application/pdf"));
569
570        let parts = msg.media_parts();
571        assert_eq!(parts.len(), 4);
572        assert_eq!(parts[0].modality(), Modality::Image);
573        assert_eq!(parts[1].modality(), Modality::Audio);
574        assert_eq!(parts[2].modality(), Modality::Video);
575        assert_eq!(parts[3].modality(), Modality::File);
576        assert_eq!(parts[2].url(), "https://e.com/a.mp4");
577        assert_eq!(parts[3].name(), None);
578        assert_eq!(parts[3].mime_type(), Some("application/pdf"));
579        assert_eq!(parts[3].base64_data(), Some("abc"));
580    }
581
582    #[test]
583    fn test_media_part_data_uri_mime() {
584        let img = ImageContent::from_base64_with_mime("zzz", "image/webp");
585        let msg = Message::human("h").with_image(img);
586        let part = &msg.media_parts()[0];
587        assert!(part.is_data_uri());
588        assert_eq!(part.mime_type(), Some("image/webp"));
589        assert_eq!(part.base64_data(), Some("zzz"));
590
591        // Plain URL: no embedded MIME, no base64.
592        let plain = Message::human_with_image("h", "https://example.com/x.jpg");
593        let p = &plain.media_parts()[0];
594        assert!(!p.is_data_uri());
595        assert_eq!(p.mime_type(), None);
596    }
597
598    #[test]
599    fn test_file_part_explicit_mime_and_name() {
600        let file = FileContent::from_url_with_mime("https://e.com/d.pdf", "application/pdf")
601            .with_name("d.pdf");
602        let msg = Message::human_with_file("读文件", file);
603        let part = &msg.media_parts()[0];
604        assert_eq!(part.mime_type(), Some("application/pdf"));
605        assert_eq!(part.name(), Some("d.pdf"));
606    }
607}
lc_schema/messages/message.rs

lc_schema/messages/
message.rs
