mira/
content.rs

1//! Multimodal content: a typed vocabulary for non-text inputs and outputs.
2//!
3//! Mira's text fields — [`Sample::input`](crate::Sample::input) and
4//! [`Transcript::final_response`](crate::Transcript::final_response) — stay the
5//! canonical path for the common, text-only case. A [`Part`] generalizes a turn
6//! or a response into one typed piece of content — text, an image, audio, a
7//! file, or a structured JSON value — so multimodal subjects fit a first-class
8//! shape rather than smuggling media through `files`/`metadata`/`events`.
9//!
10//! Design decisions (kept here, on the type):
11//! * **Media is referenced, not embedded as bytes.** A media [`Part`] carries a
12//!   `media_type` (an IANA type like `image/png`) plus *either* a `uri` (an
13//!   `http(s)://` URL or a `data:` URI) *or* inline base64 `data`. Never raw
14//!   bytes — so a `Part` is plain JSON that serializes on the wire and into
15//!   JSONL datasets unchanged, and the core stays dependency-free (no image/
16//!   audio codecs).
17//! * **Open by construction, closed in shape.** The variant set is small and
18//!   typed (scorers can match on it), while `Json` is the escape hatch for any
19//!   structured output the typed variants don't cover.
20
21use serde::{Deserialize, Serialize};
22
23/// Where a media [`Part`]'s bytes come from — **exactly one** of a referenced
24/// `uri` (URL or `data:` URI) or inline base64 `data`.
25///
26/// Modelled as an enum, not two `Option`s, so the "exactly one" invariant is
27/// enforced by the type: a media part with *neither* source fails to
28/// deserialize (rather than silently becoming ambiguous), and *both* is
29/// unrepresentable. Serializes externally-tagged: `{"uri":"…"}` / `{"data":"…"}`.
30#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
31#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
32#[serde(rename_all = "snake_case")]
33pub enum Source {
34    /// A URL or `data:` URI the consumer fetches/decodes.
35    Uri(String),
36    /// Inline base64-encoded bytes.
37    Data(String),
38}
39
40impl Source {
41    /// The referenced URI, if this is a [`Source::Uri`].
42    pub fn uri(&self) -> Option<&str> {
43        match self {
44            Source::Uri(u) => Some(u),
45            Source::Data(_) => None,
46        }
47    }
48
49    /// The inline base64 data, if this is a [`Source::Data`].
50    pub fn data(&self) -> Option<&str> {
51        match self {
52            Source::Data(d) => Some(d),
53            Source::Uri(_) => None,
54        }
55    }
56}
57
58/// One piece of multimodal content. The discriminant serializes as a `kind`
59/// tag (`text` / `image` / `audio` / `file` / `json`), so a part is a single
60/// self-describing JSON object. Media variants reference their bytes through a
61/// [`Source`] (exactly one of `uri` / `data`).
62#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
63#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
64#[serde(tag = "kind", rename_all = "snake_case")]
65pub enum Part {
66    /// Plain text — the same content `final_response` / `input` carry, but as a
67    /// part so it can sit alongside media in one ordered list.
68    Text { text: String },
69    /// An image.
70    Image { media_type: String, source: Source },
71    /// Audio.
72    Audio { media_type: String, source: Source },
73    /// An arbitrary file (a document, an archive, …), with an optional display
74    /// `name`.
75    File {
76        #[serde(default, skip_serializing_if = "String::is_empty")]
77        name: String,
78        media_type: String,
79        source: Source,
80    },
81    /// A structured JSON value — for tool-style / structured outputs that aren't
82    /// free text and aren't a media blob.
83    Json { json: serde_json::Value },
84}
85
86impl Part {
87    /// A text part.
88    pub fn text(text: impl Into<String>) -> Self {
89        Part::Text { text: text.into() }
90    }
91
92    /// An image referenced by a `uri` (a URL or a `data:` URI).
93    pub fn image_uri(media_type: impl Into<String>, uri: impl Into<String>) -> Self {
94        Part::Image {
95            media_type: media_type.into(),
96            source: Source::Uri(uri.into()),
97        }
98    }
99
100    /// An image carried inline as base64.
101    pub fn image_data(media_type: impl Into<String>, data: impl Into<String>) -> Self {
102        Part::Image {
103            media_type: media_type.into(),
104            source: Source::Data(data.into()),
105        }
106    }
107
108    /// Audio referenced by a `uri`.
109    pub fn audio_uri(media_type: impl Into<String>, uri: impl Into<String>) -> Self {
110        Part::Audio {
111            media_type: media_type.into(),
112            source: Source::Uri(uri.into()),
113        }
114    }
115
116    /// Audio carried inline as base64.
117    pub fn audio_data(media_type: impl Into<String>, data: impl Into<String>) -> Self {
118        Part::Audio {
119            media_type: media_type.into(),
120            source: Source::Data(data.into()),
121        }
122    }
123
124    /// A file referenced by a `uri`, with a display `name`.
125    pub fn file_uri(
126        name: impl Into<String>,
127        media_type: impl Into<String>,
128        uri: impl Into<String>,
129    ) -> Self {
130        Part::File {
131            name: name.into(),
132            media_type: media_type.into(),
133            source: Source::Uri(uri.into()),
134        }
135    }
136
137    /// A file carried inline as base64, with a display `name` (the `data`
138    /// counterpart of [`file_uri`](Self::file_uri)).
139    pub fn file_data(
140        name: impl Into<String>,
141        media_type: impl Into<String>,
142        data: impl Into<String>,
143    ) -> Self {
144        Part::File {
145            name: name.into(),
146            media_type: media_type.into(),
147            source: Source::Data(data.into()),
148        }
149    }
150
151    /// A structured JSON part.
152    pub fn json(value: impl Into<serde_json::Value>) -> Self {
153        Part::Json { json: value.into() }
154    }
155
156    /// The discriminant as a stable string: `text` / `image` / `audio` /
157    /// `file` / `json`. Matches the serialized `kind` tag, so scorers and
158    /// reports can name a modality without re-deriving it.
159    pub fn kind(&self) -> &'static str {
160        match self {
161            Part::Text { .. } => "text",
162            Part::Image { .. } => "image",
163            Part::Audio { .. } => "audio",
164            Part::File { .. } => "file",
165            Part::Json { .. } => "json",
166        }
167    }
168
169    /// True for a text part.
170    pub fn is_text(&self) -> bool {
171        matches!(self, Part::Text { .. })
172    }
173
174    /// The text of a [`Part::Text`], else `None`.
175    pub fn as_text(&self) -> Option<&str> {
176        match self {
177            Part::Text { text } => Some(text),
178            _ => None,
179        }
180    }
181
182    /// The IANA `media_type` of a media part (image/audio/file), else `None`.
183    pub fn media_type(&self) -> Option<&str> {
184        match self {
185            Part::Image { media_type, .. }
186            | Part::Audio { media_type, .. }
187            | Part::File { media_type, .. } => Some(media_type),
188            _ => None,
189        }
190    }
191
192    /// The [`Source`] of a media part (image/audio/file), else `None`.
193    pub fn source(&self) -> Option<&Source> {
194        match self {
195            Part::Image { source, .. } | Part::Audio { source, .. } | Part::File { source, .. } => {
196                Some(source)
197            }
198            _ => None,
199        }
200    }
201
202    /// The referenced URI of a media part, if it carries one.
203    pub fn uri(&self) -> Option<&str> {
204        self.source().and_then(Source::uri)
205    }
206
207    /// The inline base64 data of a media part, if it carries any.
208    pub fn data(&self) -> Option<&str> {
209        self.source().and_then(Source::data)
210    }
211}
212
213/// Who authored a [`Message`] in a multi-turn conversation.
214#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
215#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
216#[serde(rename_all = "snake_case")]
217pub enum Role {
218    /// The (real or simulated) user driving the conversation.
219    User,
220    /// The subject under evaluation.
221    Assistant,
222}
223
224/// One turn in a multi-turn conversation: a [`Role`] plus its multimodal
225/// [`Part`]s. Interactive evals exchange these — a [`Responder`] supplies the
226/// `User` turns, the subject produces the `Assistant` ones.
227///
228/// [`Responder`]: crate::eval::Responder
229#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
230#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
231pub struct Message {
232    pub role: Role,
233    pub content: Vec<Part>,
234}
235
236impl Message {
237    /// A message with the given role and parts.
238    pub fn new(role: Role, content: impl IntoIterator<Item = Part>) -> Self {
239        Self {
240            role,
241            content: content.into_iter().collect(),
242        }
243    }
244
245    /// A user message carrying a single text part.
246    pub fn user(text: impl Into<String>) -> Self {
247        Self::new(Role::User, [Part::text(text)])
248    }
249
250    /// An assistant message carrying a single text part.
251    pub fn assistant(text: impl Into<String>) -> Self {
252        Self::new(Role::Assistant, [Part::text(text)])
253    }
254
255    /// The concatenated text of this message's parts.
256    pub fn text(&self) -> String {
257        text_of(&self.content)
258    }
259}
260
261/// The concatenated text of all [`Part::Text`] parts, joined by newlines.
262/// Non-text parts are skipped — the text projection a text-only scorer sees.
263pub fn text_of(parts: &[Part]) -> String {
264    parts
265        .iter()
266        .filter_map(Part::as_text)
267        .collect::<Vec<_>>()
268        .join("\n")
269}
270
271/// The distinct modalities present, in first-seen order (e.g.
272/// `["text", "image"]`). Drives modality-aware scorers and reporting.
273pub fn modalities(parts: &[Part]) -> Vec<&'static str> {
274    let mut seen: Vec<&'static str> = Vec::new();
275    for p in parts {
276        let k = p.kind();
277        if !seen.contains(&k) {
278            seen.push(k);
279        }
280    }
281    seen
282}
283
284#[cfg(test)]
285mod tests {
286    use super::*;
287
288    #[test]
289    fn text_and_modalities() {
290        let parts = vec![
291            Part::text("describe this"),
292            Part::image_uri("image/png", "https://x/y.png"),
293            Part::text("and this"),
294            Part::image_data("image/jpeg", "QUJD"),
295        ];
296        assert_eq!(text_of(&parts), "describe this\nand this");
297        // Distinct, first-seen order — `image` not repeated.
298        assert_eq!(modalities(&parts), vec!["text", "image"]);
299    }
300
301    #[test]
302    fn accessors() {
303        let img = Part::image_uri("image/png", "u");
304        assert_eq!(img.kind(), "image");
305        assert!(!img.is_text());
306        assert_eq!(img.as_text(), None);
307        assert_eq!(img.media_type(), Some("image/png"));
308        assert_eq!(img.uri(), Some("u"));
309        assert_eq!(img.data(), None);
310
311        let blob = Part::image_data("image/png", "QUJD");
312        assert_eq!(blob.uri(), None);
313        assert_eq!(blob.data(), Some("QUJD"));
314
315        let txt = Part::text("hi");
316        assert_eq!(txt.kind(), "text");
317        assert!(txt.is_text());
318        assert_eq!(txt.as_text(), Some("hi"));
319        assert_eq!(txt.media_type(), None);
320        assert_eq!(txt.source(), None);
321    }
322
323    #[test]
324    fn serializes_with_kind_tag_and_round_trips() {
325        let parts = vec![
326            Part::text("hello"),
327            Part::image_data("image/png", "QkFTRTY0"),
328            Part::file_uri("notes.pdf", "application/pdf", "https://x/n.pdf"),
329            Part::file_data("blob.bin", "application/octet-stream", "QUJD"),
330            Part::json(serde_json::json!({"label": "cat", "p": 0.9})),
331        ];
332        let line = serde_json::to_string(&parts).unwrap();
333        // The discriminant rides as `kind`; the source is externally-tagged.
334        assert!(line.contains(r#""kind":"text""#));
335        assert!(line.contains(r#""kind":"image""#));
336        assert!(line.contains(r#""source":{"data":"QkFTRTY0"}"#));
337        let back: Vec<Part> = serde_json::from_str(&line).unwrap();
338        assert_eq!(back, parts);
339    }
340
341    #[test]
342    fn media_part_requires_a_source() {
343        // A media part with no source is invalid and fails to deserialize fast,
344        // rather than parsing into an ambiguous "neither uri nor data" state.
345        let err = serde_json::from_str::<Part>(r#"{"kind":"image","media_type":"image/png"}"#);
346        assert!(err.is_err());
347        // A well-formed one parses.
348        let ok = serde_json::from_str::<Part>(
349            r#"{"kind":"image","media_type":"image/png","source":{"uri":"u"}}"#,
350        );
351        assert_eq!(ok.unwrap(), Part::image_uri("image/png", "u"));
352    }
353}
mira/content.rs

mira/
content.rs
