agentai_core/
message.rs

1//! # Conversation messages
2//!
3//! The conversation history is a `Vec<Message>` that you own. Pass it to a run with
4//! `AgentRequest::with_history` in `agentai` and append `AgentResult::new_messages` to it to
5//! continue the conversation.
6//!
7//! Messages implement `Serialize` and `Deserialize` with a format defined by this crate, so a
8//! stored history keeps working when the LLM client used inside `agentai` changes.
9
10use serde::{Deserialize, Serialize};
11use serde_json::Value;
12
13/// Who wrote a [`Message`].
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
15#[serde(rename_all = "snake_case")]
16#[non_exhaustive]
17pub enum Role {
18    /// The user, or your code speaking for the user.
19    User,
20    /// The model.
21    Assistant,
22    /// Results of tool calls, sent back to the model.
23    Tool,
24}
25
26/// One message of the conversation history.
27///
28/// The system prompt is not a message: the `Agent` of `agentai` sends its own with every
29/// request.
30#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
31#[non_exhaustive]
32pub struct Message {
33    /// Who wrote the message.
34    pub role: Role,
35    /// Content of the message, in order.
36    pub parts: Vec<Part>,
37}
38
39impl Message {
40    /// Creates a message from its parts.
41    pub fn new(role: Role, parts: Vec<Part>) -> Self {
42        Self { role, parts }
43    }
44
45    /// Creates a user message with text.
46    pub fn user(text: impl Into<String>) -> Self {
47        Self::new(Role::User, vec![Part::Text(text.into())])
48    }
49
50    /// Creates an assistant message with text.
51    pub fn assistant(text: impl Into<String>) -> Self {
52        Self::new(Role::Assistant, vec![Part::Text(text.into())])
53    }
54
55    /// Creates a message with the result of a tool call.
56    pub fn tool_response(response: ToolResponse) -> Self {
57        Self::new(Role::Tool, vec![Part::ToolResponse(response)])
58    }
59
60    /// Returns the text parts joined together, or `None` when the message has no text.
61    pub fn text(&self) -> Option<String> {
62        joined_text(&self.parts)
63    }
64
65    /// Returns the tool calls in the message.
66    pub fn tool_calls(&self) -> Vec<&ToolCall> {
67        self.parts
68            .iter()
69            .filter_map(|part| match part {
70                Part::ToolCall(call) => Some(call),
71                _ => None,
72            })
73            .collect()
74    }
75}
76
77/// A piece of a [`Message`].
78#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
79#[serde(tag = "type", content = "value", rename_all = "snake_case")]
80#[non_exhaustive]
81pub enum Part {
82    /// Text.
83    Text(String),
84    /// A tool call requested by the model.
85    ToolCall(ToolCall),
86    /// The result of a tool call.
87    ToolResponse(ToolResponse),
88    /// Reasoning (thinking) text of the model.
89    Reasoning(String),
90    /// Signature of the model's reasoning. Some providers (Anthropic, Gemini) need it back to
91    /// continue a conversation with reasoning; keep it in the history as it is.
92    ReasoningSignature(String),
93    /// An image, audio or other file.
94    Media(Media),
95    /// Provider-specific data that has to be sent back unchanged, for example reasoning
96    /// details. Keep it in the history as it is.
97    ProviderData(ProviderData),
98}
99
100/// A tool call requested by the model.
101#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
102#[non_exhaustive]
103pub struct ToolCall {
104    /// Id of the call. The matching [`ToolResponse`] has the same id.
105    pub id: String,
106    /// Name of the tool.
107    pub name: String,
108    /// Arguments chosen by the model, as JSON.
109    pub arguments: Value,
110    /// Reasoning signatures some providers attach to the call.
111    #[serde(default, skip_serializing_if = "Vec::is_empty")]
112    signatures: Vec<String>,
113}
114
115impl ToolCall {
116    /// Creates a tool call.
117    pub fn new(id: impl Into<String>, name: impl Into<String>, arguments: Value) -> Self {
118        Self {
119            id: id.into(),
120            name: name.into(),
121            arguments,
122            signatures: Vec::new(),
123        }
124    }
125
126    /// Reasoning signatures some providers attach to the call. Keep them as they are, the
127    /// provider needs them back.
128    pub fn signatures(&self) -> &[String] {
129        &self.signatures
130    }
131
132    /// Sets the reasoning signatures.
133    pub fn with_signatures(mut self, signatures: Vec<String>) -> Self {
134        self.signatures = signatures;
135        self
136    }
137}
138
139/// The result of a tool call, sent back to the model.
140#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
141#[non_exhaustive]
142pub struct ToolResponse {
143    /// Id of the [`ToolCall`] this result answers.
144    pub call_id: String,
145    /// Name of the tool.
146    pub name: String,
147    /// Result sent to the model: [`Part::Text`] and [`Part::Media`] parts, in order.
148    ///
149    /// Media reaches every vision model, see [`ToolOutput`].
150    #[serde(deserialize_with = "deserialize_tool_content")]
151    pub content: Vec<Part>,
152}
153
154impl ToolResponse {
155    /// Creates the result of `call`.
156    pub fn new(call: &ToolCall, content: impl Into<ToolOutput>) -> Self {
157        Self {
158            call_id: call.id.clone(),
159            name: call.name.clone(),
160            content: content.into().parts,
161        }
162    }
163
164    /// Returns the text parts joined together, or `None` when the result has no text.
165    pub fn text(&self) -> Option<String> {
166        joined_text(&self.content)
167    }
168}
169
170/// Reads `content` as a list of parts, or as a plain string (the format before 0.5.0).
171fn deserialize_tool_content<'de, D>(deserializer: D) -> Result<Vec<Part>, D::Error>
172where
173    D: serde::Deserializer<'de>,
174{
175    #[derive(Deserialize)]
176    #[serde(untagged)]
177    enum Content {
178        Text(String),
179        Parts(Vec<Part>),
180    }
181    Ok(match Content::deserialize(deserializer)? {
182        Content::Text(text) => vec![Part::Text(text)],
183        Content::Parts(parts) => parts,
184    })
185}
186
187/// Joins the text parts, `None` when there are none.
188fn joined_text(parts: &[Part]) -> Option<String> {
189    let texts: Vec<&str> = parts
190        .iter()
191        .filter_map(|part| match part {
192            Part::Text(text) => Some(text.as_str()),
193            _ => None,
194        })
195        .collect();
196    (!texts.is_empty()).then(|| texts.concat())
197}
198
199/// An image, audio or other file in a message.
200#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
201#[non_exhaustive]
202pub struct Media {
203    /// MIME type, for example `image/png`.
204    pub content_type: String,
205    /// Where the bytes are.
206    pub source: MediaSource,
207    /// File name, if known.
208    #[serde(default, skip_serializing_if = "Option::is_none")]
209    pub name: Option<String>,
210}
211
212impl Media {
213    /// Creates media from its MIME type and source.
214    pub fn new(content_type: impl Into<String>, source: MediaSource) -> Self {
215        Self {
216            content_type: content_type.into(),
217            source,
218            name: None,
219        }
220    }
221
222    /// Creates media the provider downloads from `url`. Not every provider accepts URLs; use
223    /// [`Media::from_bytes`] when yours does not.
224    pub fn from_url(content_type: impl Into<String>, url: impl Into<String>) -> Self {
225        Self::new(content_type, MediaSource::Url(url.into()))
226    }
227
228    /// Creates an image the provider downloads from `url`. The MIME type is guessed from the
229    /// file extension (`png`, `jpg`, `jpeg`, `gif`, `webp`); other URLs are sent as
230    /// `image/jpeg`. Use [`Media::from_url`] to set it yourself.
231    pub fn image_url(url: impl Into<String>) -> Self {
232        let url = url.into();
233        Self::from_url(image_type_from_url(&url), url)
234    }
235
236    /// Creates media from its bytes, for example an image read from a file. The bytes are
237    /// sent base64 encoded inside the request.
238    pub fn from_bytes(content_type: impl Into<String>, bytes: impl AsRef<[u8]>) -> Self {
239        use base64::Engine;
240        let data = base64::engine::general_purpose::STANDARD.encode(bytes);
241        Self::from_base64(content_type, data)
242    }
243
244    /// Creates media from bytes that are already base64 encoded.
245    pub fn from_base64(content_type: impl Into<String>, data: impl Into<String>) -> Self {
246        Self::new(content_type, MediaSource::Base64(data.into()))
247    }
248
249    /// Sets the file name. Some providers show it to the model.
250    pub fn with_name(mut self, name: impl Into<String>) -> Self {
251        self.name = Some(name.into());
252        self
253    }
254
255    /// Returns `true` when the MIME type is an image type.
256    pub fn is_image(&self) -> bool {
257        self.content_type.starts_with("image/")
258    }
259}
260
261/// Guesses the MIME type of an image from the extension of its URL.
262fn image_type_from_url(url: &str) -> &'static str {
263    let path = url.split(['?', '#']).next().unwrap_or_default();
264    let extension = path
265        .rsplit_once('.')
266        .map(|(_, ext)| ext.to_ascii_lowercase())
267        .unwrap_or_default();
268    match extension.as_str() {
269        "png" => "image/png",
270        "gif" => "image/gif",
271        "webp" => "image/webp",
272        _ => "image/jpeg",
273    }
274}
275
276/// Where the bytes of [`Media`] are.
277#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
278#[serde(rename_all = "snake_case")]
279pub enum MediaSource {
280    /// A URL. Few providers accept it.
281    Url(String),
282    /// The bytes, base64 encoded.
283    Base64(String),
284}
285
286/// Provider-specific data in a message. It has no meaning outside the provider that produced it.
287#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
288#[non_exhaustive]
289pub struct ProviderData {
290    /// Provider that produced the data, if known.
291    #[serde(default, skip_serializing_if = "Option::is_none")]
292    pub provider: Option<String>,
293    /// Model that produced the data, if known.
294    #[serde(default, skip_serializing_if = "Option::is_none")]
295    pub model: Option<String>,
296    /// The data.
297    pub data: Value,
298}
299
300impl ProviderData {
301    /// Creates provider data without the provider and model.
302    pub fn new(data: Value) -> Self {
303        Self {
304            provider: None,
305            model: None,
306            data,
307        }
308    }
309
310    /// Sets the provider.
311    pub fn with_provider(mut self, provider: impl Into<String>) -> Self {
312        self.provider = Some(provider.into());
313        self
314    }
315
316    /// Sets the model.
317    pub fn with_model(mut self, model: impl Into<String>) -> Self {
318        self.model = Some(model.into());
319        self
320    }
321}
322
323/// What a tool sends back to the model: text and media (for example a generated image), in
324/// order.
325///
326/// Text converts into it, so a tool that only returns text can keep returning a `String`.
327/// The model gets the text in the tool result and the media in a user message right after the
328/// tool results, for every provider for now. A later release puts the media in the tool result
329/// itself for the providers that accept it (Anthropic, Gemini, OpenAI Responses, Bedrock and
330/// Ollama).
331///
332/// There is no structured (JSON) output: a tool that produces data converts it to text itself,
333/// for example with `serde_json::to_string`, in the format it wants the model to read.
334///
335/// ```rust
336/// # use agentai_core::message::{Media, ToolOutput};
337/// let png = [0x89, b'P', b'N', b'G'];
338/// let output = ToolOutput::from_text("Here is the chart")
339///     .with_media(Media::from_bytes("image/png", png));
340/// ```
341#[derive(Debug, Clone, Default, PartialEq)]
342#[non_exhaustive]
343pub struct ToolOutput {
344    /// [`Part::Text`] and [`Part::Media`] parts.
345    pub parts: Vec<Part>,
346}
347
348impl ToolOutput {
349    /// Creates an output with text.
350    pub fn from_text(text: impl Into<String>) -> Self {
351        Self::default().with_text(text)
352    }
353
354    /// Creates an output with one media part.
355    pub fn from_media(media: Media) -> Self {
356        Self::default().with_media(media)
357    }
358
359    /// Appends text.
360    pub fn with_text(mut self, text: impl Into<String>) -> Self {
361        self.parts.push(Part::Text(text.into()));
362        self
363    }
364
365    /// Appends media.
366    pub fn with_media(mut self, media: Media) -> Self {
367        self.parts.push(Part::Media(media));
368        self
369    }
370
371    /// Returns the text parts joined together, or `None` when there is no text.
372    pub fn text(&self) -> Option<String> {
373        joined_text(&self.parts)
374    }
375
376    /// Returns the media parts.
377    pub fn media(&self) -> Vec<&Media> {
378        self.parts
379            .iter()
380            .filter_map(|part| match part {
381                Part::Media(media) => Some(media),
382                _ => None,
383            })
384            .collect()
385    }
386}
387
388/// Text parts as they are, media as `[content type]`, for example `[image/png]`.
389impl std::fmt::Display for ToolOutput {
390    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
391        for part in &self.parts {
392            match part {
393                Part::Text(text) => f.write_str(text)?,
394                Part::Media(media) => write!(f, "[{}]", media.content_type)?,
395                _ => {}
396            }
397        }
398        Ok(())
399    }
400}
401
402impl From<String> for ToolOutput {
403    fn from(text: String) -> Self {
404        Self::from_text(text)
405    }
406}
407
408impl From<&str> for ToolOutput {
409    fn from(text: &str) -> Self {
410        Self::from_text(text)
411    }
412}
413
414impl From<Media> for ToolOutput {
415    fn from(media: Media) -> Self {
416        Self::from_media(media)
417    }
418}
419
420#[cfg(test)]
421mod tests {
422    use super::*;
423    use serde_json::json;
424
425    fn every_part() -> Message {
426        let mut call = ToolCall::new("call_1", "echo", json!({"text": "hi"}));
427        call.signatures = vec!["sig".to_string()];
428        Message::new(
429            Role::Assistant,
430            vec![
431                Part::ReasoningSignature("signature".into()),
432                Part::Reasoning("thinking".into()),
433                Part::Text("hello".into()),
434                Part::ToolCall(call.clone()),
435                Part::ToolResponse(ToolResponse::new(&call, "result")),
436                Part::ToolResponse(ToolResponse::new(
437                    &call,
438                    ToolOutput::from_text("an image")
439                        .with_media(Media::new("image/png", MediaSource::Base64("aGk=".into()))),
440                )),
441                Part::Media(Media::new("image/png", MediaSource::Base64("aGk=".into()))),
442                Part::Media(Media::new(
443                    "image/png",
444                    MediaSource::Url("https://example.com/a.png".into()),
445                )),
446                Part::ProviderData(ProviderData {
447                    provider: Some("OpenRouter".into()),
448                    model: Some("m".into()),
449                    data: json!({"type": "reasoning.encrypted"}),
450                }),
451            ],
452        )
453    }
454
455    #[test]
456    fn messages_round_trip_through_json() {
457        let message = every_part();
458        let json = serde_json::to_string(&message).unwrap();
459        assert_eq!(serde_json::from_str::<Message>(&json).unwrap(), message);
460    }
461
462    #[test]
463    fn tool_responses_stored_before_0_5_still_load() {
464        let json = r#"{"role":"tool","parts":[{"type":"tool_response",
465            "value":{"call_id":"call_1","name":"echo","content":"result"}}]}"#;
466        let message: Message = serde_json::from_str(json).unwrap();
467        let call = ToolCall::new("call_1", "echo", json!({}));
468        assert_eq!(
469            message,
470            Message::tool_response(ToolResponse::new(&call, "result"))
471        );
472    }
473
474    #[test]
475    fn image_url_guesses_the_type_from_the_extension() {
476        let types: Vec<String> = [
477            "https://example.com/a.png",
478            "https://example.com/a.PNG?size=2#top",
479            "https://example.com/a.webp",
480            "https://example.com/a.gif",
481            "https://example.com/a.jpeg",
482            "https://example.com/image",
483        ]
484        .into_iter()
485        .map(|url| Media::image_url(url).content_type)
486        .collect();
487        assert_eq!(
488            types,
489            [
490                "image/png",
491                "image/png",
492                "image/webp",
493                "image/gif",
494                "image/jpeg",
495                "image/jpeg"
496            ]
497        );
498        assert_eq!(
499            Media::image_url("https://example.com/a.png").source,
500            MediaSource::Url("https://example.com/a.png".into())
501        );
502    }
503
504    #[test]
505    fn json_format_is_stable() {
506        let message = Message::user("hi");
507        assert_eq!(
508            serde_json::to_value(&message).unwrap(),
509            json!({"role": "user", "parts": [{"type": "text", "value": "hi"}]})
510        );
511    }
512}
agentai_core/message.rs

agentai_core/
message.rs
