agentai_core/
message.rs

1//! # Conversation messages
2//!
3//! The conversation history is a `Vec<Message>` that you own. Pass it to a run with
4//! `AgentRequest::with_history` in `agentai` and append `AgentResult::new_messages` to it to
5//! continue the conversation.
6//!
7//! Messages implement `Serialize` and `Deserialize` with a format defined by this crate, so a
8//! stored history keeps working when the LLM client used inside `agentai` changes.
9
10use crate::http::{HttpClient, HttpError, HttpRequest};
11use serde::{Deserialize, Serialize};
12use serde_json::Value;
13
14/// Who wrote a [`Message`].
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
16#[serde(rename_all = "snake_case")]
17#[non_exhaustive]
18pub enum Role {
19    /// The user, or your code speaking for the user.
20    User,
21    /// The model.
22    Assistant,
23    /// Results of tool calls, sent back to the model.
24    Tool,
25}
26
27/// One message of the conversation history.
28///
29/// The system prompt is not a message: the `Agent` of `agentai` sends its own with every
30/// request.
31#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
32#[non_exhaustive]
33pub struct Message {
34    /// Who wrote the message.
35    pub role: Role,
36    /// Content of the message, in order.
37    pub parts: Vec<Part>,
38}
39
40impl Message {
41    /// Creates a message from its parts.
42    pub fn new(role: Role, parts: Vec<Part>) -> Self {
43        Self { role, parts }
44    }
45
46    /// Creates a user message with text.
47    pub fn user(text: impl Into<String>) -> Self {
48        Self::new(Role::User, vec![Part::Text(text.into())])
49    }
50
51    /// Creates an assistant message with text.
52    pub fn assistant(text: impl Into<String>) -> Self {
53        Self::new(Role::Assistant, vec![Part::Text(text.into())])
54    }
55
56    /// Creates a message with the result of a tool call.
57    pub fn tool_response(response: ToolResponse) -> Self {
58        Self::new(Role::Tool, vec![Part::ToolResponse(response)])
59    }
60
61    /// Returns the text parts joined together, or `None` when the message has no text.
62    pub fn text(&self) -> Option<String> {
63        joined_text(&self.parts)
64    }
65
66    /// Returns the tool calls in the message.
67    pub fn tool_calls(&self) -> Vec<&ToolCall> {
68        self.parts
69            .iter()
70            .filter_map(|part| match part {
71                Part::ToolCall(call) => Some(call),
72                _ => None,
73            })
74            .collect()
75    }
76}
77
78/// A piece of a [`Message`].
79#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
80#[serde(tag = "type", content = "value", rename_all = "snake_case")]
81#[non_exhaustive]
82pub enum Part {
83    /// Text.
84    Text(String),
85    /// A tool call requested by the model.
86    ToolCall(ToolCall),
87    /// The result of a tool call.
88    ToolResponse(ToolResponse),
89    /// Reasoning (thinking) text of the model.
90    Reasoning(String),
91    /// Signature of the model's reasoning. Some providers (Anthropic, Gemini) need it back to
92    /// continue a conversation with reasoning; keep it in the history as it is.
93    ReasoningSignature(String),
94    /// An image, audio or other file.
95    Media(Media),
96    /// Provider-specific data that has to be sent back unchanged, for example reasoning
97    /// details. Keep it in the history as it is.
98    ProviderData(ProviderData),
99}
100
101/// A tool call requested by the model.
102#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
103#[non_exhaustive]
104pub struct ToolCall {
105    /// Id of the call. The matching [`ToolResponse`] has the same id.
106    pub id: String,
107    /// Name of the tool.
108    pub name: String,
109    /// Arguments chosen by the model, as JSON.
110    pub arguments: Value,
111    /// Reasoning signatures some providers attach to the call.
112    #[serde(default, skip_serializing_if = "Vec::is_empty")]
113    signatures: Vec<String>,
114}
115
116impl ToolCall {
117    /// Creates a tool call.
118    pub fn new(id: impl Into<String>, name: impl Into<String>, arguments: Value) -> Self {
119        Self {
120            id: id.into(),
121            name: name.into(),
122            arguments,
123            signatures: Vec::new(),
124        }
125    }
126
127    /// Reasoning signatures some providers attach to the call. Keep them as they are, the
128    /// provider needs them back.
129    pub fn signatures(&self) -> &[String] {
130        &self.signatures
131    }
132
133    /// Sets the reasoning signatures.
134    pub fn with_signatures(mut self, signatures: Vec<String>) -> Self {
135        self.signatures = signatures;
136        self
137    }
138}
139
140/// The result of a tool call, sent back to the model.
141#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
142#[non_exhaustive]
143pub struct ToolResponse {
144    /// Id of the [`ToolCall`] this result answers.
145    pub call_id: String,
146    /// Name of the tool.
147    pub name: String,
148    /// Result sent to the model: [`Part::Text`] and [`Part::Media`] parts, in order.
149    ///
150    /// Media reaches every vision model, see [`ToolOutput`].
151    #[serde(deserialize_with = "deserialize_tool_content")]
152    pub content: Vec<Part>,
153}
154
155impl ToolResponse {
156    /// Creates the result of `call`.
157    pub fn new(call: &ToolCall, content: impl Into<ToolOutput>) -> Self {
158        Self {
159            call_id: call.id.clone(),
160            name: call.name.clone(),
161            content: content.into().parts,
162        }
163    }
164
165    /// Returns the text parts joined together, or `None` when the result has no text.
166    pub fn text(&self) -> Option<String> {
167        joined_text(&self.content)
168    }
169}
170
171/// Reads `content` as a list of parts, or as a plain string (the format before 0.5.0).
172fn deserialize_tool_content<'de, D>(deserializer: D) -> Result<Vec<Part>, D::Error>
173where
174    D: serde::Deserializer<'de>,
175{
176    #[derive(Deserialize)]
177    #[serde(untagged)]
178    enum Content {
179        Text(String),
180        Parts(Vec<Part>),
181    }
182    Ok(match Content::deserialize(deserializer)? {
183        Content::Text(text) => vec![Part::Text(text)],
184        Content::Parts(parts) => parts,
185    })
186}
187
188/// Joins the text parts, `None` when there are none.
189fn joined_text(parts: &[Part]) -> Option<String> {
190    let texts: Vec<&str> = parts
191        .iter()
192        .filter_map(|part| match part {
193            Part::Text(text) => Some(text.as_str()),
194            _ => None,
195        })
196        .collect();
197    (!texts.is_empty()).then(|| texts.concat())
198}
199
200/// An image, audio or other file in a message.
201#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
202#[non_exhaustive]
203pub struct Media {
204    /// MIME type, for example `image/png`.
205    pub content_type: String,
206    /// Where the bytes are.
207    pub source: MediaSource,
208    /// File name, if known.
209    #[serde(default, skip_serializing_if = "Option::is_none")]
210    pub name: Option<String>,
211}
212
213impl Media {
214    /// Creates media from its MIME type and source.
215    pub fn new(content_type: impl Into<String>, source: MediaSource) -> Self {
216        Self {
217            content_type: content_type.into(),
218            source,
219            name: None,
220        }
221    }
222
223    /// Creates media the provider downloads from `url`. Not every provider accepts URLs; use
224    /// [`Media::from_bytes`] when yours does not.
225    pub fn from_url(content_type: impl Into<String>, url: impl Into<String>) -> Self {
226        Self::new(content_type, MediaSource::Url(url.into()))
227    }
228
229    /// Creates an image the provider downloads from `url`. The MIME type is guessed from the
230    /// file extension (`png`, `jpg`, `jpeg`, `gif`, `webp`); other URLs are sent as
231    /// `image/jpeg`. Use [`Media::from_url`] to set it yourself.
232    pub fn image_url(url: impl Into<String>) -> Self {
233        let url = url.into();
234        Self::from_url(image_type_from_url(&url), url)
235    }
236
237    /// Creates media from its bytes, for example an image read from a file. The bytes are
238    /// sent base64 encoded inside the request.
239    pub fn from_bytes(content_type: impl Into<String>, bytes: impl AsRef<[u8]>) -> Self {
240        use base64::Engine;
241        let data = base64::engine::general_purpose::STANDARD.encode(bytes);
242        Self::from_base64(content_type, data)
243    }
244
245    /// Creates media from bytes that are already base64 encoded.
246    pub fn from_base64(content_type: impl Into<String>, data: impl Into<String>) -> Self {
247        Self::new(content_type, MediaSource::Base64(data.into()))
248    }
249
250    /// Sets the file name. Some providers show it to the model.
251    pub fn with_name(mut self, name: impl Into<String>) -> Self {
252        self.name = Some(name.into());
253        self
254    }
255
256    /// Returns `true` when the MIME type is an image type.
257    pub fn is_image(&self) -> bool {
258        self.content_type.starts_with("image/")
259    }
260
261    /// Size of the media in bytes. `None` for a URL or invalid base64.
262    pub fn byte_len(&self) -> Option<usize> {
263        use base64::Engine;
264        match &self.source {
265            MediaSource::Base64(data) => base64::engine::general_purpose::STANDARD
266                .decode(data)
267                .ok()
268                .map(|bytes| bytes.len()),
269            MediaSource::Url(_) => None,
270        }
271    }
272
273    /// Returns the bytes of the media: decoded base64, or downloaded from its URL with
274    /// `client`.
275    pub async fn bytes(&self, client: &dyn HttpClient) -> Result<Vec<u8>, HttpError> {
276        use base64::Engine;
277        match &self.source {
278            MediaSource::Base64(data) => base64::engine::general_purpose::STANDARD
279                .decode(data)
280                .map_err(|e| HttpError::InvalidMedia(e.to_string())),
281            MediaSource::Url(url) => Ok(client
282                .send(HttpRequest::get(url))
283                .await?
284                .error_for_status()?
285                .body),
286        }
287    }
288}
289
290/// Guesses the MIME type of an image from the extension of its URL.
291fn image_type_from_url(url: &str) -> &'static str {
292    let path = url.split(['?', '#']).next().unwrap_or_default();
293    let extension = path
294        .rsplit_once('.')
295        .map(|(_, ext)| ext.to_ascii_lowercase())
296        .unwrap_or_default();
297    match extension.as_str() {
298        "png" => "image/png",
299        "gif" => "image/gif",
300        "webp" => "image/webp",
301        _ => "image/jpeg",
302    }
303}
304
305/// Where the bytes of [`Media`] are.
306#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
307#[serde(rename_all = "snake_case")]
308pub enum MediaSource {
309    /// A URL. Few providers accept it.
310    Url(String),
311    /// The bytes, base64 encoded.
312    Base64(String),
313}
314
315/// Provider-specific data in a message. It has no meaning outside the provider that produced it.
316#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
317#[non_exhaustive]
318pub struct ProviderData {
319    /// Provider that produced the data, if known.
320    #[serde(default, skip_serializing_if = "Option::is_none")]
321    pub provider: Option<String>,
322    /// Model that produced the data, if known.
323    #[serde(default, skip_serializing_if = "Option::is_none")]
324    pub model: Option<String>,
325    /// The data.
326    pub data: Value,
327}
328
329impl ProviderData {
330    /// Creates provider data without the provider and model.
331    pub fn new(data: Value) -> Self {
332        Self {
333            provider: None,
334            model: None,
335            data,
336        }
337    }
338
339    /// Sets the provider.
340    pub fn with_provider(mut self, provider: impl Into<String>) -> Self {
341        self.provider = Some(provider.into());
342        self
343    }
344
345    /// Sets the model.
346    pub fn with_model(mut self, model: impl Into<String>) -> Self {
347        self.model = Some(model.into());
348        self
349    }
350}
351
352/// What a tool sends back to the model: text and media (for example a generated image), in
353/// order.
354///
355/// Text converts into it, so a tool that only returns text can keep returning a `String`.
356/// The model gets the text in the tool result and the media in a user message right after the
357/// tool results, for every provider for now. A later release puts the media in the tool result
358/// itself for the providers that accept it (Anthropic, Gemini, OpenAI Responses, Bedrock and
359/// Ollama).
360///
361/// There is no structured (JSON) output: a tool that produces data converts it to text itself,
362/// for example with `serde_json::to_string`, in the format it wants the model to read.
363///
364/// ```rust
365/// # use agentai_core::message::{Media, ToolOutput};
366/// let png = [0x89, b'P', b'N', b'G'];
367/// let output = ToolOutput::from_text("Here is the chart")
368///     .with_media(Media::from_bytes("image/png", png));
369/// ```
370#[derive(Debug, Clone, Default, PartialEq)]
371#[non_exhaustive]
372pub struct ToolOutput {
373    /// [`Part::Text`] and [`Part::Media`] parts.
374    pub parts: Vec<Part>,
375}
376
377impl ToolOutput {
378    /// Creates an output with text.
379    pub fn from_text(text: impl Into<String>) -> Self {
380        Self::default().with_text(text)
381    }
382
383    /// Creates an output with one media part.
384    pub fn from_media(media: Media) -> Self {
385        Self::default().with_media(media)
386    }
387
388    /// Appends text.
389    pub fn with_text(mut self, text: impl Into<String>) -> Self {
390        self.parts.push(Part::Text(text.into()));
391        self
392    }
393
394    /// Appends media.
395    pub fn with_media(mut self, media: Media) -> Self {
396        self.parts.push(Part::Media(media));
397        self
398    }
399
400    /// Returns the text parts joined together, or `None` when there is no text.
401    pub fn text(&self) -> Option<String> {
402        joined_text(&self.parts)
403    }
404
405    /// Returns the media parts.
406    pub fn media(&self) -> Vec<&Media> {
407        self.parts
408            .iter()
409            .filter_map(|part| match part {
410                Part::Media(media) => Some(media),
411                _ => None,
412            })
413            .collect()
414    }
415}
416
417/// Text parts as they are, media as `[content type]`, for example `[image/png]`.
418impl std::fmt::Display for ToolOutput {
419    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
420        for part in &self.parts {
421            match part {
422                Part::Text(text) => f.write_str(text)?,
423                Part::Media(media) => write!(f, "[{}]", media.content_type)?,
424                _ => {}
425            }
426        }
427        Ok(())
428    }
429}
430
431impl From<String> for ToolOutput {
432    fn from(text: String) -> Self {
433        Self::from_text(text)
434    }
435}
436
437impl From<&str> for ToolOutput {
438    fn from(text: &str) -> Self {
439        Self::from_text(text)
440    }
441}
442
443impl From<Media> for ToolOutput {
444    fn from(media: Media) -> Self {
445        Self::from_media(media)
446    }
447}
448
449#[cfg(test)]
450mod tests {
451    use super::*;
452    use serde_json::json;
453
454    fn every_part() -> Message {
455        let mut call = ToolCall::new("call_1", "echo", json!({"text": "hi"}));
456        call.signatures = vec!["sig".to_string()];
457        Message::new(
458            Role::Assistant,
459            vec![
460                Part::ReasoningSignature("signature".into()),
461                Part::Reasoning("thinking".into()),
462                Part::Text("hello".into()),
463                Part::ToolCall(call.clone()),
464                Part::ToolResponse(ToolResponse::new(&call, "result")),
465                Part::ToolResponse(ToolResponse::new(
466                    &call,
467                    ToolOutput::from_text("an image")
468                        .with_media(Media::new("image/png", MediaSource::Base64("aGk=".into()))),
469                )),
470                Part::Media(Media::new("image/png", MediaSource::Base64("aGk=".into()))),
471                Part::Media(Media::new(
472                    "image/png",
473                    MediaSource::Url("https://example.com/a.png".into()),
474                )),
475                Part::ProviderData(ProviderData {
476                    provider: Some("OpenRouter".into()),
477                    model: Some("m".into()),
478                    data: json!({"type": "reasoning.encrypted"}),
479                }),
480            ],
481        )
482    }
483
484    #[test]
485    fn messages_round_trip_through_json() {
486        let message = every_part();
487        let json = serde_json::to_string(&message).unwrap();
488        assert_eq!(serde_json::from_str::<Message>(&json).unwrap(), message);
489    }
490
491    #[test]
492    fn tool_responses_stored_before_0_5_still_load() {
493        let json = r#"{"role":"tool","parts":[{"type":"tool_response",
494            "value":{"call_id":"call_1","name":"echo","content":"result"}}]}"#;
495        let message: Message = serde_json::from_str(json).unwrap();
496        let call = ToolCall::new("call_1", "echo", json!({}));
497        assert_eq!(
498            message,
499            Message::tool_response(ToolResponse::new(&call, "result"))
500        );
501    }
502
503    #[test]
504    fn image_url_guesses_the_type_from_the_extension() {
505        let types: Vec<String> = [
506            "https://example.com/a.png",
507            "https://example.com/a.PNG?size=2#top",
508            "https://example.com/a.webp",
509            "https://example.com/a.gif",
510            "https://example.com/a.jpeg",
511            "https://example.com/image",
512        ]
513        .into_iter()
514        .map(|url| Media::image_url(url).content_type)
515        .collect();
516        assert_eq!(
517            types,
518            [
519                "image/png",
520                "image/png",
521                "image/webp",
522                "image/gif",
523                "image/jpeg",
524                "image/jpeg"
525            ]
526        );
527        assert_eq!(
528            Media::image_url("https://example.com/a.png").source,
529            MediaSource::Url("https://example.com/a.png".into())
530        );
531    }
532
533    #[test]
534    fn json_format_is_stable() {
535        let message = Message::user("hi");
536        assert_eq!(
537            serde_json::to_value(&message).unwrap(),
538            json!({"role": "user", "parts": [{"type": "text", "value": "hi"}]})
539        );
540    }
541}
agentai_core/message.rs

agentai_core/
message.rs
