1use crate::error::ProviderError;
2use serde::{Deserialize, Serialize};
3use std::{convert::Infallible, str::FromStr};
4use thiserror::Error;
5
6#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
14#[serde(tag = "role", rename_all = "lowercase")]
15pub enum Message {
16 System { content: String },
18
19 User { content: Vec<UserContent> },
21
22 Assistant(AssistantMessage),
25}
26
27pub mod citation;
28mod identity;
29mod native;
30
31pub use citation::{Citation, DocumentRange, Source, SourceLocation, Span};
32
33pub use identity::{CallId, EmptyCallId, EmptyToolName, LocalCallId, ProviderCallId, ToolName};
34pub(crate) use native::Canonical;
35pub use native::{Api, Blob, Fingerprint, Native, Opaque, Origin, StopReason};
36
37#[non_exhaustive]
44#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
45pub struct AssistantMessage {
46 pub content: Vec<AssistantContent>,
48 #[serde(default, skip_serializing_if = "Option::is_none")]
50 pub origin: Option<Origin>,
51 #[serde(default, skip_serializing_if = "Option::is_none")]
53 pub stop: Option<StopReason>,
54}
55
56impl AssistantMessage {
57 pub fn new(content: Vec<AssistantContent>) -> Self {
59 Self {
60 content,
61 ..Self::default()
62 }
63 }
64
65 pub fn rolled_back(origin: Option<Origin>, content: Vec<AssistantContent>) -> Self {
69 Self {
70 content,
71 origin,
72 stop: Some(StopReason::ToolUse),
73 }
74 }
75
76 pub fn aborted(
79 origin: Option<Origin>,
80 content: Vec<AssistantContent>,
81 reason: impl Into<String>,
82 ) -> Self {
83 Self {
84 content,
85 origin,
86 stop: Some(StopReason::Aborted(reason.into())),
87 }
88 }
89
90 pub fn with_content(mut self, content: Vec<AssistantContent>) -> Self {
92 self.content = content;
93 self
94 }
95
96 pub fn with_origin(mut self, origin: impl Into<Option<Origin>>) -> Self {
98 self.origin = origin.into();
99 self
100 }
101
102 pub fn with_stop(mut self, stop: impl Into<Option<StopReason>>) -> Self {
104 self.stop = stop.into();
105 self
106 }
107
108 pub fn tool_calls(&self) -> impl Iterator<Item = &ToolCall> {
110 self.content.iter().filter_map(|part| match part {
111 AssistantContent::ToolCall(call) => Some(call),
112 _ => None,
113 })
114 }
115
116 pub fn text(&self) -> String {
118 self.content
119 .iter()
120 .filter_map(|part| match part {
121 AssistantContent::Text(text) => Some(text.text.as_str()),
122 _ => None,
123 })
124 .collect()
125 }
126
127 pub fn is_empty_turn(&self) -> bool {
131 self.content.iter().all(AssistantContent::is_blank)
132 }
133}
134
135pub fn turn_delivered_no_answer(choice: &[AssistantContent]) -> bool {
139 !choice.iter().any(AssistantContent::is_answer)
140}
141
142pub fn turn_failure(
152 choice: &[AssistantContent],
153 stop: Option<&StopReason>,
154 finish: Option<&crate::completion::FinishReason>,
155) -> Option<String> {
156 let failed = stop.filter(|stop| stop.is_failure());
157 let calls = choice
158 .iter()
159 .any(|block| matches!(block, AssistantContent::ToolCall(_)));
160 if let Some(StopReason::Error(reason) | StopReason::Aborted(reason)) = failed
161 && calls
162 {
163 return Some(format!(
164 "the turn failed, so none of its tool calls ran: {reason}"
165 ));
166 }
167 let answered = !turn_delivered_no_answer(choice);
168 if !answered && let Some(finish) = finish.filter(|finish| finish.truncated_output()) {
169 return Some(finish.no_answer_message());
170 }
171 match failed {
172 Some(StopReason::Error(reason)) if answered => {
173 Some(format!("the provider failed the turn: {reason}"))
174 }
175 Some(StopReason::Error(reason)) => Some(format!(
176 "the provider failed the turn without an answer: {reason}"
177 )),
178 _ => None,
179 }
180}
181
182#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
185#[serde(tag = "type", rename_all = "lowercase")]
186pub enum UserContent {
187 Text(Text),
189 ToolResult(ToolResult),
191 Image(Image),
193 Audio(Audio),
195 Video(Video),
197 Document(Document),
199}
200
201#[non_exhaustive]
204#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
205#[serde(tag = "type", rename_all = "lowercase")]
206pub enum AssistantContent {
207 Text(Text),
209 ToolCall(ToolCall),
211 Reasoning(Reasoning),
213 Image(Image),
215 Opaque(Opaque),
217}
218
219impl AssistantContent {
220 pub fn canonical(&self) -> Self {
223 match self {
224 Self::Text(text) => Self::Text(Text {
225 text: text.text.clone(),
226 citations: text.citations.clone(),
227 native: None,
228 }),
229 Self::ToolCall(call) => {
230 Self::ToolCall(ToolCall::new(call.id.clone(), call.function.clone()))
231 }
232 Self::Reasoning(reasoning) => Self::Reasoning(Reasoning {
233 text: reasoning.text.clone(),
234 redacted: reasoning.redacted,
235 native: None,
236 }),
237 Self::Image(image) => Self::Image(Image {
238 data: image.data.clone(),
239 media_type: image.media_type.clone(),
240 detail: image.detail.clone(),
241 native: None,
242 }),
243 Self::Opaque(opaque) => Self::Opaque(opaque.clone()),
244 }
245 }
246
247 pub fn into_canonical(mut self) -> Self {
249 if let Some(native) = self.native_slot() {
250 *native = None;
251 }
252 self
253 }
254
255 pub fn is_answer(&self) -> bool {
258 match self {
259 Self::Text(text) => !text.text.trim().is_empty(),
260 Self::ToolCall(_) | Self::Image(_) => true,
261 Self::Reasoning(_) | Self::Opaque(_) => false,
262 }
263 }
264
265 pub fn is_blank(&self) -> bool {
271 match self {
272 Self::Text(Text { text, .. }) => text.trim().is_empty() && self.native_item().is_none(),
273 Self::Reasoning(Reasoning { text, redacted, .. }) => {
274 (*redacted || text.trim().is_empty()) && self.native_item().is_none()
275 }
276 Self::Opaque(opaque) => !opaque.replay,
277 Self::ToolCall(_) | Self::Image(_) => false,
278 }
279 }
280
281 pub fn fingerprint(&self) -> Fingerprint {
287 match self {
292 Self::Text(text) => Fingerprint::of_canonical(&("v1", "text", &text.text)),
293 Self::Reasoning(reasoning) => {
294 Fingerprint::of_canonical(&("v1", "reasoning", &reasoning.text, reasoning.redacted))
295 }
296 Self::ToolCall(call) => call.fingerprint(),
297 Self::Image(image) => Fingerprint::of_canonical(&(
298 "v1",
299 "image",
300 &image.media_type,
301 &image.detail,
302 &image.data,
303 )),
304 Self::Opaque(_) => Fingerprint::of_canonical(&("v1", "opaque")),
305 }
306 }
307
308 pub(crate) fn native_slot(&mut self) -> Option<&mut Option<Native>> {
309 match self {
310 Self::Text(text) => Some(&mut text.native),
311 Self::ToolCall(call) => Some(&mut call.native),
312 Self::Reasoning(reasoning) => Some(&mut reasoning.native),
313 Self::Image(image) => Some(&mut image.native),
314 Self::Opaque(_) => None,
315 }
316 }
317
318 pub fn with_native(mut self, item: serde_json::Value) -> Self {
321 let fingerprint = self.fingerprint();
322 let baseline = match &self {
323 Self::ToolCall(call) => call.baseline(),
324 _ => None,
325 };
326 if let Some(native) = self.native_slot() {
327 *native = Some(Native {
328 item,
329 fingerprint,
330 baseline,
331 edited: false,
332 blob_slot: None,
333 });
334 }
335 self
336 }
337
338 pub(crate) fn native(&self) -> Option<(&serde_json::Value, bool)> {
341 let native = match self {
342 Self::Text(text) => text.native.as_ref(),
343 Self::ToolCall(call) => call.native.as_ref(),
344 Self::Reasoning(reasoning) => reasoning.native.as_ref(),
345 Self::Image(image) => image.native.as_ref(),
346 Self::Opaque(_) => None,
347 }?;
348 let current = match self {
349 Self::ToolCall(call) => call.native_is_current(native),
350 _ => !native.edited && native.fingerprint == self.fingerprint(),
351 };
352 Some((&native.item, current))
353 }
354
355 pub(crate) fn stale_item(&self) -> Option<&serde_json::Value> {
358 self.native()
359 .and_then(|(item, current)| (!current).then_some(item))
360 }
361
362 pub fn native_item(&self) -> Option<&serde_json::Value> {
365 self.native()
366 .and_then(|(item, current)| current.then_some(item))
367 }
368}
369
370#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
374pub struct Reasoning {
375 pub text: String,
377 #[serde(default, skip_serializing_if = "crate::json_utils::is_false")]
379 pub redacted: bool,
380 #[serde(
382 default,
383 skip_serializing_if = "Option::is_none",
384 deserialize_with = "native::lenient"
385 )]
386 pub native: Option<Native>,
387}
388
389impl Reasoning {
390 pub fn new(text: impl Into<String>) -> Self {
392 Self {
393 text: text.into(),
394 ..Self::default()
395 }
396 }
397}
398
399#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
404pub struct ToolResult {
405 pub call: CallId,
407 pub name: ToolName,
410 pub content: Vec<ToolResultContent>,
412 #[serde(default, skip_serializing_if = "crate::json_utils::is_false")]
415 pub is_error: bool,
416}
417
418#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
420#[serde(tag = "type", rename_all = "lowercase")]
421pub enum ToolResultContent {
422 Text(Text),
424 Image(Image),
426 Json {
428 value: serde_json::Value,
430 },
431}
432
433impl ToolResultContent {
434 pub fn as_text(&self) -> Option<&str> {
436 match self {
437 Self::Text(text) => Some(&text.text),
438 Self::Image(_) | Self::Json { .. } => None,
439 }
440 }
441
442 pub fn as_json(&self) -> Option<&serde_json::Value> {
444 match self {
445 Self::Json { value } => Some(value),
446 Self::Text(_) | Self::Image(_) => None,
447 }
448 }
449
450 pub fn deserialize_json<T>(&self) -> Result<T, serde_json::Error>
466 where
467 T: serde::de::DeserializeOwned,
468 {
469 match self {
470 Self::Json { value } => T::deserialize(value),
471 Self::Text(text) => serde_json::from_str(&text.text),
472 Self::Image(_) => Err(<serde_json::Error as serde::de::Error>::custom(
473 "an image tool result is not JSON",
474 )),
475 }
476 }
477}
478
479#[derive(Clone, Debug, Deserialize, PartialEq)]
481pub struct ToolCall {
482 pub id: CallId,
485 pub function: ToolFunction,
487 #[serde(
489 default,
490 skip_serializing_if = "Option::is_none",
491 deserialize_with = "native::lenient"
492 )]
493 pub native: Option<Native>,
494}
495
496impl ToolCall {
497 fn fingerprint(&self) -> Fingerprint {
498 let id = match &self.id {
499 CallId::Provider(id) => id.as_str(),
500 CallId::Local(_) => "~local",
501 };
502 Fingerprint::of_canonical(&(
503 "v1",
504 "call",
505 id,
506 self.function.name.as_str(),
507 native::Canonical(&self.function.arguments),
508 &self.function.invalid_arguments,
509 ))
510 }
511
512 fn native_is_current(&self, native: &Native) -> bool {
513 !native.edited
514 && match &native.baseline {
515 Some(baseline) => native::matches_baseline(baseline, &self.projection()),
516 None => native.fingerprint == self.fingerprint(),
517 }
518 }
519
520 fn baseline(&self) -> Option<serde_json::Value> {
521 fn has_float(value: &serde_json::Value) -> bool {
522 match value {
523 serde_json::Value::Number(number) => number.is_f64(),
524 serde_json::Value::Array(values) => values.iter().any(has_float),
525 serde_json::Value::Object(fields) => fields.values().any(has_float),
526 _ => false,
527 }
528 }
529 self.function
530 .arguments
531 .values()
532 .any(has_float)
533 .then(|| native::baseline(self.projection()))
534 }
535
536 fn projection(&self) -> serde_json::Value {
537 let id = match &self.id {
538 CallId::Provider(id) => id.as_str(),
539 CallId::Local(_) => "~local",
540 };
541 serde_json::json!([
542 "v1",
543 "call",
544 id,
545 self.function.name.as_str(),
546 self.function.arguments,
547 self.function.invalid_arguments,
548 ])
549 }
550
551 pub fn new(id: CallId, function: ToolFunction) -> Self {
553 Self {
554 id,
555 function,
556 native: None,
557 }
558 }
559
560 pub fn from_wire(wire_id: impl Into<String>, function: ToolFunction) -> Self {
563 Self::new(CallId::from_wire(wire_id), function)
564 }
565
566 pub fn result(&self, content: Vec<ToolResultContent>) -> ToolResult {
568 ToolResult {
569 call: self.id.clone(),
570 name: self.function.name.clone(),
571 content,
572 is_error: false,
573 }
574 }
575
576 pub fn answer(&self, result: &crate::tool::ToolResult) -> ToolResult {
579 ToolResult {
580 is_error: !result.is_success(),
581 ..self.result(result.output().clone().into_content())
582 }
583 }
584}
585
586impl Serialize for ToolCall {
587 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
588 #[derive(Serialize)]
589 struct Stored<'a> {
590 id: &'a CallId,
591 function: &'a ToolFunction,
592 #[serde(skip_serializing_if = "Option::is_none")]
593 native: Option<std::borrow::Cow<'a, Native>>,
594 }
595
596 let native = self.native.as_ref().map(|native| {
597 if self.native_is_current(native) {
600 std::borrow::Cow::Borrowed(native)
601 } else {
602 let mut stale = native.clone();
603 stale.edited = true;
604 std::borrow::Cow::Owned(stale)
605 }
606 });
607 Stored {
608 id: &self.id,
609 function: &self.function,
610 native,
611 }
612 .serialize(serializer)
613 }
614}
615
616#[derive(Clone, Debug, Serialize, PartialEq)]
638pub struct ToolFunction {
639 pub name: ToolName,
641 pub arguments: serde_json::Map<String, serde_json::Value>,
643 #[serde(default, skip_serializing_if = "Option::is_none")]
646 pub invalid_arguments: Option<String>,
647}
648
649impl ToolFunction {
650 pub fn new(name: ToolName, arguments: serde_json::Value) -> Self {
654 use serde_json::Value;
655 let (arguments, invalid_arguments) = match arguments {
656 Value::Object(arguments) => (arguments, None),
657 Value::Null => (serde_json::Map::new(), None),
658 Value::String(text) => {
659 return Self::parse(name, &text);
660 }
661 other => (serde_json::Map::new(), Some(other.to_string())),
662 };
663 Self {
664 name,
665 arguments,
666 invalid_arguments,
667 }
668 }
669
670 pub fn parse(name: ToolName, text: &str) -> Self {
675 use serde_json::Value;
676 let parsed = crate::json_utils::parse_tool_arguments(text);
677 let (arguments, invalid) = match parsed {
678 Ok(Value::Object(arguments)) => (arguments, false),
679 Ok(Value::Null) => (serde_json::Map::new(), false),
680 Ok(Value::String(inner)) => match serde_json::from_str(&inner) {
681 Ok(Value::Object(arguments)) => (arguments, false),
682 _ => (serde_json::Map::new(), true),
683 },
684 Ok(_) => (serde_json::Map::new(), true),
685 Err(_) => (
686 crate::json_utils::parse_partial_object(text).unwrap_or_default(),
687 true,
688 ),
689 };
690 Self {
691 name,
692 arguments,
693 invalid_arguments: invalid.then(|| text.to_owned()),
694 }
695 }
696
697 pub fn arguments_value(&self) -> serde_json::Value {
699 serde_json::Value::Object(self.arguments.clone())
700 }
701
702 pub fn arguments_json(&self) -> String {
705 serde_json::to_string(&self.arguments).unwrap_or_default()
706 }
707
708 pub fn raw_arguments(&self) -> String {
711 self.invalid_arguments
712 .clone()
713 .unwrap_or_else(|| self.arguments_json())
714 }
715}
716
717impl<'de> Deserialize<'de> for ToolFunction {
718 fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
722 #[derive(Deserialize)]
723 struct Repr {
724 name: ToolName,
725 #[serde(default)]
726 arguments: serde_json::Value,
727 #[serde(default)]
728 invalid_arguments: Option<String>,
729 }
730 let Repr {
731 name,
732 arguments,
733 invalid_arguments,
734 } = Repr::deserialize(deserializer)?;
735 let mut function = Self::new(name, arguments);
736 if invalid_arguments.is_some() {
737 function.invalid_arguments = invalid_arguments;
738 }
739 Ok(function)
740 }
741}
742
743#[non_exhaustive]
747#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq)]
748#[serde(from = "TextRepr")]
749pub struct Text {
750 pub text: String,
752 #[serde(default, skip_serializing_if = "Option::is_none")]
755 citations: Option<citation::Citations>,
756 #[serde(
758 default,
759 skip_serializing_if = "Option::is_none",
760 deserialize_with = "native::lenient"
761 )]
762 pub native: Option<Native>,
763}
764
765#[derive(Deserialize)]
768struct TextRepr {
769 text: String,
770 #[serde(default, deserialize_with = "citation::lenient")]
771 citations: Option<citation::Citations>,
772 #[serde(default, deserialize_with = "native::lenient")]
773 native: Option<Native>,
774}
775
776impl From<TextRepr> for Text {
777 fn from(repr: TextRepr) -> Self {
778 let TextRepr {
779 text,
780 citations,
781 native,
782 } = repr;
783 Self {
784 text,
785 citations,
786 native,
787 }
788 .checked()
789 }
790}
791
792impl Text {
793 pub fn new(text: impl Into<String>) -> Self {
795 Self {
796 text: text.into(),
797 citations: None,
798 native: None,
799 }
800 }
801
802 pub fn text(&self) -> &str {
804 &self.text
805 }
806}
807
808impl std::fmt::Display for Text {
809 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
810 let Self { text, .. } = self;
811 write!(f, "{text}")
812 }
813}
814
815#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq)]
818pub struct Image {
819 pub data: DocumentSourceKind,
821 #[serde(skip_serializing_if = "Option::is_none")]
823 pub media_type: Option<ImageMediaType>,
824 #[serde(skip_serializing_if = "Option::is_none")]
826 pub detail: Option<ImageDetail>,
827 #[serde(
829 default,
830 skip_serializing_if = "Option::is_none",
831 deserialize_with = "native::lenient"
832 )]
833 pub native: Option<Native>,
834}
835
836#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
838#[serde(tag = "type", content = "value", rename_all = "camelCase")]
839pub enum DocumentSourceKind {
840 Url(String),
842 #[serde(alias = "string")]
844 Base64(String),
845 FileId(String),
847 Raw(Vec<u8>),
849 #[default]
850 Unknown,
852}
853
854impl DocumentSourceKind {
855 pub fn url(url: impl Into<String>) -> Self {
857 Self::Url(url.into())
858 }
859
860 pub fn base64(base64_string: impl Into<String>) -> Self {
862 Self::Base64(base64_string.into())
863 }
864
865 pub fn file_id(file_id: impl Into<String>) -> Self {
867 Self::FileId(file_id.into())
868 }
869}
870
871impl std::fmt::Display for DocumentSourceKind {
872 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
873 match self {
874 Self::Url(string) => write!(f, "{string}"),
875 Self::Base64(string) => write!(f, "{string}"),
876 Self::FileId(string) => write!(f, "{string}"),
877 Self::Raw(_) => write!(f, "<binary data>"),
878 Self::Unknown => write!(f, "<unknown>"),
879 }
880 }
881}
882
883#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq)]
885pub struct Audio {
886 pub data: DocumentSourceKind,
888 #[serde(skip_serializing_if = "Option::is_none")]
890 pub media_type: Option<AudioMediaType>,
891}
892
893#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq)]
895pub struct Video {
896 pub data: DocumentSourceKind,
898 #[serde(skip_serializing_if = "Option::is_none")]
900 pub media_type: Option<VideoMediaType>,
901 #[serde(default, skip_serializing_if = "Option::is_none")]
904 pub additional_params: Option<serde_json::Value>,
905}
906
907#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
909#[serde(tag = "type", content = "value", rename_all = "camelCase")]
910pub enum DocumentData {
911 #[serde(alias = "string")]
913 Text(String),
914 #[serde(untagged)]
916 File(DocumentSourceKind),
917}
918
919impl Default for DocumentData {
920 fn default() -> Self {
921 Self::File(DocumentSourceKind::Unknown)
922 }
923}
924
925impl From<DocumentSourceKind> for DocumentData {
926 fn from(source: DocumentSourceKind) -> Self {
927 Self::File(source)
928 }
929}
930
931#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq)]
933pub struct Document {
934 pub data: DocumentData,
936 #[serde(skip_serializing_if = "Option::is_none")]
938 pub media_type: Option<DocumentMediaType>,
939 #[serde(default, skip_serializing_if = "Option::is_none")]
942 pub additional_params: Option<serde_json::Value>,
943}
944
945#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
947pub enum MediaType {
948 Image(ImageMediaType),
949 Audio(AudioMediaType),
950 Document(DocumentMediaType),
951 Video(VideoMediaType),
952}
953
954#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
957#[serde(rename_all = "lowercase")]
958pub enum ImageMediaType {
959 JPEG,
960 PNG,
961 GIF,
962 WEBP,
963 HEIC,
964 HEIF,
965 SVG,
966}
967
968#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
972#[serde(rename_all = "lowercase")]
973pub enum DocumentMediaType {
974 PDF,
975 TXT,
976 RTF,
977 HTML,
978 CSS,
979 MARKDOWN,
980 CSV,
981 XML,
982 Javascript,
983 Python,
984}
985
986impl ImageMediaType {
987 pub fn sniff(bytes: &[u8]) -> Option<Self> {
998 if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
999 Some(Self::PNG)
1000 } else if bytes.starts_with(b"\xff\xd8\xff") {
1001 Some(Self::JPEG)
1002 } else if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") {
1003 Some(Self::GIF)
1004 } else if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP".as_slice()) {
1005 Some(Self::WEBP)
1006 } else {
1007 None
1008 }
1009 }
1010
1011 pub fn extension(&self) -> &'static str {
1013 match self {
1014 Self::JPEG => "jpg",
1015 Self::PNG => "png",
1016 Self::GIF => "gif",
1017 Self::WEBP => "webp",
1018 Self::HEIC => "heic",
1019 Self::HEIF => "heif",
1020 Self::SVG => "svg",
1021 }
1022 }
1023}
1024
1025#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
1028#[serde(rename_all = "lowercase")]
1029pub enum AudioMediaType {
1030 WAV,
1031 MP3,
1032 AIFF,
1033 AAC,
1034 OGG,
1035 FLAC,
1036 M4A,
1037 PCM16,
1038 PCM24,
1039}
1040
1041#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
1044#[serde(rename_all = "lowercase")]
1045pub enum VideoMediaType {
1046 AVI,
1047 MP4,
1048 MPEG,
1049 MOV,
1050 WEBM,
1051}
1052
1053#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq)]
1056#[serde(rename_all = "lowercase")]
1057pub enum ImageDetail {
1058 Low,
1059 High,
1060 #[default]
1061 Auto,
1062}
1063
1064impl Message {
1065 pub fn final_answer(&self) -> Option<String> {
1076 let Message::Assistant(reply) = self else {
1077 return None;
1078 };
1079 let mut parts = Vec::new();
1080 for item in reply.content.iter() {
1081 match item {
1082 AssistantContent::Text(text) => parts.push(text.text.as_str()),
1083 AssistantContent::ToolCall(_) => return None,
1084 _ => {}
1085 }
1086 }
1087 let text = parts.join("\n\n");
1088 let text = text.trim();
1089 (!text.is_empty()).then(|| text.to_owned())
1090 }
1091
1092 pub fn rag_text(&self) -> Option<String> {
1094 match self {
1095 Message::User { content } => {
1096 for item in content.iter() {
1097 if let UserContent::Text(Text { text, .. }) = item {
1098 return Some(text.clone());
1099 }
1100 }
1101 None
1102 }
1103 Message::System { .. } => None,
1104 _ => None,
1105 }
1106 }
1107
1108 pub fn system(text: impl Into<String>) -> Self {
1110 Message::System {
1111 content: text.into(),
1112 }
1113 }
1114
1115 pub fn user(text: impl Into<String>) -> Self {
1117 Message::User {
1118 content: vec![UserContent::text(text)],
1119 }
1120 }
1121
1122 pub fn assistant(text: impl Into<String>) -> Self {
1124 Message::Assistant(AssistantMessage::new(vec![AssistantContent::text(text)]))
1125 }
1126
1127 pub fn tool_result(call: CallId, name: ToolName, content: impl Into<String>) -> Self {
1131 Message::User {
1132 content: vec![UserContent::tool_result(
1133 call,
1134 name,
1135 vec![ToolResultContent::text(content)],
1136 )],
1137 }
1138 }
1139
1140 pub fn tool_results(results: Vec<ToolResult>) -> Self {
1142 Message::User {
1143 content: results.into_iter().map(UserContent::ToolResult).collect(),
1144 }
1145 }
1146
1147 pub fn images(&self) -> impl Iterator<Item = &Image> {
1150 let (user, turn): (&[UserContent], &[AssistantContent]) = match self {
1151 Message::User { content } => (content, &[]),
1152 Message::Assistant(turn) => (&[], &turn.content),
1153 Message::System { .. } => (&[], &[]),
1154 };
1155 let user = user.iter().flat_map(|item| {
1156 let (image, results): (_, &[ToolResultContent]) = match item {
1157 UserContent::Image(image) => (Some(image), &[]),
1158 UserContent::ToolResult(result) => (None, &result.content),
1159 _ => (None, &[]),
1160 };
1161 let results = results.iter().filter_map(|part| match part {
1162 ToolResultContent::Image(image) => Some(image),
1163 _ => None,
1164 });
1165 image.into_iter().chain(results)
1166 });
1167 let turn = turn.iter().filter_map(|block| match block {
1168 AssistantContent::Image(image) => Some(image),
1169 _ => None,
1170 });
1171 user.chain(turn)
1172 }
1173
1174 pub fn images_mut(&mut self) -> impl Iterator<Item = &mut Image> {
1176 let (user, turn): (&mut [UserContent], &mut [AssistantContent]) = match self {
1177 Message::User { content } => (content, Default::default()),
1178 Message::Assistant(turn) => (Default::default(), &mut turn.content),
1179 Message::System { .. } => Default::default(),
1180 };
1181 let user = user.iter_mut().flat_map(|item| {
1182 let (image, results): (_, &mut [ToolResultContent]) = match item {
1183 UserContent::Image(image) => (Some(image), Default::default()),
1184 UserContent::ToolResult(result) => (None, &mut result.content),
1185 _ => Default::default(),
1186 };
1187 let results = results.iter_mut().filter_map(|part| match part {
1188 ToolResultContent::Image(image) => Some(image),
1189 _ => None,
1190 });
1191 image.into_iter().chain(results)
1192 });
1193 let turn = turn.iter_mut().filter_map(|block| match block {
1194 AssistantContent::Image(image) => Some(image),
1195 _ => None,
1196 });
1197 user.chain(turn)
1198 }
1199}
1200
1201macro_rules! media_ctors {
1203 () => {};
1204 (
1205 $(#[$meta:meta])* $name:ident => Image($kind:ident: $data:ty);
1206 $($rest:tt)*
1207 ) => {
1208 $(#[$meta])*
1209 pub fn $name(
1210 data: impl Into<$data>,
1211 media_type: Option<ImageMediaType>,
1212 detail: Option<ImageDetail>,
1213 ) -> Self {
1214 Self::Image(Image {
1215 data: DocumentSourceKind::$kind(data.into()),
1216 media_type,
1217 detail,
1218 native: None,
1219 })
1220 }
1221 media_ctors! { $($rest)* }
1222 };
1223 (
1224 $(#[$meta:meta])* $name:ident => $variant:ident(params $mt:ty, $kind:path: $data:ty);
1225 $($rest:tt)*
1226 ) => {
1227 $(#[$meta])*
1228 pub fn $name(data: impl Into<$data>, media_type: Option<$mt>) -> Self {
1229 Self::$variant($variant {
1230 data: $kind(data.into()).into(),
1231 media_type,
1232 additional_params: None,
1233 })
1234 }
1235 media_ctors! { $($rest)* }
1236 };
1237 (
1238 $(#[$meta:meta])* $name:ident => $variant:ident($mt:ty, $kind:ident: $data:ty);
1239 $($rest:tt)*
1240 ) => {
1241 $(#[$meta])*
1242 pub fn $name(data: impl Into<$data>, media_type: Option<$mt>) -> Self {
1243 Self::$variant($variant {
1244 data: DocumentSourceKind::$kind(data.into()),
1245 media_type,
1246 })
1247 }
1248 media_ctors! { $($rest)* }
1249 };
1250}
1251
1252impl UserContent {
1253 pub fn text(text: impl Into<String>) -> Self {
1255 UserContent::Text(text.into().into())
1256 }
1257
1258 media_ctors! {
1259 image_base64 => Image(Base64: String);
1261 image_raw => Image(Raw: Vec<u8>);
1263 image_url => Image(Url: String);
1265 audio_base64 => Audio(AudioMediaType, Base64: String);
1267 audio_raw => Audio(AudioMediaType, Raw: Vec<u8>);
1269 audio_url => Audio(AudioMediaType, Url: String);
1271 video_base64 => Video(params VideoMediaType, DocumentSourceKind::Base64: String);
1273 video_raw => Video(params VideoMediaType, DocumentSourceKind::Raw: Vec<u8>);
1275 video_url => Video(params VideoMediaType, DocumentSourceKind::Url: String);
1277 document_base64 => Document(params DocumentMediaType, DocumentSourceKind::Base64: String);
1279 document_raw => Document(params DocumentMediaType, DocumentSourceKind::Raw: Vec<u8>);
1281 document_url => Document(params DocumentMediaType, DocumentSourceKind::Url: String);
1283 document_text => Document(params DocumentMediaType, DocumentData::Text: String);
1287 }
1288
1289 pub fn tool_result(call: CallId, name: ToolName, content: Vec<ToolResultContent>) -> Self {
1291 UserContent::ToolResult(ToolResult {
1292 call,
1293 name,
1294 content,
1295 is_error: false,
1296 })
1297 }
1298}
1299
1300impl AssistantContent {
1301 pub fn text(text: impl Into<String>) -> Self {
1303 AssistantContent::Text(text.into().into())
1304 }
1305
1306 media_ctors! {
1307 image_base64 => Image(Base64: String);
1309 }
1310
1311 pub fn tool_call(id: impl Into<String>, name: ToolName, arguments: serde_json::Value) -> Self {
1314 AssistantContent::ToolCall(ToolCall::from_wire(id, ToolFunction::new(name, arguments)))
1315 }
1316
1317 pub fn reasoning(reasoning: impl Into<String>) -> Self {
1319 AssistantContent::Reasoning(Reasoning::new(reasoning))
1320 }
1321}
1322
1323impl ToolResultContent {
1324 pub fn text(text: impl Into<String>) -> Self {
1326 ToolResultContent::Text(text.into().into())
1327 }
1328
1329 pub fn json(value: serde_json::Value) -> Self {
1331 ToolResultContent::Json { value }
1332 }
1333
1334 media_ctors! {
1335 image_base64 => Image(Base64: String);
1337 image_raw => Image(Raw: Vec<u8>);
1339 image_url => Image(Url: String);
1341 }
1342}
1343
1344pub trait MimeType {
1346 fn from_mime_type(mime_type: &str) -> Option<Self>
1347 where
1348 Self: Sized;
1349 fn to_mime_type(&self) -> &'static str;
1350}
1351
1352impl MimeType for MediaType {
1353 fn from_mime_type(mime_type: &str) -> Option<Self> {
1354 ImageMediaType::from_mime_type(mime_type)
1355 .map(MediaType::Image)
1356 .or_else(|| DocumentMediaType::from_mime_type(mime_type).map(MediaType::Document))
1357 .or_else(|| AudioMediaType::from_mime_type(mime_type).map(MediaType::Audio))
1358 .or_else(|| VideoMediaType::from_mime_type(mime_type).map(MediaType::Video))
1359 }
1360
1361 fn to_mime_type(&self) -> &'static str {
1362 match self {
1363 MediaType::Image(media_type) => media_type.to_mime_type(),
1364 MediaType::Audio(media_type) => media_type.to_mime_type(),
1365 MediaType::Document(media_type) => media_type.to_mime_type(),
1366 MediaType::Video(media_type) => media_type.to_mime_type(),
1367 }
1368 }
1369}
1370
1371macro_rules! impl_mime_type {
1376 ($ty:ident { $($variant:ident => $canonical:literal $(| $alias:literal)*),+ $(,)? }) => {
1377 impl MimeType for $ty {
1378 fn from_mime_type(mime_type: &str) -> Option<Self> {
1379 match mime_type {
1380 $($canonical $(| $alias)* => Some($ty::$variant),)+
1381 _ => None,
1382 }
1383 }
1384
1385 fn to_mime_type(&self) -> &'static str {
1386 match self {
1387 $($ty::$variant => $canonical,)+
1388 }
1389 }
1390 }
1391 };
1392}
1393
1394impl_mime_type!(ImageMediaType {
1395 JPEG => "image/jpeg",
1396 PNG => "image/png",
1397 GIF => "image/gif",
1398 WEBP => "image/webp",
1399 HEIC => "image/heic",
1400 HEIF => "image/heif",
1401 SVG => "image/svg+xml",
1402});
1403
1404impl_mime_type!(DocumentMediaType {
1405 PDF => "application/pdf",
1406 TXT => "text/plain",
1407 RTF => "text/rtf",
1408 HTML => "text/html",
1409 CSS => "text/css",
1410 MARKDOWN => "text/markdown" | "text/md",
1411 CSV => "text/csv",
1412 XML => "text/xml",
1413 Javascript => "application/x-javascript" | "text/x-javascript",
1414 Python => "application/x-python" | "text/x-python",
1415});
1416
1417impl_mime_type!(AudioMediaType {
1418 WAV => "audio/wav",
1419 MP3 => "audio/mp3",
1420 AIFF => "audio/aiff",
1421 AAC => "audio/aac",
1422 OGG => "audio/ogg",
1423 FLAC => "audio/flac",
1424 M4A => "audio/m4a",
1425 PCM16 => "audio/pcm16",
1426 PCM24 => "audio/pcm24",
1427});
1428
1429impl_mime_type!(VideoMediaType {
1430 AVI => "video/avi",
1431 MP4 => "video/mp4",
1432 MPEG => "video/mpeg",
1433 MOV => "video/mov",
1434 WEBM => "video/webm",
1435});
1436
1437macro_rules! text_from {
1439 ($($src:ty),+ $(,)?) => {$(
1440 impl From<$src> for Text {
1441 fn from(text: $src) -> Self {
1442 Text::new(text)
1443 }
1444 }
1445 )+};
1446}
1447
1448text_from!(String, &String, &str);
1449
1450macro_rules! text_content_from_string {
1452 ($($ty:ident),+ $(,)?) => {$(
1453 impl From<String> for $ty {
1454 fn from(text: String) -> Self {
1455 $ty::text(text)
1456 }
1457 }
1458 )+};
1459}
1460
1461text_content_from_string!(ToolResultContent, AssistantContent, UserContent);
1462
1463macro_rules! single_content_message_from {
1466 (User { $($src:ty => $variant:ident),+ $(,)? }) => {$(
1467 impl From<$src> for Message {
1468 fn from(value: $src) -> Self {
1469 Message::User {
1470 content: vec![UserContent::$variant(value.into())],
1471 }
1472 }
1473 }
1474 )+};
1475 (Assistant { $($src:ty => $variant:ident),+ $(,)? }) => {$(
1476 impl From<$src> for Message {
1477 fn from(value: $src) -> Self {
1478 Message::Assistant(AssistantMessage::new(vec![AssistantContent::$variant(
1479 value.into(),
1480 )]))
1481 }
1482 }
1483 )+};
1484}
1485
1486single_content_message_from!(User {
1487 String => Text,
1488 &str => Text,
1489 &String => Text,
1490 Text => Text,
1491 Image => Image,
1492 Audio => Audio,
1493 Document => Document,
1494 ToolResult => ToolResult,
1495});
1496
1497single_content_message_from!(Assistant {
1498 ToolCall => ToolCall,
1499});
1500
1501impl FromStr for Text {
1502 type Err = Infallible;
1503
1504 fn from_str(s: &str) -> Result<Self, Self::Err> {
1505 Ok(s.into())
1506 }
1507}
1508
1509impl From<&Message> for Message {
1510 fn from(msg: &Message) -> Self {
1511 msg.clone()
1512 }
1513}
1514
1515impl From<std::sync::Arc<Message>> for Message {
1518 fn from(msg: std::sync::Arc<Message>) -> Self {
1519 std::sync::Arc::unwrap_or_clone(msg)
1520 }
1521}
1522
1523impl From<AssistantContent> for Message {
1524 fn from(content: AssistantContent) -> Self {
1525 Message::Assistant(AssistantMessage::new(vec![content]))
1526 }
1527}
1528
1529impl From<AssistantMessage> for Message {
1530 fn from(message: AssistantMessage) -> Self {
1531 Message::Assistant(message)
1532 }
1533}
1534
1535impl From<UserContent> for Message {
1536 fn from(content: UserContent) -> Self {
1537 Message::User {
1538 content: vec![content],
1539 }
1540 }
1541}
1542
1543impl From<Vec<AssistantContent>> for Message {
1544 fn from(content: Vec<AssistantContent>) -> Self {
1545 Message::Assistant(AssistantMessage::new(content))
1546 }
1547}
1548
1549impl From<Vec<UserContent>> for Message {
1550 fn from(content: Vec<UserContent>) -> Self {
1551 Message::User { content }
1552 }
1553}
1554
1555#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq)]
1557#[serde(rename_all = "snake_case")]
1558pub enum ToolChoice {
1559 #[default]
1561 Auto,
1562 None,
1564 Required,
1566 Specific {
1568 function_names: Vec<ToolName>,
1570 },
1571}
1572
1573#[derive(Debug, Error)]
1575pub enum MessageError {
1576 #[error("Message conversion error: {0}")]
1577 ConversionError(String),
1578}
1579
1580impl From<MessageError> for ProviderError {
1581 fn from(error: MessageError) -> Self {
1582 ProviderError::request(error)
1583 }
1584}
1585
1586#[cfg(test)]
1587mod tests;