1use std::collections::{HashMap, HashSet};
61
62use base64::Engine as _;
63use base64::prelude::{BASE64_STANDARD, BASE64_STANDARD_NO_PAD};
64
65use crate::message::{
66 Api, AssistantContent, AssistantMessage, CallId, DocumentMediaType, DocumentSourceKind,
67 ImageMediaType, Message, Origin, Text, ToolCall, ToolResult, ToolResultContent, UserContent,
68};
69use crate::wasm_compat::WasmCompatSync;
70
71pub const NO_RESULT_PROVIDED: &str = "No result provided";
73
74pub const USER_IMAGE_OMITTED: &str = "(image omitted: model does not support images)";
76
77pub const ASSISTANT_IMAGE_OMITTED: &str = "(image omitted: model does not support images)";
80
81pub const TOOL_IMAGE_OMITTED: &str = "(tool image omitted: model does not support images)";
83
84pub const IMAGE_UNSENDABLE: &str = "(image omitted: the provider cannot receive it in this form)";
86
87pub const AUDIO_UNSENDABLE: &str = "(audio omitted: the provider cannot receive it in this form)";
89
90pub const VIDEO_UNSENDABLE: &str = "(video omitted: the provider cannot receive it in this form)";
92
93pub const DOCUMENT_UNSENDABLE: &str =
96 "(document omitted: the provider cannot receive it in this form)";
97
98pub const TOOL_IMAGE_ATTACHED: &str = "(see attached image)";
101
102pub const TOOL_IMAGES_HEADING: &str = "Attached image(s) from tool result:";
104
105#[derive(Debug, Clone, Copy, PartialEq, Eq)]
107pub struct Accepts {
108 pub user_images: bool,
110 pub assistant_images: bool,
112 pub tool_result_images: bool,
114 pub tools: bool,
116}
117
118impl Accepts {
119 pub const ALL: Self = Self {
121 user_images: true,
122 assistant_images: true,
123 tool_result_images: true,
124 tools: true,
125 };
126
127 pub const TEXT: Self = Self {
129 user_images: false,
130 assistant_images: false,
131 tool_result_images: false,
132 tools: true,
133 };
134}
135
136#[derive(Debug, Clone, Copy, PartialEq, Eq)]
138pub enum Place {
139 User,
141 ToolResult,
143 Assistant,
145}
146
147#[derive(Debug, Clone, Copy)]
151pub enum Media<'a> {
152 Image(&'a crate::message::Image, Place),
154 Audio(&'a crate::message::Audio),
156 Video(&'a crate::message::Video),
158 Document(&'a crate::message::Document),
160}
161
162pub trait ReplayTarget: std::fmt::Debug + WasmCompatSync {
165 fn api(&self) -> Api;
167
168 fn provider(&self) -> &str;
170
171 fn model(&self) -> &str;
173
174 fn accepts(&self, model: &str) -> Accepts;
177
178 fn map_options(
188 &self,
189 request: &crate::completion::CompletionRequest,
190 fields: crate::completion::options::OptionFields<'_>,
191 ) -> crate::completion::options::OptionMap;
192
193 fn encodes(&self, model: &str, media: Media<'_>) -> bool {
200 let _ = (model, media);
201 true
202 }
203
204 fn normalize_tool_call_id(&self, id: &str, model: &str, source: Option<&Origin>) -> String {
209 let _ = (model, source);
210 id.to_owned()
211 }
212
213 fn continues_stored(&self, request: &crate::completion::CompletionRequest) -> bool {
218 let _ = request;
219 false
220 }
221
222 fn declares_tools(&self, request: &crate::completion::CompletionRequest) -> bool {
228 declares_tools(request)
229 }
230
231 fn identity(&self, item: &serde_json::Value) -> serde_json::Map<String, serde_json::Value> {
235 let _ = item;
236 serde_json::Map::new()
237 }
238
239 fn call_id_slot(&self) -> Option<&'static str> {
244 None
245 }
246
247 fn states_finish_reason(&self) -> bool {
251 true
252 }
253
254 fn binds_context(&self, model: &str) -> bool {
258 let _ = model;
259 false
260 }
261
262 fn drops_unbound_items(&self, request: &crate::completion::CompletionRequest) -> bool {
266 let _ = request;
267 false
268 }
269
270 fn route(&self, request: &crate::completion::CompletionRequest) -> Option<&dyn ReplayTarget> {
275 let _ = request;
276 None
277 }
278
279 fn takes_documents(&self) -> bool {
282 false
283 }
284
285 fn needs_next(&self, item: &serde_json::Value) -> bool {
289 let _ = item;
290 false
291 }
292
293 fn hosted_pair(&self, item: &serde_json::Value) -> Option<(Pairing, String)> {
299 let _ = item;
300 None
301 }
302
303 fn starts_with_user(&self) -> bool {
307 false
308 }
309
310 fn later_system(&self, model: &str) -> LaterSystem {
314 let _ = model;
315 LaterSystem::InPlace
316 }
317
318 fn alternates_roles(&self) -> bool {
322 false
323 }
324
325 fn hosted_needs_tools(&self) -> bool {
328 false
329 }
330
331 fn result_parts(&self, model: &str) -> bool {
335 let _ = model;
336 false
337 }
338
339 fn sends_alone(&self, block: &AssistantContent) -> bool {
344 let _ = block;
345 true
346 }
347}
348
349#[derive(Debug, Clone, Copy, PartialEq, Eq)]
352pub enum LaterSystem {
353 InPlace,
355 Leading,
358 UserText,
361}
362
363#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
365pub enum Pairing {
366 Use,
368 Result,
370}
371
372#[derive(Debug, Clone, PartialEq)]
374pub enum Replay<'a> {
375 Item(std::borrow::Cow<'a, serde_json::Value>),
377 Identity(serde_json::Map<String, serde_json::Value>),
380 Rebuild,
382}
383
384impl AssistantContent {
385 pub fn replay(
388 &self,
389 target: &dyn ReplayTarget,
390 ids: &crate::providers::internal::wire_ids::WireIds,
391 ) -> Replay<'_> {
392 if let Some(item) = self.native_item() {
393 return match (self, target.call_id_slot()) {
394 (AssistantContent::ToolCall(call), Some(slot)) => {
395 let mut item = item.clone();
396 if let Some(id) = ids.of(&call.id) {
397 set_pointer(&mut item, slot, serde_json::Value::String(id.to_owned()));
398 }
399 Replay::Item(std::borrow::Cow::Owned(item))
400 }
401 _ => Replay::Item(std::borrow::Cow::Borrowed(item)),
402 };
403 }
404 let identity = self
405 .stale_item()
406 .map(|item| target.identity(item))
407 .unwrap_or_default();
408 if identity.is_empty() {
409 Replay::Rebuild
410 } else {
411 Replay::Identity(identity)
412 }
413 }
414}
415
416fn set_pointer(item: &mut serde_json::Value, pointer: &str, value: serde_json::Value) {
418 let mut at = item;
419 let mut keys = pointer.split('/').skip(1).peekable();
420 while let Some(key) = keys.next() {
421 if !at.is_object() {
422 *at = serde_json::Value::Object(serde_json::Map::new());
423 }
424 let serde_json::Value::Object(fields) = at else {
425 return;
426 };
427 if keys.peek().is_none() {
428 fields.insert(key.to_owned(), value);
429 return;
430 }
431 at = fields
432 .entry(key.to_owned())
433 .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()));
434 }
435}
436
437pub fn adapt(history: &[Message], target: &dyn ReplayTarget) -> Vec<Message> {
463 adapt_for(history, target, &Request::default())
464}
465
466#[derive(Debug, Clone, Copy)]
468pub(crate) struct Request<'a> {
469 pub(crate) model: Option<&'a str>,
471 pub(crate) stored: bool,
473 pub(crate) tools: bool,
476 pub(crate) context: Option<crate::message::Fingerprint>,
478}
479
480impl Default for Request<'_> {
481 fn default() -> Self {
482 Self {
483 model: None,
484 stored: false,
485 tools: true,
486 context: None,
487 }
488 }
489}
490
491pub(crate) fn context_of(
497 request: &crate::completion::CompletionRequest,
498 target: &dyn ReplayTarget,
499 model: &str,
500) -> crate::message::Fingerprint {
501 let mut tools: Vec<_> = request.tools.iter().collect();
502 tools.sort_by(|left, right| left.name.cmp(&right.name));
503 let raw = raw_tools(request);
504 let folds = target.later_system(model) == LaterSystem::Leading;
505 let mut system: Vec<&str> = request
506 .chat_history
507 .iter()
508 .map_while(|message| match message {
509 Message::System { content } => Some(Some(content.as_str())),
510 Message::User { .. } | Message::Assistant(_) => folds.then_some(None),
511 })
512 .flatten()
513 .filter(|content| !content.trim().is_empty())
514 .collect();
515 let joined;
516 if folds && system.len() > 1 {
517 joined = system.join("\n\n");
518 system = vec![joined.as_str()];
519 }
520 let mut fields = vec![serde_json::json!("context"), serde_json::json!(tools)];
521 fields.push(serde_json::json!(system));
522 if !raw.is_empty() {
523 fields.push(serde_json::json!(raw));
524 }
525 crate::message::Fingerprint::of(&serde_json::Value::Array(fields))
526}
527
528pub(crate) fn declares_tools(request: &crate::completion::CompletionRequest) -> bool {
531 !request.tools.is_empty() || !raw_tools(request).is_empty()
532}
533
534pub(crate) fn raw_tools(request: &crate::completion::CompletionRequest) -> &[serde_json::Value] {
537 use crate::json_utils::Lenient;
538 request
539 .additional_params
540 .as_ref()
541 .map_or(&[][..], |params| params.arr("tools"))
542}
543
544pub(crate) fn adapt_for(
546 history: &[Message],
547 target: &dyn ReplayTarget,
548 request: &Request<'_>,
549) -> Vec<Message> {
550 let model = request.model.unwrap_or(target.model());
551 let stored = request.stored;
552 let same = Same {
553 api: target.api(),
554 provider: target.provider(),
555 model,
556 context: request.context.filter(|_| target.binds_context(model)),
557 };
558 let mut accepts = target.accepts(model);
559 accepts.tools &= request.tools;
560 let hosted = hosted_pairs(history, target, &same);
561 let last_turn = history
562 .iter()
563 .rposition(|message| matches!(message, Message::Assistant(_)));
564 let mut ids = Renamed::default();
565 let mut shaped = Vec::with_capacity(history.len());
566 for (at, message) in history.iter().enumerate() {
567 match message {
568 Message::System { content } => {
569 if !content.trim().is_empty() {
570 shaped.push(Some(message.clone()));
571 }
572 }
573 Message::User { content } => {
574 let form = Form {
575 target,
576 model,
577 accepts,
578 };
579 shaped.extend(user(content, &mut ids, &form).into_iter().map(Some));
580 }
581 Message::Assistant(turn) => {
582 let last = Some(at) == last_turn
585 && (turn
586 .content
587 .iter()
588 .any(|block| matches!(block, AssistantContent::ToolCall(_)))
589 || history
590 .get(at + 1..)
591 .into_iter()
592 .flatten()
593 .all(|message| matches!(message, Message::System { .. })));
594 let here: HashSet<usize> = hosted
595 .iter()
596 .filter(|(message, _)| *message == at)
597 .map(|(_, block)| *block)
598 .collect();
599 let adapted = assistant(turn, target, &same, accepts, &mut ids, &here, last);
600 let adapted = AssistantMessage {
601 content: adapted
602 .content
603 .into_iter()
604 .map(|block| match block {
605 AssistantContent::Image(image) if image.native.is_none() => {
606 let image = sendable_image(image);
607 if matches!(image.data, DocumentSourceKind::Unknown)
608 || !target
609 .encodes(model, Media::Image(&image, Place::Assistant))
610 {
611 AssistantContent::Text(Text::new(IMAGE_UNSENDABLE))
612 } else {
613 AssistantContent::Image(image)
614 }
615 }
616 block => block,
617 })
618 .collect(),
619 ..adapted
620 };
621 let emptied = !turn.content.is_empty()
622 && !adapted
623 .content
624 .iter()
625 .any(|block| target.sends_alone(block));
626 shaped.push((!emptied).then_some(Message::Assistant(adapted)));
627 }
628 }
629 }
630 let shaped = merge_users(answer_calls(shaped, stored, accepts.tools));
634 let shaped = if accepts.tools {
635 shaped
636 } else {
637 tools_as_text(shaped)
638 };
639 let shaped = match target.later_system(model) {
640 LaterSystem::InPlace => shaped,
641 LaterSystem::Leading => leading_system(shaped),
642 LaterSystem::UserText => system_as_user_text(shaped),
643 };
644 let shaped = if target.starts_with_user() && !stored {
645 from_first_user(shaped)
646 } else {
647 shaped
648 };
649 if target.alternates_roles() {
650 alternated(shaped)
651 } else {
652 shaped
653 }
654}
655
656fn tools_as_text(history: Vec<Message>) -> Vec<Message> {
660 let history = history
661 .into_iter()
662 .map(|message| match message {
663 Message::Assistant(mut turn) => {
664 for block in &mut turn.content {
665 if let AssistantContent::ToolCall(call) = block {
666 *block = AssistantContent::Text(Text::new(format!(
667 "[called tool {} with {}]",
668 call.function.name,
669 crate::json_utils::to_canonical_string(
670 &call.function.arguments_value()
671 )
672 )));
673 }
674 }
675 Message::Assistant(turn)
676 }
677 Message::User { content } => Message::User {
678 content: content
679 .into_iter()
680 .map(|part| match part {
681 UserContent::ToolResult(result) => UserContent::text(result_text(&result)),
682 part => part,
683 })
684 .collect(),
685 },
686 message => message,
687 })
688 .collect();
689 merge_users(history)
690}
691
692fn from_first_user(mut history: Vec<Message>) -> Vec<Message> {
696 let lead = history
697 .iter()
698 .take_while(|message| matches!(message, Message::System { .. }))
699 .count();
700 while let Some(Message::Assistant(turn)) = history.get(lead) {
701 let calls: HashSet<CallId> = turn.tool_calls().map(|call| call.id.clone()).collect();
702 history.remove(lead);
703 if let Some(Message::User { content }) = history.get_mut(lead) {
704 content.retain(|part| {
705 !matches!(part, UserContent::ToolResult(result) if calls.contains(&result.call))
706 });
707 if content.is_empty() {
708 history.remove(lead);
709 }
710 }
711 }
712 history
713}
714
715fn has_later_system(history: &[Message]) -> bool {
718 history
719 .iter()
720 .skip_while(|message| matches!(message, Message::System { .. }))
721 .any(|message| matches!(message, Message::System { .. }))
722}
723
724fn system_as_user_text(history: Vec<Message>) -> Vec<Message> {
726 let mut leading = true;
727 let history = history
728 .into_iter()
729 .map(|message| match message {
730 Message::System { content } if !leading => Message::User {
731 content: vec![UserContent::text(content)],
732 },
733 message => {
734 leading &= matches!(message, Message::System { .. });
735 message
736 }
737 })
738 .collect();
739 merge_users(history)
740}
741
742fn alternated(history: Vec<Message>) -> Vec<Message> {
746 let mut alternated: Vec<Message> = Vec::with_capacity(history.len());
747 let mut held: Vec<Message> = Vec::new();
748 for message in history {
749 let started = alternated
750 .iter()
751 .any(|message| !matches!(message, Message::System { .. }));
752 match (message, alternated.last_mut()) {
753 (message @ Message::System { .. }, _) if started => held.push(message),
754 (Message::User { content }, Some(Message::User { content: previous })) => {
755 previous.extend(content);
756 }
757 (Message::Assistant(turn), Some(Message::Assistant(previous))) => {
758 if previous.origin != turn.origin {
759 previous.origin = None;
760 }
761 previous.stop = turn.stop;
762 previous.content.extend(turn.content);
763 }
764 (message, _) => {
765 alternated.append(&mut held);
766 alternated.push(message);
767 }
768 }
769 }
770 alternated.append(&mut held);
771 alternated
772}
773
774fn leading_system(history: Vec<Message>) -> Vec<Message> {
777 if !has_later_system(&history) {
778 return history;
779 }
780 let (system, rest): (Vec<Message>, Vec<Message>) = history
781 .into_iter()
782 .partition(|message| matches!(message, Message::System { .. }));
783 let prompt: Vec<String> = system
784 .into_iter()
785 .filter_map(|message| match message {
786 Message::System { content } => Some(content),
787 Message::User { .. } | Message::Assistant(_) => None,
788 })
789 .collect();
790 let mut history = Vec::with_capacity(rest.len() + 1);
791 if !prompt.is_empty() {
792 history.push(Message::system(prompt.join("\n\n")));
793 }
794 history.extend(merge_users(rest));
796 history
797}
798
799#[derive(Default)]
805struct Renamed {
806 to: HashMap<CallId, std::collections::VecDeque<CallId>>,
808 taken: HashSet<String>,
809}
810
811impl Renamed {
812 fn turn(&mut self) {
814 self.to.clear();
815 }
816
817 fn claim(&mut self, source: &CallId, wanted: String) -> CallId {
825 let id = match source {
826 CallId::Local(_) => source.clone(),
827 CallId::Provider(_) => {
828 let mut id = wanted.clone();
829 let mut attempt: u64 = 1;
830 while self.taken.contains(&id) {
831 id = with_counter(&wanted, attempt);
832 attempt += 1;
833 }
834 self.taken.insert(id.clone());
835 if id == source.wire() {
836 source.clone()
837 } else {
838 CallId::from_wire(id)
839 }
840 }
841 };
842 self.to
843 .entry(source.clone())
844 .or_default()
845 .push_back(id.clone());
846 id
847 }
848
849 fn answer(&mut self, source: &CallId) -> Option<CallId> {
852 let ids = self.to.get_mut(source)?;
853 if ids.len() > 1 {
854 ids.pop_front()
855 } else {
856 ids.front().cloned()
857 }
858 }
859}
860
861fn with_counter(id: &str, attempt: u64) -> String {
863 let mut digits = Vec::new();
864 let mut value = attempt;
865 while value > 0 {
866 digits.push(char::from_digit((value % 36) as u32, 36).unwrap_or('0'));
867 value /= 36;
868 }
869 digits.reverse();
870 let keep = id.chars().count().saturating_sub(digits.len());
871 id.chars().take(keep).chain(digits).collect()
872}
873
874struct Same<'a> {
876 api: Api,
877 provider: &'a str,
878 model: &'a str,
879 context: Option<crate::message::Fingerprint>,
881}
882
883impl Same<'_> {
884 fn is(&self, origin: &Origin) -> bool {
887 origin.same_model(&self.api, self.provider, self.model)
888 && self
889 .context
890 .is_none_or(|context| origin.context == Some(context))
891 }
892}
893
894fn assistant(
896 turn: &AssistantMessage,
897 target: &dyn ReplayTarget,
898 same_model: &Same<'_>,
899 accepts: Accepts,
900 ids: &mut Renamed,
901 hosted: &HashSet<usize>,
902 last: bool,
903) -> AssistantMessage {
904 let model = same_model.model;
905 let same = turn
906 .origin
907 .as_ref()
908 .is_some_and(|origin| same_model.is(origin));
909 if turn.stop.as_ref().is_some_and(|stop| stop.is_failure()) {
911 return turn.clone();
912 }
913 ids.turn();
914 let content: Vec<Option<AssistantContent>> = turn
915 .content
916 .iter()
917 .map(|block| {
918 let block = if same {
919 match block.clone() {
920 AssistantContent::ToolCall(mut call) => {
921 let wanted = call.id.wire().into_owned();
922 let item = AssistantContent::ToolCall(call.clone())
923 .native_item()
924 .cloned()
925 .filter(|_| target.call_id_slot().is_some());
926 call.id = ids.claim(&call.id, wanted);
927 let block = AssistantContent::ToolCall(call);
928 match item {
931 Some(item) if block.native_item().is_none() => {
932 block.canonical().with_native(item)
933 }
934 _ => block,
935 }
936 }
937 AssistantContent::Image(_) if !accepts.assistant_images => return None,
940 block => block,
941 }
942 } else {
943 match block.canonical() {
944 AssistantContent::Reasoning(reasoning) => {
945 if reasoning.redacted || reasoning.text.trim().is_empty() {
946 return None;
947 }
948 AssistantContent::Text(Text::new(reasoning.text))
949 }
950 AssistantContent::Opaque(_) => return None,
951 AssistantContent::Image(_) if !accepts.assistant_images => {
952 AssistantContent::Text(Text::new(ASSISTANT_IMAGE_OMITTED))
953 }
954 AssistantContent::ToolCall(mut call) => {
955 let normalized = target.normalize_tool_call_id(
956 &call.id.wire(),
957 model,
958 turn.origin.as_ref(),
959 );
960 call.id = ids.claim(&call.id, normalized);
961 AssistantContent::ToolCall(call)
962 }
963 block => block,
964 }
965 };
966 let block = match block {
969 AssistantContent::Opaque(opaque)
970 if !accepts.tools
971 && target.hosted_needs_tools()
972 && target.hosted_pair(&opaque.item).is_some() =>
973 {
974 return None;
975 }
976 block => block,
977 };
978 (!block.is_blank()).then_some(block)
979 })
980 .collect();
981 let content = if same {
982 paired(content, target, accepts.tools, hosted, last)
983 } else {
984 content.into_iter().flatten().collect()
985 };
986 AssistantMessage {
987 content,
988 origin: turn.origin.clone(),
989 stop: turn.stop.clone(),
990 }
991}
992
993fn hosted_pairs(
999 history: &[Message],
1000 target: &dyn ReplayTarget,
1001 same: &Same<'_>,
1002) -> HashSet<(usize, usize)> {
1003 let mut open: HashMap<String, (usize, usize)> = HashMap::new();
1004 let mut paired = HashSet::new();
1005 for (at, message) in history.iter().enumerate() {
1006 let Message::Assistant(turn) = message else {
1007 continue;
1008 };
1009 if turn.stop.as_ref().is_some_and(|stop| stop.is_failure())
1010 || !turn.origin.as_ref().is_some_and(|origin| same.is(origin))
1011 {
1012 continue;
1013 }
1014 for (index, block) in turn.content.iter().enumerate() {
1015 let AssistantContent::Opaque(opaque) = block else {
1016 continue;
1017 };
1018 if !opaque.replay {
1019 continue;
1020 }
1021 match target.hosted_pair(&opaque.item) {
1022 Some((Pairing::Use, id)) => {
1023 open.insert(id, (at, index));
1024 }
1025 Some((Pairing::Result, id)) => {
1026 if let Some(used) = open.remove(&id) {
1027 paired.insert(used);
1028 paired.insert((at, index));
1029 }
1030 }
1031 None => {}
1032 }
1033 }
1034 }
1035 paired
1036}
1037
1038fn paired(
1046 mut content: Vec<Option<AssistantContent>>,
1047 target: &dyn ReplayTarget,
1048 tools: bool,
1049 hosted: &HashSet<usize>,
1050 last: bool,
1051) -> Vec<AssistantContent> {
1052 let pair = |block: &AssistantContent| match block {
1053 AssistantContent::Opaque(opaque) if opaque.replay => target.hosted_pair(&opaque.item),
1054 _ => None,
1055 };
1056 for at in 0..content.len() {
1057 let Some((side, _)) = content.get(at).and_then(Option::as_ref).and_then(pair) else {
1058 continue;
1059 };
1060 let running = last
1061 && side == Pairing::Use
1062 && content
1063 .get(at + 1..)
1064 .into_iter()
1065 .flatten()
1066 .flatten()
1067 .all(|block| {
1068 matches!(
1069 block,
1070 AssistantContent::ToolCall(_) | AssistantContent::Opaque(_)
1071 )
1072 });
1073 if !hosted.contains(&at)
1074 && !running
1075 && let Some(slot) = content.get_mut(at)
1076 {
1077 *slot = None;
1078 }
1079 }
1080 for at in (0..content.len()).rev() {
1081 let needs = content
1082 .get(at)
1083 .and_then(Option::as_ref)
1084 .and_then(|block| match block {
1085 AssistantContent::Opaque(opaque) if opaque.replay => Some(&opaque.item),
1086 block => block.native_item().or_else(|| {
1089 block
1090 .stale_item()
1091 .filter(|item| !target.identity(item).is_empty())
1092 }),
1093 })
1094 .is_some_and(|item| target.needs_next(item));
1095 let next_gone =
1098 content
1099 .get(at + 1)
1100 .and_then(Option::as_ref)
1101 .is_none_or(|next| match next {
1102 AssistantContent::Opaque(opaque) => !opaque.replay,
1103 AssistantContent::ToolCall(_) if !tools => true,
1105 next => {
1106 next.native_item().is_none()
1107 && next
1108 .stale_item()
1109 .is_none_or(|item| target.identity(item).is_empty())
1110 }
1111 });
1112 if needs
1113 && next_gone
1114 && let Some(slot) = content.get_mut(at)
1115 {
1116 *slot = None;
1117 }
1118 }
1119 content.into_iter().flatten().collect()
1120}
1121
1122struct Form<'a> {
1124 target: &'a dyn ReplayTarget,
1125 model: &'a str,
1126 accepts: Accepts,
1127}
1128
1129impl Form<'_> {
1130 fn sends(&self, image: &crate::message::Image, place: Place) -> bool {
1132 let reads = match place {
1133 Place::User => self.accepts.user_images,
1134 Place::ToolResult => self.accepts.tool_result_images,
1135 Place::Assistant => self.accepts.assistant_images,
1136 };
1137 reads
1138 && !matches!(image.data, DocumentSourceKind::Unknown)
1139 && self.target.encodes(self.model, Media::Image(image, place))
1140 }
1141}
1142
1143fn user(content: &[UserContent], ids: &mut Renamed, form: &Form<'_>) -> Vec<Message> {
1146 let mut shaped: Vec<UserContent> = Vec::with_capacity(content.len());
1147 let mut attached = Vec::new();
1148 for part in content {
1149 let placeholder = match part {
1150 UserContent::Image(image) => {
1151 let image = sendable_image(image.clone());
1152 if form.sends(&image, Place::User) {
1153 shaped.push(UserContent::Image(image));
1154 continue;
1155 }
1156 if form.accepts.user_images {
1157 IMAGE_UNSENDABLE
1158 } else {
1159 USER_IMAGE_OMITTED
1160 }
1161 }
1162 UserContent::Audio(audio) => {
1163 let mut audio = audio.clone();
1164 audio.data = sendable(audio.data);
1165 if !matches!(audio.data, DocumentSourceKind::Unknown)
1166 && form.target.encodes(form.model, Media::Audio(&audio))
1167 {
1168 shaped.push(UserContent::Audio(audio));
1169 continue;
1170 }
1171 AUDIO_UNSENDABLE
1172 }
1173 UserContent::Video(video) => {
1174 let mut video = video.clone();
1175 video.data = sendable(video.data);
1176 if !matches!(video.data, DocumentSourceKind::Unknown)
1177 && form.target.encodes(form.model, Media::Video(&video))
1178 {
1179 shaped.push(UserContent::Video(video));
1180 continue;
1181 }
1182 VIDEO_UNSENDABLE
1183 }
1184 UserContent::Document(document) => {
1185 let mut document = document.clone();
1186 document.data = sendable(document.data);
1187 if !matches!(document.data, DocumentSourceKind::Unknown)
1188 && form.target.encodes(form.model, Media::Document(&document))
1189 {
1190 shaped.push(UserContent::Document(document));
1191 continue;
1192 }
1193 if let Some(text) = document_text(&document) {
1194 shaped.push(UserContent::text(text));
1195 continue;
1196 }
1197 DOCUMENT_UNSENDABLE
1198 }
1199 UserContent::ToolResult(result) => {
1200 let mut result = result.clone();
1201 if let Some(id) = ids.answer(&result.call) {
1202 result.call = id;
1203 }
1204 result.content = result_images(result.content, form, &mut attached);
1205 let parts = form.accepts.tool_result_images || form.target.result_parts(form.model);
1206 result.content = result_text_parts(result.content, parts, result.is_error);
1207 shaped.push(UserContent::ToolResult(result));
1208 continue;
1209 }
1210 UserContent::Text(text) if text.text.trim().is_empty() => continue,
1212 UserContent::Text(_) => {
1213 shaped.push(part.clone());
1214 continue;
1215 }
1216 };
1217 let repeated = matches!(
1218 shaped.last(),
1219 Some(UserContent::Text(text)) if text.text == placeholder
1220 );
1221 if !repeated {
1222 shaped.push(UserContent::text(placeholder));
1223 }
1224 }
1225 let mut messages = Vec::new();
1226 if !shaped.is_empty() {
1227 messages.push(Message::User { content: shaped });
1228 }
1229 if !attached.is_empty() {
1230 let mut content = vec![UserContent::text(TOOL_IMAGES_HEADING)];
1231 content.extend(attached.into_iter().map(UserContent::Image));
1232 messages.push(Message::User { content });
1233 }
1234 messages
1235}
1236
1237pub const NO_TOOL_OUTPUT: &str = "(no tool output)";
1240
1241fn result_text_parts(
1245 content: Vec<ToolResultContent>,
1246 parts: bool,
1247 is_error: bool,
1248) -> Vec<ToolResultContent> {
1249 let blank = content.iter().all(|part| match part {
1250 ToolResultContent::Text(text) => text.text.trim().is_empty(),
1251 ToolResultContent::Json { .. } | ToolResultContent::Image(_) => false,
1252 });
1253 if blank {
1254 let text = if is_error {
1255 format!("[tool error] {NO_TOOL_OUTPUT}")
1256 } else {
1257 NO_TOOL_OUTPUT.to_owned()
1258 };
1259 return vec![ToolResultContent::text(text)];
1260 }
1261 let texts = content
1262 .iter()
1263 .filter(|part| !matches!(part, ToolResultContent::Image(_)))
1264 .count();
1265 if parts || texts < 2 {
1266 return content;
1267 }
1268 let mut joined: Vec<String> = Vec::new();
1269 let mut images = Vec::new();
1270 for part in content {
1271 match part {
1272 ToolResultContent::Text(text) => joined.push(text.text),
1273 ToolResultContent::Json { value } => joined.push(value.to_string()),
1274 image @ ToolResultContent::Image(_) => images.push(image),
1275 }
1276 }
1277 std::iter::once(ToolResultContent::text(joined.join("\n")))
1278 .chain(images)
1279 .collect()
1280}
1281
1282fn result_text(result: &ToolResult) -> String {
1284 let text: Vec<String> = result
1285 .content
1286 .iter()
1287 .map(|part| match part {
1288 ToolResultContent::Text(text) => text.text.clone(),
1289 ToolResultContent::Json { value } => value.to_string(),
1290 ToolResultContent::Image(_) => TOOL_IMAGE_OMITTED.to_owned(),
1291 })
1292 .collect();
1293 let kind = if result.is_error { "error" } else { "result" };
1294 format!("[tool {} {kind}] {}", result.name, text.join("\n"))
1295}
1296
1297fn result_images(
1301 content: Vec<ToolResultContent>,
1302 form: &Form<'_>,
1303 attached: &mut Vec<crate::message::Image>,
1304) -> Vec<ToolResultContent> {
1305 let mut shaped: Vec<ToolResultContent> = Vec::with_capacity(content.len());
1306 for part in content {
1307 match part {
1308 ToolResultContent::Image(image) => {
1309 let image = sendable_image(image);
1310 if form.sends(&image, Place::ToolResult) {
1311 shaped.push(ToolResultContent::Image(image));
1312 continue;
1313 }
1314 let placeholder = if form.sends(&image, Place::User) {
1315 attached.push(image);
1316 TOOL_IMAGE_ATTACHED
1317 } else {
1318 TOOL_IMAGE_OMITTED
1319 };
1320 if shaped.last().and_then(ToolResultContent::as_text) != Some(placeholder) {
1321 shaped.push(ToolResultContent::text(placeholder));
1322 }
1323 }
1324 part => shaped.push(part),
1325 }
1326 }
1327 shaped
1328}
1329
1330fn sendable(source: DocumentSourceKind) -> DocumentSourceKind {
1333 match source {
1334 DocumentSourceKind::Raw(bytes) => DocumentSourceKind::Base64(BASE64_STANDARD.encode(bytes)),
1335 source => source,
1336 }
1337}
1338
1339fn sendable_image(mut image: crate::message::Image) -> crate::message::Image {
1342 image.data = sendable(image.data);
1343 if image.media_type.is_none()
1344 && let DocumentSourceKind::Base64(data) = &image.data
1345 {
1346 image.media_type = sniffed(data);
1347 }
1348 image
1349}
1350
1351fn sniffed(data: &str) -> Option<ImageMediaType> {
1353 let head: String = data.chars().take(24).collect();
1354 let bytes = BASE64_STANDARD
1355 .decode(head.as_bytes())
1356 .or_else(|_| BASE64_STANDARD_NO_PAD.decode(head.as_bytes()))
1357 .ok()?;
1358 match bytes.as_slice() {
1359 [0x89, b'P', b'N', b'G', ..] => Some(ImageMediaType::PNG),
1360 [0xFF, 0xD8, 0xFF, ..] => Some(ImageMediaType::JPEG),
1361 [b'G', b'I', b'F', b'8', ..] => Some(ImageMediaType::GIF),
1362 [
1363 b'R',
1364 b'I',
1365 b'F',
1366 b'F',
1367 _,
1368 _,
1369 _,
1370 _,
1371 b'W',
1372 b'E',
1373 b'B',
1374 b'P',
1375 ..,
1376 ] => Some(ImageMediaType::WEBP),
1377 _ => None,
1378 }
1379}
1380
1381fn document_text(document: &crate::message::Document) -> Option<String> {
1384 match &document.data {
1385 DocumentSourceKind::String(text) => Some(text.clone()),
1386 DocumentSourceKind::Base64(data)
1387 if document
1388 .media_type
1389 .as_ref()
1390 .is_some_and(|media_type| *media_type != DocumentMediaType::PDF) =>
1391 {
1392 let bytes = BASE64_STANDARD.decode(data.as_bytes()).ok()?;
1393 String::from_utf8(bytes).ok()
1394 }
1395 _ => None,
1396 }
1397}
1398
1399fn merge_users(history: Vec<Message>) -> Vec<Message> {
1403 let mut merged: Vec<Message> = Vec::with_capacity(history.len());
1404 for message in history {
1405 match (merged.last_mut(), message) {
1406 (Some(Message::User { content: previous }), Message::User { content })
1407 if !previous.is_empty() && !content.is_empty() =>
1408 {
1409 previous.extend(content);
1410 }
1411 (_, message) => merged.push(message),
1412 }
1413 }
1414 merged
1415}
1416
1417fn answer_calls(history: Vec<Option<Message>>, stored: bool, answers: bool) -> Vec<Message> {
1424 let mut stored = stored;
1427 let mut shaped = Vec::with_capacity(history.len());
1428 let mut waiting: Vec<ToolCall> = Vec::new();
1429 let mut held = Vec::new();
1430 let mut gap = false;
1431 let mut pending: Vec<UserContent> = Vec::new();
1434 let mut pending_gap = false;
1435 for message in adjacent_users_merged(history) {
1436 let Some(message) = message else {
1437 close(
1438 &mut shaped,
1439 &mut waiting,
1440 &mut held,
1441 answers,
1442 std::mem::take(&mut pending),
1443 std::mem::take(&mut pending_gap),
1444 );
1445 stored = false;
1446 gap = true;
1447 continue;
1448 };
1449 match message {
1450 Message::Assistant(turn) => {
1451 close(
1452 &mut shaped,
1453 &mut waiting,
1454 &mut held,
1455 answers,
1456 std::mem::take(&mut pending),
1457 std::mem::take(&mut pending_gap),
1458 );
1459 stored = false;
1460 if turn.stop.as_ref().is_some_and(|stop| stop.is_failure()) {
1461 gap = true;
1462 continue;
1463 }
1464 gap = false;
1465 waiting = distinct(turn.tool_calls());
1466 shaped.push(Message::Assistant(turn));
1467 }
1468 Message::User { mut content } => {
1469 if content.is_empty() {
1470 close(
1471 &mut shaped,
1472 &mut waiting,
1473 &mut held,
1474 answers,
1475 std::mem::take(&mut pending),
1476 std::mem::take(&mut pending_gap),
1477 );
1478 shaped.push(Message::User { content });
1479 gap = false;
1480 continue;
1481 }
1482 let mut answered: HashSet<CallId> = pending
1484 .iter()
1485 .filter_map(|part| match part {
1486 UserContent::ToolResult(result) => Some(result.call.clone()),
1487 _ => None,
1488 })
1489 .collect();
1490 content.retain(|part| match part {
1491 UserContent::ToolResult(result) => {
1492 (stored || waiting.iter().any(|call| call.id == result.call))
1493 && answered.insert(result.call.clone())
1494 }
1495 UserContent::Text(_)
1496 | UserContent::Image(_)
1497 | UserContent::Audio(_)
1498 | UserContent::Video(_)
1499 | UserContent::Document(_) => true,
1500 });
1501 if content.is_empty() && waiting.is_empty() {
1502 continue;
1504 }
1505 let only_results = !content.is_empty()
1506 && content
1507 .iter()
1508 .all(|part| matches!(part, UserContent::ToolResult(_)));
1509 if pending.is_empty() {
1510 pending_gap = gap;
1511 }
1512 pending.extend(content);
1513 gap = false;
1514 if only_results && waiting.iter().any(|call| !answered.contains(&call.id)) {
1517 continue;
1518 }
1519 close(
1520 &mut shaped,
1521 &mut waiting,
1522 &mut held,
1523 answers,
1524 std::mem::take(&mut pending),
1525 std::mem::take(&mut pending_gap),
1526 );
1527 }
1528 Message::System { .. } if !waiting.is_empty() => held.push(message),
1529 system => {
1530 gap = false;
1531 shaped.push(system);
1532 }
1533 }
1534 }
1535 close(
1536 &mut shaped,
1537 &mut waiting,
1538 &mut held,
1539 answers,
1540 pending,
1541 pending_gap,
1542 );
1543 shaped
1544}
1545
1546fn adjacent_users_merged(history: Vec<Option<Message>>) -> Vec<Option<Message>> {
1549 let mut merged: Vec<Option<Message>> = Vec::with_capacity(history.len());
1550 for message in history {
1551 match (merged.last_mut(), message) {
1552 (Some(Some(Message::User { content: previous })), Some(Message::User { content }))
1553 if !previous.is_empty() && !content.is_empty() =>
1554 {
1555 previous.extend(content)
1556 }
1557 (_, message) => merged.push(message),
1558 }
1559 }
1560 merged
1561}
1562
1563fn close(
1569 shaped: &mut Vec<Message>,
1570 waiting: &mut Vec<ToolCall>,
1571 held: &mut Vec<Message>,
1572 answers: bool,
1573 mut content: Vec<UserContent>,
1574 merge: bool,
1575) {
1576 if !answers {
1577 waiting.clear();
1578 }
1579 let missing: Vec<UserContent> = waiting
1580 .drain(..)
1581 .filter(|call| {
1582 !content.iter().any(
1583 |part| matches!(part, UserContent::ToolResult(result) if result.call == call.id),
1584 )
1585 })
1586 .map(|call| {
1587 UserContent::ToolResult(ToolResult {
1588 call: call.id,
1589 name: call.function.name,
1590 content: vec![ToolResultContent::text(NO_RESULT_PROVIDED)],
1591 is_error: true,
1592 })
1593 })
1594 .collect();
1595 let at = content
1596 .iter()
1597 .position(|part| !matches!(part, UserContent::ToolResult(_)))
1598 .unwrap_or(content.len());
1599 content.splice(at..at, missing);
1600 content.sort_by_key(|part| !matches!(part, UserContent::ToolResult(_)));
1603 let at = content
1606 .iter()
1607 .position(|part| !matches!(part, UserContent::ToolResult(_)))
1608 .unwrap_or(content.len());
1609 let text = if held.is_empty() {
1610 Vec::new()
1611 } else {
1612 content.split_off(at)
1613 };
1614 if !content.is_empty() {
1615 match shaped.last_mut() {
1616 Some(Message::User { content: previous }) if merge => previous.extend(content),
1617 _ => shaped.push(Message::User { content }),
1618 }
1619 }
1620 shaped.append(held);
1621 if !text.is_empty() {
1622 shaped.push(Message::User { content: text });
1623 }
1624}
1625
1626fn distinct<'a>(calls: impl Iterator<Item = &'a ToolCall>) -> Vec<ToolCall> {
1628 let mut seen = HashSet::new();
1629 calls
1630 .filter(|call| seen.insert(call.id.clone()))
1631 .cloned()
1632 .collect()
1633}
1634
1635#[cfg(test)]
1636mod tests;