1use std::borrow::Cow;
63use std::collections::{HashMap, HashSet};
64use std::sync::Arc;
65
66use base64::Engine as _;
67use base64::prelude::{BASE64_STANDARD, BASE64_STANDARD_NO_PAD};
68
69use crate::conversation::pairing::is_result;
70use crate::message::{
71 Api, AssistantContent, AssistantMessage, CallId, DocumentData, DocumentMediaType,
72 DocumentSourceKind, ImageMediaType, Message, Origin, StopReason, Text, ToolResult,
73 ToolResultContent, UserContent,
74};
75use crate::tool::feedback;
76use crate::wasm_compat::WasmCompatSync;
77
78pub const IMAGE_OMITTED: &str = "(image omitted: model does not support images)";
82
83pub const TOOL_IMAGE_OMITTED: &str = "(tool image omitted: model does not support images)";
85
86pub const IMAGE_UNSENDABLE: &str = "(image omitted: the provider cannot receive it in this form)";
88
89pub const AUDIO_UNSENDABLE: &str = "(audio omitted: the provider cannot receive it in this form)";
91
92pub const VIDEO_UNSENDABLE: &str = "(video omitted: the provider cannot receive it in this form)";
94
95pub const DOCUMENT_UNSENDABLE: &str =
98 "(document omitted: the provider cannot receive it in this form)";
99
100pub const TOOL_IMAGE_ATTACHED: &str = "(see attached image)";
103
104pub const TOOL_IMAGES_HEADING: &str = "Attached image(s) from tool result:";
106
107#[derive(Debug, Clone, Copy, PartialEq, Eq)]
109pub struct Accepts {
110 pub user_images: bool,
112 pub assistant_images: bool,
114 pub tool_result_images: bool,
116 pub tools: bool,
118 pub foreign_reasoning: bool,
125}
126
127impl Accepts {
128 pub const ALL: Self = Self {
131 user_images: true,
132 assistant_images: true,
133 tool_result_images: true,
134 tools: true,
135 foreign_reasoning: false,
136 };
137
138 pub const TEXT: Self = Self {
141 user_images: false,
142 assistant_images: false,
143 tool_result_images: false,
144 tools: true,
145 foreign_reasoning: false,
146 };
147}
148
149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
151pub enum Place {
152 User,
154 ToolResult,
156 Assistant,
158}
159
160#[derive(Debug, Clone, Copy)]
164pub enum Media<'a> {
165 Image(&'a crate::message::Image, Place),
167 Audio(&'a crate::message::Audio),
169 Video(&'a crate::message::Video),
171 Document(&'a crate::message::DocumentContent),
173}
174
175pub trait ReplayTarget: std::fmt::Debug + WasmCompatSync {
178 fn api(&self) -> Api;
180
181 fn provider(&self) -> &str;
183
184 fn model(&self) -> &str;
186
187 fn accepts(&self, model: &str) -> Accepts;
190
191 fn map_options(
201 &self,
202 request: &crate::completion::CompletionRequest,
203 fields: crate::completion::options::OptionFields<'_>,
204 ) -> crate::completion::options::OptionMap;
205
206 fn facts(&self) -> Option<&crate::catalog::ModelFacts> {
210 None
211 }
212
213 fn encodes(&self, model: &str, media: Media<'_>) -> bool {
220 let _ = (model, media);
221 true
222 }
223
224 fn normalize_tool_call_id(&self, id: &str, model: &str, source: Option<&Origin>) -> String {
229 let _ = (model, source);
230 id.to_owned()
231 }
232
233 fn continues_stored(&self, request: &crate::completion::CompletionRequest) -> bool {
238 let _ = request;
239 false
240 }
241
242 fn declares_tools(&self, request: &crate::completion::CompletionRequest) -> bool {
248 declares_tools(request)
249 }
250
251 fn identity(&self, item: &serde_json::Value) -> serde_json::Map<String, serde_json::Value> {
255 let _ = item;
256 serde_json::Map::new()
257 }
258
259 fn call_id_slot(&self) -> Option<&'static str> {
264 None
265 }
266
267 fn blob_slots(&self) -> &'static [&'static str] {
273 &[]
274 }
275
276 fn states_finish_reason(&self) -> bool {
280 true
281 }
282
283 fn catalog_priced(&self) -> bool {
287 true
288 }
289
290 fn binds_context(&self, model: &str) -> bool {
294 let _ = model;
295 false
296 }
297
298 fn drops_unbound_items(&self, request: &crate::completion::CompletionRequest) -> bool {
302 let _ = request;
303 false
304 }
305
306 fn takes_documents(&self) -> bool {
309 false
310 }
311
312 fn needs_next(&self, item: &serde_json::Value) -> bool {
316 let _ = item;
317 false
318 }
319
320 fn hosted_pair(&self, item: &serde_json::Value) -> Option<(Pairing, String)> {
326 let _ = item;
327 None
328 }
329
330 fn starts_with_user(&self) -> bool {
334 false
335 }
336
337 fn later_system(&self, model: &str) -> LaterSystem {
341 let _ = model;
342 LaterSystem::InPlace
343 }
344
345 fn alternates_roles(&self) -> bool {
349 false
350 }
351
352 fn hosted_needs_tools(&self) -> bool {
355 false
356 }
357
358 fn result_parts(&self, model: &str) -> bool {
362 let _ = model;
363 false
364 }
365
366 fn sends_alone(&self, block: &AssistantContent) -> bool {
371 let _ = block;
372 true
373 }
374}
375
376#[derive(Debug, Clone, Copy, PartialEq, Eq)]
379pub enum LaterSystem {
380 InPlace,
382 Leading,
384 UserText,
387}
388
389#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
391pub enum Pairing {
392 Use,
394 Result,
396}
397
398#[derive(Debug, Clone, PartialEq)]
400pub enum Replay<'a> {
401 Item(std::borrow::Cow<'a, serde_json::Value>),
403 Identity(serde_json::Map<String, serde_json::Value>),
406 Rebuild,
408}
409
410impl AssistantContent {
411 pub fn replay(
414 &self,
415 target: &dyn ReplayTarget,
416 ids: &crate::providers::internal::wire_ids::WireIds,
417 ) -> Replay<'_> {
418 if let Some(item) = self.native_item() {
419 return match (self, target.call_id_slot()) {
420 (AssistantContent::ToolCall(call), Some(slot)) => {
421 let mut item = item.clone();
422 if let Some(id) = ids.of(&call.id) {
423 set_pointer(&mut item, slot, serde_json::Value::String(id.to_owned()));
424 }
425 Replay::Item(std::borrow::Cow::Owned(item))
426 }
427 _ => Replay::Item(std::borrow::Cow::Borrowed(item)),
428 };
429 }
430 let identity = self
431 .stale_item()
432 .map(|item| target.identity(item))
433 .unwrap_or_default();
434 if identity.is_empty() {
435 Replay::Rebuild
436 } else {
437 Replay::Identity(identity)
438 }
439 }
440}
441
442fn set_pointer(item: &mut serde_json::Value, pointer: &str, value: serde_json::Value) {
444 let mut at = item;
445 let mut keys = pointer.split('/').skip(1).peekable();
446 while let Some(key) = keys.next() {
447 if !at.is_object() {
448 *at = serde_json::Value::Object(serde_json::Map::new());
449 }
450 let serde_json::Value::Object(fields) = at else {
451 return;
452 };
453 if keys.peek().is_none() {
454 fields.insert(key.to_owned(), value);
455 return;
456 }
457 at = fields
458 .entry(key.to_owned())
459 .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()));
460 }
461}
462
463pub fn adapt(history: &[Message], target: &dyn ReplayTarget) -> Vec<Message> {
492 let history = history.iter().cloned().map(Arc::new).collect();
493 let adapted = adapt_for(history, target, &Request::default(), &mut Memo::default());
494 adapted.into_iter().map(Arc::unwrap_or_clone).collect()
495}
496
497#[derive(Debug, Default)]
504pub(crate) struct Memo {
505 shape: Option<Shape>,
506 given: Vec<Arc<Message>>,
507 shaped: Vec<Option<Arc<Message>>>,
508 taken: HashSet<String>,
509}
510
511#[derive(Debug, PartialEq)]
513struct Shape {
514 api: Api,
515 provider: String,
516 model: String,
517 context: Option<crate::message::Fingerprint>,
518 accepts: Accepts,
519 from_user: bool,
520}
521
522#[derive(Debug, Clone, Copy)]
524pub(crate) struct Request<'a> {
525 pub(crate) model: Option<&'a str>,
527 pub(crate) stored: bool,
529 pub(crate) tools: bool,
532 pub(crate) context: Option<crate::message::Fingerprint>,
534 pub(crate) foreign_reasoning: bool,
539}
540
541impl Default for Request<'_> {
542 fn default() -> Self {
543 Self {
544 model: None,
545 stored: false,
546 tools: true,
547 context: None,
548 foreign_reasoning: false,
549 }
550 }
551}
552
553pub(crate) fn context_of(
559 request: &crate::completion::CompletionRequest,
560 target: &dyn ReplayTarget,
561 model: &str,
562) -> crate::message::Fingerprint {
563 let mut tools: Vec<_> = request.tools.iter().collect();
564 tools.sort_by(|left, right| left.name.cmp(&right.name));
565 let raw = raw_tools(request);
566 let folds = target.later_system(model) == LaterSystem::Leading;
567 let mut system: Vec<&str> = request
568 .chat_history
569 .iter()
570 .map_while(|message| match &**message {
571 Message::System { content } => Some(Some(content.as_str())),
572 Message::User { .. } | Message::Assistant(_) => folds.then_some(None),
573 })
574 .flatten()
575 .filter(|content| !content.trim().is_empty())
576 .collect();
577 let joined;
578 if folds && system.len() > 1 {
579 joined = system.join("\n\n");
580 system = vec![joined.as_str()];
581 }
582 let mut fields = vec![serde_json::json!("context"), serde_json::json!(tools)];
583 fields.push(serde_json::json!(system));
584 if !raw.is_empty() {
585 fields.push(serde_json::json!(raw));
586 }
587 let fields = crate::message::Canonical(&serde_json::Value::Array(fields));
589 crate::message::Fingerprint::of_canonical(&fields)
590}
591
592pub(crate) fn declares_tools(request: &crate::completion::CompletionRequest) -> bool {
595 !request.tools.is_empty() || !raw_tools(request).is_empty()
596}
597
598pub(crate) fn raw_tools(request: &crate::completion::CompletionRequest) -> &[serde_json::Value] {
601 use crate::json_utils::Lenient;
602 request
603 .additional_params
604 .as_ref()
605 .map_or(&[][..], |params| params.arr("tools"))
606}
607
608pub(crate) fn adapt_for(
610 history: Vec<Arc<Message>>,
611 target: &dyn ReplayTarget,
612 request: &Request<'_>,
613 memo: &mut Memo,
614) -> Vec<Arc<Message>> {
615 let model = request.model.unwrap_or(target.model());
616 let stored = request.stored;
617 let same = Same {
618 api: target.api(),
619 provider: target.provider(),
620 model,
621 context: request.context.filter(|_| target.binds_context(model)),
622 };
623 let mut accepts = target.accepts(model);
624 accepts.tools &= request.tools;
625 accepts.foreign_reasoning &= request.foreign_reasoning;
626 let starts_with_user = target.starts_with_user() && !stored;
627 let shape = Shape {
628 api: same.api.clone(),
629 provider: same.provider.to_owned(),
630 model: model.to_owned(),
631 context: same.context,
632 accepts,
633 from_user: starts_with_user,
634 };
635 let resumed = memo.shape.as_ref() == Some(&shape)
636 && history.get(..memo.given.len()).is_some_and(|given| {
637 given
638 .iter()
639 .zip(&memo.given)
640 .all(|(message, was)| Arc::ptr_eq(message, was) || message == was)
641 });
642 if !resumed {
643 *memo = Memo::default();
644 }
645 let from = memo.given.len().saturating_sub(1);
647 let last_turn = history
648 .iter()
649 .rposition(|message| matches!(&**message, Message::Assistant(_)));
650 let (hosted, open) = hosted_pairs(&history, from, last_turn, target, &same);
651 let mut ids = Renamed {
652 to: Vec::new(),
653 before: &memo.taken,
654 taken: HashSet::new(),
655 };
656 let mut claimed = Vec::new();
657 let mut kept = 0;
658 let form = Form {
659 target,
660 model,
661 accepts,
662 };
663 let mut shaped: Vec<Option<Arc<Message>>> = Vec::with_capacity(history.len());
665 shaped.extend(memo.shaped.iter().cloned());
666 for (at, message) in history.iter().enumerate().skip(from) {
667 if Some(at) == last_turn {
668 claimed = ids.taken.iter().map(|id| id.to_string()).collect();
669 kept = shaped.len();
670 }
671 match &**message {
672 Message::System { content } => {
673 if !content.trim().is_empty() {
674 shaped.push(Some(message.clone()));
675 }
676 }
677 Message::User { content } => {
678 shaped.extend(
679 user(message, content, &mut ids, &form)
680 .into_iter()
681 .map(Some),
682 );
683 }
684 Message::Assistant(turn) if matches!(turn.stop, Some(StopReason::Aborted(_))) => {
685 shaped.push(stopped(turn, &results_after(&history, at)));
686 }
687 Message::Assistant(turn)
689 if turn.stop.as_ref().is_some_and(|stop| stop.is_failure()) =>
690 {
691 shaped.push(None);
692 }
693 Message::Assistant(turn) => {
694 let last = Some(at) == last_turn
697 && (turn
698 .content
699 .iter()
700 .any(|block| matches!(block, AssistantContent::ToolCall(_)))
701 || history
702 .get(at + 1..)
703 .into_iter()
704 .flatten()
705 .all(|message| matches!(&**message, Message::System { .. })));
706 let here: HashSet<usize> = hosted
707 .iter()
708 .filter(|(message, _)| *message == at)
709 .map(|(_, block)| *block)
710 .collect();
711 shaped.push(assistant(
712 message, turn, &form, &same, &mut ids, &here, last,
713 ));
714 }
715 }
716 }
717 let (before, tail) = shaped.split_at(memo.shaped.len().min(shaped.len()));
718 let last = before.last().and_then(Option::as_deref);
719 if settled(tail, last, starts_with_user, accepts.tools) {
720 drop(ids);
721 match last_turn.filter(|_| !open) {
724 Some(upto) => {
725 let given = history.get(memo.given.len()..=upto).unwrap_or_default();
726 memo.given.extend(given.iter().cloned());
727 let shaped = shaped.get(memo.shaped.len()..kept).unwrap_or_default();
728 memo.shaped.extend(shaped.iter().cloned());
729 memo.taken.extend(claimed);
730 memo.shape = Some(shape);
731 }
732 None => *memo = Memo::default(),
733 }
734 let settled: Vec<Arc<Message>> = shaped.into_iter().flatten().collect();
735 #[cfg(test)]
736 {
737 let shaped = settled.iter().map(|message| Some(Message::clone(message)));
738 let rebuilt = reshaped(shaped.collect(), target, model, stored, accepts.tools);
739 let settled: Vec<&Message> = settled.iter().map(Arc::as_ref).collect();
740 assert_eq!(
741 settled,
742 rebuilt.iter().collect::<Vec<_>>(),
743 "a settled history changed"
744 );
745 }
746 return settled;
747 }
748 drop(ids);
749 *memo = Memo::default();
750 let shaped = shaped
751 .into_iter()
752 .map(|message| message.map(Arc::unwrap_or_clone));
753 let shaped = reshaped(shaped.collect(), target, model, stored, accepts.tools);
754 shaped.into_iter().map(Arc::new).collect()
755}
756
757fn settled<'a>(
764 history: &'a [Option<Arc<Message>>],
765 mut last: Option<&'a Message>,
766 starts_with_user: bool,
767 tools: bool,
768) -> bool {
769 let mut waiting: Vec<&CallId> = Vec::new();
770 for message in history {
771 let Some(message) = message.as_deref() else {
772 return false;
773 };
774 match (message, last) {
775 (Message::System { .. }, None) => continue,
776 (Message::System { .. }, Some(_))
777 | (Message::User { .. }, Some(Message::User { .. }))
778 | (Message::Assistant(_), Some(Message::Assistant(_))) => return false,
779 (Message::Assistant(_), None) if starts_with_user => return false,
780 (Message::Assistant(turn), _) => {
783 waiting.extend(turn.tool_calls().map(|call| &call.id));
784 if !tools && !waiting.is_empty() {
785 return false;
786 }
787 }
788 (Message::User { content }, _) => {
789 let results = content.iter().take_while(|part| is_result(part)).count();
790 let answered = content.iter().take(results).all(|part| match part {
791 UserContent::ToolResult(result) => {
792 let at = waiting.iter().position(|id| **id == result.call);
793 at.map(|at| waiting.remove(at)).is_some()
794 }
795 _ => false,
796 });
797 if content.is_empty()
798 || (!tools && results > 0)
799 || !answered
800 || !waiting.is_empty()
801 || content.iter().skip(results).any(is_result)
802 {
803 return false;
804 }
805 }
806 }
807 last = Some(message);
808 }
809 waiting.is_empty()
810}
811
812fn reshaped(
815 shaped: Vec<Option<Message>>,
816 target: &dyn ReplayTarget,
817 model: &str,
818 stored: bool,
819 tools: bool,
820) -> Vec<Message> {
821 let pairing = crate::conversation::pairing::Pairing {
825 stored,
826 answers: tools,
827 };
828 let shaped = merge_users(crate::conversation::pairing::pair(shaped, pairing).messages);
829 let shaped = if tools { shaped } else { tools_as_text(shaped) };
830 let shaped = match target.later_system(model) {
831 LaterSystem::InPlace => shaped,
832 LaterSystem::Leading => leading_system(shaped),
833 LaterSystem::UserText => system_as_user_text(shaped),
834 };
835 let shaped = if target.starts_with_user() && !stored {
836 from_first_user(shaped)
837 } else {
838 shaped
839 };
840 if target.alternates_roles() {
841 alternated(shaped)
842 } else {
843 shaped
844 }
845}
846
847pub(crate) fn cache_points(
855 points: &[crate::completion::CachePoint],
856 written: &[Arc<Message>],
857 documents: Option<&Message>,
858 adapted: &[Arc<Message>],
859 target: &dyn ReplayTarget,
860 request: &Request<'_>,
861) -> Vec<crate::completion::CachePoint> {
862 use crate::completion::CachePoint;
863 let prefix_of = |turns: usize, items: Option<usize>| -> Option<Vec<Arc<Message>>> {
864 let mut prefix = Vec::new();
865 let mut counted = 0;
866 for message in written {
867 if counted == turns {
868 break;
869 }
870 let mut message = Arc::clone(message);
871 if !matches!(*message, Message::System { .. }) {
872 counted += 1;
873 }
874 if let Some(items) = items.filter(|_| counted == turns) {
876 match Arc::make_mut(&mut message) {
877 Message::User { content } => content.truncate(items),
878 Message::Assistant(turn) => turn.content.truncate(items),
879 Message::System { .. } => {}
880 }
881 }
882 prefix.push(message);
883 }
884 (counted == turns).then_some(prefix)
885 };
886 points
887 .iter()
888 .filter_map(|point| {
889 let prefix = match *point {
890 CachePoint::Tools | CachePoint::System | CachePoint::Messages(0) => {
891 return Some(*point);
892 }
893 CachePoint::Messages(turns) => prefix_of(turns, None),
894 CachePoint::Within { message, items } => message
895 .checked_add(1)
896 .and_then(|turns| prefix_of(turns, Some(items))),
897 };
898 let mut prefix = prefix?;
899 if let Some(documents) = documents {
900 crate::completion::request::insert_after_leading_system(
901 &mut prefix,
902 documents.clone(),
903 );
904 }
905 let prefix = adapt_for(prefix, target, request, &mut Memo::default());
906 Some(shared_prefix(adapted, &prefix))
907 })
908 .collect()
909}
910
911fn shared_prefix(
914 history: &[Arc<Message>],
915 prefix: &[Arc<Message>],
916) -> crate::completion::CachePoint {
917 use crate::completion::CachePoint;
918 fn turns(history: &[Arc<Message>]) -> impl Iterator<Item = &Message> {
919 history
920 .iter()
921 .map(Arc::as_ref)
922 .filter(|message| !matches!(message, Message::System { .. }))
923 }
924 fn lead<T: PartialEq>(whole: &[T], part: &[T]) -> (usize, usize, usize) {
925 let shared = whole
926 .iter()
927 .zip(part)
928 .take_while(|(whole, part)| whole == part)
929 .count();
930 (shared, whole.len(), part.len())
931 }
932 let mut message = 0;
933 for (whole, part) in turns(history).zip(turns(prefix)) {
934 let (shared, whole, part) = match (whole, part) {
935 (Message::User { content: whole }, Message::User { content: part }) => {
936 lead(whole, part)
937 }
938 (Message::Assistant(whole), Message::Assistant(part)) => {
939 lead(&whole.content, &part.content)
940 }
941 _ => (0, 1, 1),
942 };
943 if shared == whole && shared == part {
944 message += 1;
945 continue;
946 }
947 return if shared == 0 {
948 CachePoint::Messages(message)
949 } else {
950 CachePoint::Within {
951 message,
952 items: shared,
953 }
954 };
955 }
956 CachePoint::Messages(message)
957}
958
959fn tools_as_text(history: Vec<Message>) -> Vec<Message> {
963 let history = history
964 .into_iter()
965 .map(|message| match message {
966 Message::Assistant(mut turn) => {
967 for block in &mut turn.content {
968 if let AssistantContent::ToolCall(call) = block {
969 *block = AssistantContent::Text(Text::new(format!(
970 "[called tool {} with {}]",
971 call.function.name,
972 crate::json_utils::to_canonical_string(
973 &call.function.arguments_value()
974 )
975 )));
976 }
977 }
978 Message::Assistant(turn)
979 }
980 Message::User { content } => Message::User {
981 content: content
982 .into_iter()
983 .map(|part| match part {
984 UserContent::ToolResult(result) => UserContent::text(result_text(&result)),
985 part => part,
986 })
987 .collect(),
988 },
989 message => message,
990 })
991 .collect();
992 merge_users(history)
993}
994
995fn from_first_user(mut history: Vec<Message>) -> Vec<Message> {
999 let lead = history
1000 .iter()
1001 .take_while(|message| matches!(message, Message::System { .. }))
1002 .count();
1003 while let Some(Message::Assistant(turn)) = history.get(lead) {
1004 let calls: HashSet<CallId> = turn.tool_calls().map(|call| call.id.clone()).collect();
1005 history.remove(lead);
1006 if let Some(Message::User { content }) = history.get_mut(lead) {
1007 content.retain(|part| {
1008 !matches!(part, UserContent::ToolResult(result) if calls.contains(&result.call))
1009 });
1010 if content.is_empty() {
1011 history.remove(lead);
1012 }
1013 }
1014 }
1015 history
1016}
1017
1018fn has_later_system(history: &[Message]) -> bool {
1021 history
1022 .iter()
1023 .skip_while(|message| matches!(message, Message::System { .. }))
1024 .any(|message| matches!(message, Message::System { .. }))
1025}
1026
1027fn system_as_user_text(history: Vec<Message>) -> Vec<Message> {
1029 let mut leading = true;
1030 let history = history
1031 .into_iter()
1032 .map(|message| match message {
1033 Message::System { content } if !leading => Message::User {
1034 content: vec![UserContent::text(content)],
1035 },
1036 message => {
1037 leading &= matches!(message, Message::System { .. });
1038 message
1039 }
1040 })
1041 .collect();
1042 merge_users(history)
1043}
1044
1045fn alternated(history: Vec<Message>) -> Vec<Message> {
1049 let mut alternated: Vec<Message> = Vec::with_capacity(history.len());
1050 let mut held: Vec<Message> = Vec::new();
1051 for message in history {
1052 let started = alternated
1053 .iter()
1054 .any(|message| !matches!(message, Message::System { .. }));
1055 match (message, alternated.last_mut()) {
1056 (message @ Message::System { .. }, _) if started => held.push(message),
1057 (Message::User { content }, Some(Message::User { content: previous })) => {
1058 previous.extend(content);
1059 }
1060 (Message::Assistant(turn), Some(Message::Assistant(previous))) => {
1061 if previous.origin != turn.origin {
1062 previous.origin = None;
1063 }
1064 previous.stop = turn.stop;
1065 previous.content.extend(turn.content);
1066 }
1067 (message, _) => {
1068 alternated.append(&mut held);
1069 alternated.push(message);
1070 }
1071 }
1072 }
1073 alternated.append(&mut held);
1074 alternated
1075}
1076
1077fn leading_system(history: Vec<Message>) -> Vec<Message> {
1080 if !has_later_system(&history) {
1081 return history;
1082 }
1083 let (system, rest): (Vec<Message>, Vec<Message>) = history
1084 .into_iter()
1085 .partition(|message| matches!(message, Message::System { .. }));
1086 let prompt: Vec<String> = system
1087 .into_iter()
1088 .filter_map(|message| match message {
1089 Message::System { content } => Some(content),
1090 Message::User { .. } | Message::Assistant(_) => None,
1091 })
1092 .collect();
1093 let mut history = Vec::with_capacity(rest.len() + 1);
1094 if !prompt.is_empty() {
1095 history.push(Message::system(prompt.join("\n\n")));
1096 }
1097 history.extend(merge_users(rest));
1099 history
1100}
1101
1102struct Renamed<'a> {
1108 to: Vec<(Cow<'a, CallId>, Cow<'a, CallId>)>,
1111 before: &'a HashSet<String>,
1113 taken: HashSet<Cow<'a, str>>,
1114}
1115
1116impl<'a> Renamed<'a> {
1117 fn turn(&mut self) {
1119 self.to.clear();
1120 }
1121
1122 fn claim(&mut self, source: Cow<'a, CallId>, wanted: Cow<'a, str>) -> Cow<'a, CallId> {
1130 let id = match &*source {
1131 CallId::Local(_) => source.clone(),
1132 CallId::Provider(_) => {
1133 let mut id = wanted.clone();
1134 let mut attempt: u64 = 1;
1135 while self.before.contains(id.as_ref()) || self.taken.contains(&id) {
1136 id = Cow::Owned(with_counter(&wanted, attempt));
1137 attempt += 1;
1138 }
1139 self.taken.insert(id.clone());
1140 if id == source.wire() {
1141 source.clone()
1142 } else {
1143 Cow::Owned(CallId::from_wire(id))
1144 }
1145 }
1146 };
1147 self.to.push((source, id.clone()));
1148 id
1149 }
1150
1151 fn answer(&mut self, source: &CallId) -> Option<Cow<'a, CallId>> {
1154 let mut calls = (self.to.iter().enumerate())
1155 .filter(|(_, (from, _))| from.as_ref() == source)
1156 .map(|(at, _)| at);
1157 let first = calls.next()?;
1158 if calls.next().is_some() {
1159 Some(self.to.remove(first).1)
1160 } else {
1161 self.to.get(first).map(|(_, id)| id.clone())
1162 }
1163 }
1164}
1165
1166fn with_counter(id: &str, attempt: u64) -> String {
1168 let mut digits = Vec::new();
1169 let mut value = attempt;
1170 while value > 0 {
1171 digits.push(char::from_digit((value % 36) as u32, 36).unwrap_or('0'));
1172 value /= 36;
1173 }
1174 digits.reverse();
1175 let keep = id.chars().count().saturating_sub(digits.len());
1176 id.chars().take(keep).chain(digits).collect()
1177}
1178
1179struct Same<'a> {
1181 api: Api,
1182 provider: &'a str,
1183 model: &'a str,
1184 context: Option<crate::message::Fingerprint>,
1186}
1187
1188impl Same<'_> {
1189 fn is(&self, origin: &Origin) -> bool {
1192 origin.same_model(&self.api, self.provider, self.model)
1193 && self
1194 .context
1195 .is_none_or(|context| origin.context == Some(context))
1196 }
1197}
1198
1199fn assistant<'a>(
1203 message: &Arc<Message>,
1204 turn: &'a AssistantMessage,
1205 form: &Form<'_>,
1206 same_model: &Same<'_>,
1207 ids: &mut Renamed<'a>,
1208 hosted: &HashSet<usize>,
1209 last: bool,
1210) -> Option<Arc<Message>> {
1211 let Form {
1212 target,
1213 model,
1214 accepts,
1215 } = *form;
1216 let same = turn
1217 .origin
1218 .as_ref()
1219 .is_some_and(|origin| same_model.is(origin));
1220 ids.turn();
1221 let content: Vec<Option<Cow<'a, AssistantContent>>> = turn
1222 .content
1223 .iter()
1224 .map(|block| {
1225 let block = if same {
1226 match block {
1227 AssistantContent::ToolCall(call) => {
1228 let id = ids.claim(Cow::Borrowed(&call.id), call.id.wire());
1229 if *id == call.id {
1230 Cow::Borrowed(block)
1231 } else {
1232 let keep =
1236 block.native_item().is_some() && target.call_id_slot().is_some();
1237 let id = id.into_owned();
1238 let renamed = crate::message::ToolCall { id, ..call.clone() };
1239 let mut renamed = AssistantContent::ToolCall(renamed);
1240 if keep
1241 && let Some(native) = renamed.native_slot().and_then(Option::take)
1242 {
1243 let blob_slot = native.blob_slot;
1244 renamed = renamed.with_native(native.item);
1245 if let Some(Some(native)) = renamed.native_slot() {
1246 native.blob_slot = blob_slot;
1247 }
1248 } else if let Some(Some(native)) = renamed.native_slot() {
1249 native.edited = true;
1250 }
1251 Cow::Owned(renamed)
1252 }
1253 }
1254 AssistantContent::Image(_) if !accepts.assistant_images => return None,
1257 block => Cow::Borrowed(block),
1258 }
1259 } else {
1260 Cow::Owned(match block.canonical() {
1261 AssistantContent::Reasoning(reasoning) => {
1262 if reasoning.redacted || reasoning.text.trim().is_empty() {
1263 return None;
1264 }
1265 if accepts.foreign_reasoning {
1266 AssistantContent::Reasoning(reasoning)
1267 } else {
1268 AssistantContent::Text(Text::new(reasoning.text))
1269 }
1270 }
1271 AssistantContent::Opaque(_) => return None,
1272 AssistantContent::Image(_) if !accepts.assistant_images => {
1273 AssistantContent::Text(Text::new(IMAGE_OMITTED))
1274 }
1275 AssistantContent::ToolCall(mut call) => {
1276 let normalized = target.normalize_tool_call_id(
1277 &call.id.wire(),
1278 model,
1279 turn.origin.as_ref(),
1280 );
1281 let source = Cow::Owned(call.id);
1282 call.id = ids.claim(source, Cow::Owned(normalized)).into_owned();
1283 AssistantContent::ToolCall(call)
1284 }
1285 block => block,
1286 })
1287 };
1288 let block = match &*block {
1291 AssistantContent::Opaque(opaque)
1292 if !accepts.tools
1293 && target.hosted_needs_tools()
1294 && target.hosted_pair(&opaque.item).is_some() =>
1295 {
1296 return None;
1297 }
1298 _ => block,
1299 };
1300 (!block.is_blank()).then_some(block)
1301 })
1302 .collect();
1303 let content: Vec<Cow<'a, AssistantContent>> = if same {
1304 paired(content, target, accepts.tools, hosted, last)
1305 } else {
1306 content.into_iter().flatten().collect()
1307 };
1308 let content: Vec<Cow<'a, AssistantContent>> = content
1309 .into_iter()
1310 .map(|block| match &*block {
1311 AssistantContent::Image(image) if image.native.is_none() => {
1312 if sendable_as_is(image) && form.sends(image, Place::Assistant) {
1313 return block;
1314 }
1315 let image = sendable_image(image.clone());
1316 Cow::Owned(
1317 if matches!(image.data, DocumentSourceKind::Unknown)
1318 || !target.encodes(model, Media::Image(&image, Place::Assistant))
1319 {
1320 AssistantContent::Text(Text::new(IMAGE_UNSENDABLE))
1321 } else {
1322 AssistantContent::Image(image)
1323 },
1324 )
1325 }
1326 _ => block,
1327 })
1328 .collect();
1329 if !turn.content.is_empty() && !content.iter().any(|block| target.sends_alone(block)) {
1330 return None;
1331 }
1332 if content.len() == turn.content.len()
1333 && content
1334 .iter()
1335 .all(|block| matches!(block, Cow::Borrowed(_)))
1336 {
1337 return Some(message.clone());
1338 }
1339 Some(Arc::new(Message::Assistant(AssistantMessage {
1340 content: content.into_iter().map(Cow::into_owned).collect(),
1341 origin: turn.origin.clone(),
1342 stop: turn.stop.clone(),
1343 })))
1344}
1345
1346fn stopped(turn: &AssistantMessage, answered: &HashSet<&CallId>) -> Option<Arc<Message>> {
1352 let text = turn
1353 .content
1354 .iter()
1355 .filter(|block| matches!(block, AssistantContent::Text(_)))
1356 .map(AssistantContent::canonical)
1357 .filter(|block| !block.is_blank());
1358 let notes = turn
1359 .tool_calls()
1360 .filter(|call| !answered.contains(&call.id))
1361 .map(|call| AssistantContent::text(feedback::stopped_call_note(call)));
1362 let content: Vec<AssistantContent> = text.chain(notes).collect();
1363 (!content.is_empty()).then(|| Arc::new(Message::Assistant(AssistantMessage::new(content))))
1364}
1365
1366fn results_after(history: &[Arc<Message>], at: usize) -> HashSet<&CallId> {
1370 let next = history.get(at.saturating_add(1)).map(|message| &**message);
1371 next.into_iter()
1372 .flat_map(Message::results)
1373 .map(|result| &result.call)
1374 .collect()
1375}
1376
1377fn hosted_pairs(
1385 history: &[Arc<Message>],
1386 from: usize,
1387 boundary: Option<usize>,
1388 target: &dyn ReplayTarget,
1389 same: &Same<'_>,
1390) -> (HashSet<(usize, usize)>, bool) {
1391 let mut open: HashMap<String, (usize, usize)> = HashMap::new();
1392 let mut paired = HashSet::new();
1393 let mut open_at_boundary = false;
1394 for (at, message) in history.iter().enumerate().skip(from) {
1395 if Some(at) == boundary {
1396 open_at_boundary = !open.is_empty();
1397 }
1398 let Message::Assistant(turn) = &**message else {
1399 continue;
1400 };
1401 if turn.stop.as_ref().is_some_and(|stop| stop.is_failure())
1402 || !turn.origin.as_ref().is_some_and(|origin| same.is(origin))
1403 {
1404 continue;
1405 }
1406 for (index, block) in turn.content.iter().enumerate() {
1407 let AssistantContent::Opaque(opaque) = block else {
1408 continue;
1409 };
1410 if !opaque.replay {
1411 continue;
1412 }
1413 match target.hosted_pair(&opaque.item) {
1414 Some((Pairing::Use, id)) => {
1415 open.insert(id, (at, index));
1416 }
1417 Some((Pairing::Result, id)) => {
1418 if let Some(used) = open.remove(&id) {
1419 paired.insert(used);
1420 paired.insert((at, index));
1421 }
1422 }
1423 None => {}
1424 }
1425 }
1426 }
1427 (paired, open_at_boundary)
1428}
1429
1430fn paired<'a>(
1438 mut content: Vec<Option<Cow<'a, AssistantContent>>>,
1439 target: &dyn ReplayTarget,
1440 tools: bool,
1441 hosted: &HashSet<usize>,
1442 last: bool,
1443) -> Vec<Cow<'a, AssistantContent>> {
1444 let pair = |block: &AssistantContent| match block {
1445 AssistantContent::Opaque(opaque) if opaque.replay => target.hosted_pair(&opaque.item),
1446 _ => None,
1447 };
1448 for at in 0..content.len() {
1449 let Some((side, _)) = content.get(at).and_then(Option::as_deref).and_then(pair) else {
1450 continue;
1451 };
1452 let running = last
1453 && side == Pairing::Use
1454 && content
1455 .get(at + 1..)
1456 .into_iter()
1457 .flatten()
1458 .flatten()
1459 .all(|block| {
1460 matches!(
1461 **block,
1462 AssistantContent::ToolCall(_) | AssistantContent::Opaque(_)
1463 )
1464 });
1465 if !hosted.contains(&at)
1466 && !running
1467 && let Some(slot) = content.get_mut(at)
1468 {
1469 *slot = None;
1470 }
1471 }
1472 let (kept, needs): (Vec<bool>, Vec<bool>) = content
1477 .iter()
1478 .map(|block| {
1479 let item = block.as_deref().and_then(|block| match block {
1480 AssistantContent::Opaque(opaque) => opaque.replay.then_some(&opaque.item),
1481 block => block
1482 .native()
1483 .filter(|(item, current)| *current || !target.identity(item).is_empty())
1484 .map(|(item, _)| item),
1485 });
1486 (
1487 item.is_some(),
1488 item.is_some_and(|item| target.needs_next(item)),
1489 )
1490 })
1491 .unzip();
1492 for at in (0..content.len()).rev() {
1493 let next_gone = content
1497 .get(at + 1)
1498 .and_then(Option::as_deref)
1499 .is_none_or(|next| {
1500 !kept.get(at + 1).copied().unwrap_or_default()
1501 || (!tools && matches!(next, AssistantContent::ToolCall(_)))
1502 });
1503 if needs.get(at).copied().unwrap_or_default()
1504 && next_gone
1505 && let Some(slot) = content.get_mut(at)
1506 {
1507 *slot = None;
1508 }
1509 }
1510 content.into_iter().flatten().collect()
1511}
1512
1513struct Form<'a> {
1515 target: &'a dyn ReplayTarget,
1516 model: &'a str,
1517 accepts: Accepts,
1518}
1519
1520impl Form<'_> {
1521 fn sends(&self, image: &crate::message::Image, place: Place) -> bool {
1523 let reads = match place {
1524 Place::User => self.accepts.user_images,
1525 Place::ToolResult => self.accepts.tool_result_images,
1526 Place::Assistant => self.accepts.assistant_images,
1527 };
1528 reads
1529 && !matches!(image.data, DocumentSourceKind::Unknown)
1530 && self.target.encodes(self.model, Media::Image(image, place))
1531 }
1532}
1533
1534fn user<'a>(
1538 message: &Arc<Message>,
1539 content: &'a [UserContent],
1540 ids: &mut Renamed<'a>,
1541 form: &Form<'_>,
1542) -> Vec<Arc<Message>> {
1543 let mut shaped: Vec<Cow<'_, UserContent>> = Vec::with_capacity(content.len());
1544 let mut attached = Vec::new();
1545 for part in content {
1546 let placeholder = match part {
1547 UserContent::Image(image)
1548 if sendable_as_is(image) && form.sends(image, Place::User) =>
1549 {
1550 shaped.push(Cow::Borrowed(part));
1551 continue;
1552 }
1553 UserContent::Image(image) => {
1554 let image = sendable_image(image.clone());
1555 if form.sends(&image, Place::User) {
1556 shaped.push(Cow::Owned(UserContent::Image(image)));
1557 continue;
1558 }
1559 if form.accepts.user_images {
1560 IMAGE_UNSENDABLE
1561 } else {
1562 IMAGE_OMITTED
1563 }
1564 }
1565 UserContent::Audio(audio) => {
1566 let mut audio = audio.clone();
1567 audio.data = sendable(audio.data);
1568 if !matches!(audio.data, DocumentSourceKind::Unknown)
1569 && form.target.encodes(form.model, Media::Audio(&audio))
1570 {
1571 shaped.push(Cow::Owned(UserContent::Audio(audio)));
1572 continue;
1573 }
1574 AUDIO_UNSENDABLE
1575 }
1576 UserContent::Video(video) => {
1577 let mut video = video.clone();
1578 video.data = sendable(video.data);
1579 if !matches!(video.data, DocumentSourceKind::Unknown)
1580 && form.target.encodes(form.model, Media::Video(&video))
1581 {
1582 shaped.push(Cow::Owned(UserContent::Video(video)));
1583 continue;
1584 }
1585 VIDEO_UNSENDABLE
1586 }
1587 UserContent::Document(document) => {
1588 let mut document = document.clone();
1589 if let DocumentData::File(source) = document.data {
1590 document.data = DocumentData::File(sendable(source));
1591 }
1592 if document.data != DocumentData::File(DocumentSourceKind::Unknown)
1593 && form.target.encodes(form.model, Media::Document(&document))
1594 {
1595 shaped.push(Cow::Owned(UserContent::Document(document)));
1596 continue;
1597 }
1598 let text = match document.data {
1601 DocumentData::Text(text) => Some(text),
1602 DocumentData::File(DocumentSourceKind::Base64(data))
1603 if !matches!(document.media_type, None | Some(DocumentMediaType::PDF)) =>
1604 {
1605 BASE64_STANDARD
1606 .decode(data.as_bytes())
1607 .ok()
1608 .and_then(|bytes| String::from_utf8(bytes).ok())
1609 }
1610 DocumentData::File(_) => None,
1611 };
1612 if let Some(text) = text {
1613 shaped.push(Cow::Owned(UserContent::text(text)));
1614 continue;
1615 }
1616 DOCUMENT_UNSENDABLE
1617 }
1618 UserContent::ToolResult(result) => {
1619 let answer = ids.answer(&result.call);
1620 let parts = form.accepts.tool_result_images || form.target.result_parts(form.model);
1621 if answer.as_ref().is_none_or(|id| **id == result.call)
1622 && result_as_is(result, form, parts)
1623 {
1624 shaped.push(Cow::Borrowed(part));
1625 continue;
1626 }
1627 let mut result = result.clone();
1628 if let Some(id) = answer {
1629 result.call = id.into_owned();
1630 }
1631 result.content = result_images(result.content, form, &mut attached);
1632 result.content = result_text_parts(result.content, parts, result.is_error);
1633 shaped.push(Cow::Owned(UserContent::ToolResult(result)));
1634 continue;
1635 }
1636 UserContent::Text(text) if text.text.trim().is_empty() => continue,
1638 UserContent::Text(_) => {
1639 shaped.push(Cow::Borrowed(part));
1640 continue;
1641 }
1642 };
1643 let repeated = matches!(
1644 shaped.last().map(|part| &**part),
1645 Some(UserContent::Text(text)) if text.text == placeholder
1646 );
1647 if !repeated {
1648 shaped.push(Cow::Owned(UserContent::text(placeholder)));
1649 }
1650 }
1651 let unchanged =
1652 shaped.len() == content.len() && shaped.iter().all(|part| matches!(part, Cow::Borrowed(_)));
1653 if !shaped.is_empty() && unchanged && attached.is_empty() {
1654 return vec![message.clone()];
1655 }
1656 let mut messages = Vec::new();
1657 if !shaped.is_empty() {
1658 let content = shaped.into_iter().map(Cow::into_owned).collect();
1659 messages.push(Arc::new(Message::User { content }));
1660 }
1661 if !attached.is_empty() {
1662 let mut content = vec![UserContent::text(TOOL_IMAGES_HEADING)];
1663 content.extend(attached.into_iter().map(UserContent::Image));
1664 messages.push(Arc::new(Message::User { content }));
1665 }
1666 messages
1667}
1668
1669fn result_as_is(result: &ToolResult, form: &Form<'_>, parts: bool) -> bool {
1673 let images = result.content.iter().all(|part| match part {
1674 ToolResultContent::Image(image) => {
1675 sendable_as_is(image) && form.sends(image, Place::ToolResult)
1676 }
1677 ToolResultContent::Text(_) | ToolResultContent::Json { .. } => true,
1678 });
1679 images && !says_nothing(&result.content) && (parts || texts(&result.content) < 2)
1680}
1681
1682pub const NO_TOOL_OUTPUT: &str = "(no tool output)";
1685
1686fn result_text_parts(
1690 content: Vec<ToolResultContent>,
1691 parts: bool,
1692 is_error: bool,
1693) -> Vec<ToolResultContent> {
1694 if says_nothing(&content) {
1695 let text = if is_error {
1696 format!("[tool error] {NO_TOOL_OUTPUT}")
1697 } else {
1698 NO_TOOL_OUTPUT.to_owned()
1699 };
1700 return vec![ToolResultContent::text(text)];
1701 }
1702 if parts || texts(&content) < 2 {
1703 return content;
1704 }
1705 let mut joined: Vec<String> = Vec::new();
1706 let mut images = Vec::new();
1707 for part in content {
1708 match part {
1709 ToolResultContent::Text(text) => joined.push(text.text),
1710 ToolResultContent::Json { value } => joined.push(value.to_string()),
1711 image @ ToolResultContent::Image(_) => images.push(image),
1712 }
1713 }
1714 std::iter::once(ToolResultContent::text(joined.join("\n")))
1715 .chain(images)
1716 .collect()
1717}
1718
1719fn says_nothing(content: &[ToolResultContent]) -> bool {
1721 content.iter().all(|part| match part {
1722 ToolResultContent::Text(text) => text.text.trim().is_empty(),
1723 ToolResultContent::Json { .. } | ToolResultContent::Image(_) => false,
1724 })
1725}
1726
1727fn texts(content: &[ToolResultContent]) -> usize {
1729 content
1730 .iter()
1731 .filter(|part| !matches!(part, ToolResultContent::Image(_)))
1732 .count()
1733}
1734
1735fn result_text(result: &ToolResult) -> String {
1737 let text: Vec<String> = result
1738 .content
1739 .iter()
1740 .map(|part| part.to_text(TOOL_IMAGE_OMITTED))
1741 .collect();
1742 let kind = if result.is_error { "error" } else { "result" };
1743 format!("[tool {} {kind}] {}", result.name, text.join("\n"))
1744}
1745
1746fn result_images(
1750 content: Vec<ToolResultContent>,
1751 form: &Form<'_>,
1752 attached: &mut Vec<crate::message::Image>,
1753) -> Vec<ToolResultContent> {
1754 let mut shaped: Vec<ToolResultContent> = Vec::with_capacity(content.len());
1755 for part in content {
1756 match part {
1757 ToolResultContent::Image(image) => {
1758 let image = sendable_image(image);
1759 if form.sends(&image, Place::ToolResult) {
1760 shaped.push(ToolResultContent::Image(image));
1761 continue;
1762 }
1763 let placeholder = if form.sends(&image, Place::User) {
1764 attached.push(image);
1765 TOOL_IMAGE_ATTACHED
1766 } else {
1767 TOOL_IMAGE_OMITTED
1768 };
1769 if shaped.last().and_then(ToolResultContent::as_text) != Some(placeholder) {
1770 shaped.push(ToolResultContent::text(placeholder));
1771 }
1772 }
1773 part => shaped.push(part),
1774 }
1775 }
1776 shaped
1777}
1778
1779fn sendable(source: DocumentSourceKind) -> DocumentSourceKind {
1782 match source {
1783 DocumentSourceKind::Raw(bytes) => DocumentSourceKind::Base64(BASE64_STANDARD.encode(bytes)),
1784 source => source,
1785 }
1786}
1787
1788fn sendable_as_is(image: &crate::message::Image) -> bool {
1791 match &image.data {
1792 DocumentSourceKind::Raw(_) => false,
1793 DocumentSourceKind::Base64(data) => image.media_type.is_some() || sniffed(data).is_none(),
1794 _ => true,
1795 }
1796}
1797
1798fn sendable_image(mut image: crate::message::Image) -> crate::message::Image {
1801 image.data = sendable(image.data);
1802 if image.media_type.is_none()
1803 && let DocumentSourceKind::Base64(data) = &image.data
1804 {
1805 image.media_type = sniffed(data);
1806 }
1807 image
1808}
1809
1810fn sniffed(data: &str) -> Option<ImageMediaType> {
1812 let head: String = data.chars().take(24).collect();
1813 let bytes = BASE64_STANDARD
1814 .decode(head.as_bytes())
1815 .or_else(|_| BASE64_STANDARD_NO_PAD.decode(head.as_bytes()))
1816 .ok()?;
1817 ImageMediaType::sniff(&bytes)
1818}
1819
1820fn merge_users(history: Vec<Message>) -> Vec<Message> {
1824 let mut merged: Vec<Message> = Vec::with_capacity(history.len());
1825 for message in history {
1826 match (merged.last_mut(), message) {
1827 (Some(Message::User { content: previous }), Message::User { content })
1828 if !previous.is_empty() && !content.is_empty() =>
1829 {
1830 previous.extend(content);
1831 }
1832 (_, message) => merged.push(message),
1833 }
1834 }
1835 merged
1836}
1837
1838#[cfg(test)]
1839mod tests;