use std::collections::{HashMap, HashSet};
use base64::Engine as _;
use base64::prelude::{BASE64_STANDARD, BASE64_STANDARD_NO_PAD};
use crate::message::{
Api, AssistantContent, AssistantMessage, CallId, DocumentMediaType, DocumentSourceKind,
ImageMediaType, Message, Origin, Text, ToolCall, ToolResult, ToolResultContent, UserContent,
};
use crate::wasm_compat::WasmCompatSync;
pub const NO_RESULT_PROVIDED: &str = "No result provided";
pub const USER_IMAGE_OMITTED: &str = "(image omitted: model does not support images)";
pub const ASSISTANT_IMAGE_OMITTED: &str = "(image omitted: model does not support images)";
pub const TOOL_IMAGE_OMITTED: &str = "(tool image omitted: model does not support images)";
pub const IMAGE_UNSENDABLE: &str = "(image omitted: the provider cannot receive it in this form)";
pub const AUDIO_UNSENDABLE: &str = "(audio omitted: the provider cannot receive it in this form)";
pub const VIDEO_UNSENDABLE: &str = "(video omitted: the provider cannot receive it in this form)";
pub const DOCUMENT_UNSENDABLE: &str =
"(document omitted: the provider cannot receive it in this form)";
pub const TOOL_IMAGE_ATTACHED: &str = "(see attached image)";
pub const TOOL_IMAGES_HEADING: &str = "Attached image(s) from tool result:";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Accepts {
pub user_images: bool,
pub assistant_images: bool,
pub tool_result_images: bool,
pub tools: bool,
}
impl Accepts {
pub const ALL: Self = Self {
user_images: true,
assistant_images: true,
tool_result_images: true,
tools: true,
};
pub const TEXT: Self = Self {
user_images: false,
assistant_images: false,
tool_result_images: false,
tools: true,
};
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Place {
User,
ToolResult,
Assistant,
}
#[derive(Debug, Clone, Copy)]
pub enum Media<'a> {
Image(&'a crate::message::Image, Place),
Audio(&'a crate::message::Audio),
Video(&'a crate::message::Video),
Document(&'a crate::message::Document),
}
pub trait ReplayTarget: std::fmt::Debug + WasmCompatSync {
fn api(&self) -> Api;
fn provider(&self) -> &str;
fn model(&self) -> &str;
fn accepts(&self, model: &str) -> Accepts;
fn map_options(
&self,
request: &crate::completion::CompletionRequest,
fields: crate::completion::options::OptionFields<'_>,
) -> crate::completion::options::OptionMap;
fn encodes(&self, model: &str, media: Media<'_>) -> bool {
let _ = (model, media);
true
}
fn normalize_tool_call_id(&self, id: &str, model: &str, source: Option<&Origin>) -> String {
let _ = (model, source);
id.to_owned()
}
fn continues_stored(&self, request: &crate::completion::CompletionRequest) -> bool {
let _ = request;
false
}
fn declares_tools(&self, request: &crate::completion::CompletionRequest) -> bool {
declares_tools(request)
}
fn identity(&self, item: &serde_json::Value) -> serde_json::Map<String, serde_json::Value> {
let _ = item;
serde_json::Map::new()
}
fn call_id_slot(&self) -> Option<&'static str> {
None
}
fn states_finish_reason(&self) -> bool {
true
}
fn binds_context(&self, model: &str) -> bool {
let _ = model;
false
}
fn drops_unbound_items(&self, request: &crate::completion::CompletionRequest) -> bool {
let _ = request;
false
}
fn route(&self, request: &crate::completion::CompletionRequest) -> Option<&dyn ReplayTarget> {
let _ = request;
None
}
fn takes_documents(&self) -> bool {
false
}
fn needs_next(&self, item: &serde_json::Value) -> bool {
let _ = item;
false
}
fn hosted_pair(&self, item: &serde_json::Value) -> Option<(Pairing, String)> {
let _ = item;
None
}
fn starts_with_user(&self) -> bool {
false
}
fn later_system(&self, model: &str) -> LaterSystem {
let _ = model;
LaterSystem::InPlace
}
fn alternates_roles(&self) -> bool {
false
}
fn hosted_needs_tools(&self) -> bool {
false
}
fn result_parts(&self, model: &str) -> bool {
let _ = model;
false
}
fn sends_alone(&self, block: &AssistantContent) -> bool {
let _ = block;
true
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LaterSystem {
InPlace,
Leading,
UserText,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Pairing {
Use,
Result,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Replay<'a> {
Item(std::borrow::Cow<'a, serde_json::Value>),
Identity(serde_json::Map<String, serde_json::Value>),
Rebuild,
}
impl AssistantContent {
pub fn replay(
&self,
target: &dyn ReplayTarget,
ids: &crate::providers::internal::wire_ids::WireIds,
) -> Replay<'_> {
if let Some(item) = self.native_item() {
return match (self, target.call_id_slot()) {
(AssistantContent::ToolCall(call), Some(slot)) => {
let mut item = item.clone();
if let Some(id) = ids.of(&call.id) {
set_pointer(&mut item, slot, serde_json::Value::String(id.to_owned()));
}
Replay::Item(std::borrow::Cow::Owned(item))
}
_ => Replay::Item(std::borrow::Cow::Borrowed(item)),
};
}
let identity = self
.stale_item()
.map(|item| target.identity(item))
.unwrap_or_default();
if identity.is_empty() {
Replay::Rebuild
} else {
Replay::Identity(identity)
}
}
}
fn set_pointer(item: &mut serde_json::Value, pointer: &str, value: serde_json::Value) {
let mut at = item;
let mut keys = pointer.split('/').skip(1).peekable();
while let Some(key) = keys.next() {
if !at.is_object() {
*at = serde_json::Value::Object(serde_json::Map::new());
}
let serde_json::Value::Object(fields) = at else {
return;
};
if keys.peek().is_none() {
fields.insert(key.to_owned(), value);
return;
}
at = fields
.entry(key.to_owned())
.or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()));
}
}
pub fn adapt(history: &[Message], target: &dyn ReplayTarget) -> Vec<Message> {
adapt_for(history, target, &Request::default())
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct Request<'a> {
pub(crate) model: Option<&'a str>,
pub(crate) stored: bool,
pub(crate) tools: bool,
pub(crate) context: Option<crate::message::Fingerprint>,
}
impl Default for Request<'_> {
fn default() -> Self {
Self {
model: None,
stored: false,
tools: true,
context: None,
}
}
}
pub(crate) fn context_of(
request: &crate::completion::CompletionRequest,
target: &dyn ReplayTarget,
model: &str,
) -> crate::message::Fingerprint {
let mut tools: Vec<_> = request.tools.iter().collect();
tools.sort_by(|left, right| left.name.cmp(&right.name));
let raw = raw_tools(request);
let folds = target.later_system(model) == LaterSystem::Leading;
let mut system: Vec<&str> = request
.chat_history
.iter()
.map_while(|message| match message {
Message::System { content } => Some(Some(content.as_str())),
Message::User { .. } | Message::Assistant(_) => folds.then_some(None),
})
.flatten()
.filter(|content| !content.trim().is_empty())
.collect();
let joined;
if folds && system.len() > 1 {
joined = system.join("\n\n");
system = vec![joined.as_str()];
}
let mut fields = vec![serde_json::json!("context"), serde_json::json!(tools)];
fields.push(serde_json::json!(system));
if !raw.is_empty() {
fields.push(serde_json::json!(raw));
}
crate::message::Fingerprint::of(&serde_json::Value::Array(fields))
}
pub(crate) fn declares_tools(request: &crate::completion::CompletionRequest) -> bool {
!request.tools.is_empty() || !raw_tools(request).is_empty()
}
pub(crate) fn raw_tools(request: &crate::completion::CompletionRequest) -> &[serde_json::Value] {
use crate::json_utils::Lenient;
request
.additional_params
.as_ref()
.map_or(&[][..], |params| params.arr("tools"))
}
pub(crate) fn adapt_for(
history: &[Message],
target: &dyn ReplayTarget,
request: &Request<'_>,
) -> Vec<Message> {
let model = request.model.unwrap_or(target.model());
let stored = request.stored;
let same = Same {
api: target.api(),
provider: target.provider(),
model,
context: request.context.filter(|_| target.binds_context(model)),
};
let mut accepts = target.accepts(model);
accepts.tools &= request.tools;
let hosted = hosted_pairs(history, target, &same);
let last_turn = history
.iter()
.rposition(|message| matches!(message, Message::Assistant(_)));
let mut ids = Renamed::default();
let mut shaped = Vec::with_capacity(history.len());
for (at, message) in history.iter().enumerate() {
match message {
Message::System { content } => {
if !content.trim().is_empty() {
shaped.push(Some(message.clone()));
}
}
Message::User { content } => {
let form = Form {
target,
model,
accepts,
};
shaped.extend(user(content, &mut ids, &form).into_iter().map(Some));
}
Message::Assistant(turn) => {
let last = Some(at) == last_turn
&& (turn
.content
.iter()
.any(|block| matches!(block, AssistantContent::ToolCall(_)))
|| history
.get(at + 1..)
.into_iter()
.flatten()
.all(|message| matches!(message, Message::System { .. })));
let here: HashSet<usize> = hosted
.iter()
.filter(|(message, _)| *message == at)
.map(|(_, block)| *block)
.collect();
let adapted = assistant(turn, target, &same, accepts, &mut ids, &here, last);
let adapted = AssistantMessage {
content: adapted
.content
.into_iter()
.map(|block| match block {
AssistantContent::Image(image) if image.native.is_none() => {
let image = sendable_image(image);
if matches!(image.data, DocumentSourceKind::Unknown)
|| !target
.encodes(model, Media::Image(&image, Place::Assistant))
{
AssistantContent::Text(Text::new(IMAGE_UNSENDABLE))
} else {
AssistantContent::Image(image)
}
}
block => block,
})
.collect(),
..adapted
};
let emptied = !turn.content.is_empty()
&& !adapted
.content
.iter()
.any(|block| target.sends_alone(block));
shaped.push((!emptied).then_some(Message::Assistant(adapted)));
}
}
}
let shaped = merge_users(answer_calls(shaped, stored, accepts.tools));
let shaped = if accepts.tools {
shaped
} else {
tools_as_text(shaped)
};
let shaped = match target.later_system(model) {
LaterSystem::InPlace => shaped,
LaterSystem::Leading => leading_system(shaped),
LaterSystem::UserText => system_as_user_text(shaped),
};
let shaped = if target.starts_with_user() && !stored {
from_first_user(shaped)
} else {
shaped
};
if target.alternates_roles() {
alternated(shaped)
} else {
shaped
}
}
fn tools_as_text(history: Vec<Message>) -> Vec<Message> {
let history = history
.into_iter()
.map(|message| match message {
Message::Assistant(mut turn) => {
for block in &mut turn.content {
if let AssistantContent::ToolCall(call) = block {
*block = AssistantContent::Text(Text::new(format!(
"[called tool {} with {}]",
call.function.name,
crate::json_utils::to_canonical_string(
&call.function.arguments_value()
)
)));
}
}
Message::Assistant(turn)
}
Message::User { content } => Message::User {
content: content
.into_iter()
.map(|part| match part {
UserContent::ToolResult(result) => UserContent::text(result_text(&result)),
part => part,
})
.collect(),
},
message => message,
})
.collect();
merge_users(history)
}
fn from_first_user(mut history: Vec<Message>) -> Vec<Message> {
let lead = history
.iter()
.take_while(|message| matches!(message, Message::System { .. }))
.count();
while let Some(Message::Assistant(turn)) = history.get(lead) {
let calls: HashSet<CallId> = turn.tool_calls().map(|call| call.id.clone()).collect();
history.remove(lead);
if let Some(Message::User { content }) = history.get_mut(lead) {
content.retain(|part| {
!matches!(part, UserContent::ToolResult(result) if calls.contains(&result.call))
});
if content.is_empty() {
history.remove(lead);
}
}
}
history
}
fn has_later_system(history: &[Message]) -> bool {
history
.iter()
.skip_while(|message| matches!(message, Message::System { .. }))
.any(|message| matches!(message, Message::System { .. }))
}
fn system_as_user_text(history: Vec<Message>) -> Vec<Message> {
let mut leading = true;
let history = history
.into_iter()
.map(|message| match message {
Message::System { content } if !leading => Message::User {
content: vec![UserContent::text(content)],
},
message => {
leading &= matches!(message, Message::System { .. });
message
}
})
.collect();
merge_users(history)
}
fn alternated(history: Vec<Message>) -> Vec<Message> {
let mut alternated: Vec<Message> = Vec::with_capacity(history.len());
let mut held: Vec<Message> = Vec::new();
for message in history {
let started = alternated
.iter()
.any(|message| !matches!(message, Message::System { .. }));
match (message, alternated.last_mut()) {
(message @ Message::System { .. }, _) if started => held.push(message),
(Message::User { content }, Some(Message::User { content: previous })) => {
previous.extend(content);
}
(Message::Assistant(turn), Some(Message::Assistant(previous))) => {
if previous.origin != turn.origin {
previous.origin = None;
}
previous.stop = turn.stop;
previous.content.extend(turn.content);
}
(message, _) => {
alternated.append(&mut held);
alternated.push(message);
}
}
}
alternated.append(&mut held);
alternated
}
fn leading_system(history: Vec<Message>) -> Vec<Message> {
if !has_later_system(&history) {
return history;
}
let (system, rest): (Vec<Message>, Vec<Message>) = history
.into_iter()
.partition(|message| matches!(message, Message::System { .. }));
let prompt: Vec<String> = system
.into_iter()
.filter_map(|message| match message {
Message::System { content } => Some(content),
Message::User { .. } | Message::Assistant(_) => None,
})
.collect();
let mut history = Vec::with_capacity(rest.len() + 1);
if !prompt.is_empty() {
history.push(Message::system(prompt.join("\n\n")));
}
history.extend(merge_users(rest));
history
}
#[derive(Default)]
struct Renamed {
to: HashMap<CallId, std::collections::VecDeque<CallId>>,
taken: HashSet<String>,
}
impl Renamed {
fn turn(&mut self) {
self.to.clear();
}
fn claim(&mut self, source: &CallId, wanted: String) -> CallId {
let id = match source {
CallId::Local(_) => source.clone(),
CallId::Provider(_) => {
let mut id = wanted.clone();
let mut attempt: u64 = 1;
while self.taken.contains(&id) {
id = with_counter(&wanted, attempt);
attempt += 1;
}
self.taken.insert(id.clone());
if id == source.wire() {
source.clone()
} else {
CallId::from_wire(id)
}
}
};
self.to
.entry(source.clone())
.or_default()
.push_back(id.clone());
id
}
fn answer(&mut self, source: &CallId) -> Option<CallId> {
let ids = self.to.get_mut(source)?;
if ids.len() > 1 {
ids.pop_front()
} else {
ids.front().cloned()
}
}
}
fn with_counter(id: &str, attempt: u64) -> String {
let mut digits = Vec::new();
let mut value = attempt;
while value > 0 {
digits.push(char::from_digit((value % 36) as u32, 36).unwrap_or('0'));
value /= 36;
}
digits.reverse();
let keep = id.chars().count().saturating_sub(digits.len());
id.chars().take(keep).chain(digits).collect()
}
struct Same<'a> {
api: Api,
provider: &'a str,
model: &'a str,
context: Option<crate::message::Fingerprint>,
}
impl Same<'_> {
fn is(&self, origin: &Origin) -> bool {
origin.same_model(&self.api, self.provider, self.model)
&& self
.context
.is_none_or(|context| origin.context == Some(context))
}
}
fn assistant(
turn: &AssistantMessage,
target: &dyn ReplayTarget,
same_model: &Same<'_>,
accepts: Accepts,
ids: &mut Renamed,
hosted: &HashSet<usize>,
last: bool,
) -> AssistantMessage {
let model = same_model.model;
let same = turn
.origin
.as_ref()
.is_some_and(|origin| same_model.is(origin));
if turn.stop.as_ref().is_some_and(|stop| stop.is_failure()) {
return turn.clone();
}
ids.turn();
let content: Vec<Option<AssistantContent>> = turn
.content
.iter()
.map(|block| {
let block = if same {
match block.clone() {
AssistantContent::ToolCall(mut call) => {
let wanted = call.id.wire().into_owned();
let item = AssistantContent::ToolCall(call.clone())
.native_item()
.cloned()
.filter(|_| target.call_id_slot().is_some());
call.id = ids.claim(&call.id, wanted);
let block = AssistantContent::ToolCall(call);
match item {
Some(item) if block.native_item().is_none() => {
block.canonical().with_native(item)
}
_ => block,
}
}
AssistantContent::Image(_) if !accepts.assistant_images => return None,
block => block,
}
} else {
match block.canonical() {
AssistantContent::Reasoning(reasoning) => {
if reasoning.redacted || reasoning.text.trim().is_empty() {
return None;
}
AssistantContent::Text(Text::new(reasoning.text))
}
AssistantContent::Opaque(_) => return None,
AssistantContent::Image(_) if !accepts.assistant_images => {
AssistantContent::Text(Text::new(ASSISTANT_IMAGE_OMITTED))
}
AssistantContent::ToolCall(mut call) => {
let normalized = target.normalize_tool_call_id(
&call.id.wire(),
model,
turn.origin.as_ref(),
);
call.id = ids.claim(&call.id, normalized);
AssistantContent::ToolCall(call)
}
block => block,
}
};
let block = match block {
AssistantContent::Opaque(opaque)
if !accepts.tools
&& target.hosted_needs_tools()
&& target.hosted_pair(&opaque.item).is_some() =>
{
return None;
}
block => block,
};
(!block.is_blank()).then_some(block)
})
.collect();
let content = if same {
paired(content, target, accepts.tools, hosted, last)
} else {
content.into_iter().flatten().collect()
};
AssistantMessage {
content,
origin: turn.origin.clone(),
stop: turn.stop.clone(),
}
}
fn hosted_pairs(
history: &[Message],
target: &dyn ReplayTarget,
same: &Same<'_>,
) -> HashSet<(usize, usize)> {
let mut open: HashMap<String, (usize, usize)> = HashMap::new();
let mut paired = HashSet::new();
for (at, message) in history.iter().enumerate() {
let Message::Assistant(turn) = message else {
continue;
};
if turn.stop.as_ref().is_some_and(|stop| stop.is_failure())
|| !turn.origin.as_ref().is_some_and(|origin| same.is(origin))
{
continue;
}
for (index, block) in turn.content.iter().enumerate() {
let AssistantContent::Opaque(opaque) = block else {
continue;
};
if !opaque.replay {
continue;
}
match target.hosted_pair(&opaque.item) {
Some((Pairing::Use, id)) => {
open.insert(id, (at, index));
}
Some((Pairing::Result, id)) => {
if let Some(used) = open.remove(&id) {
paired.insert(used);
paired.insert((at, index));
}
}
None => {}
}
}
}
paired
}
fn paired(
mut content: Vec<Option<AssistantContent>>,
target: &dyn ReplayTarget,
tools: bool,
hosted: &HashSet<usize>,
last: bool,
) -> Vec<AssistantContent> {
let pair = |block: &AssistantContent| match block {
AssistantContent::Opaque(opaque) if opaque.replay => target.hosted_pair(&opaque.item),
_ => None,
};
for at in 0..content.len() {
let Some((side, _)) = content.get(at).and_then(Option::as_ref).and_then(pair) else {
continue;
};
let running = last
&& side == Pairing::Use
&& content
.get(at + 1..)
.into_iter()
.flatten()
.flatten()
.all(|block| {
matches!(
block,
AssistantContent::ToolCall(_) | AssistantContent::Opaque(_)
)
});
if !hosted.contains(&at)
&& !running
&& let Some(slot) = content.get_mut(at)
{
*slot = None;
}
}
for at in (0..content.len()).rev() {
let needs = content
.get(at)
.and_then(Option::as_ref)
.and_then(|block| match block {
AssistantContent::Opaque(opaque) if opaque.replay => Some(&opaque.item),
block => block.native_item().or_else(|| {
block
.stale_item()
.filter(|item| !target.identity(item).is_empty())
}),
})
.is_some_and(|item| target.needs_next(item));
let next_gone =
content
.get(at + 1)
.and_then(Option::as_ref)
.is_none_or(|next| match next {
AssistantContent::Opaque(opaque) => !opaque.replay,
AssistantContent::ToolCall(_) if !tools => true,
next => {
next.native_item().is_none()
&& next
.stale_item()
.is_none_or(|item| target.identity(item).is_empty())
}
});
if needs
&& next_gone
&& let Some(slot) = content.get_mut(at)
{
*slot = None;
}
}
content.into_iter().flatten().collect()
}
struct Form<'a> {
target: &'a dyn ReplayTarget,
model: &'a str,
accepts: Accepts,
}
impl Form<'_> {
fn sends(&self, image: &crate::message::Image, place: Place) -> bool {
let reads = match place {
Place::User => self.accepts.user_images,
Place::ToolResult => self.accepts.tool_result_images,
Place::Assistant => self.accepts.assistant_images,
};
reads
&& !matches!(image.data, DocumentSourceKind::Unknown)
&& self.target.encodes(self.model, Media::Image(image, place))
}
}
fn user(content: &[UserContent], ids: &mut Renamed, form: &Form<'_>) -> Vec<Message> {
let mut shaped: Vec<UserContent> = Vec::with_capacity(content.len());
let mut attached = Vec::new();
for part in content {
let placeholder = match part {
UserContent::Image(image) => {
let image = sendable_image(image.clone());
if form.sends(&image, Place::User) {
shaped.push(UserContent::Image(image));
continue;
}
if form.accepts.user_images {
IMAGE_UNSENDABLE
} else {
USER_IMAGE_OMITTED
}
}
UserContent::Audio(audio) => {
let mut audio = audio.clone();
audio.data = sendable(audio.data);
if !matches!(audio.data, DocumentSourceKind::Unknown)
&& form.target.encodes(form.model, Media::Audio(&audio))
{
shaped.push(UserContent::Audio(audio));
continue;
}
AUDIO_UNSENDABLE
}
UserContent::Video(video) => {
let mut video = video.clone();
video.data = sendable(video.data);
if !matches!(video.data, DocumentSourceKind::Unknown)
&& form.target.encodes(form.model, Media::Video(&video))
{
shaped.push(UserContent::Video(video));
continue;
}
VIDEO_UNSENDABLE
}
UserContent::Document(document) => {
let mut document = document.clone();
document.data = sendable(document.data);
if !matches!(document.data, DocumentSourceKind::Unknown)
&& form.target.encodes(form.model, Media::Document(&document))
{
shaped.push(UserContent::Document(document));
continue;
}
if let Some(text) = document_text(&document) {
shaped.push(UserContent::text(text));
continue;
}
DOCUMENT_UNSENDABLE
}
UserContent::ToolResult(result) => {
let mut result = result.clone();
if let Some(id) = ids.answer(&result.call) {
result.call = id;
}
result.content = result_images(result.content, form, &mut attached);
let parts = form.accepts.tool_result_images || form.target.result_parts(form.model);
result.content = result_text_parts(result.content, parts, result.is_error);
shaped.push(UserContent::ToolResult(result));
continue;
}
UserContent::Text(text) if text.text.trim().is_empty() => continue,
UserContent::Text(_) => {
shaped.push(part.clone());
continue;
}
};
let repeated = matches!(
shaped.last(),
Some(UserContent::Text(text)) if text.text == placeholder
);
if !repeated {
shaped.push(UserContent::text(placeholder));
}
}
let mut messages = Vec::new();
if !shaped.is_empty() {
messages.push(Message::User { content: shaped });
}
if !attached.is_empty() {
let mut content = vec![UserContent::text(TOOL_IMAGES_HEADING)];
content.extend(attached.into_iter().map(UserContent::Image));
messages.push(Message::User { content });
}
messages
}
pub const NO_TOOL_OUTPUT: &str = "(no tool output)";
fn result_text_parts(
content: Vec<ToolResultContent>,
parts: bool,
is_error: bool,
) -> Vec<ToolResultContent> {
let blank = content.iter().all(|part| match part {
ToolResultContent::Text(text) => text.text.trim().is_empty(),
ToolResultContent::Json { .. } | ToolResultContent::Image(_) => false,
});
if blank {
let text = if is_error {
format!("[tool error] {NO_TOOL_OUTPUT}")
} else {
NO_TOOL_OUTPUT.to_owned()
};
return vec![ToolResultContent::text(text)];
}
let texts = content
.iter()
.filter(|part| !matches!(part, ToolResultContent::Image(_)))
.count();
if parts || texts < 2 {
return content;
}
let mut joined: Vec<String> = Vec::new();
let mut images = Vec::new();
for part in content {
match part {
ToolResultContent::Text(text) => joined.push(text.text),
ToolResultContent::Json { value } => joined.push(value.to_string()),
image @ ToolResultContent::Image(_) => images.push(image),
}
}
std::iter::once(ToolResultContent::text(joined.join("\n")))
.chain(images)
.collect()
}
fn result_text(result: &ToolResult) -> String {
let text: Vec<String> = result
.content
.iter()
.map(|part| match part {
ToolResultContent::Text(text) => text.text.clone(),
ToolResultContent::Json { value } => value.to_string(),
ToolResultContent::Image(_) => TOOL_IMAGE_OMITTED.to_owned(),
})
.collect();
let kind = if result.is_error { "error" } else { "result" };
format!("[tool {} {kind}] {}", result.name, text.join("\n"))
}
fn result_images(
content: Vec<ToolResultContent>,
form: &Form<'_>,
attached: &mut Vec<crate::message::Image>,
) -> Vec<ToolResultContent> {
let mut shaped: Vec<ToolResultContent> = Vec::with_capacity(content.len());
for part in content {
match part {
ToolResultContent::Image(image) => {
let image = sendable_image(image);
if form.sends(&image, Place::ToolResult) {
shaped.push(ToolResultContent::Image(image));
continue;
}
let placeholder = if form.sends(&image, Place::User) {
attached.push(image);
TOOL_IMAGE_ATTACHED
} else {
TOOL_IMAGE_OMITTED
};
if shaped.last().and_then(ToolResultContent::as_text) != Some(placeholder) {
shaped.push(ToolResultContent::text(placeholder));
}
}
part => shaped.push(part),
}
}
shaped
}
fn sendable(source: DocumentSourceKind) -> DocumentSourceKind {
match source {
DocumentSourceKind::Raw(bytes) => DocumentSourceKind::Base64(BASE64_STANDARD.encode(bytes)),
source => source,
}
}
fn sendable_image(mut image: crate::message::Image) -> crate::message::Image {
image.data = sendable(image.data);
if image.media_type.is_none()
&& let DocumentSourceKind::Base64(data) = &image.data
{
image.media_type = sniffed(data);
}
image
}
fn sniffed(data: &str) -> Option<ImageMediaType> {
let head: String = data.chars().take(24).collect();
let bytes = BASE64_STANDARD
.decode(head.as_bytes())
.or_else(|_| BASE64_STANDARD_NO_PAD.decode(head.as_bytes()))
.ok()?;
match bytes.as_slice() {
[0x89, b'P', b'N', b'G', ..] => Some(ImageMediaType::PNG),
[0xFF, 0xD8, 0xFF, ..] => Some(ImageMediaType::JPEG),
[b'G', b'I', b'F', b'8', ..] => Some(ImageMediaType::GIF),
[
b'R',
b'I',
b'F',
b'F',
_,
_,
_,
_,
b'W',
b'E',
b'B',
b'P',
..,
] => Some(ImageMediaType::WEBP),
_ => None,
}
}
fn document_text(document: &crate::message::Document) -> Option<String> {
match &document.data {
DocumentSourceKind::String(text) => Some(text.clone()),
DocumentSourceKind::Base64(data)
if document
.media_type
.as_ref()
.is_some_and(|media_type| *media_type != DocumentMediaType::PDF) =>
{
let bytes = BASE64_STANDARD.decode(data.as_bytes()).ok()?;
String::from_utf8(bytes).ok()
}
_ => None,
}
}
fn merge_users(history: Vec<Message>) -> Vec<Message> {
let mut merged: Vec<Message> = Vec::with_capacity(history.len());
for message in history {
match (merged.last_mut(), message) {
(Some(Message::User { content: previous }), Message::User { content })
if !previous.is_empty() && !content.is_empty() =>
{
previous.extend(content);
}
(_, message) => merged.push(message),
}
}
merged
}
fn answer_calls(history: Vec<Option<Message>>, stored: bool, answers: bool) -> Vec<Message> {
let mut stored = stored;
let mut shaped = Vec::with_capacity(history.len());
let mut waiting: Vec<ToolCall> = Vec::new();
let mut held = Vec::new();
let mut gap = false;
let mut pending: Vec<UserContent> = Vec::new();
let mut pending_gap = false;
for message in adjacent_users_merged(history) {
let Some(message) = message else {
close(
&mut shaped,
&mut waiting,
&mut held,
answers,
std::mem::take(&mut pending),
std::mem::take(&mut pending_gap),
);
stored = false;
gap = true;
continue;
};
match message {
Message::Assistant(turn) => {
close(
&mut shaped,
&mut waiting,
&mut held,
answers,
std::mem::take(&mut pending),
std::mem::take(&mut pending_gap),
);
stored = false;
if turn.stop.as_ref().is_some_and(|stop| stop.is_failure()) {
gap = true;
continue;
}
gap = false;
waiting = distinct(turn.tool_calls());
shaped.push(Message::Assistant(turn));
}
Message::User { mut content } => {
if content.is_empty() {
close(
&mut shaped,
&mut waiting,
&mut held,
answers,
std::mem::take(&mut pending),
std::mem::take(&mut pending_gap),
);
shaped.push(Message::User { content });
gap = false;
continue;
}
let mut answered: HashSet<CallId> = pending
.iter()
.filter_map(|part| match part {
UserContent::ToolResult(result) => Some(result.call.clone()),
_ => None,
})
.collect();
content.retain(|part| match part {
UserContent::ToolResult(result) => {
(stored || waiting.iter().any(|call| call.id == result.call))
&& answered.insert(result.call.clone())
}
UserContent::Text(_)
| UserContent::Image(_)
| UserContent::Audio(_)
| UserContent::Video(_)
| UserContent::Document(_) => true,
});
if content.is_empty() && waiting.is_empty() {
continue;
}
let only_results = !content.is_empty()
&& content
.iter()
.all(|part| matches!(part, UserContent::ToolResult(_)));
if pending.is_empty() {
pending_gap = gap;
}
pending.extend(content);
gap = false;
if only_results && waiting.iter().any(|call| !answered.contains(&call.id)) {
continue;
}
close(
&mut shaped,
&mut waiting,
&mut held,
answers,
std::mem::take(&mut pending),
std::mem::take(&mut pending_gap),
);
}
Message::System { .. } if !waiting.is_empty() => held.push(message),
system => {
gap = false;
shaped.push(system);
}
}
}
close(
&mut shaped,
&mut waiting,
&mut held,
answers,
pending,
pending_gap,
);
shaped
}
fn adjacent_users_merged(history: Vec<Option<Message>>) -> Vec<Option<Message>> {
let mut merged: Vec<Option<Message>> = Vec::with_capacity(history.len());
for message in history {
match (merged.last_mut(), message) {
(Some(Some(Message::User { content: previous })), Some(Message::User { content }))
if !previous.is_empty() && !content.is_empty() =>
{
previous.extend(content)
}
(_, message) => merged.push(message),
}
}
merged
}
fn close(
shaped: &mut Vec<Message>,
waiting: &mut Vec<ToolCall>,
held: &mut Vec<Message>,
answers: bool,
mut content: Vec<UserContent>,
merge: bool,
) {
if !answers {
waiting.clear();
}
let missing: Vec<UserContent> = waiting
.drain(..)
.filter(|call| {
!content.iter().any(
|part| matches!(part, UserContent::ToolResult(result) if result.call == call.id),
)
})
.map(|call| {
UserContent::ToolResult(ToolResult {
call: call.id,
name: call.function.name,
content: vec![ToolResultContent::text(NO_RESULT_PROVIDED)],
is_error: true,
})
})
.collect();
let at = content
.iter()
.position(|part| !matches!(part, UserContent::ToolResult(_)))
.unwrap_or(content.len());
content.splice(at..at, missing);
content.sort_by_key(|part| !matches!(part, UserContent::ToolResult(_)));
let at = content
.iter()
.position(|part| !matches!(part, UserContent::ToolResult(_)))
.unwrap_or(content.len());
let text = if held.is_empty() {
Vec::new()
} else {
content.split_off(at)
};
if !content.is_empty() {
match shaped.last_mut() {
Some(Message::User { content: previous }) if merge => previous.extend(content),
_ => shaped.push(Message::User { content }),
}
}
shaped.append(held);
if !text.is_empty() {
shaped.push(Message::User { content: text });
}
}
fn distinct<'a>(calls: impl Iterator<Item = &'a ToolCall>) -> Vec<ToolCall> {
let mut seen = HashSet::new();
calls
.filter(|call| seen.insert(call.id.clone()))
.cloned()
.collect()
}
#[cfg(test)]
mod tests;