ferrin_google/
convert_prompt.rs

1//! Conversion of the specification prompt to `systemInstruction` and
2//! `contents`.
3//!
4//! Derived from the Vercel AI SDK (Apache-2.0, Copyright 2023 Vercel, Inc.),
5//! translated from TypeScript to Rust and modified; see `NOTICE`.
6
7use base64::Engine;
8use ferrin_provider_util::media_type::resolve_full_media_type;
9use ferrin_provider_util::tool_name_mapping::ToolNameMapping;
10use ferrin_spec::FileData;
11use ferrin_spec::JsonObject;
12use ferrin_spec::JsonValue;
13use ferrin_spec::MediaType;
14use ferrin_spec::ProviderOptions;
15use ferrin_spec::ProviderReference;
16use ferrin_spec::error::NoSuchProviderReferenceError;
17use ferrin_spec::error::ProviderError;
18use ferrin_spec::error::UnsupportedFunctionalityError;
19use ferrin_spec::language_model::PromptMessage;
20use ferrin_spec::language_model::prompt::AssistantPromptPart;
21use ferrin_spec::language_model::prompt::ToolPromptPart;
22use ferrin_spec::language_model::prompt::ToolResultContentPart;
23use ferrin_spec::language_model::prompt::ToolResultOutput;
24use ferrin_spec::language_model::prompt::ToolResultPart;
25use ferrin_spec::language_model::prompt::UserPromptPart;
26use ferrin_spec::shared::Warning;
27use serde_json::json;
28
29use crate::capabilities::ModelCapabilities;
30use crate::config::CANONICAL_OPTIONS_KEY;
31use crate::config::GoogleConfig;
32use crate::options::read_part_options;
33
34/// Signature sent for Gemini 3 tool calls that lack one.
35pub const SKIP_THOUGHT_SIGNATURE: &str = "skip_thought_signature_validator";
36
37/// Result of the prompt conversion.
38#[derive(Debug, Clone, Default, PartialEq)]
39pub struct ConvertedPrompt {
40    /// `systemInstruction` (`{parts: [{text}]}`), absent without system
41    /// messages or for Gemma models (whose system text is prepended to the
42    /// first user part).
43    pub system_instruction: Option<JsonObject>,
44    /// `contents`.
45    pub contents: Vec<JsonValue>,
46    /// Warnings collected during the conversion.
47    pub warnings: Vec<Warning>,
48}
49
50/// Resolves a file reference under the configured name, then `google`.
51///
52/// # Errors
53///
54/// Returns [`NoSuchProviderReferenceError`] when neither key is present.
55pub fn resolve_reference<'a>(
56    config: &GoogleConfig,
57    reference: &'a ProviderReference,
58) -> Result<&'a str, NoSuchProviderReferenceError> {
59    reference
60        .get(config.options_key())
61        .or_else(|| reference.get(CANONICAL_OPTIONS_KEY))
62        .map(String::as_str)
63        .ok_or_else(|| NoSuchProviderReferenceError::new(config.name.clone(), reference.clone()))
64}
65
66fn encode_base64(bytes: &[u8]) -> String {
67    base64::engine::general_purpose::STANDARD.encode(bytes)
68}
69
70fn inline_data(media_type: &str, data: &str) -> JsonValue {
71    json!({"inlineData": {"mimeType": media_type, "data": data}})
72}
73
74fn file_data(media_type: &str, uri: &str) -> JsonValue {
75    json!({"fileData": {"mimeType": media_type, "fileUri": uri}})
76}
77
78fn full_media_type(media_type: &MediaType, bytes: Option<&[u8]>) -> Result<String, ProviderError> {
79    Ok(resolve_full_media_type(media_type, bytes)?.into_string())
80}
81
82/// Converts a file to a `fileData` or `inlineData` part.
83fn file_part(
84    config: &GoogleConfig,
85    data: &FileData,
86    media_type: &MediaType,
87    in_assistant: bool,
88) -> Result<JsonValue, ProviderError> {
89    match data {
90        FileData::Url { url } => {
91            if in_assistant {
92                return Err(UnsupportedFunctionalityError::new(
93                    "File data URLs in assistant messages are not supported",
94                )
95                .into());
96            }
97            Ok(file_data(&full_media_type(media_type, None)?, url.as_str()))
98        }
99        FileData::Reference { reference } => Ok(file_data(
100            &full_media_type(media_type, None)?,
101            resolve_reference(config, reference)?,
102        )),
103        FileData::Bytes { data } => Ok(inline_data(
104            &full_media_type(media_type, Some(data.as_ref()))?,
105            &encode_base64(data.as_ref()),
106        )),
107        FileData::Text { text } => {
108            let media_type = if media_type.is_full() {
109                media_type.as_str().to_owned()
110            } else {
111                "text/plain".to_owned()
112            };
113            Ok(inline_data(&media_type, &encode_base64(text.as_bytes())))
114        }
115        #[allow(unreachable_patterns, reason = "FileData is non-exhaustive")]
116        _ => Err(UnsupportedFunctionalityError::new("file data variant").into()),
117    }
118}
119
120fn with_signature(mut part: JsonValue, signature: Option<&str>) -> JsonValue {
121    if let (Some(signature), Some(object)) = (signature, part.as_object_mut()) {
122        object.insert("thoughtSignature".to_owned(), JsonValue::from(signature));
123    }
124    part
125}
126
127fn with_thought(mut part: JsonValue) -> JsonValue {
128    if let Some(object) = part.as_object_mut() {
129        object.insert("thought".to_owned(), JsonValue::Bool(true));
130    }
131    part
132}
133
134fn tool_result_value(output: &ToolResultOutput) -> JsonValue {
135    match output {
136        ToolResultOutput::Text { value, .. } | ToolResultOutput::ErrorText { value, .. } => {
137            JsonValue::from(value.as_str())
138        }
139        ToolResultOutput::Json { value, .. } | ToolResultOutput::ErrorJson { value, .. } => {
140            value.clone()
141        }
142        ToolResultOutput::ExecutionDenied { reason, .. } => JsonValue::from(
143            reason
144                .clone()
145                .unwrap_or_else(|| "Tool call execution denied.".to_owned()),
146        ),
147        ToolResultOutput::Content { value, .. } => JsonValue::Array(
148            value
149                .iter()
150                .filter_map(|part| match part {
151                    ToolResultContentPart::Text { text, .. } => {
152                        Some(JsonValue::from(text.as_str()))
153                    }
154                    _ => None,
155                })
156                .collect(),
157        ),
158        #[allow(unreachable_patterns, reason = "ToolResultOutput is non-exhaustive")]
159        _ => JsonValue::Null,
160    }
161}
162
163fn function_response(id: Option<&str>, name: &str, response: JsonValue) -> JsonObject {
164    let mut function_response = JsonObject::new();
165    if let Some(id) = id.filter(|id| !id.is_empty()) {
166        function_response.insert("id".to_owned(), JsonValue::from(id));
167    }
168    function_response.insert("name".to_owned(), JsonValue::from(name));
169    function_response.insert("response".to_owned(), response);
170    function_response
171}
172
173/// Decodes a base64 `data:` URL into `(media type, base64 payload)`.
174fn parse_data_url(url: &url::Url) -> Option<(String, String)> {
175    let rest = url.as_str().strip_prefix("data:")?;
176    let (header, payload) = rest.split_once(',')?;
177    let mut segments = header.split(';');
178    let media_type = segments.next().unwrap_or_default().to_owned();
179    if segments.any(|segment| segment.eq_ignore_ascii_case("base64")) {
180        Some((media_type, payload.to_owned()))
181    } else {
182        let decoded = percent_decode(payload);
183        Some((media_type, encode_base64(decoded.as_bytes())))
184    }
185}
186
187fn percent_decode(text: &str) -> String {
188    let bytes = text.as_bytes();
189    let mut decoded = Vec::with_capacity(bytes.len());
190    let mut index = 0;
191    while index < bytes.len() {
192        if bytes[index] == b'%'
193            && let Some(hex) = text.get(index + 1..index + 3)
194            && let Ok(byte) = u8::from_str_radix(hex, 16)
195        {
196            decoded.push(byte);
197            index += 3;
198        } else {
199            decoded.push(bytes[index]);
200            index += 1;
201        }
202    }
203    String::from_utf8_lossy(&decoded).into_owned()
204}
205
206struct Converter<'a> {
207    config: &'a GoogleConfig,
208    capabilities: ModelCapabilities,
209    mapping: &'a ToolNameMapping,
210    warnings: Vec<Warning>,
211    contents: Vec<JsonValue>,
212}
213
214impl Converter<'_> {
215    fn push_content(&mut self, role: &str, parts: Vec<JsonValue>) {
216        if !parts.is_empty() {
217            self.contents.push(json!({"role": role, "parts": parts}));
218        }
219    }
220
221    /// Appends `part` to the last model content, or opens a new one.
222    fn append_to_model(&mut self, part: JsonValue) {
223        if let Some(JsonValue::Object(last)) = self.contents.last_mut()
224            && last.get("role").and_then(JsonValue::as_str) == Some("model")
225            && let Some(JsonValue::Array(parts)) = last.get_mut("parts")
226        {
227            parts.push(part);
228            return;
229        }
230        self.contents
231            .push(json!({"role": "model", "parts": [part]}));
232    }
233
234    fn user_message(&mut self, content: &[UserPromptPart]) -> Result<(), ProviderError> {
235        let mut parts = Vec::new();
236        for part in content {
237            match part {
238                UserPromptPart::Text(text) => {
239                    parts.push(json!({"text": text.text}));
240                }
241                UserPromptPart::File(file) => {
242                    parts.push(file_part(self.config, &file.data, &file.media_type, false)?);
243                }
244                #[allow(unreachable_patterns, reason = "UserPromptPart is non-exhaustive")]
245                _ => self
246                    .warnings
247                    .push(Warning::other("unknown user part ignored")),
248            }
249        }
250        self.push_content("user", parts);
251        Ok(())
252    }
253
254    #[allow(clippy::too_many_lines, reason = "one arm per assistant part kind")]
255    fn assistant_message(&mut self, content: &[AssistantPromptPart]) -> Result<(), ProviderError> {
256        let mut parts = Vec::new();
257        let mut has_signed_call = false;
258        let mut unsigned_calls: Vec<String> = Vec::new();
259        for part in content {
260            match part {
261                AssistantPromptPart::Text(text) => {
262                    if text.text.is_empty() {
263                        continue;
264                    }
265                    let options = read_part_options(self.config, text.provider_options.as_ref());
266                    parts.push(with_signature(
267                        json!({"text": text.text}),
268                        options.thought_signature.as_deref(),
269                    ));
270                }
271                AssistantPromptPart::Reasoning(reasoning) => {
272                    let options =
273                        read_part_options(self.config, reasoning.provider_options.as_ref());
274                    parts.push(with_signature(
275                        json!({"text": reasoning.text, "thought": true}),
276                        options.thought_signature.as_deref(),
277                    ));
278                }
279                AssistantPromptPart::ReasoningFile(file) => {
280                    parts.push(with_thought(file_part(
281                        self.config,
282                        &file.data,
283                        &file.media_type,
284                        true,
285                    )?));
286                }
287                AssistantPromptPart::File(file) => {
288                    let options = read_part_options(self.config, file.provider_options.as_ref());
289                    let converted = file_part(self.config, &file.data, &file.media_type, true)?;
290                    parts.push(if options.thought == Some(true) {
291                        with_thought(converted)
292                    } else {
293                        converted
294                    });
295                }
296                AssistantPromptPart::Custom(_) => {
297                    self.warnings.push(Warning::other(
298                        "custom assistant parts are not supported and were ignored",
299                    ));
300                }
301                AssistantPromptPart::ToolCall(call) => {
302                    let options = read_part_options(self.config, call.provider_options.as_ref());
303                    if let Some(tool_type) = &options.server_tool_type {
304                        let mut tool_call = JsonObject::new();
305                        tool_call
306                            .insert("toolType".to_owned(), JsonValue::from(tool_type.as_str()));
307                        tool_call.insert("args".to_owned(), call.input.clone());
308                        if let Some(id) = &options.server_tool_call_id {
309                            tool_call.insert("id".to_owned(), JsonValue::from(id.as_str()));
310                        }
311                        parts.push(with_signature(
312                            json!({"toolCall": tool_call}),
313                            options.thought_signature.as_deref(),
314                        ));
315                        continue;
316                    }
317                    let mut function_call = JsonObject::new();
318                    if !call.tool_call_id.as_str().is_empty() {
319                        function_call
320                            .insert("id".to_owned(), JsonValue::from(call.tool_call_id.as_str()));
321                    }
322                    function_call.insert(
323                        "name".to_owned(),
324                        JsonValue::from(
325                            self.mapping.to_provider_tool_name(call.tool_name.as_str()),
326                        ),
327                    );
328                    function_call.insert("args".to_owned(), call.input.clone());
329                    let signature = match &options.thought_signature {
330                        Some(signature) => {
331                            has_signed_call = true;
332                            Some(signature.clone())
333                        }
334                        None if self.capabilities.uses_gemini3_features && !has_signed_call => {
335                            unsigned_calls.push(call.tool_name.as_str().to_owned());
336                            Some(SKIP_THOUGHT_SIGNATURE.to_owned())
337                        }
338                        None => None,
339                    };
340                    parts.push(with_signature(
341                        json!({"functionCall": function_call}),
342                        signature.as_deref(),
343                    ));
344                }
345                AssistantPromptPart::ToolResult(result) => {
346                    let options = read_part_options(self.config, result.provider_options.as_ref());
347                    if let Some(tool_type) = &options.server_tool_type {
348                        let mut response = JsonObject::new();
349                        response.insert("toolType".to_owned(), JsonValue::from(tool_type.as_str()));
350                        if let Some(id) = &options.server_tool_call_id {
351                            response.insert("id".to_owned(), JsonValue::from(id.as_str()));
352                        }
353                        response.insert("response".to_owned(), tool_result_value(&result.output));
354                        parts.push(json!({"toolResponse": response}));
355                    }
356                }
357                #[allow(unreachable_patterns, reason = "AssistantPromptPart is non-exhaustive")]
358                _ => self
359                    .warnings
360                    .push(Warning::other("unknown assistant part ignored")),
361            }
362        }
363        if !unsigned_calls.is_empty() {
364            self.warnings.push(Warning::other(format!(
365                "thought signatures are missing for tool call(s) {}; \"{SKIP_THOUGHT_SIGNATURE}\" was sent instead, which may reduce response quality",
366                unsigned_calls.join(", ")
367            )));
368        }
369        self.push_content("model", parts);
370        Ok(())
371    }
372
373    fn tool_message(&mut self, content: &[ToolPromptPart]) -> Result<(), ProviderError> {
374        let mut parts = Vec::new();
375        for part in content {
376            match part {
377                ToolPromptPart::ToolApprovalResponse(_) => {}
378                ToolPromptPart::ToolResult(result) => {
379                    let options = read_part_options(self.config, result.provider_options.as_ref());
380                    if let Some(tool_type) = &options.server_tool_type {
381                        let mut response = JsonObject::new();
382                        response.insert("toolType".to_owned(), JsonValue::from(tool_type.as_str()));
383                        if let Some(id) = &options.server_tool_call_id {
384                            response.insert("id".to_owned(), JsonValue::from(id.as_str()));
385                        }
386                        response.insert("response".to_owned(), tool_result_value(&result.output));
387                        self.append_to_model(json!({"toolResponse": response}));
388                        continue;
389                    }
390                    self.function_result(result, &mut parts)?;
391                }
392                #[allow(unreachable_patterns, reason = "ToolPromptPart is non-exhaustive")]
393                _ => self
394                    .warnings
395                    .push(Warning::other("unknown tool part ignored")),
396            }
397        }
398        self.push_content("user", parts);
399        Ok(())
400    }
401
402    fn function_result(
403        &mut self,
404        result: &ToolResultPart,
405        parts: &mut Vec<JsonValue>,
406    ) -> Result<(), ProviderError> {
407        let name = self
408            .mapping
409            .to_provider_tool_name(result.tool_name.as_str())
410            .to_owned();
411        let id = Some(result.tool_call_id.as_str());
412        let ToolResultOutput::Content { value, .. } = &result.output else {
413            let response = json!({"name": name, "content": tool_result_value(&result.output)});
414            parts.push(json!({"functionResponse": function_response(id, &name, response)}));
415            return Ok(());
416        };
417        let mut texts: Vec<&str> = Vec::new();
418        let mut files: Vec<(String, String)> = Vec::new();
419        for part in value {
420            match part {
421                ToolResultContentPart::Text { text, .. } => texts.push(text),
422                ToolResultContentPart::File {
423                    data, media_type, ..
424                } => match data {
425                    FileData::Bytes { data } => files.push((
426                        full_media_type(media_type, Some(data.as_ref()))?,
427                        encode_base64(data.as_ref()),
428                    )),
429                    FileData::Url { url } if url.scheme() == "data" => {
430                        if let Some((mime, payload)) = parse_data_url(url) {
431                            files.push((mime, payload));
432                        }
433                    }
434                    FileData::Text { text } => {
435                        files.push(("text/plain".to_owned(), encode_base64(text.as_bytes())));
436                    }
437                    _ => self.warnings.push(Warning::unsupported_with_details(
438                        "tool result file",
439                        "tool result files must be provided as bytes or data URLs; the part was ignored",
440                    )),
441                },
442                ToolResultContentPart::Custom { .. } => {}
443                #[allow(unreachable_patterns, reason = "ToolResultContentPart is non-exhaustive")]
444                _ => {}
445            }
446        }
447        if self.capabilities.uses_gemini3_features {
448            let content = if texts.is_empty() {
449                "Tool executed successfully.".to_owned()
450            } else {
451                texts.join("\n")
452            };
453            let mut function_response =
454                function_response(id, &name, json!({"name": name, "content": content}));
455            if !files.is_empty() {
456                function_response.insert(
457                    "parts".to_owned(),
458                    JsonValue::Array(
459                        files
460                            .iter()
461                            .map(|(mime, data)| inline_data(mime, data))
462                            .collect(),
463                    ),
464                );
465            }
466            parts.push(json!({"functionResponse": function_response}));
467            return Ok(());
468        }
469        for text in texts {
470            parts.push(json!({"functionResponse": function_response(
471                id,
472                &name,
473                json!({"name": name, "content": text}),
474            )}));
475        }
476        for (mime, data) in files {
477            let kind = if mime.starts_with("image/") {
478                "image"
479            } else {
480                "file"
481            };
482            parts.push(inline_data(&mime, &data));
483            parts.push(json!({
484                "text": format!("Tool executed successfully and returned this {kind} as a response")
485            }));
486        }
487        Ok(())
488    }
489}
490
491fn message_options(message: &PromptMessage) -> Option<&ProviderOptions> {
492    message.provider_options()
493}
494
495/// Converts `prompt` to Gemini `contents` and `systemInstruction`.
496///
497/// # Errors
498///
499/// Returns [`ProviderError::UnsupportedFunctionality`] for system messages
500/// after the first non-system message, for assistant file URLs and for
501/// media types without a resolvable subtype, and
502/// [`ProviderError::NoSuchProviderReference`] for file references without a
503/// key for this provider.
504pub fn convert_prompt(
505    config: &GoogleConfig,
506    prompt: &[PromptMessage],
507    capabilities: ModelCapabilities,
508    mapping: &ToolNameMapping,
509) -> Result<ConvertedPrompt, ProviderError> {
510    let mut converter = Converter {
511        config,
512        capabilities,
513        mapping,
514        warnings: Vec::new(),
515        contents: Vec::new(),
516    };
517    let mut system_parts: Vec<JsonValue> = Vec::new();
518    let mut system_allowed = true;
519    for message in prompt {
520        let _ = message_options(message);
521        match message {
522            PromptMessage::System { content, .. } => {
523                if !system_allowed {
524                    return Err(UnsupportedFunctionalityError::new(
525                        "system messages are only supported at the beginning of the conversation",
526                    )
527                    .into());
528                }
529                system_parts.push(json!({"text": content}));
530            }
531            PromptMessage::User { content, .. } => {
532                system_allowed = false;
533                converter.user_message(content)?;
534            }
535            PromptMessage::Assistant { content, .. } => {
536                system_allowed = false;
537                converter.assistant_message(content)?;
538            }
539            PromptMessage::Tool { content, .. } => {
540                system_allowed = false;
541                converter.tool_message(content)?;
542            }
543            #[allow(unreachable_patterns, reason = "PromptMessage is non-exhaustive")]
544            _ => converter
545                .warnings
546                .push(Warning::other("unknown message role ignored")),
547        }
548    }
549    let mut system_instruction = None;
550    if !system_parts.is_empty() {
551        if capabilities.is_gemma {
552            prepend_system_text(&mut converter.contents, &system_parts);
553        } else {
554            let mut instruction = JsonObject::new();
555            instruction.insert("parts".to_owned(), JsonValue::Array(system_parts));
556            system_instruction = Some(instruction);
557        }
558    }
559    Ok(ConvertedPrompt {
560        system_instruction,
561        contents: converter.contents,
562        warnings: converter.warnings,
563    })
564}
565
566/// Gemma models reject `systemInstruction`: the system text is prepended to
567/// the first user text part instead.
568fn prepend_system_text(contents: &mut [JsonValue], system_parts: &[JsonValue]) {
569    let system_text = system_parts
570        .iter()
571        .filter_map(|part| part.get("text").and_then(JsonValue::as_str))
572        .collect::<Vec<_>>()
573        .join("\n\n");
574    let Some(first_user) = contents
575        .iter_mut()
576        .find(|content| content.get("role").and_then(JsonValue::as_str) == Some("user"))
577    else {
578        return;
579    };
580    let Some(JsonValue::Array(parts)) = first_user.get_mut("parts") else {
581        return;
582    };
583    match parts.first_mut().and_then(|part| part.get_mut("text")) {
584        Some(JsonValue::String(text)) => {
585            *text = format!("{system_text}\n\n{text}");
586        }
587        _ => parts.insert(0, json!({"text": system_text})),
588    }
589}
ferrin_google/convert_prompt.rs

ferrin_google/
convert_prompt.rs
