ferrin_google/
convert_prompt.rs

1//! Conversion of the specification prompt to `systemInstruction` and
2//! `contents`.
3//!
4//! Derived from the Vercel AI SDK (Apache-2.0, Copyright 2023 Vercel, Inc.),
5//! translated from TypeScript to Rust and modified; see `NOTICE`.
6
7use base64::Engine;
8use ferrin_provider_util::media_type::resolve_full_media_type;
9use ferrin_provider_util::tool_name_mapping::ToolNameMapping;
10use ferrin_spec::FileData;
11use ferrin_spec::JsonObject;
12use ferrin_spec::JsonValue;
13use ferrin_spec::MediaType;
14use ferrin_spec::ProviderOptions;
15use ferrin_spec::ProviderReference;
16use ferrin_spec::error::NoSuchProviderReferenceError;
17use ferrin_spec::error::ProviderError;
18use ferrin_spec::error::UnsupportedFunctionalityError;
19use ferrin_spec::language_model::PromptMessage;
20use ferrin_spec::language_model::prompt::AssistantPromptPart;
21use ferrin_spec::language_model::prompt::ToolPromptPart;
22use ferrin_spec::language_model::prompt::ToolResultContentPart;
23use ferrin_spec::language_model::prompt::ToolResultOutput;
24use ferrin_spec::language_model::prompt::ToolResultPart;
25use ferrin_spec::language_model::prompt::UserPromptPart;
26use ferrin_spec::shared::Warning;
27use serde_json::json;
28
29use crate::capabilities::ModelCapabilities;
30use crate::config::CANONICAL_OPTIONS_KEY;
31use crate::config::GoogleConfig;
32use crate::options::read_part_options;
33
34/// Signature sent for Gemini 3 tool calls that lack one.
35pub const SKIP_THOUGHT_SIGNATURE: &str = "skip_thought_signature_validator";
36
37/// Result of the prompt conversion.
38#[derive(Debug, Clone, Default, PartialEq)]
39pub struct ConvertedPrompt {
40    /// `systemInstruction` (`{parts: [{text}]}`), absent without system
41    /// messages or for Gemma models (whose system text is prepended to the
42    /// first user part).
43    pub system_instruction: Option<JsonObject>,
44    /// `contents`.
45    pub contents: Vec<JsonValue>,
46    /// Warnings collected during the conversion.
47    pub warnings: Vec<Warning>,
48}
49
50/// Resolves a file reference under the configured name, then `google`.
51///
52/// # Errors
53///
54/// Returns [`NoSuchProviderReferenceError`] when neither key is present.
55pub fn resolve_reference<'a>(
56    config: &GoogleConfig,
57    reference: &'a ProviderReference,
58) -> Result<&'a str, NoSuchProviderReferenceError> {
59    reference
60        .get(config.options_key())
61        .or_else(|| reference.get(CANONICAL_OPTIONS_KEY))
62        .map(String::as_str)
63        .ok_or_else(|| NoSuchProviderReferenceError::new(config.name.clone(), reference.clone()))
64}
65
66fn encode_base64(bytes: &[u8]) -> String {
67    base64::engine::general_purpose::STANDARD.encode(bytes)
68}
69
70fn inline_data(media_type: &str, data: &str) -> JsonValue {
71    json!({"inlineData": {"mimeType": media_type, "data": data}})
72}
73
74fn file_data(media_type: &str, uri: &str) -> JsonValue {
75    json!({"fileData": {"mimeType": media_type, "fileUri": uri}})
76}
77
78fn full_media_type(media_type: &MediaType, bytes: Option<&[u8]>) -> Result<String, ProviderError> {
79    Ok(resolve_full_media_type(media_type, bytes)?.into_string())
80}
81
82/// Converts a file to a `fileData` or `inlineData` part.
83fn file_part(
84    config: &GoogleConfig,
85    data: &FileData,
86    media_type: &MediaType,
87    in_assistant: bool,
88) -> Result<JsonValue, ProviderError> {
89    match data {
90        FileData::Url { url } => {
91            if in_assistant {
92                return Err(UnsupportedFunctionalityError::new(
93                    "File data URLs in assistant messages are not supported",
94                )
95                .into());
96            }
97            Ok(file_data(&full_media_type(media_type, None)?, url.as_str()))
98        }
99        FileData::Reference { reference } => Ok(file_data(
100            &full_media_type(media_type, None)?,
101            resolve_reference(config, reference)?,
102        )),
103        FileData::Bytes { data } => Ok(inline_data(
104            &full_media_type(media_type, Some(data.as_ref()))?,
105            &encode_base64(data.as_ref()),
106        )),
107        FileData::Text { text } => {
108            let media_type = if media_type.is_full() {
109                media_type.as_str().to_owned()
110            } else {
111                "text/plain".to_owned()
112            };
113            Ok(inline_data(&media_type, &encode_base64(text.as_bytes())))
114        }
115        #[allow(unreachable_patterns, reason = "FileData is non-exhaustive")]
116        _ => Err(UnsupportedFunctionalityError::new("file data variant").into()),
117    }
118}
119
120fn with_signature(mut part: JsonValue, signature: Option<&str>) -> JsonValue {
121    if let (Some(signature), Some(object)) = (signature, part.as_object_mut()) {
122        object.insert("thoughtSignature".to_owned(), JsonValue::from(signature));
123    }
124    part
125}
126
127fn with_thought(mut part: JsonValue) -> JsonValue {
128    if let Some(object) = part.as_object_mut() {
129        object.insert("thought".to_owned(), JsonValue::Bool(true));
130    }
131    part
132}
133
134fn tool_result_value(output: &ToolResultOutput) -> JsonValue {
135    match output {
136        ToolResultOutput::Text { value, .. } | ToolResultOutput::ErrorText { value, .. } => {
137            JsonValue::from(value.as_str())
138        }
139        ToolResultOutput::Json { value, .. } | ToolResultOutput::ErrorJson { value, .. } => {
140            value.clone()
141        }
142        ToolResultOutput::ExecutionDenied { reason, .. } => JsonValue::from(
143            reason
144                .clone()
145                .unwrap_or_else(|| "Tool call execution denied.".to_owned()),
146        ),
147        ToolResultOutput::Content { value, .. } => JsonValue::Array(
148            value
149                .iter()
150                .filter_map(|part| match part {
151                    ToolResultContentPart::Text { text, .. } => {
152                        Some(JsonValue::from(text.as_str()))
153                    }
154                    _ => None,
155                })
156                .collect(),
157        ),
158        #[allow(unreachable_patterns, reason = "ToolResultOutput is non-exhaustive")]
159        _ => JsonValue::Null,
160    }
161}
162
163fn function_response(id: Option<&str>, name: &str, response: JsonValue) -> JsonObject {
164    let mut function_response = JsonObject::new();
165    if let Some(id) = id.filter(|id| !id.is_empty()) {
166        function_response.insert("id".to_owned(), JsonValue::from(id));
167    }
168    function_response.insert("name".to_owned(), JsonValue::from(name));
169    function_response.insert("response".to_owned(), response);
170    function_response
171}
172
173/// Decodes a base64 `data:` URL into `(media type, base64 payload)`.
174fn parse_data_url(url: &url::Url) -> Option<(String, String)> {
175    let rest = url.as_str().strip_prefix("data:")?;
176    let (header, payload) = rest.split_once(',')?;
177    let mut segments = header.split(';');
178    let media_type = segments.next().unwrap_or_default().to_owned();
179    if segments.any(|segment| segment.eq_ignore_ascii_case("base64")) {
180        Some((media_type, payload.to_owned()))
181    } else {
182        let decoded = percent_decode(payload);
183        Some((media_type, encode_base64(decoded.as_bytes())))
184    }
185}
186
187fn percent_decode(text: &str) -> String {
188    let bytes = text.as_bytes();
189    let mut decoded = Vec::with_capacity(bytes.len());
190    let mut index = 0;
191    while index < bytes.len() {
192        if bytes[index] == b'%'
193            && let Some(hex) = text.get(index + 1..index + 3)
194            && let Ok(byte) = u8::from_str_radix(hex, 16)
195        {
196            decoded.push(byte);
197            index += 3;
198        } else {
199            decoded.push(bytes[index]);
200            index += 1;
201        }
202    }
203    String::from_utf8_lossy(&decoded).into_owned()
204}
205
206struct Converter<'a> {
207    config: &'a GoogleConfig,
208    capabilities: ModelCapabilities,
209    mapping: &'a ToolNameMapping,
210    warnings: Vec<Warning>,
211    contents: Vec<JsonValue>,
212}
213
214impl Converter<'_> {
215    fn push_content(&mut self, role: &str, parts: Vec<JsonValue>) {
216        if !parts.is_empty() {
217            self.contents.push(json!({"role": role, "parts": parts}));
218        }
219    }
220
221    /// Appends `part` to the last model content, or opens a new one.
222    fn append_to_model(&mut self, part: JsonValue) {
223        if let Some(JsonValue::Object(last)) = self.contents.last_mut()
224            && last.get("role").and_then(JsonValue::as_str) == Some("model")
225            && let Some(JsonValue::Array(parts)) = last.get_mut("parts")
226        {
227            parts.push(part);
228            return;
229        }
230        self.contents
231            .push(json!({"role": "model", "parts": [part]}));
232    }
233
234    fn user_message(&mut self, content: &[UserPromptPart]) -> Result<(), ProviderError> {
235        let mut parts = Vec::new();
236        for part in content {
237            match part {
238                UserPromptPart::Text(text) => {
239                    parts.push(json!({"text": text.text}));
240                }
241                UserPromptPart::File(file) => {
242                    parts.push(file_part(self.config, &file.data, &file.media_type, false)?);
243                }
244                #[allow(unreachable_patterns, reason = "UserPromptPart is non-exhaustive")]
245                _ => self
246                    .warnings
247                    .push(Warning::other("unknown user part ignored")),
248            }
249        }
250        self.push_content("user", parts);
251        Ok(())
252    }
253
254    #[allow(clippy::too_many_lines, reason = "one arm per assistant part kind")]
255    fn assistant_message(&mut self, content: &[AssistantPromptPart]) -> Result<(), ProviderError> {
256        let mut parts = Vec::new();
257        let mut has_signed_call = false;
258        let mut unsigned_calls: Vec<String> = Vec::new();
259        for part in content {
260            match part {
261                AssistantPromptPart::Text(text) => {
262                    if text.text.is_empty() {
263                        continue;
264                    }
265                    let options = read_part_options(self.config, text.provider_options.as_ref());
266                    parts.push(with_signature(
267                        json!({"text": text.text}),
268                        options.thought_signature.as_deref(),
269                    ));
270                }
271                AssistantPromptPart::Reasoning(reasoning) => {
272                    let options =
273                        read_part_options(self.config, reasoning.provider_options.as_ref());
274                    parts.push(with_signature(
275                        json!({"text": reasoning.text, "thought": true}),
276                        options.thought_signature.as_deref(),
277                    ));
278                }
279                AssistantPromptPart::ReasoningFile(file) => {
280                    let options = read_part_options(self.config, file.provider_options.as_ref());
281                    parts.push(with_signature(
282                        with_thought(file_part(self.config, &file.data, &file.media_type, true)?),
283                        options.thought_signature.as_deref(),
284                    ));
285                }
286                AssistantPromptPart::File(file) => {
287                    let options = read_part_options(self.config, file.provider_options.as_ref());
288                    let converted = file_part(self.config, &file.data, &file.media_type, true)?;
289                    let converted = if options.thought == Some(true) {
290                        with_thought(converted)
291                    } else {
292                        converted
293                    };
294                    parts.push(with_signature(
295                        converted,
296                        options.thought_signature.as_deref(),
297                    ));
298                }
299                AssistantPromptPart::Custom(_) => {
300                    self.warnings.push(Warning::other(
301                        "custom assistant parts are not supported and were ignored",
302                    ));
303                }
304                AssistantPromptPart::ToolCall(call) => {
305                    let options = read_part_options(self.config, call.provider_options.as_ref());
306                    if options.server_tool_type.as_deref() == Some("code_execution") {
307                        parts.push(with_signature(
308                            json!({"executableCode": call.input}),
309                            options.thought_signature.as_deref(),
310                        ));
311                        continue;
312                    }
313                    if let Some(tool_type) = &options.server_tool_type {
314                        let mut tool_call = JsonObject::new();
315                        tool_call
316                            .insert("toolType".to_owned(), JsonValue::from(tool_type.as_str()));
317                        tool_call.insert("args".to_owned(), call.input.clone());
318                        if let Some(id) = &options.server_tool_call_id {
319                            tool_call.insert("id".to_owned(), JsonValue::from(id.as_str()));
320                        }
321                        parts.push(with_signature(
322                            json!({"toolCall": tool_call}),
323                            options.thought_signature.as_deref(),
324                        ));
325                        continue;
326                    }
327                    let mut function_call = JsonObject::new();
328                    if !call.tool_call_id.as_str().is_empty() {
329                        function_call
330                            .insert("id".to_owned(), JsonValue::from(call.tool_call_id.as_str()));
331                    }
332                    function_call.insert(
333                        "name".to_owned(),
334                        JsonValue::from(
335                            self.mapping.to_provider_tool_name(call.tool_name.as_str()),
336                        ),
337                    );
338                    function_call.insert("args".to_owned(), call.input.clone());
339                    let signature = match &options.thought_signature {
340                        Some(signature) => {
341                            has_signed_call = true;
342                            Some(signature.clone())
343                        }
344                        None if self.capabilities.uses_gemini3_features && !has_signed_call => {
345                            unsigned_calls.push(call.tool_name.as_str().to_owned());
346                            Some(SKIP_THOUGHT_SIGNATURE.to_owned())
347                        }
348                        None => None,
349                    };
350                    parts.push(with_signature(
351                        json!({"functionCall": function_call}),
352                        signature.as_deref(),
353                    ));
354                }
355                AssistantPromptPart::ToolResult(result) => {
356                    let options = read_part_options(self.config, result.provider_options.as_ref());
357                    if options.server_tool_type.as_deref() == Some("code_execution") {
358                        parts.push(with_signature(
359                            json!({"codeExecutionResult": tool_result_value(&result.output)}),
360                            options.thought_signature.as_deref(),
361                        ));
362                        continue;
363                    }
364                    if let Some(tool_type) = &options.server_tool_type {
365                        let mut response = JsonObject::new();
366                        response.insert("toolType".to_owned(), JsonValue::from(tool_type.as_str()));
367                        if let Some(id) = &options.server_tool_call_id {
368                            response.insert("id".to_owned(), JsonValue::from(id.as_str()));
369                        }
370                        response.insert("response".to_owned(), tool_result_value(&result.output));
371                        parts.push(json!({"toolResponse": response}));
372                    }
373                }
374                #[allow(unreachable_patterns, reason = "AssistantPromptPart is non-exhaustive")]
375                _ => self
376                    .warnings
377                    .push(Warning::other("unknown assistant part ignored")),
378            }
379        }
380        if !unsigned_calls.is_empty() {
381            self.warnings.push(Warning::other(format!(
382                "thought signatures are missing for tool call(s) {}; \"{SKIP_THOUGHT_SIGNATURE}\" was sent instead, which may reduce response quality",
383                unsigned_calls.join(", ")
384            )));
385        }
386        self.push_content("model", parts);
387        Ok(())
388    }
389
390    fn tool_message(&mut self, content: &[ToolPromptPart]) -> Result<(), ProviderError> {
391        let mut parts = Vec::new();
392        for part in content {
393            match part {
394                ToolPromptPart::ToolApprovalResponse(_) => {}
395                ToolPromptPart::ToolResult(result) => {
396                    let options = read_part_options(self.config, result.provider_options.as_ref());
397                    if options.server_tool_type.as_deref() == Some("code_execution") {
398                        self.append_to_model(with_signature(
399                            json!({"codeExecutionResult": tool_result_value(&result.output)}),
400                            options.thought_signature.as_deref(),
401                        ));
402                        continue;
403                    }
404                    if let Some(tool_type) = &options.server_tool_type {
405                        let mut response = JsonObject::new();
406                        response.insert("toolType".to_owned(), JsonValue::from(tool_type.as_str()));
407                        if let Some(id) = &options.server_tool_call_id {
408                            response.insert("id".to_owned(), JsonValue::from(id.as_str()));
409                        }
410                        response.insert("response".to_owned(), tool_result_value(&result.output));
411                        self.append_to_model(json!({"toolResponse": response}));
412                        continue;
413                    }
414                    self.function_result(result, &mut parts)?;
415                }
416                #[allow(unreachable_patterns, reason = "ToolPromptPart is non-exhaustive")]
417                _ => self
418                    .warnings
419                    .push(Warning::other("unknown tool part ignored")),
420            }
421        }
422        self.push_content("user", parts);
423        Ok(())
424    }
425
426    fn function_result(
427        &mut self,
428        result: &ToolResultPart,
429        parts: &mut Vec<JsonValue>,
430    ) -> Result<(), ProviderError> {
431        let name = self
432            .mapping
433            .to_provider_tool_name(result.tool_name.as_str())
434            .to_owned();
435        let id = Some(result.tool_call_id.as_str());
436        let ToolResultOutput::Content { value, .. } = &result.output else {
437            let response = json!({"name": name, "content": tool_result_value(&result.output)});
438            parts.push(json!({"functionResponse": function_response(id, &name, response)}));
439            return Ok(());
440        };
441        let mut texts: Vec<&str> = Vec::new();
442        let mut files: Vec<(String, String)> = Vec::new();
443        for part in value {
444            match part {
445                ToolResultContentPart::Text { text, .. } => texts.push(text),
446                ToolResultContentPart::File {
447                    data, media_type, ..
448                } => match data {
449                    FileData::Bytes { data } => files.push((
450                        full_media_type(media_type, Some(data.as_ref()))?,
451                        encode_base64(data.as_ref()),
452                    )),
453                    FileData::Url { url } if url.scheme() == "data" => {
454                        if let Some((mime, payload)) = parse_data_url(url) {
455                            files.push((mime, payload));
456                        }
457                    }
458                    FileData::Text { text } => {
459                        files.push(("text/plain".to_owned(), encode_base64(text.as_bytes())));
460                    }
461                    _ => self.warnings.push(Warning::unsupported_with_details(
462                        "tool result file",
463                        "tool result files must be provided as bytes or data URLs; the part was ignored",
464                    )),
465                },
466                ToolResultContentPart::Custom { .. } => {}
467                #[allow(unreachable_patterns, reason = "ToolResultContentPart is non-exhaustive")]
468                _ => {}
469            }
470        }
471        if self.capabilities.uses_gemini3_features {
472            let content = if texts.is_empty() {
473                "Tool executed successfully.".to_owned()
474            } else {
475                texts.join("\n")
476            };
477            let mut function_response =
478                function_response(id, &name, json!({"name": name, "content": content}));
479            if !files.is_empty() {
480                function_response.insert(
481                    "parts".to_owned(),
482                    JsonValue::Array(
483                        files
484                            .iter()
485                            .map(|(mime, data)| inline_data(mime, data))
486                            .collect(),
487                    ),
488                );
489            }
490            parts.push(json!({"functionResponse": function_response}));
491            return Ok(());
492        }
493        for text in texts {
494            parts.push(json!({"functionResponse": function_response(
495                id,
496                &name,
497                json!({"name": name, "content": text}),
498            )}));
499        }
500        for (mime, data) in files {
501            let kind = if mime.starts_with("image/") {
502                "image"
503            } else {
504                "file"
505            };
506            parts.push(inline_data(&mime, &data));
507            parts.push(json!({
508                "text": format!("Tool executed successfully and returned this {kind} as a response")
509            }));
510        }
511        Ok(())
512    }
513}
514
515fn message_options(message: &PromptMessage) -> Option<&ProviderOptions> {
516    message.provider_options()
517}
518
519/// Converts `prompt` to Gemini `contents` and `systemInstruction`.
520///
521/// # Errors
522///
523/// Returns [`ProviderError::UnsupportedFunctionality`] for system messages
524/// after the first non-system message, for assistant file URLs and for
525/// media types without a resolvable subtype, and
526/// [`ProviderError::NoSuchProviderReference`] for file references without a
527/// key for this provider.
528pub fn convert_prompt(
529    config: &GoogleConfig,
530    prompt: &[PromptMessage],
531    capabilities: ModelCapabilities,
532    mapping: &ToolNameMapping,
533) -> Result<ConvertedPrompt, ProviderError> {
534    let mut converter = Converter {
535        config,
536        capabilities,
537        mapping,
538        warnings: Vec::new(),
539        contents: Vec::new(),
540    };
541    let mut system_parts: Vec<JsonValue> = Vec::new();
542    let mut system_allowed = true;
543    for message in prompt {
544        let _ = message_options(message);
545        match message {
546            PromptMessage::System { content, .. } => {
547                if !system_allowed {
548                    return Err(UnsupportedFunctionalityError::new(
549                        "system messages are only supported at the beginning of the conversation",
550                    )
551                    .into());
552                }
553                system_parts.push(json!({"text": content}));
554            }
555            PromptMessage::User { content, .. } => {
556                system_allowed = false;
557                converter.user_message(content)?;
558            }
559            PromptMessage::Assistant { content, .. } => {
560                system_allowed = false;
561                converter.assistant_message(content)?;
562            }
563            PromptMessage::Tool { content, .. } => {
564                system_allowed = false;
565                converter.tool_message(content)?;
566            }
567            #[allow(unreachable_patterns, reason = "PromptMessage is non-exhaustive")]
568            _ => converter
569                .warnings
570                .push(Warning::other("unknown message role ignored")),
571        }
572    }
573    let mut system_instruction = None;
574    if !system_parts.is_empty() {
575        if capabilities.is_gemma {
576            prepend_system_text(&mut converter.contents, &system_parts);
577        } else {
578            let mut instruction = JsonObject::new();
579            instruction.insert("parts".to_owned(), JsonValue::Array(system_parts));
580            system_instruction = Some(instruction);
581        }
582    }
583    Ok(ConvertedPrompt {
584        system_instruction,
585        contents: converter.contents,
586        warnings: converter.warnings,
587    })
588}
589
590/// Gemma models reject `systemInstruction`: the system text is prepended to
591/// the first user text part instead.
592fn prepend_system_text(contents: &mut [JsonValue], system_parts: &[JsonValue]) {
593    let system_text = system_parts
594        .iter()
595        .filter_map(|part| part.get("text").and_then(JsonValue::as_str))
596        .collect::<Vec<_>>()
597        .join("\n\n");
598    let Some(first_user) = contents
599        .iter_mut()
600        .find(|content| content.get("role").and_then(JsonValue::as_str) == Some("user"))
601    else {
602        return;
603    };
604    let Some(JsonValue::Array(parts)) = first_user.get_mut("parts") else {
605        return;
606    };
607    match parts.first_mut().and_then(|part| part.get_mut("text")) {
608        Some(JsonValue::String(text)) => {
609            *text = format!("{system_text}\n\n{text}");
610        }
611        _ => parts.insert(0, json!({"text": system_text})),
612    }
613}
ferrin_google/convert_prompt.rs

ferrin_google/
convert_prompt.rs
