1use std::path::Path;
8
9use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
10use serde::{Deserialize, Serialize};
11
12use super::message::{Content, ContentPart, FileRef, ImageUrl, InputAudio, Message, Role};
13use super::{FilePdfConfig, FilePluginConfig, Plugin};
14use crate::error::{Error, Result};
15
16#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
22#[serde(rename_all = "lowercase")]
23pub enum ImageDetail {
24 Low,
26 High,
28 Auto,
30}
31
32impl ImageDetail {
33 fn as_str(self) -> &'static str {
34 match self {
35 ImageDetail::Low => "low",
36 ImageDetail::High => "high",
37 ImageDetail::Auto => "auto",
38 }
39 }
40}
41
42pub fn create_user_message_with_image(text: impl Into<String>, url: impl Into<String>) -> Message {
44 user_message_with_parts(
45 text,
46 vec![ContentPart::ImageUrl {
47 image_url: ImageUrl {
48 url: url.into(),
49 detail: None,
50 },
51 }],
52 )
53}
54
55pub fn create_user_message_with_images<I, S>(text: impl Into<String>, urls: I) -> Message
57where
58 I: IntoIterator<Item = S>,
59 S: Into<String>,
60{
61 let extras = urls
62 .into_iter()
63 .map(|u| ContentPart::ImageUrl {
64 image_url: ImageUrl {
65 url: u.into(),
66 detail: None,
67 },
68 })
69 .collect();
70 user_message_with_parts(text, extras)
71}
72
73pub fn create_user_message_with_image_detail(
75 text: impl Into<String>,
76 url: impl Into<String>,
77 detail: ImageDetail,
78) -> Message {
79 user_message_with_parts(
80 text,
81 vec![ContentPart::ImageUrl {
82 image_url: ImageUrl {
83 url: url.into(),
84 detail: Some(detail.as_str().to_string()),
85 },
86 }],
87 )
88}
89
90pub fn create_user_message_with_base64_image(
93 text: impl Into<String>,
94 path: impl AsRef<Path>,
95) -> Result<Message> {
96 let data_url = encode_image_to_base64(path)?;
97 Ok(user_message_with_parts(
98 text,
99 vec![ContentPart::ImageUrl {
100 image_url: ImageUrl {
101 url: data_url,
102 detail: None,
103 },
104 }],
105 ))
106}
107
108pub fn create_user_message_with_base64_image_bytes(
111 text: impl Into<String>,
112 bytes: &[u8],
113 mime: &str,
114) -> Message {
115 let data_url = encode_image_bytes_to_base64(bytes, mime);
116 user_message_with_parts(
117 text,
118 vec![ContentPart::ImageUrl {
119 image_url: ImageUrl {
120 url: data_url,
121 detail: None,
122 },
123 }],
124 )
125}
126
127pub fn encode_image_to_base64(path: impl AsRef<Path>) -> Result<String> {
129 let path = path.as_ref();
130 let mime = image_mime_from_path(path)?;
131 let bytes =
132 std::fs::read(path).map_err(|_| Error::InvalidInput("failed to read image file"))?;
133 Ok(encode_image_bytes_to_base64(&bytes, mime))
134}
135
136pub fn encode_image_bytes_to_base64(bytes: &[u8], mime: &str) -> String {
138 format!("data:{};base64,{}", mime, BASE64_STANDARD.encode(bytes))
139}
140
141fn image_mime_from_path(path: &Path) -> Result<&'static str> {
142 let ext = path
143 .extension()
144 .and_then(|s| s.to_str())
145 .map(|s| s.to_ascii_lowercase());
146 match ext.as_deref() {
147 Some("png") => Ok("image/png"),
148 Some("jpg" | "jpeg") => Ok("image/jpeg"),
149 Some("webp") => Ok("image/webp"),
150 Some("gif") => Ok("image/gif"),
151 _ => Err(Error::InvalidInput("unsupported image format")),
152 }
153}
154
155#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
161pub enum FileParserEngine {
162 PdfText,
164 MistralOcr,
166 Native,
168 Auto,
170}
171
172impl FileParserEngine {
173 fn as_str(self) -> Option<&'static str> {
174 match self {
175 FileParserEngine::PdfText => Some("pdf-text"),
176 FileParserEngine::MistralOcr => Some("mistral-ocr"),
177 FileParserEngine::Native => Some("native"),
178 FileParserEngine::Auto => None,
179 }
180 }
181}
182
183#[derive(Clone, Debug, PartialEq)]
191pub struct File {
192 pub filename: String,
194 pub file_data: String,
197}
198
199impl File {
200 pub fn from_url(filename: impl Into<String>, url: impl Into<String>) -> Self {
203 Self {
204 filename: filename.into(),
205 file_data: url.into(),
206 }
207 }
208
209 pub fn from_data_url(filename: impl Into<String>, data_url: impl Into<String>) -> Self {
211 Self {
212 filename: filename.into(),
213 file_data: data_url.into(),
214 }
215 }
216
217 pub fn from_pdf_path(filename: impl Into<String>, path: impl AsRef<Path>) -> Result<Self> {
219 let bytes =
220 std::fs::read(path).map_err(|_| Error::InvalidInput("failed to read pdf file"))?;
221 let data_url = format!(
222 "data:application/pdf;base64,{}",
223 BASE64_STANDARD.encode(&bytes)
224 );
225 Ok(Self::from_data_url(filename, data_url))
226 }
227
228 fn into_part(self) -> ContentPart {
229 ContentPart::File {
230 file: FileRef {
231 filename: Some(self.filename),
232 file_data: Some(self.file_data),
233 file_url: None,
234 },
235 }
236 }
237}
238
239pub fn create_user_message_with_pdf(
241 text: impl Into<String>,
242 url: impl Into<String>,
243 filename: impl Into<String>,
244) -> Message {
245 user_message_with_parts(text, vec![File::from_url(filename, url).into_part()])
246}
247
248pub fn create_user_message_with_base64_pdf(
250 text: impl Into<String>,
251 path: impl AsRef<Path>,
252 filename: impl Into<String>,
253) -> Result<Message> {
254 let file = File::from_pdf_path(filename, path)?;
255 Ok(user_message_with_parts(text, vec![file.into_part()]))
256}
257
258pub fn create_user_message_with_files(text: impl Into<String>, files: Vec<File>) -> Message {
260 let parts = files.into_iter().map(File::into_part).collect();
261 user_message_with_parts(text, parts)
262}
263
264pub fn create_file_parser_plugin(engine: FileParserEngine) -> Plugin {
268 let pdf = engine.as_str().map(|e| FilePdfConfig {
269 engine: Some(e.to_string()),
270 });
271 Plugin::File(FilePluginConfig { pdf })
272}
273
274#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
280pub enum AudioFormat {
281 Wav,
283 Mp3,
285}
286
287impl AudioFormat {
288 fn as_str(self) -> &'static str {
289 match self {
290 AudioFormat::Wav => "wav",
291 AudioFormat::Mp3 => "mp3",
292 }
293 }
294
295 fn from_path(path: &Path) -> Result<Self> {
296 let ext = path
297 .extension()
298 .and_then(|s| s.to_str())
299 .map(|s| s.to_ascii_lowercase());
300 match ext.as_deref() {
301 Some("wav") => Ok(AudioFormat::Wav),
302 Some("mp3") => Ok(AudioFormat::Mp3),
303 _ => Err(Error::InvalidInput("unsupported audio format")),
304 }
305 }
306}
307
308pub fn create_user_message_with_audio(
311 text: impl Into<String>,
312 path: impl AsRef<Path>,
313) -> Result<Message> {
314 let path = path.as_ref();
315 let format = AudioFormat::from_path(path)?;
316 let bytes =
317 std::fs::read(path).map_err(|_| Error::InvalidInput("failed to read audio file"))?;
318 Ok(create_user_message_with_audio_bytes(text, &bytes, format))
319}
320
321pub fn create_user_message_with_audio_bytes(
323 text: impl Into<String>,
324 bytes: &[u8],
325 format: AudioFormat,
326) -> Message {
327 user_message_with_parts(
328 text,
329 vec![ContentPart::InputAudio {
330 input_audio: InputAudio {
331 data: BASE64_STANDARD.encode(bytes),
332 format: format.as_str().to_string(),
333 },
334 }],
335 )
336}
337
338const ALLOWED_TEXT_EXTENSIONS: &[&str] = &[
343 "txt", "md", "json", "yaml", "yml", "toml", "csv", "log", "xml", "html", "css", "js", "ts",
344 "py", "rs", "go", "java", "c", "cpp", "h", "sh",
345];
346
347fn check_text_extension(path: &Path) -> Result<()> {
348 let ext = path
349 .extension()
350 .and_then(|s| s.to_str())
351 .map(|s| s.to_ascii_lowercase());
352 match ext {
353 Some(e) if ALLOWED_TEXT_EXTENSIONS.contains(&e.as_str()) => Ok(()),
354 _ => Err(Error::InvalidInput("unsupported text file extension")),
355 }
356}
357
358fn format_text_file(filename: &str, content: &str) -> String {
359 format!("--- filename: {filename} ---\n{content}")
360}
361
362pub fn create_user_message_with_text_file(
364 text: impl Into<String>,
365 path: impl AsRef<Path>,
366) -> Result<Message> {
367 let path = path.as_ref();
368 check_text_extension(path)?;
369 let content = std::fs::read_to_string(path)
370 .map_err(|_| Error::InvalidInput("failed to read text file (non-UTF-8?)"))?;
371 let filename = path
372 .file_name()
373 .and_then(|s| s.to_str())
374 .unwrap_or("file")
375 .to_string();
376 Ok(create_user_message_with_text_content(
377 text, content, filename,
378 ))
379}
380
381pub fn create_user_message_with_text_files(
383 text: impl Into<String>,
384 paths: &[impl AsRef<Path>],
385) -> Result<Message> {
386 let mut combined = String::new();
387 for path in paths {
388 let path = path.as_ref();
389 check_text_extension(path)?;
390 let content = std::fs::read_to_string(path)
391 .map_err(|_| Error::InvalidInput("failed to read text file (non-UTF-8?)"))?;
392 let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("file");
393 if !combined.is_empty() {
394 combined.push_str("\n\n");
395 }
396 combined.push_str(&format_text_file(filename, &content));
397 }
398 let text = text.into();
399 let body = if text.is_empty() {
400 combined
401 } else {
402 format!("{text}\n\n{combined}")
403 };
404 Ok(Message::user(body))
405}
406
407pub fn create_user_message_with_text_content(
409 text: impl Into<String>,
410 content: impl Into<String>,
411 filename: impl Into<String>,
412) -> Message {
413 let text = text.into();
414 let formatted = format_text_file(&filename.into(), &content.into());
415 let body = if text.is_empty() {
416 formatted
417 } else {
418 format!("{text}\n\n{formatted}")
419 };
420 Message::user(body)
421}
422
423#[derive(Clone, Debug, Default)]
441pub struct ContentBuilder {
442 parts: Vec<ContentPart>,
443}
444
445impl ContentBuilder {
446 pub fn new() -> Self {
448 Self::default()
449 }
450
451 pub fn add_text(mut self, text: impl Into<String>) -> Self {
453 self.parts.push(ContentPart::Text { text: text.into() });
454 self
455 }
456
457 pub fn add_image(mut self, url: impl Into<String>) -> Self {
459 self.parts.push(ContentPart::ImageUrl {
460 image_url: ImageUrl {
461 url: url.into(),
462 detail: None,
463 },
464 });
465 self
466 }
467
468 pub fn add_image_with_detail(mut self, url: impl Into<String>, detail: ImageDetail) -> Self {
470 self.parts.push(ContentPart::ImageUrl {
471 image_url: ImageUrl {
472 url: url.into(),
473 detail: Some(detail.as_str().to_string()),
474 },
475 });
476 self
477 }
478
479 pub fn add_base64_image(mut self, path: impl AsRef<Path>) -> Result<Self> {
481 let data_url = encode_image_to_base64(path)?;
482 self.parts.push(ContentPart::ImageUrl {
483 image_url: ImageUrl {
484 url: data_url,
485 detail: None,
486 },
487 });
488 Ok(self)
489 }
490
491 pub fn add_pdf(mut self, url: impl Into<String>, filename: impl Into<String>) -> Self {
493 self.parts.push(File::from_url(filename, url).into_part());
494 self
495 }
496
497 pub fn add_base64_pdf(
499 mut self,
500 path: impl AsRef<Path>,
501 filename: impl Into<String>,
502 ) -> Result<Self> {
503 self.parts
504 .push(File::from_pdf_path(filename, path)?.into_part());
505 Ok(self)
506 }
507
508 pub fn add_audio(mut self, path: impl AsRef<Path>) -> Result<Self> {
510 let path = path.as_ref();
511 let format = AudioFormat::from_path(path)?;
512 let bytes =
513 std::fs::read(path).map_err(|_| Error::InvalidInput("failed to read audio file"))?;
514 self.parts.push(ContentPart::InputAudio {
515 input_audio: InputAudio {
516 data: BASE64_STANDARD.encode(&bytes),
517 format: format.as_str().to_string(),
518 },
519 });
520 Ok(self)
521 }
522
523 pub fn add_text_file(mut self, path: impl AsRef<Path>) -> Result<Self> {
525 let path = path.as_ref();
526 check_text_extension(path)?;
527 let content = std::fs::read_to_string(path)
528 .map_err(|_| Error::InvalidInput("failed to read text file (non-UTF-8?)"))?;
529 let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("file");
530 self.parts.push(ContentPart::Text {
531 text: format_text_file(filename, &content),
532 });
533 Ok(self)
534 }
535
536 pub fn add_part(mut self, part: ContentPart) -> Self {
538 self.parts.push(part);
539 self
540 }
541
542 pub fn build_message(self, role: Role) -> Message {
544 Message {
545 role,
546 content: Content::Parts(self.parts),
547 name: None,
548 tool_calls: None,
549 tool_call_id: None,
550 reasoning: None,
551 annotations: None,
552 }
553 }
554
555 pub fn build_parts(self) -> Vec<ContentPart> {
557 self.parts
558 }
559}
560
561fn user_message_with_parts(text: impl Into<String>, extras: Vec<ContentPart>) -> Message {
566 let text = text.into();
567 let mut parts: Vec<ContentPart> = Vec::with_capacity(1 + extras.len());
568 if !text.is_empty() {
569 parts.push(ContentPart::Text { text });
570 }
571 parts.extend(extras);
572 Message {
573 role: Role::User,
574 content: Content::Parts(parts),
575 name: None,
576 tool_calls: None,
577 tool_call_id: None,
578 reasoning: None,
579 annotations: None,
580 }
581}
582
583#[cfg(test)]
584mod tests {
585 use super::*;
586 use crate::types::{Annotation, FileAnnotation};
587 use pretty_assertions::assert_eq;
588 use serde_json::json;
589
590 #[test]
591 fn image_url_message_shape() {
592 let m = create_user_message_with_image("look", "https://x/y.png");
593 let v = serde_json::to_value(&m).unwrap();
594 assert_eq!(
595 v,
596 json!({
597 "role": "user",
598 "content": [
599 {"type": "text", "text": "look"},
600 {"type": "image_url", "image_url": {"url": "https://x/y.png"}},
601 ]
602 })
603 );
604 }
605
606 #[test]
607 fn image_detail_serializes_lowercase() {
608 let m = create_user_message_with_image_detail("x", "https://x/y.png", ImageDetail::High);
609 let v = serde_json::to_value(&m).unwrap();
610 let parts = v["content"].as_array().unwrap();
611 assert_eq!(parts[1]["image_url"]["detail"], json!("high"));
612 }
613
614 #[test]
615 fn multiple_images_attach_all() {
616 let m = create_user_message_with_images("look", ["a", "b", "c"]);
617 if let Content::Parts(parts) = &m.content {
618 assert_eq!(parts.len(), 4); } else {
620 panic!("expected parts");
621 }
622 }
623
624 #[test]
625 fn encode_image_bytes_produces_data_url() {
626 let url = encode_image_bytes_to_base64(&[0x89, 0x50, 0x4e, 0x47], "image/png");
627 assert!(url.starts_with("data:image/png;base64,"));
628 assert!(url.ends_with("iVBORw=="));
629 }
630
631 #[test]
632 fn unsupported_image_extension_rejected() {
633 let err = encode_image_to_base64("/tmp/foo.bmp").unwrap_err();
634 match err {
635 Error::InvalidInput(m) => assert_eq!(m, "unsupported image format"),
636 _ => panic!("expected InvalidInput"),
637 }
638 }
639
640 #[test]
641 fn pdf_url_message_shape() {
642 let m = create_user_message_with_pdf("summarize", "https://x/p.pdf", "p.pdf");
643 let v = serde_json::to_value(&m).unwrap();
644 assert_eq!(
645 v,
646 json!({
647 "role": "user",
648 "content": [
649 {"type": "text", "text": "summarize"},
650 {"type": "file", "file": {"filename": "p.pdf", "file_data": "https://x/p.pdf"}},
651 ]
652 })
653 );
654 }
655
656 #[test]
657 fn file_parser_plugin_pdf_text() {
658 let plugin = create_file_parser_plugin(FileParserEngine::PdfText);
659 let v = serde_json::to_value(&plugin).unwrap();
660 assert_eq!(
661 v,
662 json!({"id": "file-parser", "pdf": {"engine": "pdf-text"}})
663 );
664 }
665
666 #[test]
667 fn file_parser_plugin_auto_omits_engine() {
668 let plugin = create_file_parser_plugin(FileParserEngine::Auto);
669 let v = serde_json::to_value(&plugin).unwrap();
670 assert_eq!(v, json!({"id": "file-parser"}));
671 }
672
673 #[test]
674 fn file_parser_plugin_mistral_ocr() {
675 let plugin = create_file_parser_plugin(FileParserEngine::MistralOcr);
676 let v = serde_json::to_value(&plugin).unwrap();
677 assert_eq!(v["pdf"]["engine"], json!("mistral-ocr"));
678 }
679
680 #[test]
681 fn file_annotation_roundtrip() {
682 let ann = Annotation::File {
683 file: FileAnnotation {
684 filename: "p.pdf".into(),
685 file_data: "data:application/pdf;base64,AAA=".into(),
686 },
687 };
688 let v = serde_json::to_value(&ann).unwrap();
689 assert_eq!(
690 v,
691 json!({"type": "file", "file": {"filename": "p.pdf", "file_data": "data:application/pdf;base64,AAA="}})
692 );
693 let back: Annotation = serde_json::from_value(v).unwrap();
694 assert_eq!(back, ann);
695 }
696
697 #[test]
698 fn url_citation_annotation_still_works() {
699 let v = json!({"type": "url_citation", "url_citation": {"url": "https://x"}});
700 let a: Annotation = serde_json::from_value(v).unwrap();
701 match a {
702 Annotation::UrlCitation { url_citation } => assert_eq!(url_citation.url, "https://x"),
703 _ => panic!("expected UrlCitation"),
704 }
705 }
706
707 #[test]
708 fn audio_bytes_message_shape() {
709 let m = create_user_message_with_audio_bytes("transcribe", &[1, 2, 3], AudioFormat::Wav);
710 let v = serde_json::to_value(&m).unwrap();
711 let parts = v["content"].as_array().unwrap();
712 assert_eq!(parts[1]["type"], json!("input_audio"));
713 assert_eq!(parts[1]["input_audio"]["format"], json!("wav"));
714 assert_eq!(parts[1]["input_audio"]["data"], json!("AQID"));
715 }
716
717 #[test]
718 fn text_content_helper_no_io() {
719 let m = create_user_message_with_text_content("ctx", "hello", "g.txt");
720 let body = m.content_text().unwrap();
721 assert!(body.starts_with("ctx\n\n--- filename: g.txt ---\n"));
722 assert!(body.ends_with("hello"));
723 }
724
725 #[test]
726 fn text_content_helper_empty_prefix() {
727 let m = create_user_message_with_text_content("", "hello", "g.txt");
728 assert_eq!(m.content_text().unwrap(), "--- filename: g.txt ---\nhello");
729 }
730
731 #[test]
732 fn text_extension_whitelist_rejects_exe() {
733 let err = create_user_message_with_text_file("x", "/tmp/foo.exe").unwrap_err();
734 match err {
735 Error::InvalidInput(m) => assert_eq!(m, "unsupported text file extension"),
736 _ => panic!("expected InvalidInput"),
737 }
738 }
739
740 #[test]
741 fn content_builder_chains() {
742 let m = ContentBuilder::new()
743 .add_text("look at")
744 .add_image("https://x/y.png")
745 .add_image_with_detail("https://x/z.png", ImageDetail::Low)
746 .build_message(Role::User);
747 let v = serde_json::to_value(&m).unwrap();
748 let parts = v["content"].as_array().unwrap();
749 assert_eq!(parts.len(), 3);
750 assert_eq!(parts[0]["type"], json!("text"));
751 assert_eq!(parts[1]["type"], json!("image_url"));
752 assert_eq!(parts[2]["image_url"]["detail"], json!("low"));
753 }
754
755 #[test]
756 fn content_builder_pdf_url() {
757 let parts = ContentBuilder::new()
758 .add_text("summarize")
759 .add_pdf("https://x/p.pdf", "p.pdf")
760 .build_parts();
761 assert_eq!(parts.len(), 2);
762 match &parts[1] {
763 ContentPart::File { file } => {
764 assert_eq!(file.filename.as_deref(), Some("p.pdf"));
765 assert_eq!(file.file_data.as_deref(), Some("https://x/p.pdf"));
766 assert!(file.file_url.is_none());
767 }
768 _ => panic!("expected file part"),
769 }
770 }
771
772 #[test]
773 fn multi_files_helper() {
774 let m = create_user_message_with_files(
775 "compare",
776 vec![
777 File::from_url("a.pdf", "https://x/a.pdf"),
778 File::from_url("b.pdf", "https://x/b.pdf"),
779 ],
780 );
781 if let Content::Parts(parts) = &m.content {
782 assert_eq!(parts.len(), 3); } else {
784 panic!("expected parts");
785 }
786 }
787}