nexus_core/app/
transcribe.rs

1//! Image handling: encode clipboard images to PNG, attach them as markdown
2//! `![alt](filename.ext)` in message content, and describe images for
3//! non-vision models.
4
5// Casts here are on bounded values: token counts, byte sizes, and
6// selection indices — never on unbounded input. JSON-derived indices in
7// provider/tools go through try_from instead.
8#![allow(
9    clippy::cast_possible_truncation,
10    clippy::cast_possible_wrap,
11    clippy::cast_precision_loss,
12    clippy::cast_sign_loss
13)]
14use anyhow::{Context, Result};
15use base64::Engine;
16
17/// Encode raw RGBA pixels as PNG bytes.
18pub fn encode_png(width: usize, height: usize, rgba: &[u8]) -> Result<Vec<u8>> {
19    let mut bytes = Vec::new();
20    {
21        let mut enc = png::Encoder::new(&mut bytes, width as u32, height as u32);
22        enc.set_color(png::ColorType::Rgba);
23        enc.set_depth(png::BitDepth::Eight);
24        let mut writer = enc.write_header().context("png header")?;
25        writer.write_image_data(rgba).context("png data")?;
26    }
27    Ok(bytes)
28}
29
30/// `data:image/png;base64,…` URL for PNG bytes.
31pub fn png_bytes_data_url(bytes: &[u8]) -> String {
32    format!(
33        "data:image/png;base64,{}",
34        base64::engine::general_purpose::STANDARD.encode(bytes)
35    )
36}
37
38/// Encode raw RGBA pixels as a `data:image/png;base64,…` URL. Only exercised
39/// directly by tests now — production code goes through `encode_png` +
40/// `png_bytes_data_url` separately to avoid re-decoding the PNG it just wrote.
41#[cfg(test)]
42pub fn png_data_url(width: usize, height: usize, rgba: &[u8]) -> Result<String> {
43    let bytes = encode_png(width, height, rgba)?;
44    Ok(png_bytes_data_url(&bytes))
45}
46
47impl super::App {
48    /// Save a clipboard image to the space's images dir and return a markdown
49    /// snippet `![pasted image](filename.ext)` that can be inserted into the
50    /// composer text.
51    pub fn save_clipboard_image(
52        &mut self,
53        width: usize,
54        height: usize,
55        bytes: &[u8],
56    ) -> Option<String> {
57        let bytes = match encode_png(width, height, bytes) {
58            Ok(b) => b,
59            Err(e) => {
60                self.push_status(format!("could not encode image: {e}"));
61                return None;
62            }
63        };
64        let (dir, filename) = if self.incognito {
65            let d = self.incognito_img_dir.get_or_insert_with(|| {
66                let p =
67                    std::env::temp_dir().join(format!("nexus-incognito-{}", uuid::Uuid::new_v4()));
68                let _ = std::fs::create_dir_all(&p);
69                p
70            });
71            (d.clone(), format!("{}.png", uuid::Uuid::new_v4()))
72        } else {
73            let dir = self.space.files_dir(&self.active_space.name);
74            if let Err(e) = std::fs::create_dir_all(&dir) {
75                self.push_status(format!("could not create {}: {e}", dir.display()));
76                return None;
77            }
78            (dir, format!("{}.png", uuid::Uuid::new_v4()))
79        };
80        let path = dir.join(&filename);
81        if let Err(e) = std::fs::write(&path, &bytes) {
82            self.push_status(format!("could not write {}: {e}", path.display()));
83            return None;
84        }
85        if !self.incognito {
86            // Also save as a space file so the model can see/OCR/search it.
87            let files_dir = self.space.files_dir(&self.active_space.name);
88            if std::fs::create_dir_all(&files_dir).is_ok() {
89                let _ = std::fs::write(files_dir.join(&filename), &bytes);
90            }
91            self.rescan_files();
92        }
93        Some(format!("![pasted image]({filename})"))
94    }
95
96    /// Remove the incognito temp image directory if it exists.
97    pub fn cleanup_incognito_images(&mut self) {
98        if let Some(d) = self.incognito_img_dir.take() {
99            let _ = std::fs::remove_dir_all(&d);
100        }
101    }
102}
103
104#[cfg(test)]
105mod tests {
106    use super::*;
107
108    #[test]
109    fn encodes_rgba_as_png_data_url() {
110        use base64::Engine;
111        // 2x1 image: red pixel, transparent pixel.
112        let rgba = [255, 0, 0, 255, 0, 0, 0, 0];
113        let url = png_data_url(2, 1, &rgba).unwrap();
114        assert!(url.starts_with("data:image/png;base64,"));
115        // Round-trip: the payload decodes as a real 2x1 PNG.
116        let bytes = base64::engine::general_purpose::STANDARD
117            .decode(url.strip_prefix("data:image/png;base64,").unwrap())
118            .unwrap();
119        let decoder = png::Decoder::new(std::io::Cursor::new(bytes));
120        let reader = decoder.read_info().unwrap();
121        assert_eq!(reader.info().width, 2);
122        assert_eq!(reader.info().height, 1);
123    }
124
125    #[tokio::test]
126    async fn save_clipboard_returns_markdown_snippet() {
127        let mut a = crate::app::tests::app_with_key();
128        a.incognito = true; // skip rescan_files -> tokio::spawn
129        let rgba = vec![255, 0, 0, 255, 0, 0, 0, 0];
130        let result = a.save_clipboard_image(2, 1, &rgba);
131        assert!(result.is_some());
132        let md = result.unwrap();
133        assert!(md.starts_with("![pasted image]("));
134        assert!(md.ends_with(".png)"));
135    }
136}
nexus_core/app/transcribe.rs

nexus_core/app/
transcribe.rs
