llama_cpp_bindings/mtmd/
mtmd_bitmap.rs

1use std::ffi::{CStr, CString, c_char};
2use std::path::PathBuf;
3use std::ptr::NonNull;
4use std::slice;
5
6use crate::ffi_error_reader::read_and_free_cpp_error;
7
8use super::mtmd_bitmap_error::MtmdBitmapError;
9use super::mtmd_context::MtmdContext;
10
11fn cstr_ptr_to_optional_string(ptr: *const c_char) -> Option<String> {
12    if ptr.is_null() {
13        None
14    } else {
15        let id = unsafe { CStr::from_ptr(ptr) }
16            .to_string_lossy()
17            .into_owned();
18
19        Some(id)
20    }
21}
22
23/// Safe wrapper around `mtmd_bitmap`.
24///
25/// Represents bitmap data for images or audio that can be processed
26/// by the multimodal system. For images, data is stored in RGB format.
27/// For audio, data is stored as PCM F32 samples.
28#[derive(Debug, Clone)]
29pub struct MtmdBitmap {
30    /// Raw pointer to the underlying `mtmd_bitmap`.
31    pub bitmap: NonNull<llama_cpp_bindings_sys::mtmd_bitmap>,
32}
33
34unsafe impl Send for MtmdBitmap {}
35unsafe impl Sync for MtmdBitmap {}
36
37impl MtmdBitmap {
38    /// Create a bitmap from image data in RGB format.
39    ///
40    /// # Errors
41    ///
42    /// * `InvalidDataSize` - Data length doesn't match `nx * ny * 3`
43    /// * `NullResult` - Underlying C function returned null
44    ///
45    /// # Examples
46    ///
47    /// ```
48    /// use llama_cpp_bindings::mtmd::MtmdBitmap;
49    ///
50    /// // Create a 2x2 red image
51    /// let red_pixel = [255, 0, 0]; // RGB values for red
52    /// let image_data = red_pixel.repeat(4); // 2x2 = 4 pixels
53    ///
54    /// let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data);
55    /// assert!(bitmap.is_ok());
56    /// ```
57    pub fn from_image_data(nx: u32, ny: u32, data: &[u8]) -> Result<Self, MtmdBitmapError> {
58        if nx < 2 || ny < 2 {
59            return Err(MtmdBitmapError::ImageDimensionsTooSmall(nx, ny));
60        }
61
62        if data.len() != (nx * ny * 3) as usize {
63            return Err(MtmdBitmapError::InvalidDataSize);
64        }
65
66        let bitmap = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_init(nx, ny, data.as_ptr()) };
67
68        let bitmap = NonNull::new(bitmap).ok_or(MtmdBitmapError::BitmapDecodeFailed)?;
69
70        Ok(Self { bitmap })
71    }
72
73    /// Create a bitmap from audio data in PCM F32 format.
74    ///
75    /// # Errors
76    ///
77    /// * `NullResult` - Underlying C function returned null
78    ///
79    /// # Examples
80    ///
81    /// ```
82    /// use llama_cpp_bindings::mtmd::MtmdBitmap;
83    ///
84    /// // Create a simple sine wave audio sample
85    /// let audio_data: Vec<f32> = (0..100)
86    ///     .map(|sample_index| (sample_index as f32 * 0.1).sin())
87    ///     .collect();
88    ///
89    /// let bitmap = MtmdBitmap::from_audio_data(&audio_data);
90    /// // Note: This will likely fail without proper MTMD context setup
91    /// ```
92    pub fn from_audio_data(data: &[f32]) -> Result<Self, MtmdBitmapError> {
93        let bitmap = unsafe {
94            llama_cpp_bindings_sys::mtmd_bitmap_init_from_audio(data.len(), data.as_ptr())
95        };
96
97        let bitmap = NonNull::new(bitmap).ok_or(MtmdBitmapError::BitmapDecodeFailed)?;
98
99        Ok(Self { bitmap })
100    }
101
102    /// Create a bitmap from a file.
103    ///
104    /// Supported formats:
105    /// - Images: formats supported by `stb_image` (jpg, png, bmp, gif, etc.)
106    /// - Audio: formats supported by miniaudio (wav, mp3, flac)
107    ///
108    /// # Errors
109    ///
110    /// Returns an [`MtmdBitmapError`] variant matching the wrapper's status code.
111    pub fn from_file(ctx: &MtmdContext, path: &str) -> Result<Self, MtmdBitmapError> {
112        let path_cstr = CString::new(path)?;
113        let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = std::ptr::null_mut();
114        let mut out_error: *mut c_char = std::ptr::null_mut();
115
116        let status = unsafe {
117            llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_file(
118                ctx.context.as_ptr(),
119                path_cstr.as_ptr(),
120                &raw mut out_bitmap,
121                &raw mut out_error,
122            )
123        };
124
125        match status {
126            llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_OK => {
127                let bitmap = NonNull::new(out_bitmap).ok_or_else(|| {
128                    MtmdBitmapError::FileUnreadable {
129                        path: PathBuf::from(path),
130                    }
131                })?;
132                Ok(Self { bitmap })
133            }
134            llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_VENDORED_RETURNED_NULL => {
135                Err(MtmdBitmapError::FileUnreadable {
136                    path: PathBuf::from(path),
137                })
138            }
139            llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_ERROR_STRING_ALLOCATION_FAILED => {
140                Err(MtmdBitmapError::NotEnoughMemory)
141            }
142            llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_VENDORED_THREW_CXX_EXCEPTION => {
143                let message = unsafe { read_and_free_cpp_error(out_error) };
144                Err(MtmdBitmapError::Reported { message })
145            }
146            other => unreachable!(
147                "llama_rs_mtmd_bitmap_init_from_file returned unrecognized status: {other}"
148            ),
149        }
150    }
151
152    /// Create a bitmap from a buffer containing file data.
153    ///
154    /// Supported formats:
155    /// - Images: formats supported by `stb_image` (jpg, png, bmp, gif, etc.)
156    /// - Audio: formats supported by miniaudio (wav, mp3, flac)
157    ///
158    /// # Errors
159    ///
160    /// * `NullResult` - Buffer could not be processed
161    pub fn from_buffer(ctx: &MtmdContext, data: &[u8]) -> Result<Self, MtmdBitmapError> {
162        let bitmap = unsafe {
163            llama_cpp_bindings_sys::mtmd_helper_bitmap_init_from_buf(
164                ctx.context.as_ptr(),
165                data.as_ptr(),
166                data.len(),
167            )
168        };
169
170        let bitmap = NonNull::new(bitmap).ok_or(MtmdBitmapError::BitmapDecodeFailed)?;
171
172        Ok(Self { bitmap })
173    }
174
175    /// Get bitmap width in pixels.
176    #[must_use]
177    pub fn nx(&self) -> u32 {
178        unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_nx(self.bitmap.as_ptr()) }
179    }
180
181    /// Get bitmap height in pixels.
182    #[must_use]
183    pub fn ny(&self) -> u32 {
184        unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_ny(self.bitmap.as_ptr()) }
185    }
186
187    /// Get bitmap data as a byte slice.
188    ///
189    /// For images: RGB format with length `nx * ny * 3`
190    /// For audio: PCM F32 format with length `n_samples * 4`
191    #[must_use]
192    pub fn data(&self) -> &[u8] {
193        let ptr = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_data(self.bitmap.as_ptr()) };
194        let len = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_n_bytes(self.bitmap.as_ptr()) };
195        unsafe { slice::from_raw_parts(ptr, len) }
196    }
197
198    /// Check if this bitmap contains audio data (vs image data).
199    #[must_use]
200    pub fn is_audio(&self) -> bool {
201        unsafe { llama_cpp_bindings_sys::mtmd_bitmap_is_audio(self.bitmap.as_ptr()) }
202    }
203
204    /// Get the bitmap's optional ID string.
205    #[must_use]
206    pub fn id(&self) -> Option<String> {
207        let ptr = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_id(self.bitmap.as_ptr()) };
208
209        cstr_ptr_to_optional_string(ptr)
210    }
211
212    /// Set the bitmap's ID string.
213    ///
214    /// # Errors
215    ///
216    /// Returns an error if the ID string contains null bytes.
217    ///
218    /// # Examples
219    ///
220    /// ```no_run
221    /// # use llama_cpp_bindings::mtmd::MtmdBitmap;
222    /// # fn example(bitmap: &MtmdBitmap) -> Result<(), Box<dyn std::error::Error>> {
223    /// bitmap.set_id("image_001")?;
224    /// assert_eq!(bitmap.id(), Some("image_001".to_string()));
225    /// # Ok(())
226    /// # }
227    /// ```
228    pub fn set_id(&self, id: &str) -> Result<(), std::ffi::NulError> {
229        let id_cstr = CString::new(id)?;
230        unsafe {
231            llama_cpp_bindings_sys::mtmd_bitmap_set_id(self.bitmap.as_ptr(), id_cstr.as_ptr());
232        }
233
234        Ok(())
235    }
236}
237
238impl Drop for MtmdBitmap {
239    fn drop(&mut self) {
240        unsafe { llama_cpp_bindings_sys::mtmd_bitmap_free(self.bitmap.as_ptr()) }
241    }
242}
243
244#[cfg(test)]
245mod tests {
246    use super::MtmdBitmap;
247    use super::MtmdBitmapError;
248
249    #[test]
250    fn cstr_ptr_to_optional_string_returns_none_for_null() {
251        assert!(super::cstr_ptr_to_optional_string(std::ptr::null()).is_none());
252    }
253
254    #[test]
255    fn cstr_ptr_to_optional_string_returns_some_for_valid() {
256        let cstr = std::ffi::CString::new("hello").unwrap();
257        let result = super::cstr_ptr_to_optional_string(cstr.as_ptr());
258
259        assert_eq!(result, Some("hello".to_string()));
260    }
261
262    #[test]
263    fn from_image_data_creates_valid_bitmap() {
264        let red_pixel: [u8; 3] = [255, 0, 0];
265        let image_data: Vec<u8> = red_pixel.repeat(4);
266        let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap();
267        assert_eq!(bitmap.nx(), 2);
268        assert_eq!(bitmap.ny(), 2);
269        assert!(!bitmap.is_audio());
270    }
271
272    #[test]
273    fn invalid_data_size_returns_error() {
274        let too_short = vec![0u8; 5];
275        let result = MtmdBitmap::from_image_data(2, 2, &too_short);
276        assert!(result.is_err());
277    }
278
279    #[test]
280    fn from_image_data_rejects_dimensions_below_minimum() {
281        let result_1x1 = MtmdBitmap::from_image_data(1, 1, &[0u8; 3]);
282        let result_1x2 = MtmdBitmap::from_image_data(1, 2, &[0u8; 6]);
283        let result_2x1 = MtmdBitmap::from_image_data(2, 1, &[0u8; 6]);
284        let result_0x0 = MtmdBitmap::from_image_data(0, 0, &[]);
285
286        assert!(matches!(
287            result_1x1,
288            Err(MtmdBitmapError::ImageDimensionsTooSmall(1, 1))
289        ));
290        assert!(matches!(
291            result_1x2,
292            Err(MtmdBitmapError::ImageDimensionsTooSmall(1, 2))
293        ));
294        assert!(matches!(
295            result_2x1,
296            Err(MtmdBitmapError::ImageDimensionsTooSmall(2, 1))
297        ));
298        assert!(matches!(
299            result_0x0,
300            Err(MtmdBitmapError::ImageDimensionsTooSmall(0, 0))
301        ));
302    }
303
304    #[test]
305    fn set_id_changes_id() {
306        let image_data = vec![0u8; 12];
307        let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap();
308        bitmap.set_id("test_image").unwrap();
309
310        assert_eq!(bitmap.id().as_deref(), Some("test_image"));
311    }
312
313    #[test]
314    fn from_audio_data_creates_valid_bitmap() {
315        #[expect(
316            clippy::cast_precision_loss,
317            reason = "test fixture casts a small i32 (0..100) to f32 to synthesise a sine wave; \
318                      the values are well within f32's exact-representation range"
319        )]
320        let audio_samples: Vec<f32> = (0..100).map(|index| (index as f32 * 0.1).sin()).collect();
321        let bitmap = MtmdBitmap::from_audio_data(&audio_samples).unwrap();
322
323        assert!(bitmap.is_audio());
324    }
325
326    #[test]
327    fn data_returns_expected_bytes_for_image() {
328        let pixel_data: Vec<u8> = vec![255, 0, 0, 0, 255, 0, 0, 0, 255, 128, 128, 128];
329        let bitmap = MtmdBitmap::from_image_data(2, 2, &pixel_data).unwrap();
330        let returned_data = bitmap.data();
331
332        assert_eq!(returned_data, &pixel_data);
333    }
334
335    #[test]
336    fn id_returns_some_by_default() {
337        let image_data = vec![0u8; 12];
338        let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap();
339
340        assert!(bitmap.id().is_some());
341    }
342
343    #[test]
344    fn id_returns_custom_value_after_set() {
345        let image_data = vec![0u8; 12];
346        let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap();
347        bitmap.set_id("my_image").unwrap();
348
349        assert_eq!(bitmap.id(), Some("my_image".to_string()));
350    }
351
352    #[test]
353    fn set_id_with_null_byte_returns_error() {
354        let image_data = vec![0u8; 12];
355        let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap();
356        let result = bitmap.set_id("id\0null");
357
358        assert!(result.is_err());
359    }
360}
llama_cpp_bindings/mtmd/mtmd_bitmap.rs

llama_cpp_bindings/mtmd/
mtmd_bitmap.rs
