eredu_core/
media.rs

1//! Portable decoded-media requests and chat-placeholder composition.
2
3use sha2::{Digest, Sha256};
4
5/// Failure while validating portable media or composing a chat request.
6#[derive(Debug, Clone, PartialEq, thiserror::Error)]
7pub enum MediaRequestError {
8    /// An input request must contain at least one segment.
9    #[error("multimodal input request is empty")]
10    EmptyRequest,
11    /// RGB dimensions or payload length are invalid.
12    #[error("RGB8 image shape {width}x{height} requires {expected} bytes, got {actual}")]
13    InvalidRgbImage {
14        /// Image width.
15        width: u32,
16        /// Image height.
17        height: u32,
18        /// Required packed RGB byte count.
19        expected: usize,
20        /// Supplied byte count.
21        actual: usize,
22    },
23    /// Audio must have a positive sample rate and finite samples.
24    #[error("decoded audio is invalid: {0}")]
25    InvalidAudio(String),
26    /// Video frame or timing metadata is invalid.
27    #[error("decoded video is invalid: {0}")]
28    InvalidVideo(String),
29    /// A chat media placeholder cannot be empty.
30    #[error("chat media binding {index} has an empty placeholder")]
31    EmptyPlaceholder {
32        /// Binding index.
33        index: usize,
34    },
35    /// Placeholder occurrences do not match supplied bindings.
36    #[error(
37        "rendered chat contains {actual} occurrence(s) of media placeholder {placeholder:?}, but {expected} binding(s) were supplied"
38    )]
39    PlaceholderCount {
40        /// Complete placeholder spelling.
41        placeholder: String,
42        /// Number of supplied bindings with that spelling.
43        expected: usize,
44        /// Number of occurrences in rendered text.
45        actual: usize,
46    },
47    /// Bindings were not ordered like their occurrences in the prompt.
48    #[error(
49        "chat media binding {index} placeholder {placeholder:?} does not occur after the preceding binding"
50    )]
51    PlaceholderOrder {
52        /// Binding index.
53        index: usize,
54        /// Complete placeholder spelling.
55        placeholder: String,
56    },
57}
58
59/// Packed decoded RGB8 image owned by a portable request.
60#[derive(Debug, Clone, PartialEq)]
61pub struct RgbImage {
62    pixels: Vec<u8>,
63    width: u32,
64    height: u32,
65}
66
67impl RgbImage {
68    /// Validates and owns one packed RGB8 image.
69    pub fn new(pixels: Vec<u8>, width: u32, height: u32) -> Result<Self, MediaRequestError> {
70        let expected = usize::try_from(width)
71            .ok()
72            .zip(usize::try_from(height).ok())
73            .and_then(|(width, height)| width.checked_mul(height))
74            .and_then(|pixels| pixels.checked_mul(3))
75            .unwrap_or(usize::MAX);
76        if width == 0 || height == 0 || pixels.len() != expected {
77            return Err(MediaRequestError::InvalidRgbImage {
78                width,
79                height,
80                expected,
81                actual: pixels.len(),
82            });
83        }
84        Ok(Self {
85            pixels,
86            width,
87            height,
88        })
89    }
90
91    /// Packed RGB channel bytes in row-major order.
92    pub fn pixels(&self) -> &[u8] {
93        &self.pixels
94    }
95
96    /// Image width in pixels.
97    pub const fn width(&self) -> u32 {
98        self.width
99    }
100
101    /// Image height in pixels.
102    pub const fn height(&self) -> u32 {
103        self.height
104    }
105}
106
107/// Decoded mono floating-point PCM audio.
108#[derive(Debug, Clone, PartialEq)]
109pub struct Audio {
110    samples: Vec<f32>,
111    sample_rate: u32,
112}
113
114impl Audio {
115    /// Validates and owns decoded mono PCM samples.
116    pub fn new(samples: Vec<f32>, sample_rate: u32) -> Result<Self, MediaRequestError> {
117        if sample_rate == 0 {
118            return Err(MediaRequestError::InvalidAudio(
119                "sample rate must be positive".into(),
120            ));
121        }
122        if samples.is_empty() {
123            return Err(MediaRequestError::InvalidAudio(
124                "waveform must contain at least one sample".into(),
125            ));
126        }
127        if samples.iter().any(|sample| !sample.is_finite()) {
128            return Err(MediaRequestError::InvalidAudio(
129                "waveform samples must be finite".into(),
130            ));
131        }
132        Ok(Self {
133            samples,
134            sample_rate,
135        })
136    }
137
138    /// Mono PCM samples.
139    pub fn samples(&self) -> &[f32] {
140        &self.samples
141    }
142
143    /// Sampling rate in hertz.
144    pub const fn sample_rate(&self) -> u32 {
145        self.sample_rate
146    }
147}
148
149/// Frame-selection policy for decoded video.
150#[derive(Debug, Clone, Copy, Default, PartialEq)]
151pub enum VideoSampling {
152    /// Uses the selected backend processor's default policy.
153    #[default]
154    ProcessorDefault,
155    /// Uniformly samples approximately this many frames per second.
156    Fps(f64),
157    /// Uniformly samples exactly this many frames, capped by source length.
158    FrameCount(usize),
159    /// Uses every decoded source frame.
160    All,
161}
162
163/// Decoded RGB8 video and portable sampling policy.
164#[derive(Debug, Clone, PartialEq)]
165pub struct Video {
166    frames: Vec<RgbImage>,
167    source_fps: Option<f64>,
168    sampling: VideoSampling,
169}
170
171impl Video {
172    /// Validates and owns decoded video frames.
173    pub fn new(
174        frames: Vec<RgbImage>,
175        source_fps: Option<f64>,
176        sampling: VideoSampling,
177    ) -> Result<Self, MediaRequestError> {
178        if frames.is_empty() {
179            return Err(MediaRequestError::InvalidVideo(
180                "video must contain at least one frame".into(),
181            ));
182        }
183        if source_fps.is_some_and(|fps| !fps.is_finite() || fps <= 0.0) {
184            return Err(MediaRequestError::InvalidVideo(
185                "source frame rate must be finite and positive".into(),
186            ));
187        }
188        match sampling {
189            VideoSampling::Fps(fps) if !fps.is_finite() || fps <= 0.0 => {
190                return Err(MediaRequestError::InvalidVideo(
191                    "sampling frame rate must be finite and positive".into(),
192                ));
193            }
194            VideoSampling::FrameCount(0) => {
195                return Err(MediaRequestError::InvalidVideo(
196                    "sampling frame count must be positive".into(),
197                ));
198            }
199            _ => {}
200        }
201        Ok(Self {
202            frames,
203            source_fps,
204            sampling,
205        })
206    }
207
208    /// Decoded frames in source order.
209    pub fn frames(&self) -> &[RgbImage] {
210        &self.frames
211    }
212
213    /// Source frame rate when known.
214    pub const fn source_fps(&self) -> Option<f64> {
215        self.source_fps
216    }
217
218    /// Requested frame-selection policy.
219    pub const fn sampling(&self) -> VideoSampling {
220        self.sampling
221    }
222}
223
224/// One decoded media item in a portable request.
225#[derive(Debug, Clone, PartialEq)]
226pub enum Media {
227    /// Decoded packed RGB8 image.
228    Image(RgbImage),
229    /// Decoded RGB8 video.
230    Video(Video),
231    /// Decoded mono PCM audio.
232    Audio(Audio),
233}
234
235/// One ordered portable model-input segment.
236#[derive(Debug, Clone, PartialEq)]
237pub enum MultimodalSegment {
238    /// Text to encode with [`MultimodalRequest::tokenize`].
239    Text(String),
240    /// Text already represented as tokenizer vocabulary IDs.
241    TokenIds(Vec<u32>),
242    /// Decoded media preprocessed by the selected backend.
243    Media(Media),
244}
245
246/// Validated ordered text and decoded-media input.
247#[derive(Debug, Clone, PartialEq)]
248pub struct MultimodalRequest {
249    segments: Vec<MultimodalSegment>,
250}
251
252impl MultimodalRequest {
253    /// Validates an ordered multimodal request.
254    pub fn new(segments: Vec<MultimodalSegment>) -> Result<Self, MediaRequestError> {
255        if segments.is_empty() {
256            return Err(MediaRequestError::EmptyRequest);
257        }
258        Ok(Self { segments })
259    }
260
261    /// Composes a rendered prompt with media bound to exact placeholder occurrences.
262    pub fn from_chat(
263        rendered_prompt: &str,
264        bindings: &[MediaBinding],
265    ) -> Result<Self, MediaRequestError> {
266        validate_bindings(rendered_prompt, bindings)?;
267        let mut segments = Vec::with_capacity(bindings.len().saturating_mul(2) + 1);
268        let mut cursor = 0;
269        for (index, binding) in bindings.iter().enumerate() {
270            let remainder = &rendered_prompt[cursor..];
271            let relative = remainder.find(binding.placeholder()).ok_or_else(|| {
272                MediaRequestError::PlaceholderOrder {
273                    index,
274                    placeholder: binding.placeholder().into(),
275                }
276            })?;
277            let start = cursor + relative;
278            if start > cursor {
279                segments.push(MultimodalSegment::Text(
280                    rendered_prompt[cursor..start].into(),
281                ));
282            }
283            segments.push(MultimodalSegment::Media(binding.media().clone()));
284            cursor = start + binding.placeholder().len();
285        }
286        if cursor < rendered_prompt.len() {
287            segments.push(MultimodalSegment::Text(rendered_prompt[cursor..].into()));
288        }
289        Self::new(segments)
290    }
291
292    /// Ordered request segments.
293    pub fn segments(&self) -> &[MultimodalSegment] {
294        &self.segments
295    }
296
297    /// Encodes text while preserving media order for backend preparation.
298    pub fn tokenize<E>(
299        &self,
300        mut encode: impl FnMut(&str) -> Result<Vec<u32>, E>,
301    ) -> Result<TokenizedMultimodalRequest, E> {
302        let mut segments = Vec::with_capacity(self.segments.len());
303        for segment in &self.segments {
304            segments.push(match segment {
305                MultimodalSegment::Text(text) => {
306                    TokenizedMultimodalSegment::TokenIds(encode(text)?)
307                }
308                MultimodalSegment::TokenIds(ids) => {
309                    TokenizedMultimodalSegment::TokenIds(ids.clone())
310                }
311                MultimodalSegment::Media(media) => TokenizedMultimodalSegment::Media(media.clone()),
312            });
313        }
314        Ok(TokenizedMultimodalRequest { segments })
315    }
316}
317
318/// One exact rendered-prompt placeholder bound to decoded media.
319#[derive(Debug, Clone, PartialEq)]
320pub struct MediaBinding {
321    placeholder: String,
322    media: Media,
323}
324
325impl MediaBinding {
326    /// Binds decoded media to one complete placeholder spelling.
327    pub fn new(placeholder: impl Into<String>, media: Media) -> Self {
328        Self {
329            placeholder: placeholder.into(),
330            media,
331        }
332    }
333
334    /// Complete placeholder spelling.
335    pub fn placeholder(&self) -> &str {
336        &self.placeholder
337    }
338
339    /// Bound decoded media.
340    pub const fn media(&self) -> &Media {
341        &self.media
342    }
343}
344
345/// One ordered segment after facade tokenization and before backend preprocessing.
346#[derive(Debug, Clone, PartialEq)]
347pub enum TokenizedMultimodalSegment {
348    /// Tokenizer vocabulary IDs.
349    TokenIds(Vec<u32>),
350    /// Decoded media.
351    Media(Media),
352}
353
354/// Backend preparation request with tokenizer work already complete.
355#[derive(Debug, Clone, PartialEq)]
356pub struct TokenizedMultimodalRequest {
357    segments: Vec<TokenizedMultimodalSegment>,
358}
359
360impl TokenizedMultimodalRequest {
361    /// Ordered token and media segments.
362    pub fn segments(&self) -> &[TokenizedMultimodalSegment] {
363        &self.segments
364    }
365
366    /// Derives a stable digest of the exact ordered tokens and decoded media.
367    pub fn semantic_content_fingerprint(&self) -> String {
368        let mut digest = Sha256::new();
369        digest.update(b"eredu-tokenized-multimodal-input-v1\0");
370        digest.update((self.segments.len() as u64).to_le_bytes());
371        for segment in &self.segments {
372            match segment {
373                TokenizedMultimodalSegment::TokenIds(tokens) => {
374                    digest.update([0]);
375                    digest.update((tokens.len() as u64).to_le_bytes());
376                    for token in tokens {
377                        digest.update(token.to_le_bytes());
378                    }
379                }
380                TokenizedMultimodalSegment::Media(media) => hash_media(&mut digest, media),
381            }
382        }
383        let bytes = digest.finalize();
384        let mut encoded = String::with_capacity(71);
385        encoded.push_str("sha256:");
386        for byte in bytes {
387            use std::fmt::Write as _;
388            write!(&mut encoded, "{byte:02x}").expect("writing to a string cannot fail");
389        }
390        encoded
391    }
392}
393
394fn hash_image(digest: &mut Sha256, image: &RgbImage) {
395    digest.update(image.width.to_le_bytes());
396    digest.update(image.height.to_le_bytes());
397    digest.update((image.pixels.len() as u64).to_le_bytes());
398    digest.update(&image.pixels);
399}
400
401fn hash_media(digest: &mut Sha256, media: &Media) {
402    match media {
403        Media::Image(image) => {
404            digest.update([1]);
405            hash_image(digest, image);
406        }
407        Media::Video(video) => {
408            digest.update([2]);
409            digest.update((video.frames.len() as u64).to_le_bytes());
410            match video.source_fps {
411                Some(fps) => {
412                    digest.update([1]);
413                    digest.update(fps.to_bits().to_le_bytes());
414                }
415                None => digest.update([0]),
416            }
417            match video.sampling {
418                VideoSampling::ProcessorDefault => digest.update([0]),
419                VideoSampling::Fps(fps) => {
420                    digest.update([1]);
421                    digest.update(fps.to_bits().to_le_bytes());
422                }
423                VideoSampling::FrameCount(count) => {
424                    digest.update([2]);
425                    digest.update((count as u64).to_le_bytes());
426                }
427                VideoSampling::All => digest.update([3]),
428            }
429            for frame in &video.frames {
430                hash_image(digest, frame);
431            }
432        }
433        Media::Audio(audio) => {
434            digest.update([3]);
435            digest.update(audio.sample_rate.to_le_bytes());
436            digest.update((audio.samples.len() as u64).to_le_bytes());
437            for sample in &audio.samples {
438                digest.update(sample.to_bits().to_le_bytes());
439            }
440        }
441    }
442}
443
444fn validate_bindings(
445    rendered_prompt: &str,
446    bindings: &[MediaBinding],
447) -> Result<(), MediaRequestError> {
448    for (index, binding) in bindings.iter().enumerate() {
449        if binding.placeholder.is_empty() {
450            return Err(MediaRequestError::EmptyPlaceholder { index });
451        }
452        if bindings[..index]
453            .iter()
454            .any(|earlier| earlier.placeholder == binding.placeholder)
455        {
456            continue;
457        }
458        let expected = bindings
459            .iter()
460            .filter(|candidate| candidate.placeholder == binding.placeholder)
461            .count();
462        let actual = rendered_prompt.matches(&binding.placeholder).count();
463        if actual != expected {
464            return Err(MediaRequestError::PlaceholderCount {
465                placeholder: binding.placeholder.clone(),
466                expected,
467                actual,
468            });
469        }
470    }
471    Ok(())
472}
473
474#[cfg(test)]
475mod tests {
476    use super::*;
477
478    fn image(value: u8) -> Media {
479        Media::Image(RgbImage::new(vec![value; 3], 1, 1).unwrap())
480    }
481
482    #[test]
483    fn chat_composition_and_tokenization_preserve_exact_order() {
484        let request = MultimodalRequest::from_chat(
485            "before<image>middle<image>after",
486            &[
487                MediaBinding::new("<image>", image(1)),
488                MediaBinding::new("<image>", image(2)),
489            ],
490        )
491        .unwrap();
492        let tokenized = request
493            .tokenize::<std::convert::Infallible>(|text| {
494                Ok(text
495                    .as_bytes()
496                    .iter()
497                    .map(|byte| u32::from(*byte))
498                    .collect())
499            })
500            .unwrap();
501        assert_eq!(tokenized.segments().len(), 5);
502        assert!(matches!(
503            &tokenized.segments()[0],
504            TokenizedMultimodalSegment::TokenIds(ids)
505                if ids == &[98, 101, 102, 111, 114, 101]
506        ));
507        assert!(matches!(
508            &tokenized.segments()[1],
509            TokenizedMultimodalSegment::Media(Media::Image(image)) if image.pixels() == [1, 1, 1]
510        ));
511        assert!(matches!(
512            &tokenized.segments()[3],
513            TokenizedMultimodalSegment::Media(Media::Image(image)) if image.pixels() == [2, 2, 2]
514        ));
515    }
516
517    #[test]
518    fn validation_rejects_bad_media_and_placeholder_contracts() {
519        assert!(matches!(
520            RgbImage::new(vec![0; 2], 1, 1),
521            Err(MediaRequestError::InvalidRgbImage { .. })
522        ));
523        assert!(Audio::new(vec![f32::NAN], 16_000).is_err());
524        assert!(Video::new(Vec::new(), None, VideoSampling::All).is_err());
525        assert!(matches!(
526            MultimodalRequest::from_chat(
527                "<image>",
528                &[
529                    MediaBinding::new("<image>", image(1)),
530                    MediaBinding::new("<image>", image(2)),
531                ],
532            ),
533            Err(MediaRequestError::PlaceholderCount { .. })
534        ));
535    }
536
537    #[test]
538    fn request_preserves_text_only_segments() {
539        let request = MultimodalRequest::new(vec![
540            MultimodalSegment::TokenIds(vec![1, 2]),
541            MultimodalSegment::Text("tail".into()),
542        ])
543        .unwrap();
544
545        assert_eq!(request.segments().len(), 2);
546        assert!(matches!(
547            &request.segments()[0],
548            MultimodalSegment::TokenIds(ids) if ids == &[1, 2]
549        ));
550    }
551
552    #[test]
553    fn semantic_fingerprint_is_stable_and_distinguishes_equal_geometry_payloads() {
554        fn request(image_value: u8, audio_value: f32) -> TokenizedMultimodalRequest {
555            MultimodalRequest::new(vec![
556                MultimodalSegment::TokenIds(vec![7, 11]),
557                MultimodalSegment::Media(Media::Image(
558                    RgbImage::new(vec![image_value; 12], 2, 2).unwrap(),
559                )),
560                MultimodalSegment::Media(Media::Audio(
561                    Audio::new(vec![audio_value; 4], 16_000).unwrap(),
562                )),
563            ])
564            .unwrap()
565            .tokenize::<std::convert::Infallible>(|_| unreachable!())
566            .unwrap()
567        }
568
569        let first = request(3, 0.25);
570        let same = request(3, 0.25);
571        let changed_image = request(4, 0.25);
572        let changed_audio = request(3, 0.5);
573
574        let fingerprint = first.semantic_content_fingerprint();
575        assert_eq!(fingerprint, same.semantic_content_fingerprint());
576        assert_eq!(fingerprint, first.semantic_content_fingerprint());
577        assert_ne!(fingerprint, changed_image.semantic_content_fingerprint());
578        assert_ne!(fingerprint, changed_audio.semantic_content_fingerprint());
579        assert_eq!(fingerprint.len(), 71);
580        assert!(fingerprint.starts_with("sha256:"));
581    }
582}
eredu_core/media.rs

eredu_core/
media.rs
