eredu_core/
media.rs

1//! Portable decoded-media requests and chat-placeholder composition.
2
3/// Failure while validating portable media or composing a chat request.
4#[derive(Debug, Clone, PartialEq, thiserror::Error)]
5pub enum MediaRequestError {
6    /// An input request must contain at least one segment.
7    #[error("multimodal input request is empty")]
8    EmptyRequest,
9    /// Multimodal preparation is meaningful only when media is present.
10    #[error("multimodal input request contains no media")]
11    MissingMedia,
12    /// RGB dimensions or payload length are invalid.
13    #[error("RGB8 image shape {width}x{height} requires {expected} bytes, got {actual}")]
14    InvalidRgbImage {
15        /// Image width.
16        width: u32,
17        /// Image height.
18        height: u32,
19        /// Required packed RGB byte count.
20        expected: usize,
21        /// Supplied byte count.
22        actual: usize,
23    },
24    /// Audio must have a positive sample rate and finite samples.
25    #[error("decoded audio is invalid: {0}")]
26    InvalidAudio(String),
27    /// Video frame or timing metadata is invalid.
28    #[error("decoded video is invalid: {0}")]
29    InvalidVideo(String),
30    /// A chat media placeholder cannot be empty.
31    #[error("chat media binding {index} has an empty placeholder")]
32    EmptyPlaceholder {
33        /// Binding index.
34        index: usize,
35    },
36    /// Placeholder occurrences do not match supplied bindings.
37    #[error(
38        "rendered chat contains {actual} occurrence(s) of media placeholder {placeholder:?}, but {expected} binding(s) were supplied"
39    )]
40    PlaceholderCount {
41        /// Complete placeholder spelling.
42        placeholder: String,
43        /// Number of supplied bindings with that spelling.
44        expected: usize,
45        /// Number of occurrences in rendered text.
46        actual: usize,
47    },
48    /// Bindings were not ordered like their occurrences in the prompt.
49    #[error(
50        "chat media binding {index} placeholder {placeholder:?} does not occur after the preceding binding"
51    )]
52    PlaceholderOrder {
53        /// Binding index.
54        index: usize,
55        /// Complete placeholder spelling.
56        placeholder: String,
57    },
58}
59
60/// Packed decoded RGB8 image owned by a portable request.
61#[derive(Debug, Clone, PartialEq)]
62pub struct RgbImage {
63    pixels: Vec<u8>,
64    width: u32,
65    height: u32,
66}
67
68impl RgbImage {
69    /// Validates and owns one packed RGB8 image.
70    pub fn new(pixels: Vec<u8>, width: u32, height: u32) -> Result<Self, MediaRequestError> {
71        let expected = usize::try_from(width)
72            .ok()
73            .zip(usize::try_from(height).ok())
74            .and_then(|(width, height)| width.checked_mul(height))
75            .and_then(|pixels| pixels.checked_mul(3))
76            .unwrap_or(usize::MAX);
77        if width == 0 || height == 0 || pixels.len() != expected {
78            return Err(MediaRequestError::InvalidRgbImage {
79                width,
80                height,
81                expected,
82                actual: pixels.len(),
83            });
84        }
85        Ok(Self {
86            pixels,
87            width,
88            height,
89        })
90    }
91
92    /// Packed RGB channel bytes in row-major order.
93    pub fn pixels(&self) -> &[u8] {
94        &self.pixels
95    }
96
97    /// Image width in pixels.
98    pub const fn width(&self) -> u32 {
99        self.width
100    }
101
102    /// Image height in pixels.
103    pub const fn height(&self) -> u32 {
104        self.height
105    }
106}
107
108/// Decoded mono floating-point PCM audio.
109#[derive(Debug, Clone, PartialEq)]
110pub struct Audio {
111    samples: Vec<f32>,
112    sample_rate: u32,
113}
114
115impl Audio {
116    /// Validates and owns decoded mono PCM samples.
117    pub fn new(samples: Vec<f32>, sample_rate: u32) -> Result<Self, MediaRequestError> {
118        if sample_rate == 0 {
119            return Err(MediaRequestError::InvalidAudio(
120                "sample rate must be positive".into(),
121            ));
122        }
123        if samples.is_empty() {
124            return Err(MediaRequestError::InvalidAudio(
125                "waveform must contain at least one sample".into(),
126            ));
127        }
128        if samples.iter().any(|sample| !sample.is_finite()) {
129            return Err(MediaRequestError::InvalidAudio(
130                "waveform samples must be finite".into(),
131            ));
132        }
133        Ok(Self {
134            samples,
135            sample_rate,
136        })
137    }
138
139    /// Mono PCM samples.
140    pub fn samples(&self) -> &[f32] {
141        &self.samples
142    }
143
144    /// Sampling rate in hertz.
145    pub const fn sample_rate(&self) -> u32 {
146        self.sample_rate
147    }
148}
149
150/// Frame-selection policy for decoded video.
151#[derive(Debug, Clone, Copy, Default, PartialEq)]
152pub enum VideoSampling {
153    /// Uses the selected backend processor's default policy.
154    #[default]
155    ProcessorDefault,
156    /// Uniformly samples approximately this many frames per second.
157    Fps(f64),
158    /// Uniformly samples exactly this many frames, capped by source length.
159    FrameCount(usize),
160    /// Uses every decoded source frame.
161    All,
162}
163
164/// Decoded RGB8 video and portable sampling policy.
165#[derive(Debug, Clone, PartialEq)]
166pub struct Video {
167    frames: Vec<RgbImage>,
168    source_fps: Option<f64>,
169    sampling: VideoSampling,
170}
171
172impl Video {
173    /// Validates and owns decoded video frames.
174    pub fn new(
175        frames: Vec<RgbImage>,
176        source_fps: Option<f64>,
177        sampling: VideoSampling,
178    ) -> Result<Self, MediaRequestError> {
179        if frames.is_empty() {
180            return Err(MediaRequestError::InvalidVideo(
181                "video must contain at least one frame".into(),
182            ));
183        }
184        if source_fps.is_some_and(|fps| !fps.is_finite() || fps <= 0.0) {
185            return Err(MediaRequestError::InvalidVideo(
186                "source frame rate must be finite and positive".into(),
187            ));
188        }
189        match sampling {
190            VideoSampling::Fps(fps) if !fps.is_finite() || fps <= 0.0 => {
191                return Err(MediaRequestError::InvalidVideo(
192                    "sampling frame rate must be finite and positive".into(),
193                ));
194            }
195            VideoSampling::FrameCount(0) => {
196                return Err(MediaRequestError::InvalidVideo(
197                    "sampling frame count must be positive".into(),
198                ));
199            }
200            _ => {}
201        }
202        Ok(Self {
203            frames,
204            source_fps,
205            sampling,
206        })
207    }
208
209    /// Decoded frames in source order.
210    pub fn frames(&self) -> &[RgbImage] {
211        &self.frames
212    }
213
214    /// Source frame rate when known.
215    pub const fn source_fps(&self) -> Option<f64> {
216        self.source_fps
217    }
218
219    /// Requested frame-selection policy.
220    pub const fn sampling(&self) -> VideoSampling {
221        self.sampling
222    }
223}
224
225/// One decoded media item in a portable request.
226#[derive(Debug, Clone, PartialEq)]
227pub enum Media {
228    /// Decoded packed RGB8 image.
229    Image(RgbImage),
230    /// Decoded RGB8 video.
231    Video(Video),
232    /// Decoded mono PCM audio.
233    Audio(Audio),
234}
235
236/// One ordered portable model-input segment.
237#[derive(Debug, Clone, PartialEq)]
238pub enum MultimodalSegment {
239    /// Text to encode with [`MultimodalRequest::tokenize`].
240    Text(String),
241    /// Text already represented as tokenizer vocabulary IDs.
242    TokenIds(Vec<u32>),
243    /// Decoded media preprocessed by the selected backend.
244    Media(Media),
245}
246
247/// Validated ordered text and decoded-media input.
248#[derive(Debug, Clone, PartialEq)]
249pub struct MultimodalRequest {
250    segments: Vec<MultimodalSegment>,
251}
252
253impl MultimodalRequest {
254    /// Validates an ordered multimodal request.
255    pub fn new(segments: Vec<MultimodalSegment>) -> Result<Self, MediaRequestError> {
256        if segments.is_empty() {
257            return Err(MediaRequestError::EmptyRequest);
258        }
259        if !segments
260            .iter()
261            .any(|segment| matches!(segment, MultimodalSegment::Media(_)))
262        {
263            return Err(MediaRequestError::MissingMedia);
264        }
265        Ok(Self { segments })
266    }
267
268    /// Composes a rendered prompt with media bound to exact placeholder occurrences.
269    pub fn from_chat(
270        rendered_prompt: &str,
271        bindings: &[MediaBinding],
272    ) -> Result<Self, MediaRequestError> {
273        validate_bindings(rendered_prompt, bindings)?;
274        let mut segments = Vec::with_capacity(bindings.len().saturating_mul(2) + 1);
275        let mut cursor = 0;
276        for (index, binding) in bindings.iter().enumerate() {
277            let remainder = &rendered_prompt[cursor..];
278            let relative = remainder.find(binding.placeholder()).ok_or_else(|| {
279                MediaRequestError::PlaceholderOrder {
280                    index,
281                    placeholder: binding.placeholder().into(),
282                }
283            })?;
284            let start = cursor + relative;
285            if start > cursor {
286                segments.push(MultimodalSegment::Text(
287                    rendered_prompt[cursor..start].into(),
288                ));
289            }
290            segments.push(MultimodalSegment::Media(binding.media().clone()));
291            cursor = start + binding.placeholder().len();
292        }
293        if cursor < rendered_prompt.len() {
294            segments.push(MultimodalSegment::Text(rendered_prompt[cursor..].into()));
295        }
296        Self::new(segments)
297    }
298
299    /// Ordered request segments.
300    pub fn segments(&self) -> &[MultimodalSegment] {
301        &self.segments
302    }
303
304    /// Encodes text while preserving media order for backend preparation.
305    pub fn tokenize<E>(
306        &self,
307        mut encode: impl FnMut(&str) -> Result<Vec<u32>, E>,
308    ) -> Result<TokenizedMultimodalRequest, E> {
309        let mut segments = Vec::with_capacity(self.segments.len());
310        for segment in &self.segments {
311            segments.push(match segment {
312                MultimodalSegment::Text(text) => {
313                    TokenizedMultimodalSegment::TokenIds(encode(text)?)
314                }
315                MultimodalSegment::TokenIds(ids) => {
316                    TokenizedMultimodalSegment::TokenIds(ids.clone())
317                }
318                MultimodalSegment::Media(media) => TokenizedMultimodalSegment::Media(media.clone()),
319            });
320        }
321        Ok(TokenizedMultimodalRequest { segments })
322    }
323}
324
325/// One exact rendered-prompt placeholder bound to decoded media.
326#[derive(Debug, Clone, PartialEq)]
327pub struct MediaBinding {
328    placeholder: String,
329    media: Media,
330}
331
332impl MediaBinding {
333    /// Binds decoded media to one complete placeholder spelling.
334    pub fn new(placeholder: impl Into<String>, media: Media) -> Self {
335        Self {
336            placeholder: placeholder.into(),
337            media,
338        }
339    }
340
341    /// Complete placeholder spelling.
342    pub fn placeholder(&self) -> &str {
343        &self.placeholder
344    }
345
346    /// Bound decoded media.
347    pub const fn media(&self) -> &Media {
348        &self.media
349    }
350}
351
352/// One ordered segment after facade tokenization and before backend preprocessing.
353#[derive(Debug, Clone, PartialEq)]
354pub enum TokenizedMultimodalSegment {
355    /// Tokenizer vocabulary IDs.
356    TokenIds(Vec<u32>),
357    /// Decoded media.
358    Media(Media),
359}
360
361/// Backend preparation request with tokenizer work already complete.
362#[derive(Debug, Clone, PartialEq)]
363pub struct TokenizedMultimodalRequest {
364    segments: Vec<TokenizedMultimodalSegment>,
365}
366
367impl TokenizedMultimodalRequest {
368    /// Ordered token and media segments.
369    pub fn segments(&self) -> &[TokenizedMultimodalSegment] {
370        &self.segments
371    }
372}
373
374fn validate_bindings(
375    rendered_prompt: &str,
376    bindings: &[MediaBinding],
377) -> Result<(), MediaRequestError> {
378    for (index, binding) in bindings.iter().enumerate() {
379        if binding.placeholder.is_empty() {
380            return Err(MediaRequestError::EmptyPlaceholder { index });
381        }
382        if bindings[..index]
383            .iter()
384            .any(|earlier| earlier.placeholder == binding.placeholder)
385        {
386            continue;
387        }
388        let expected = bindings
389            .iter()
390            .filter(|candidate| candidate.placeholder == binding.placeholder)
391            .count();
392        let actual = rendered_prompt.matches(&binding.placeholder).count();
393        if actual != expected {
394            return Err(MediaRequestError::PlaceholderCount {
395                placeholder: binding.placeholder.clone(),
396                expected,
397                actual,
398            });
399        }
400    }
401    Ok(())
402}
403
404#[cfg(test)]
405mod tests {
406    use super::*;
407
408    fn image(value: u8) -> Media {
409        Media::Image(RgbImage::new(vec![value; 3], 1, 1).unwrap())
410    }
411
412    #[test]
413    fn chat_composition_and_tokenization_preserve_exact_order() {
414        let request = MultimodalRequest::from_chat(
415            "before<image>middle<image>after",
416            &[
417                MediaBinding::new("<image>", image(1)),
418                MediaBinding::new("<image>", image(2)),
419            ],
420        )
421        .unwrap();
422        let tokenized = request
423            .tokenize::<std::convert::Infallible>(|text| {
424                Ok(text
425                    .as_bytes()
426                    .iter()
427                    .map(|byte| u32::from(*byte))
428                    .collect())
429            })
430            .unwrap();
431        assert_eq!(tokenized.segments().len(), 5);
432        assert!(matches!(
433            &tokenized.segments()[0],
434            TokenizedMultimodalSegment::TokenIds(ids)
435                if ids == &[98, 101, 102, 111, 114, 101]
436        ));
437        assert!(matches!(
438            &tokenized.segments()[1],
439            TokenizedMultimodalSegment::Media(Media::Image(image)) if image.pixels() == [1, 1, 1]
440        ));
441        assert!(matches!(
442            &tokenized.segments()[3],
443            TokenizedMultimodalSegment::Media(Media::Image(image)) if image.pixels() == [2, 2, 2]
444        ));
445    }
446
447    #[test]
448    fn validation_rejects_bad_media_and_placeholder_contracts() {
449        assert!(matches!(
450            RgbImage::new(vec![0; 2], 1, 1),
451            Err(MediaRequestError::InvalidRgbImage { .. })
452        ));
453        assert!(Audio::new(vec![f32::NAN], 16_000).is_err());
454        assert!(Video::new(Vec::new(), None, VideoSampling::All).is_err());
455        assert!(matches!(
456            MultimodalRequest::from_chat(
457                "<image>",
458                &[
459                    MediaBinding::new("<image>", image(1)),
460                    MediaBinding::new("<image>", image(2)),
461                ],
462            ),
463            Err(MediaRequestError::PlaceholderCount { .. })
464        ));
465    }
466}
eredu_core/media.rs

eredu_core/
media.rs
