edgefirst_tensor/
preprocess.rs

1// SPDX-FileCopyrightText: Copyright 2026 Au-Zone Technologies
2// SPDX-License-Identifier: Apache-2.0
3
4//! Preprocessing values shared by the image and decoder crates: how pixels
5//! are normalized, how a source is fit into a model input, where the image
6//! content lands ([`letterbox_rect`], [`ContentRegion`]), and a model's whole
7//! recipe as one value ([`PreprocessSpec`]).
8//!
9//! They live here, beside [`unletter_norm`](crate::unletter_norm), so the
10//! image crate (which applies them) and the decoder crate (which reads them
11//! from model metadata) share one definition without depending on each
12//! other. Only plain values and pure functions belong in this module.
13
14use crate::{BoundingBox, Error, Region, Result};
15
16crate::ef_vocabulary! {
17    /// Where a letterboxed image sits inside the destination. The code is the
18    /// one the C `EF_LETTERBOX_PLACEMENT_*` enumerators and the Python
19    /// `LetterboxPlacement` enum use; the string is the `edgefirst.json` name.
20    #[derive(Default)]
21    #[non_exhaustive]
22    pub enum LetterboxPlacement {
23        /// Centred, padding split between both sides (the default). On odd
24        /// slack the extra pixel goes to the right or bottom.
25        #[default]
26        Center = 0, "center", CENTER,
27        /// Anchored at the top-left corner, padding on the right and bottom.
28        TopLeft = 1, "topleft", TOP_LEFT,
29    }
30    #[doc(hidden)]
31    pub mod letterbox_placement_code;
32}
33
34crate::ef_vocabulary! {
35    /// A named [`Normalization`]. The code is the one the C
36    /// `EF_NORMALIZATION_*` enumerators and the Python `NormalizationPreset`
37    /// enum use; the names match the `edgefirst.json` presets.
38    #[derive(Default)]
39    #[non_exhaustive]
40    pub enum NormalizationPreset {
41        /// `x / 255`, giving `[0, 1]` (the default).
42        #[default]
43        Unsigned = 0, "unsigned", UNSIGNED,
44        /// `x / 127.5 - 1`, giving `[-1, 1]`.
45        Signed = 1, "signed", SIGNED,
46        /// `x`, giving `[0, 255]`.
47        Raw = 2, "raw", RAW,
48        /// ImageNet mean and standard deviation.
49        Imagenet = 3, "imagenet", IMAGENET,
50    }
51    #[doc(hidden)]
52    pub mod normalization_code;
53}
54
55crate::ef_vocabulary! {
56    /// How a source is fit into the destination, without the letterbox pad
57    /// colour. The code is the one the C `EF_FIT_*` enumerators and the
58    /// Python `Fit` enum use.
59    #[derive(Default)]
60    #[non_exhaustive]
61    pub enum FitMode {
62        /// Scale each axis independently to fill the destination (the
63        /// default).
64        #[default]
65        Stretch = 0, "stretch", STRETCH,
66        /// Keep the source aspect ratio and pad the rest.
67        Letterbox = 1, "letterbox", LETTERBOX,
68    }
69    #[doc(hidden)]
70    pub mod fit_code;
71}
72
73crate::ef_vocabulary! {
74    /// Resampling filter. The code is the one the C `EF_INTERPOLATION_*`
75    /// enumerators use; the string is the `edgefirst.json` and Python name.
76    ///
77    /// The image crate runs [`Nearest`](Self::Nearest) and
78    /// [`Bilinear`](Self::Bilinear) (half-pixel centres, two taps per axis at
79    /// every scale, matching OpenCV `INTER_LINEAR`). [`Bicubic`](Self::Bicubic)
80    /// and [`Area`](Self::Area) describe reference pipelines; asking the image
81    /// crate for them logs a warning and runs [`supported`](Self::supported).
82    #[derive(Default)]
83    #[non_exhaustive]
84    pub enum Interpolation {
85        /// Bilinear (the default).
86        #[default]
87        Bilinear = 0, "bilinear", BILINEAR,
88        /// Nearest neighbour.
89        Nearest = 1, "nearest", NEAREST,
90        /// Bicubic.
91        Bicubic = 2, "bicubic", BICUBIC,
92        /// Pixel-area averaging.
93        Area = 3, "area", AREA,
94    }
95    #[doc(hidden)]
96    pub mod interpolation_code;
97}
98
99crate::ef_vocabulary! {
100    /// The space a letterbox pad value is expressed in. The code is the one
101    /// the C `EF_PAD_SPACE_*` enumerators use; the string is the
102    /// `edgefirst.json` name.
103    #[derive(Default)]
104    #[non_exhaustive]
105    pub enum PadSpace {
106        /// A 0–255 pixel value, normalized like the image (the default).
107        #[default]
108        Pixel = 0, "pixel", PIXEL,
109        /// A value written as-is into the normalized destination, e.g. 0.0 to
110        /// pad with the mean colour after ImageNet normalization.
111        Normalized = 1, "normalized", NORMALIZED,
112    }
113    #[doc(hidden)]
114    pub mod pad_space_code;
115}
116
117impl Interpolation {
118    /// The filter the image crate runs for this one: itself for
119    /// [`Nearest`](Self::Nearest) and [`Bilinear`](Self::Bilinear), else
120    /// [`Bilinear`](Self::Bilinear).
121    pub const fn supported(self) -> Self {
122        match self {
123            Self::Nearest => Self::Nearest,
124            Self::Bilinear | Self::Bicubic | Self::Area => Self::Bilinear,
125        }
126    }
127}
128
129macro_rules! serde_by_name {
130    ($($ty:ident),+) => {$(
131        impl serde::Serialize for $ty {
132            fn serialize<S: serde::Serializer>(&self, s: S) -> std::result::Result<S::Ok, S::Error> {
133                s.serialize_str(self.as_str())
134            }
135        }
136
137        impl<'de> serde::Deserialize<'de> for $ty {
138            fn deserialize<D: serde::Deserializer<'de>>(d: D) -> std::result::Result<Self, D::Error> {
139                let s = <std::borrow::Cow<'de, str> as serde::Deserialize>::deserialize(d)?;
140                Self::from_str_code(&s).ok_or_else(|| {
141                    let names: Vec<&str> = Self::all().iter().map(|v| v.as_str()).collect();
142                    serde::de::Error::custom(format!(
143                        "unknown {} `{s}`, expected one of {names:?}",
144                        stringify!($ty)
145                    ))
146                })
147            }
148        }
149    )+};
150}
151
152serde_by_name!(
153    LetterboxPlacement,
154    NormalizationPreset,
155    FitMode,
156    Interpolation,
157    PadSpace
158);
159
160const IMAGENET_MEAN: [f32; 3] = [123.675, 116.28, 103.53];
161const IMAGENET_STD: [f32; 3] = [58.395, 57.12, 57.375];
162
163/// Per-channel `(x - mean) / std` on 0–255 pixel values, in RGB order.
164///
165/// It applies to the colour channels of the destination; an alpha channel
166/// always maps as [`NormalizationPreset::Unsigned`]. A `Grey` destination uses
167/// the first channel's coefficients.
168#[derive(Debug, Clone, Copy, PartialEq)]
169pub struct Normalization {
170    mean: [f32; 3],
171    std: [f32; 3],
172}
173
174impl Default for Normalization {
175    fn default() -> Self {
176        Self::preset(NormalizationPreset::Unsigned)
177    }
178}
179
180impl From<NormalizationPreset> for Normalization {
181    fn from(p: NormalizationPreset) -> Self {
182        Self::preset(p)
183    }
184}
185
186impl Normalization {
187    /// The coefficients of a named preset.
188    pub const fn preset(p: NormalizationPreset) -> Self {
189        let (mean, std) = match p {
190            NormalizationPreset::Unsigned => ([0.0; 3], [255.0; 3]),
191            NormalizationPreset::Signed => ([127.5; 3], [127.5; 3]),
192            NormalizationPreset::Raw => ([0.0; 3], [1.0; 3]),
193            NormalizationPreset::Imagenet => (IMAGENET_MEAN, IMAGENET_STD),
194        };
195        Self { mean, std }
196    }
197
198    /// `x / 255`.
199    pub const fn unsigned() -> Self {
200        Self::preset(NormalizationPreset::Unsigned)
201    }
202
203    /// `x / 127.5 - 1`.
204    pub const fn signed() -> Self {
205        Self::preset(NormalizationPreset::Signed)
206    }
207
208    /// `x` unchanged.
209    pub const fn raw() -> Self {
210        Self::preset(NormalizationPreset::Raw)
211    }
212
213    /// ImageNet mean `[123.675, 116.28, 103.53]`, std `[58.395, 57.12, 57.375]`.
214    pub const fn imagenet() -> Self {
215        Self::preset(NormalizationPreset::Imagenet)
216    }
217
218    /// Explicit coefficients on the 0–255 scale: one value (every channel)
219    /// or three (RGB).
220    ///
221    /// # Errors
222    ///
223    /// [`Error::InvalidArgument`] for another count, a non-finite `mean`, or
224    /// a `std` that is not finite and positive.
225    ///
226    /// ```
227    /// use edgefirst_tensor::Normalization;
228    ///
229    /// assert_eq!(Normalization::mean_std(&[0.0], &[255.0]).unwrap(), Normalization::unsigned());
230    /// assert!(Normalization::mean_std(&[0.0], &[0.0]).is_err());
231    /// assert!(Normalization::mean_std(&[0.0], &[-1.0]).is_err());
232    /// ```
233    pub fn mean_std(mean: &[f32], std: &[f32]) -> Result<Self> {
234        let count = |name: &str, v: &[f32]| {
235            broadcast_rgb(v).ok_or_else(|| {
236                Error::InvalidArgument(format!(
237                    "normalization {name} needs 1 or 3 values, got {}",
238                    v.len()
239                ))
240            })
241        };
242        let mean = count("mean", mean)?;
243        let std = count("std", std)?;
244        if mean.iter().any(|m| !m.is_finite()) {
245            return Err(Error::InvalidArgument(format!(
246                "normalization mean {mean:?} must be finite"
247            )));
248        }
249        if std.iter().any(|s| !s.is_finite() || *s <= 0.0) {
250            return Err(Error::InvalidArgument(format!(
251                "normalization std {std:?} must be finite and positive"
252            )));
253        }
254        Ok(Self { mean, std })
255    }
256
257    /// Coefficients on the 0–1 scale, as Hugging Face and torchvision write
258    /// them; scaled by `scale` (usually 255) onto the 0–255 scale in f64.
259    ///
260    /// # Errors
261    ///
262    /// As for [`mean_std`](Self::mean_std).
263    ///
264    /// ```
265    /// use edgefirst_tensor::Normalization;
266    ///
267    /// let n = Normalization::unit_mean_std(&[0.5], &[0.5], 255.0).unwrap();
268    /// assert_eq!(n, Normalization::signed());
269    /// ```
270    pub fn unit_mean_std(mean: &[f32], std: &[f32], scale: f64) -> Result<Self> {
271        let scaled =
272            |v: &[f32]| -> Vec<f32> { v.iter().map(|&x| (f64::from(x) * scale) as f32).collect() };
273        Self::mean_std(&scaled(mean), &scaled(std))
274    }
275
276    /// The per-channel mean, RGB order, on the 0–255 scale.
277    pub fn mean(&self) -> [f32; 3] {
278        self.mean
279    }
280
281    /// The per-channel standard deviation, RGB order, on the 0–255 scale.
282    pub fn std(&self) -> [f32; 3] {
283        self.std
284    }
285
286    /// The preset with exactly these coefficients, if any.
287    pub fn as_preset(&self) -> Option<NormalizationPreset> {
288        NormalizationPreset::all()
289            .iter()
290            .copied()
291            .find(|&p| Self::preset(p) == *self)
292    }
293}
294
295/// One value repeated for every RGB channel, or three values as given;
296/// `None` for any other count.
297///
298/// ```
299/// use edgefirst_tensor::broadcast_rgb;
300///
301/// assert_eq!(broadcast_rgb(&[1]), Some([1, 1, 1]));
302/// assert_eq!(broadcast_rgb(&[1, 2]), None);
303/// ```
304pub fn broadcast_rgb<T: Copy>(v: &[T]) -> Option<[T; 3]> {
305    match *v {
306        [x] => Some([x; 3]),
307        [r, g, b] => Some([r, g, b]),
308        _ => None,
309    }
310}
311
312/// How a source is fit into the requested destination shape.
313#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
314pub enum Fit {
315    /// Stretch the source (or source crop) to fill the whole destination.
316    #[default]
317    Stretch,
318    /// Keep the source aspect ratio and pad the rest of the destination with
319    /// the RGBA pixel `pad` (e.g. `[114, 114, 114, 255]` for YOLO-style
320    /// preprocessing). Where the image sits is a [`LetterboxPlacement`].
321    Letterbox {
322        /// The RGBA pad pixel.
323        pad: [u8; 4],
324    },
325}
326
327/// The rectangle an `src_w`×`src_h` source occupies when letterboxed into a
328/// `dst_w`×`dst_h` destination: the larger axis fills, the other is scaled
329/// to keep the aspect ratio (rounded, at least one pixel), and `placement`
330/// positions it. A zero-sized source fills the destination.
331///
332/// This is the one letterbox computation: `convert` paints this rectangle,
333/// and [`ContentRegion`]s are built from it.
334///
335/// ```
336/// use edgefirst_tensor::{letterbox_rect, LetterboxPlacement, Region};
337///
338/// let r = letterbox_rect(1280, 720, 640, 640, LetterboxPlacement::Center);
339/// assert_eq!(r, Region::new(0, 140, 640, 360));
340/// let r = letterbox_rect(1280, 720, 640, 640, LetterboxPlacement::TopLeft);
341/// assert_eq!(r, Region::new(0, 0, 640, 360));
342/// ```
343pub fn letterbox_rect(
344    src_w: usize,
345    src_h: usize,
346    dst_w: usize,
347    dst_h: usize,
348    placement: LetterboxPlacement,
349) -> Region {
350    if src_w == 0 || src_h == 0 || dst_w == 0 || dst_h == 0 {
351        return Region::new(0, 0, dst_w, dst_h);
352    }
353    let src_aspect = src_w as f64 / src_h as f64;
354    let dst_aspect = dst_w as f64 / dst_h as f64;
355    let (w, h) = if src_aspect > dst_aspect {
356        (dst_w, ((dst_w as f64 / src_aspect).round() as usize).max(1))
357    } else {
358        (((dst_h as f64 * src_aspect).round() as usize).max(1), dst_h)
359    };
360    let (x, y) = match placement {
361        LetterboxPlacement::Center => (dst_w.saturating_sub(w) / 2, dst_h.saturating_sub(h) / 2),
362        LetterboxPlacement::TopLeft => (0, 0),
363    };
364    Region::new(x, y, w, h)
365}
366
367/// Where the image content sits in a destination: in pixels, and as
368/// `[xmin, ymin, xmax, ymax]` normalized by the destination size, the form
369/// [`unletter_norm`](crate::unletter_norm) and the semantic decoder take.
370#[derive(Debug, Clone, Copy, PartialEq)]
371pub struct ContentRegion {
372    /// The content rectangle in destination pixels.
373    pub pixels: Region,
374    /// The content rectangle normalized by the destination size.
375    pub normalized: [f32; 4],
376}
377
378impl ContentRegion {
379    /// The region of `pixels` in a `dst_w`×`dst_h` destination.
380    pub fn new(pixels: Region, dst_w: usize, dst_h: usize) -> Self {
381        let (w, h) = (dst_w.max(1) as f32, dst_h.max(1) as f32);
382        Self {
383            pixels,
384            normalized: [
385                pixels.x as f32 / w,
386                pixels.y as f32 / h,
387                pixels.x.saturating_add(pixels.width) as f32 / w,
388                pixels.y.saturating_add(pixels.height) as f32 / h,
389            ],
390        }
391    }
392
393    /// Content that fills a `dst_w`×`dst_h` destination.
394    pub fn full(dst_w: usize, dst_h: usize) -> Self {
395        Self::new(Region::new(0, 0, dst_w, dst_h), dst_w, dst_h)
396    }
397
398    /// Whether the content fills the destination.
399    pub fn is_full(&self) -> bool {
400        self.normalized == [0.0, 0.0, 1.0, 1.0]
401    }
402
403    /// The normalized content rectangle when it does not fill the
404    /// destination (a letterbox), else `None`.
405    pub fn letterboxed(&self) -> Option<[f32; 4]> {
406        (!self.is_full()).then_some(self.normalized)
407    }
408
409    /// Map a box normalized to the destination back to the content, i.e. to
410    /// the source image (see [`unletter_norm`](crate::unletter_norm)).
411    pub fn unletter(&self, b: BoundingBox) -> BoundingBox {
412        crate::unletter_norm(b, self.normalized)
413    }
414}
415
416/// Something that knows where it puts the image content in a destination,
417/// such as the image crate's `ConvertOptions` or a [`PreprocessSpec`]. The
418/// decoder takes it to map its output back to the source image with the same
419/// geometry the convert used.
420pub trait ContentRegionSource {
421    /// The content region for an `src_w`×`src_h` source converted into a
422    /// `dst_w`×`dst_h` destination.
423    ///
424    /// # Errors
425    ///
426    /// [`Error::InvalidArgument`] when the geometry cannot be resolved, e.g.
427    /// a source crop outside the source.
428    fn content_region(
429        &self,
430        src_w: usize,
431        src_h: usize,
432        dst_w: usize,
433        dst_h: usize,
434    ) -> Result<ContentRegion>;
435}
436
437/// A model's input preprocessing as plain values: what the image crate needs
438/// to build a convert, produced by the decoder from model metadata.
439///
440/// The default is the documented `edgefirst.json` default: unsigned
441/// normalization, a centred letterbox padded with pixel 114, bilinear.
442///
443/// ```
444/// use edgefirst_tensor::{FitMode, LetterboxPlacement, PreprocessSpec};
445///
446/// let spec = PreprocessSpec::default();
447/// assert_eq!(spec.fit, FitMode::Letterbox);
448/// assert_eq!(spec.placement, LetterboxPlacement::Center);
449/// assert_eq!(spec.pad, [114.0; 3]);
450/// ```
451#[derive(Debug, Clone, Copy, PartialEq)]
452#[non_exhaustive]
453pub struct PreprocessSpec {
454    /// Pixel normalization.
455    pub normalization: Normalization,
456    /// Stretch or letterbox.
457    pub fit: FitMode,
458    /// Letterbox placement; ignored by a stretch.
459    pub placement: LetterboxPlacement,
460    /// Letterbox pad per RGB channel, in the space `pad_space` names.
461    pub pad: [f32; 3],
462    /// The space `pad` is expressed in.
463    pub pad_space: PadSpace,
464    /// Resampling filter of the reference pipeline.
465    pub interpolation: Interpolation,
466    /// Model input `(width, height)`, when known.
467    pub size: Option<(usize, usize)>,
468}
469
470impl Default for PreprocessSpec {
471    fn default() -> Self {
472        Self {
473            normalization: Normalization::unsigned(),
474            fit: FitMode::Letterbox,
475            placement: LetterboxPlacement::Center,
476            pad: [114.0; 3],
477            pad_space: PadSpace::Pixel,
478            interpolation: Interpolation::Bilinear,
479            size: None,
480        }
481    }
482}
483
484impl PreprocessSpec {
485    /// The default spec; set the public fields to change it.
486    pub fn new() -> Self {
487        Self::default()
488    }
489}
490
491impl ContentRegionSource for PreprocessSpec {
492    fn content_region(
493        &self,
494        src_w: usize,
495        src_h: usize,
496        dst_w: usize,
497        dst_h: usize,
498    ) -> Result<ContentRegion> {
499        Ok(match self.fit {
500            FitMode::Stretch => ContentRegion::full(dst_w, dst_h),
501            FitMode::Letterbox => ContentRegion::new(
502                letterbox_rect(src_w, src_h, dst_w, dst_h, self.placement),
503                dst_w,
504                dst_h,
505            ),
506        })
507    }
508}
509
510#[cfg(test)]
511mod tests {
512    use super::*;
513
514    #[test]
515    fn letterbox_rect_rounds_and_places() {
516        let c = LetterboxPlacement::Center;
517        let t = LetterboxPlacement::TopLeft;
518        // 641x479 -> width fills, height round(640 * 479 / 641) = 478.
519        assert_eq!(
520            letterbox_rect(641, 479, 640, 640, c),
521            Region::new(0, 81, 640, 478)
522        );
523        assert_eq!(
524            letterbox_rect(641, 479, 640, 640, t),
525            Region::new(0, 0, 640, 478)
526        );
527        assert_eq!(
528            letterbox_rect(479, 641, 640, 640, c),
529            Region::new(81, 0, 478, 640)
530        );
531        assert_eq!(
532            letterbox_rect(1, 3, 320, 256, c),
533            Region::new(117, 0, 85, 256)
534        );
535        assert_eq!(
536            letterbox_rect(3, 1, 320, 256, c),
537            Region::new(0, 74, 320, 107)
538        );
539        assert_eq!(
540            letterbox_rect(640, 640, 320, 320, c),
541            Region::new(0, 0, 320, 320)
542        );
543        assert_eq!(letterbox_rect(0, 5, 32, 16, c), Region::new(0, 0, 32, 16));
544        // A sliver keeps at least one pixel.
545        assert_eq!(
546            letterbox_rect(10000, 1, 64, 64, t),
547            Region::new(0, 0, 64, 1)
548        );
549    }
550
551    #[test]
552    fn content_region_normalizes_pixels() {
553        let r = ContentRegion::new(Region::new(0, 140, 640, 360), 640, 640);
554        assert_eq!(r.normalized, [0.0, 0.21875, 1.0, 0.78125]);
555        assert!(!r.is_full());
556        assert_eq!(r.letterboxed(), Some(r.normalized));
557        assert!(ContentRegion::full(32, 16).is_full());
558        assert_eq!(ContentRegion::full(32, 16).letterboxed(), None);
559    }
560
561    #[test]
562    fn interpolation_codes_keep_c_values() {
563        assert_eq!(Interpolation::Bilinear.code(), 0);
564        assert_eq!(Interpolation::Nearest.code(), 1);
565        assert_eq!(Interpolation::Bicubic.supported(), Interpolation::Bilinear);
566        assert_eq!(Interpolation::Area.supported(), Interpolation::Bilinear);
567        assert_eq!(Interpolation::Nearest.supported(), Interpolation::Nearest);
568    }
569
570    #[test]
571    fn vocab_serde_round_trips() {
572        for &p in LetterboxPlacement::all() {
573            let s = serde_json::to_string(&p).unwrap();
574            assert_eq!(serde_json::from_str::<LetterboxPlacement>(&s).unwrap(), p);
575        }
576        assert_eq!(
577            serde_json::to_string(&PadSpace::Normalized).unwrap(),
578            "\"normalized\""
579        );
580        assert_eq!(
581            serde_json::to_string(&LetterboxPlacement::TopLeft).unwrap(),
582            "\"topleft\""
583        );
584        assert!(serde_json::from_str::<LetterboxPlacement>("\"top_left\"").is_err());
585        let e = serde_json::from_str::<Interpolation>("\"lanczos\"").unwrap_err();
586        assert!(e.to_string().contains("bicubic"), "{e}");
587    }
588
589    #[test]
590    fn spec_content_region_follows_fit() {
591        let mut spec = PreprocessSpec::new();
592        let lb = spec.content_region(1280, 720, 640, 640).unwrap();
593        assert_eq!(lb.pixels, Region::new(0, 140, 640, 360));
594        spec.fit = FitMode::Stretch;
595        assert!(spec.content_region(1280, 720, 640, 640).unwrap().is_full());
596    }
597
598    #[test]
599    fn normalization_finds_its_preset() {
600        assert_eq!(
601            Normalization::imagenet().as_preset(),
602            Some(NormalizationPreset::Imagenet)
603        );
604        let custom = Normalization::mean_std(&[1.0], &[2.0]).unwrap();
605        assert_eq!(custom.as_preset(), None);
606        assert!(Normalization::mean_std(&[0.0, 0.0], &[1.0]).is_err());
607        assert!(Normalization::mean_std(&[f32::NAN], &[1.0]).is_err());
608    }
609}
edgefirst_tensor/preprocess.rs

edgefirst_tensor/
preprocess.rs
