Skip to main content

threers_codec/decode/
picture.rs

1//! Sample planes and the decoded frame the decoders hand back.
2
3/// One plane of samples. Stored as `u16` whatever the bit depth, so 8-bit and
4/// 10-bit streams share every reconstruction path.
5#[derive(Clone, Debug)]
6pub struct Plane {
7    /// Samples, row by row, `width * height` of them.
8    pub data: Vec<u16>,
9    /// Width in samples.
10    pub width: usize,
11    /// Height in samples.
12    pub height: usize,
13}
14
15impl Plane {
16    /// A `width × height` plane with every sample set to `fill`.
17    pub fn new(width: usize, height: usize, fill: u16) -> Self {
18        Plane {
19            data: vec![fill; width * height],
20            width,
21            height,
22        }
23    }
24
25    /// The sample at `(x, y)`.
26    #[inline(always)]
27    pub fn at(&self, x: usize, y: usize) -> u16 {
28        self.data[y * self.width + x]
29    }
30
31    /// Set the sample at `(x, y)`.
32    #[inline(always)]
33    pub fn set(&mut self, x: usize, y: usize, v: u16) {
34        self.data[y * self.width + x] = v;
35    }
36
37    /// The sample at `(x, y)` with the coordinates clamped into the plane —
38    /// what motion compensation reads when a vector points off the edge.
39    #[inline(always)]
40    pub fn clamped(&self, x: isize, y: isize) -> u16 {
41        let x = x.clamp(0, self.width as isize - 1) as usize;
42        let y = y.clamp(0, self.height as isize - 1) as usize;
43        self.data[y * self.width + x]
44    }
45}
46
47/// Chroma subsampling of a decoded frame.
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
49pub enum ChromaFormat {
50    /// Luma only.
51    Monochrome,
52    /// Chroma halved both ways.
53    Yuv420,
54    /// Chroma halved horizontally.
55    Yuv422,
56    /// Full-resolution chroma.
57    Yuv444,
58    /// Chroma halved vertically only (VP9 profile 1).
59    Yuv440,
60}
61
62impl ChromaFormat {
63    /// From H.264/HEVC `chroma_format_idc`, 0 to 3.
64    pub fn from_idc(idc: u32) -> Option<Self> {
65        Some(match idc {
66            0 => ChromaFormat::Monochrome,
67            1 => ChromaFormat::Yuv420,
68            2 => ChromaFormat::Yuv422,
69            3 => ChromaFormat::Yuv444,
70            _ => return None,
71        })
72    }
73
74    /// `(SubWidthC, SubHeightC)`.
75    pub fn subsampling(self) -> (usize, usize) {
76        match self {
77            ChromaFormat::Yuv420 => (2, 2),
78            ChromaFormat::Yuv422 => (2, 1),
79            ChromaFormat::Yuv440 => (1, 2),
80            _ => (1, 1),
81        }
82    }
83}
84
85/// The YCbCr → RGB matrix a stream says it was coded with.
86#[derive(Debug, Clone, Copy, PartialEq, Eq)]
87pub enum Matrix {
88    /// ITU-R BT.601 (SD).
89    Bt601,
90    /// ITU-R BT.709 (HD).
91    Bt709,
92    /// ITU-R BT.2020 (UHD, non-constant luminance).
93    Bt2020,
94}
95
96/// What the VUI says about colour, reduced to what conversion needs.
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub struct ColorInfo {
99    /// The YCbCr → RGB matrix.
100    pub matrix: Matrix,
101    /// Full-range (0–255 at 8 bits) rather than video range (16–235).
102    pub full_range: bool,
103    /// The VUI's `matrix_coefficients` as coded (`None` without a colour
104    /// description) — what ffmpeg's swscale keys its tables on.
105    pub vui_matrix: Option<u32>,
106}
107
108impl ColorInfo {
109    /// From the VUI's `matrix_coefficients` and `video_full_range_flag`.
110    ///
111    /// An unspecified matrix (2, or no VUI at all) is a guess either way;
112    /// this makes the same one players do — BT.709 for HD, BT.601 below it.
113    pub fn from_vui(matrix_coefficients: Option<u32>, full_range: bool, height: u32) -> Self {
114        let matrix = match matrix_coefficients {
115            Some(1) => Matrix::Bt709,
116            Some(5) | Some(6) => Matrix::Bt601,
117            Some(9) | Some(10) => Matrix::Bt2020,
118            _ if height >= 720 => Matrix::Bt709,
119            _ => Matrix::Bt601,
120        };
121        ColorInfo { matrix, full_range, vui_matrix: matrix_coefficients }
122    }
123}
124
125/// A decoded picture, cropped to its display size.
126#[derive(Debug, Clone)]
127pub struct DecodedFrame {
128    /// Display width in pixels.
129    pub width: u32,
130    /// Display height in pixels.
131    pub height: u32,
132    /// Bits per sample, 8 to 12.
133    pub bit_depth: u8,
134    /// Chroma subsampling.
135    pub chroma: ChromaFormat,
136    /// Luma, `width * height`, tightly packed.
137    pub y: Vec<u16>,
138    /// Chroma planes, [`chroma_size`](Self::chroma_size) each. Empty for
139    /// monochrome.
140    pub u: Vec<u16>,
141    /// Cr, laid out like `u`.
142    pub v: Vec<u16>,
143    /// What the stream says about its colour.
144    pub color: ColorInfo,
145}
146
147impl DecodedFrame {
148    /// Size of each chroma plane.
149    pub fn chroma_size(&self) -> (u32, u32) {
150        if self.chroma == ChromaFormat::Monochrome {
151            return (0, 0);
152        }
153        let (sw, sh) = self.chroma.subsampling();
154        (
155            self.width.div_ceil(sw as u32),
156            self.height.div_ceil(sh as u32),
157        )
158    }
159
160    /// Crop planes out of the coded picture.
161    pub(crate) fn from_planes(
162        planes: &[Plane; 3],
163        chroma: ChromaFormat,
164        bit_depth: u8,
165        crop: [usize; 4],
166        color: ColorInfo,
167    ) -> Self {
168        let [left, right, top, bottom] = crop;
169        let w = planes[0].width.saturating_sub(left + right);
170        let h = planes[0].height.saturating_sub(top + bottom);
171        let cut = |p: &Plane, x0: usize, y0: usize, w: usize, h: usize| {
172            let mut out = Vec::with_capacity(w * h);
173            for y in y0..y0 + h {
174                out.extend_from_slice(&p.data[y * p.width + x0..y * p.width + x0 + w]);
175            }
176            out
177        };
178        let y = cut(&planes[0], left, top, w, h);
179        let (u, v) = if chroma == ChromaFormat::Monochrome {
180            (Vec::new(), Vec::new())
181        } else {
182            let (sw, sh) = chroma.subsampling();
183            let (cw, ch) = (w.div_ceil(sw), h.div_ceil(sh));
184            (
185                cut(&planes[1], left / sw, top / sh, cw, ch),
186                cut(&planes[2], left / sw, top / sh, cw, ch),
187            )
188        };
189        DecodedFrame {
190            width: w as u32,
191            height: h as u32,
192            bit_depth,
193            chroma,
194            y,
195            u,
196            v,
197            color,
198        }
199    }
200
201    /// The planes as ffmpeg's `rawvideo` writes them for the stream's native
202    /// pixel format: one byte a sample at 8 bits, two (little-endian) above.
203    pub fn to_raw_yuv(&self) -> Vec<u8> {
204        let mut out = Vec::new();
205        for plane in [&self.y, &self.u, &self.v] {
206            if self.bit_depth <= 8 {
207                out.extend(plane.iter().map(|&s| s as u8));
208            } else {
209                for &s in plane.iter() {
210                    out.extend_from_slice(&s.to_le_bytes());
211                }
212            }
213        }
214        out
215    }
216
217    /// Convert to tightly packed 8-bit BGRA, alpha 255, with the stream's
218    /// matrix and range ([`ColorInfo`]) and bilinear chroma upsampling.
219    /// Large pictures are converted on several threads (not on `wasm32`).
220    pub fn to_bgra(&self) -> Vec<u8> {
221        self.to_rgb_order([2, 1, 0])
222    }
223
224    /// [`to_bgra`](Self::to_bgra), in RGBA order.
225    pub fn to_rgba(&self) -> Vec<u8> {
226        self.to_rgb_order([0, 1, 2])
227    }
228
229    fn to_rgb_order(&self, order: [usize; 3]) -> Vec<u8> {
230        let (w, h) = (self.width as usize, self.height as usize);
231        let mut out = vec![255u8; w * h * 4];
232        if w == 0 || h == 0 {
233            return out;
234        }
235        let k = Coefficients::new(self);
236        let threads = conversion_threads(w, h);
237        if threads <= 1 {
238            self.convert_rows(0..h, &mut out, order, &k);
239            return out;
240        }
241        let rows_each = h.div_ceil(threads);
242        std::thread::scope(|scope| {
243            for (n, chunk) in out.chunks_mut(rows_each * w * 4).enumerate() {
244                let (k, start) = (&k, n * rows_each);
245                let rows = start..(start + rows_each).min(h);
246                scope.spawn(move || self.convert_rows(rows, chunk, order, k));
247            }
248        });
249        out
250    }
251
252    /// Rows `rows` of the picture into `out`, which holds exactly those rows.
253    ///
254    /// Chroma is upsampled bilinearly from where the samples sit by default
255    /// in H.264 and HEVC (`chroma_sample_loc_type` 0): level with the even
256    /// luma columns, and halfway between luma rows for 4:2:0. Each output
257    /// pixel's chroma is a weighted sum scaled by 8 — vertical weights 3:1
258    /// (or 4), horizontal 1:1 at odd columns (or 2) — so the arithmetic stays
259    /// in integers.
260    fn convert_rows(&self, rows: std::ops::Range<usize>, out: &mut [u8], order: [usize; 3], k: &Coefficients) {
261        let w = self.width as usize;
262        let mono = self.chroma == ChromaFormat::Monochrome;
263        let (sw, sh) = self.chroma.subsampling();
264        let (cw, ch) = self.chroma_size();
265        let (cw, ch) = (cw as usize, ch as usize);
266        let mid = k.c_mid;
267        // One chroma row, interpolated vertically, scaled by 4, centred on 0.
268        let mut ru = vec![0i32; cw];
269        let mut rv = vec![0i32; cw];
270        for (row, y) in rows.enumerate() {
271            if !mono {
272                let (a, b) = if sh == 2 {
273                    let c = (y / 2).min(ch - 1);
274                    let other = if y % 2 == 0 { c.saturating_sub(1) } else { (c + 1).min(ch - 1) };
275                    (c, other)
276                } else {
277                    (y.min(ch - 1), y.min(ch - 1))
278                };
279                let (wa, wb) = if sh == 2 { (3, 1) } else { (4, 0) };
280                let (ua, ub) = (&self.u[a * cw..][..cw], &self.u[b * cw..][..cw]);
281                let (va, vb) = (&self.v[a * cw..][..cw], &self.v[b * cw..][..cw]);
282                for i in 0..cw {
283                    ru[i] = wa * ua[i] as i32 + wb * ub[i] as i32 - 4 * mid;
284                    rv[i] = wa * va[i] as i32 + wb * vb[i] as i32 - 4 * mid;
285                }
286            }
287            let luma = &self.y[y * w..][..w];
288            let line = &mut out[row * w * 4..][..w * 4];
289            for (x, (px, &l)) in line.chunks_exact_mut(4).zip(luma).enumerate() {
290                let (cb, cr) = if mono {
291                    (0, 0)
292                } else if sw == 2 {
293                    let j = x / 2;
294                    if x % 2 == 0 {
295                        (2 * ru[j], 2 * rv[j])
296                    } else {
297                        let n = (j + 1).min(cw - 1);
298                        (ru[j] + ru[n], rv[j] + rv[n])
299                    }
300                } else {
301                    (2 * ru[x], 2 * rv[x])
302                };
303                let yv = k.y * (l as i32 - k.y_off);
304                let to8 = |v: i32| ((v + (1 << 15)) >> 16).clamp(0, 255) as u8;
305                px[order[0]] = to8(yv + k.rv * cr);
306                px[order[1]] = to8(yv - k.gu * cb - k.gv * cr);
307                px[order[2]] = to8(yv + k.bu * cb);
308            }
309        }
310    }
311}
312
313/// YCbCr → 8-bit RGB in 16.16 fixed point. Chroma coefficients include the
314/// ÷8 of [`DecodedFrame::convert_rows`]' interpolation weights.
315struct Coefficients {
316    y: i32,
317    y_off: i32,
318    c_mid: i32,
319    rv: i32,
320    gu: i32,
321    gv: i32,
322    bu: i32,
323}
324
325impl Coefficients {
326    fn new(f: &DecodedFrame) -> Self {
327        let max = ((1u32 << f.bit_depth) - 1) as f64;
328        let s = (1u32 << (f.bit_depth - 8)) as f64;
329        // Y in [0, 1] and C in [-0.5, 0.5], then to 0..255.
330        let (y_off, y_scale, c_scale) = if f.color.full_range {
331            (0.0, 1.0 / max, 1.0 / max)
332        } else {
333            (16.0 * s, 1.0 / (219.0 * s), 1.0 / (224.0 * s))
334        };
335        let (kr, kb) = match f.color.matrix {
336            Matrix::Bt601 => (0.299, 0.114),
337            Matrix::Bt709 => (0.2126, 0.0722),
338            Matrix::Bt2020 => (0.2627, 0.0593),
339        };
340        let kg = 1.0 - kr - kb;
341        let fixed = |v: f64| (v * 65536.0).round() as i32;
342        let c = 255.0 * c_scale / 8.0;
343        Coefficients {
344            y: fixed(255.0 * y_scale),
345            y_off: y_off as i32,
346            c_mid: 1 << (f.bit_depth - 1),
347            rv: fixed(2.0 * (1.0 - kr) * c),
348            gu: fixed(2.0 * kb * (1.0 - kb) / kg * c),
349            gv: fixed(2.0 * kr * (1.0 - kr) / kg * c),
350            bu: fixed(2.0 * (1.0 - kb) * c),
351        }
352    }
353}
354
355/// How many threads to convert a `w × h` picture on: one for small
356/// pictures, where starting threads costs more than it saves, and always one
357/// on `wasm32`, which has none.
358fn conversion_threads(w: usize, h: usize) -> usize {
359    if cfg!(target_family = "wasm") || w * h < 1 << 17 {
360        return 1;
361    }
362    std::thread::available_parallelism().map_or(1, |n| n.get()).min(8).min(h / 16).max(1)
363}
364
365#[cfg(test)]
366mod tests {
367    use super::*;
368
369    fn frame(w: u32, h: u32, y: Vec<u16>, u: Vec<u16>, v: Vec<u16>, matrix: Matrix) -> DecodedFrame {
370        DecodedFrame {
371            width: w,
372            height: h,
373            bit_depth: 8,
374            chroma: ChromaFormat::Yuv420,
375            y,
376            u,
377            v,
378            color: ColorInfo { matrix, full_range: false, vui_matrix: None },
379        }
380    }
381
382    #[test]
383    fn flat_colours_convert_like_the_formula() {
384        // BT.601 limited: (81, 90, 240) is pure red, (16, 128, 128) black,
385        // (235, 128, 128) white.
386        for (yuv, rgb) in [((81, 90, 240), [255, 0, 0]), ((16, 128, 128), [0, 0, 0]), ((235, 128, 128), [255, 255, 255])] {
387            let f = frame(4, 4, vec![yuv.0; 16], vec![yuv.1; 4], vec![yuv.2; 4], Matrix::Bt601);
388            for px in f.to_rgba().chunks_exact(4) {
389                for c in 0..3 {
390                    assert!((px[c] as i32 - rgb[c]).abs() <= 1, "{yuv:?} → {px:?}, want {rgb:?}");
391                }
392                assert_eq!(px[3], 255);
393            }
394        }
395    }
396
397    #[test]
398    fn chroma_is_interpolated_between_samples() {
399        // Cb steps from 128 to 228 between chroma columns 0 and 1 (luma 0–1
400        // and 2–3). Column 1 sits halfway: its blue is between its
401        // neighbours', not equal to one of them.
402        let (w, h) = (4u32, 2u32);
403        let f = frame(w, h, vec![128; 8], vec![128, 228], vec![128, 128], Matrix::Bt709);
404        let bgra = f.to_bgra();
405        let blue: Vec<u8> = (0..w as usize).map(|x| bgra[x * 4]).collect();
406        assert!(blue[0] < blue[1] && blue[1] < blue[2], "{blue:?}");
407        assert_eq!(blue[2], blue[3], "past the last sample, held: {blue:?}");
408    }
409
410    #[test]
411    fn threaded_conversion_matches_one_thread() {
412        let (w, h) = (640u32, 480u32);
413        let n = (w * h) as usize;
414        let y: Vec<u16> = (0..n).map(|i| (i * 7 % 220 + 16) as u16).collect();
415        let u: Vec<u16> = (0..n / 4).map(|i| (i * 13 % 224 + 16) as u16).collect();
416        let v: Vec<u16> = (0..n / 4).map(|i| (i * 29 % 224 + 16) as u16).collect();
417        let f = frame(w, h, y, u, v, Matrix::Bt709);
418        assert!(conversion_threads(w as usize, h as usize) > 1 || cfg!(target_family = "wasm") || std::thread::available_parallelism().map_or(true, |p| p.get() == 1));
419        let mut single = vec![255u8; n * 4];
420        f.convert_rows(0..h as usize, &mut single, [2, 1, 0], &Coefficients::new(&f));
421        assert!(f.to_bgra() == single);
422    }
423}