Skip to main content

threers_codec/ffcompat/
reader.rs

1//! An MP4/MOV video read the way the ffmpeg n7.1 CLI pipes it as rgb24:
2//! [`super::mov`] timing, [`super::pipe`] frame selection, the native
3//! decoders, and [`super::swscale`].
4
5use super::mov::{self, MovVideo};
6use super::swscale::Yuv2Rgb;
7use crate::decode::NativeDecoder;
8use crate::mp4_demux::{demux_video, VideoTrack};
9use std::collections::{BTreeMap, BTreeSet};
10
11/// A video file opened for reading as `ffmpeg -i <file> -f rawvideo -pix_fmt rgb24 -`
12/// would: same frames, same order, same pixels.
13pub struct VideoReader {
14    data: Vec<u8>,
15    /// The track's timing as ffmpeg computes it.
16    pub mov: MovVideo,
17    track: VideoTrack,
18}
19
20impl VideoReader {
21    /// `Err` for anything outside the supported subset (codec, rotation,
22    /// fragmented files, multiple sample descriptions or edits).
23    pub fn open(data: Vec<u8>) -> Result<Self, String> {
24        if data.starts_with(&[0x1A, 0x45, 0xDF, 0xA3]) {
25            // Matroska / WebM: the packets are copied out back to back.
26            let (mov, track, packets) = super::mkv::parse(&data)?;
27            return Ok(Self {
28                data: packets,
29                mov,
30                track,
31            });
32        }
33        let mov = mov::parse(&data)?;
34        let track = demux_video(&data)?;
35        Ok(Self { data, mov, track })
36    }
37
38    /// Size of the piped pictures (after autorotation).
39    pub fn size(&self) -> (u32, u32) {
40        let (w, h) = (self.track.width, self.track.height);
41        match self.mov.orientation {
42            mov::Orientation::Transpose(_) => (h, w),
43            _ => (w, h),
44        }
45    }
46
47    /// Indices (into [`MovVideo::pts`]) of the frames the CLI writes for
48    /// `-ss A -i f -ss B`, `(A, B)` in microseconds.
49    /// Where decoding can start for the samples in `need` (decode index ->
50    /// frame): one sync sample before the last sync sample at or before the
51    /// first needed one. The extra GOP covers open-GOP leading pictures (a
52    /// B-frame shown before its I-frame references the previous GOP); with
53    /// fewer sync samples than that, the stream start. Without this every
54    /// call decodes from the start of the stream, which makes extracting a
55    /// long video chunk by chunk quadratic.
56    /// Whether [`decode_rgb`](Self::decode_rgb) of `wanted` starts after the
57    /// stream start (diagnostics / tests).
58    pub fn seeks_for(&self, wanted: &BTreeSet<usize>) -> bool {
59        let need: BTreeMap<usize, usize> = wanted
60            .iter()
61            .filter_map(|&i| self.mov.sample.get(i).map(|&s| (s, i)))
62            .collect();
63        self.seek_start(self.mov.decode_range.0, &need) > self.mov.decode_range.0
64    }
65
66    fn seek_start(&self, from: usize, need: &BTreeMap<usize, usize>) -> usize {
67        let Some(&first) = need.keys().next() else {
68            return from;
69        };
70        let mut last = None;
71        let mut prev = None;
72        for si in (from..=first.min(self.track.samples.len().saturating_sub(1))).rev() {
73            if self.track.samples[si].sync {
74                if last.is_none() {
75                    last = Some(si);
76                } else {
77                    prev = Some(si);
78                    break;
79                }
80            }
81        }
82        prev.unwrap_or(from)
83    }
84
85    /// Which frames the CLI writes to its pipe, as indices into `mov.pts`, for
86    /// input and output seeks `-ss A -i f -ss B` given as `(A, B)` in
87    /// microseconds.
88    pub fn pipe(&self, seek: Option<(i64, i64)>) -> Vec<usize> {
89        super::pipe::pipe_frames(&self.mov, seek)
90    }
91
92    /// Decode once, calling `f(index, rgb24)` for each wanted frame index
93    /// (presentation order, ascending).
94    pub fn decode_rgb(
95        &self,
96        wanted: &BTreeSet<usize>,
97        mut f: impl FnMut(usize, &[u8]) -> Result<(), String>,
98    ) -> Result<(), String> {
99        let Some(&last) = wanted.iter().next_back() else {
100            return Ok(());
101        };
102        // Decode-order position of every output frame we need.
103        let mut need: BTreeMap<usize, usize> = BTreeMap::new(); // sample -> frame index
104        for &i in wanted {
105            need.insert(
106                *self.mov.sample.get(i).ok_or("frame index out of range")?,
107                i,
108            );
109        }
110        let last_sample = (0..=last).map(|i| self.mov.sample[i]).max().unwrap_or(0);
111        let mut dec = NativeDecoder::new(&self.track)?;
112        let mut conv: Option<Yuv2Rgb> = None;
113        let mut generic: Option<super::sws_generic::GenericScaler> = None;
114        let mut ready: BTreeMap<usize, Vec<u8>> = BTreeMap::new();
115        let mut next_out = wanted.iter().copied();
116        let mut pending = next_out.next();
117        let (from, to) = self.mov.decode_range;
118        let start = self.seek_start(from, &need);
119        for si in start..to.min(last_sample + 1) {
120            let s = &self.track.samples[si];
121            let bytes = self
122                .data
123                .get(s.offset as usize..s.offset as usize + s.size as usize)
124                .ok_or("sample outside the file")?;
125            dec.push(bytes, s.pts, s.dts, s.duration as i64)?;
126            // (A VP9 packet holding only a hidden frame shows nothing.)
127            let Some(&idx) = need.get(&si) else {
128                let _ = dec.take_frame(s.pts);
129                continue;
130            };
131            let frame = dec.take_frame(s.pts).ok_or("decoder produced no picture")?;
132            use crate::decode::ChromaFormat;
133            if (frame.width, frame.height) != (self.track.width, self.track.height) {
134                return Err(format!(
135                    "the picture size changes mid-stream ({}x{} at sample {si}, {}x{} at the start): the CLI rescales it with swscale's bicubic filter, which is not ported",
136                    frame.width, frame.height, self.track.width, self.track.height
137                ));
138            }
139            if frame.chroma == ChromaFormat::Monochrome
140                && self.mov.orientation == mov::Orientation::Identity
141            {
142                // swscale's gray -> rgb24: R = G = B = Y, high bit depths
143                // rounded down to 8 bits (measured: 10-bit is (y + 2) >> 2).
144                let sh = frame.bit_depth as u32 - 8;
145                let rnd = if sh > 0 { 1u32 << (sh - 1) } else { 0 };
146                let mut rgb = Vec::with_capacity(frame.y.len() * 3);
147                for &v in &frame.y {
148                    let g = ((v as u32 + rnd) >> sh).min(255) as u8;
149                    rgb.extend_from_slice(&[g, g, g]);
150                }
151                ready.insert(idx, rgb);
152                while let Some(p) = pending {
153                    let Some(rgb) = ready.remove(&p) else { break };
154                    f(p, &rgb)?;
155                    pending = next_out.next();
156                }
157                continue;
158            }
159            if frame.color.vui_matrix == Some(0)
160                && frame.chroma == ChromaFormat::Yuv444
161                && self.mov.orientation == mov::Orientation::Identity
162            {
163                // Planar G, B, R (VP9 RGB): swscale's gbrp -> rgb24 is a copy.
164                if frame.bit_depth != 8 {
165                    return Err(format!(
166                        "{}-bit RGB to rgb24 is not supported yet",
167                        frame.bit_depth
168                    ));
169                }
170                let n = frame.width as usize * frame.height as usize;
171                let mut rgb = vec![0u8; 3 * n];
172                for i in 0..n {
173                    rgb[3 * i] = frame.v[i] as u8;
174                    rgb[3 * i + 1] = frame.y[i] as u8;
175                    rgb[3 * i + 2] = frame.u[i] as u8;
176                }
177                ready.insert(idx, rgb);
178                while let Some(p) = pending {
179                    let Some(rgb) = ready.remove(&p) else { break };
180                    f(p, &rgb)?;
181                    pending = next_out.next();
182                }
183                continue;
184            }
185            let (log2_cw, log2_ch) = match frame.chroma {
186                ChromaFormat::Yuv420 => (1, 1),
187                ChromaFormat::Yuv422 => (1, 0),
188                ChromaFormat::Yuv444 => (0, 0),
189                ChromaFormat::Yuv440 => (0, 1),
190                ChromaFormat::Monochrome => {
191                    return Err("monochrome to rgb24 is not supported yet".into())
192                }
193            };
194            let (w, h) = (frame.width as usize, frame.height as usize);
195            let c = conv.get_or_insert_with(|| {
196                Yuv2Rgb::new(frame.color.vui_matrix.unwrap_or(2), frame.color.full_range)
197            });
198            let (cw, ch) = (
199                (w + (1 << log2_cw) - 1) >> log2_cw,
200                (h + (1 << log2_ch) - 1) >> log2_ch,
201            );
202            let o = self.mov.orientation;
203            let transposed = matches!(o, mov::Orientation::Transpose(_));
204            if (transposed || matches!(o, mov::Orientation::Arbitrary(_))) && log2_cw != log2_ch {
205                return Err(format!(
206                    "{:?} with rotation is not supported yet",
207                    frame.chroma
208                ));
209            }
210            let (ow, oh) = if transposed { (h, w) } else { (w, h) };
211            // swscale's special converter: 8-bit 4:2:0 / 4:2:2, even height.
212            let fast = frame.bit_depth == 8 && log2_cw == 1 && oh % 2 == 0;
213            if !fast {
214                if matches!(o, mov::Orientation::Arbitrary(_)) {
215                    return Err(format!(
216                        "{}-bit {:?} with an arbitrary rotation is not supported yet",
217                        frame.bit_depth, frame.chroma
218                    ));
219                }
220                let (y, u, v) = (
221                    orient(&frame.y, w, h, o),
222                    orient(&frame.u, cw, ch, o),
223                    orient(&frame.v, cw, ch, o),
224                );
225                let fmt = super::sws_generic::SrcFormat {
226                    w: ow,
227                    h: oh,
228                    log2_cw,
229                    log2_ch,
230                    bit_depth: frame.bit_depth as u32,
231                };
232                let sc = generic.get_or_insert_with(|| super::sws_generic::GenericScaler::new(fmt));
233                ready.insert(idx, sc.convert(c, &y, &u, &v));
234            } else {
235                let to8 = |p: &[u16]| -> Vec<u8> { p.iter().map(|&v| v as u8).collect() };
236                let (y, u, v) = match o {
237                    mov::Orientation::Arbitrary(deg) => {
238                        // ff_draw_color(black): Y 16 (limited) or 0 (full), chroma 128.
239                        let fy = if frame.color.full_range { 0 } else { 16 };
240                        let a = rotate_angle(deg);
241                        (
242                            rotate_plane(&to8(&frame.y), w, h, a, fy),
243                            rotate_plane(&to8(&frame.u), cw, ch, a, 128),
244                            rotate_plane(&to8(&frame.v), cw, ch, a, 128),
245                        )
246                    }
247                    _ => (
248                        orient(&to8(&frame.y), w, h, o),
249                        orient(&to8(&frame.u), cw, ch, o),
250                        orient(&to8(&frame.v), cw, ch, o),
251                    ),
252                };
253                ready.insert(
254                    idx,
255                    if log2_ch == 1 {
256                        c.convert_420(&y, &u, &v, ow, oh)
257                    } else {
258                        c.convert_422(&y, &u, &v, ow, oh)
259                    },
260                );
261            }
262            while let Some(p) = pending {
263                let Some(rgb) = ready.remove(&p) else { break };
264                f(p, &rgb)?;
265                pending = next_out.next();
266            }
267        }
268        if pending.is_some() {
269            return Err("stream ended before every wanted frame was decoded".into());
270        }
271        Ok(())
272    }
273}
274
275/// `vf_transpose` / `vf_hflip` / `vf_vflip` on one `w x h` plane.
276fn orient<T: Copy + Default>(p: &[T], w: usize, h: usize, o: mov::Orientation) -> Vec<T> {
277    match o {
278        mov::Orientation::Identity | mov::Orientation::Arbitrary(_) => p.to_vec(),
279        mov::Orientation::Flip { h: hf, v: vf } => {
280            let mut out = vec![T::default(); w * h];
281            for y in 0..h {
282                let sy = if vf { h - 1 - y } else { y };
283                for x in 0..w {
284                    let sx = if hf { w - 1 - x } else { x };
285                    out[y * w + x] = p[sy * w + sx];
286                }
287            }
288            out
289        }
290        mov::Orientation::Transpose(dir) => {
291            // dst' = transpose(src'), src' rows read bottom-up when dir & 1,
292            // dst rows written bottom-up when dir & 2. dst is h x w (w' = h).
293            let (dw, dh) = (h, w);
294            let mut out = vec![T::default(); dw * dh];
295            for dy in 0..dh {
296                let ry = if dir & 2 != 0 { dh - 1 - dy } else { dy };
297                for dx in 0..dw {
298                    // transposed: dst'(ry, dx) = src'(dx, ry) -> src row dx, col ry.
299                    let sy = if dir & 1 != 0 { h - 1 - dx } else { dx };
300                    out[dy * dw + dx] = p[sy * w + ry];
301                }
302            }
303            out
304        }
305    }
306}
307
308const FIXP: i64 = 1 << 16;
309const FIXP2: i64 = 1 << 20;
310const INT_PI: i64 = 3294199;
311
312/// `vf_rotate`'s `int_sin`: input scaled by 2^20, output by 2^16.
313fn int_sin(mut a: i64) -> i64 {
314    if a < 0 {
315        a = INT_PI - a;
316    }
317    a %= 2 * INT_PI;
318    if a >= INT_PI * 3 / 2 {
319        a -= 2 * INT_PI;
320    }
321    if a >= INT_PI / 2 {
322        a = INT_PI - a;
323    }
324    let a2 = (a * a) / FIXP2;
325    let mut res = 0;
326    let mut i = 2;
327    while i < 11 {
328        res += a;
329        a = -a * a2 / (FIXP2 * i * (i + 1));
330        i += 2;
331    }
332    (res + 8) >> 4
333}
334
335/// The angle `av_expr` evaluates for the CLI's `"%f*PI/180"`.
336fn rotate_angle(deg: i64) -> f64 {
337    let theta: f64 = format!("{:.6}", deg as f64).parse().unwrap();
338    theta * std::f64::consts::PI / 180.0
339}
340
341/// `vf_rotate` (bilinear, output size = input size, black fill) on one plane.
342fn rotate_plane(p: &[u8], w: usize, h: usize, angle: f64, fill: u8) -> Vec<u8> {
343    let angle_int = (angle * FIXP as f64 * 16.0) as i64;
344    let s = int_sin(angle_int) as i32 as i64;
345    let c = int_sin(angle_int + INT_PI / 2) as i32 as i64;
346    let (inw, inh, outw, outh) = (w as i64, h as i64, w as i64, h as i64);
347    // C integer division truncates toward zero.
348    let xi = -(outw - 1) * c / 2;
349    let yi = (outw - 1) * s / 2;
350    let mut xprime = -(outh - 1) * s / 2;
351    let mut yprime = -(outh - 1) * c / 2;
352    let mut out = vec![fill; w * h];
353    let (max_x, max_y) = (inw - 1, inh - 1);
354    for j in 0..outh {
355        let mut x = xprime + xi + FIXP * (inw - 1) / 2;
356        let mut y = yprime + yi + FIXP * (inh - 1) / 2;
357        for i in 0..outw {
358            let (x1, y1) = (x >> 16, y >> 16);
359            if x1 >= -1 && x1 <= inw && y1 >= -1 && y1 <= inh {
360                // interpolate_bilinear8
361                let ix = (x >> 16).clamp(0, max_x);
362                let iy = (y >> 16).clamp(0, max_y);
363                let (fx, fy) = (x & 0xffff, y & 0xffff);
364                let ix1 = (ix + 1).min(max_x);
365                let iy1 = (iy + 1).min(max_y);
366                let at = |xx: i64, yy: i64| p[(yy * inw + xx) as usize] as i64;
367                let s0 = (((1 << 16) - fx) * at(ix, iy) + fx * at(ix1, iy)) as i32 as i64;
368                let s1 = (((1 << 16) - fx) * at(ix, iy1) + fx * at(ix1, iy1)) as i32 as i64;
369                out[(j * outw + i) as usize] = ((((1 << 16) - fy) * s0 + fy * s1) >> 32) as u8;
370            }
371            x += c;
372            y -= s;
373        }
374        xprime += s;
375        yprime += c;
376    }
377    out
378}