Skip to main content

threers_codec/decode/
reader.rs

1//! A video file's pictures one at a time, in display order, for playback.
2//!
3//! [`decode_video`](super::decode_video) walks a whole file and calls back
4//! with every picture, in decode order. A player wants the opposite: the next
5//! picture to show, when it is due, and nothing held that is not needed. That
6//! is [`VideoReader`]: open a file, call [`next_frame`](VideoReader::next_frame)
7//! for each picture in display order, and [`rewind`](VideoReader::rewind) to go
8//! round again.
9//!
10//! ```no_run
11//! use threers_codec::decode::VideoReader;
12//!
13//! let mut video = VideoReader::open(std::fs::read("clip.mp4").unwrap()).unwrap();
14//! while let Some((seconds, frame)) = video.next_frame().unwrap() {
15//!     let rgba = frame.to_rgba();
16//!     let _ = (seconds, rgba);
17//! }
18//! ```
19//!
20//! The containers are those [`decode_video`](super::decode_video) reads:
21//! MP4/MOV (H.264, HEVC, AV1), WebM/Matroska (VP9, AV1) and IVF (VP9, AV1).
22//! An Annex B elementary stream has no timing and no index, so it is decoded
23//! whole when opened and shown at 30 frames a second.
24//!
25//! Pictures decode in hardware where the build and machine have it:
26//! VideoToolbox on macOS (`videotoolbox`; H.264, HEVC, AV1 on M3 and later,
27//! VP9), NVDEC on NVIDIA (`nvidia`) and VAAPI on Linux (`vaapi`), in that
28//! order, with the native decoders behind them for anything the hardware will
29//! not take. [`backend`](VideoReader::backend) says which is in use;
30//! [`open_software`](VideoReader::open_software) keeps to the native ones.
31//!
32//! Display order comes from the container's timestamps, which are all known
33//! before the first picture is decoded: a picture goes out as soon as every
34//! sample that could show earlier has been decoded. A stream with B-frames
35//! holds back only the pictures it reorders, not a fixed window.
36
37use super::{Backend, DecodedFrame, NativeDecoder, Picture};
38use crate::bytes::{Bytes, FramePool};
39use crate::mp4_demux::TrackCodec;
40use std::collections::VecDeque;
41
42/// Frame rate assumed for streams that carry no timing.
43const UNTIMED_FPS: f64 = 30.0;
44
45/// One coded sample: where its bytes are, its timestamps, and when it shows.
46#[derive(Debug, Clone, Copy)]
47pub struct CodedSample {
48    /// Where its bytes start in [`Demuxed::bytes`].
49    pub start: usize,
50    /// Where they end.
51    pub end: usize,
52    /// Presentation time, in the track's units.
53    pub pts: i64,
54    /// Decode time, in the track's units.
55    pub dts: i64,
56    /// Duration, in the track's units; 0 where the container has none.
57    pub duration: i64,
58    /// When it shows, in seconds from the first picture shown.
59    pub seconds: f64,
60}
61
62type Sample = CodedSample;
63
64/// A video track as a decoder is fed it: the codec and its configuration
65/// record, and every coded sample in decode order with when it shows. What
66/// [`VideoReader`] reads, and what a platform decoder outside this crate (the
67/// browser's WebCodecs, say) can be given instead; see
68/// [`webcodecs`](super::webcodecs).
69#[derive(Debug, Clone)]
70pub struct Demuxed {
71    /// The codec.
72    pub codec: TrackCodec,
73    /// The configuration record (`avcC`, `hvcC`, `av1C`, `vpcC`) as the
74    /// container carries it; empty where it carries none (IVF, VP9 in
75    /// Matroska).
76    pub config: Vec<u8>,
77    /// Picture width the container states.
78    pub width: u32,
79    /// Picture height.
80    pub height: u32,
81    /// The samples' bytes: the file itself for MP4 (mapped, if it was
82    /// opened that way), the packets laid end to end for Matroska and IVF.
83    pub bytes: Bytes,
84    /// Every sample, in decode order.
85    pub samples: Vec<CodedSample>,
86}
87
88impl Demuxed {
89    /// Sample `i`'s bytes.
90    pub fn sample(&self, i: usize) -> &[u8] {
91        let s = &self.samples[i];
92        &self.bytes[s.start..s.end]
93    }
94
95    /// From the first picture shown to the end of the last, in seconds.
96    pub fn duration(&self) -> f64 {
97        clip_length(&self.samples)
98    }
99}
100
101/// Demux MP4/MOV, WebM/Matroska or IVF held in memory. An Annex B stream
102/// has no container to demux and is refused.
103pub fn demux(data: impl Into<Bytes>) -> Result<Demuxed, String> {
104    let data: Bytes = data.into();
105    let (codec, config, width, height, bytes, samples) = if data.starts_with(b"DKIF") {
106        let (fourcc, w, h, (num, den), packets) = crate::mkv_demux::index_ivf(&data)?;
107        let codec = match &fourcc {
108            b"VP90" => TrackCodec::Vp9,
109            b"AV01" => TrackCodec::Av1,
110            other => {
111                return Err(format!(
112                    "IVF codec {} is not supported",
113                    String::from_utf8_lossy(other)
114                ))
115            }
116        };
117        let first = packets.iter().map(|(pts, _)| *pts).min().unwrap_or(0);
118        let tick = num as f64 / den.max(1) as f64;
119        // The packets stay where they are in the file.
120        let samples = packets
121            .iter()
122            .map(|(pts, at)| Sample {
123                start: at.start,
124                end: at.end,
125                pts: *pts,
126                dts: *pts,
127                duration: 0,
128                seconds: (pts - first) as f64 * tick,
129            })
130            .collect();
131        (codec, Vec::new(), w, h, data, samples)
132    } else if data.starts_with(&[0, 0, 1]) || data.starts_with(&[0, 0, 0, 1]) {
133        return Err("an Annex B elementary stream has no container to demux".into());
134    } else {
135        let (track, bytes) = if data.starts_with(&[0x1A, 0x45, 0xDF, 0xA3]) {
136            // Matroska: the track's sample offsets point into the file,
137            // or, for a track with header stripping, into its packets laid
138            // end to end.
139            let (_, track, packets) = crate::ffcompat::mkv::parse_in_place(&data)?;
140            (track, packets.map_or(data, Bytes::from))
141        } else {
142            (crate::mp4_demux::demux_video(&data)?, data)
143        };
144        let first = track.samples.iter().map(|s| s.pts).min().unwrap_or(0);
145        let timescale = track.timescale.max(1) as f64;
146        let samples = track
147            .samples
148            .iter()
149            .map(|s| {
150                let start = s.offset as usize;
151                let end = start + s.size as usize;
152                if end > bytes.len() {
153                    return Err("sample runs past the end of the file".to_string());
154                }
155                Ok(Sample {
156                    start,
157                    end,
158                    pts: s.pts,
159                    dts: s.dts,
160                    duration: s.duration as i64,
161                    seconds: (s.pts - first) as f64 / timescale,
162                })
163            })
164            .collect::<Result<Vec<_>, String>>()?;
165        (
166            track.codec,
167            track.config,
168            track.width,
169            track.height,
170            bytes,
171            samples,
172        )
173    };
174    Ok(Demuxed {
175        codec,
176        config,
177        width,
178        height,
179        bytes,
180        samples,
181    })
182}
183
184/// What the pictures come from.
185enum Source {
186    /// A demuxed track, decoded a sample at a time.
187    Samples {
188        bytes: Bytes,
189        samples: Vec<Sample>,
190        /// The earliest display time among `samples[i..]`, and infinity past
191        /// the end: what a waiting picture has to be earlier than to go out.
192        earliest_after: Vec<f64>,
193        codec: TrackCodec,
194        config: Vec<u8>,
195        decoder: SampleDecoder,
196        next: usize,
197        /// Decoded, not yet shown, earliest first.
198        waiting: VecDeque<(f64, Picture)>,
199    },
200    /// An elementary stream, decoded whole when opened, and again on a rewind.
201    Decoded {
202        data: Bytes,
203        codec: TrackCodec,
204        frames: Vec<Option<DecodedFrame>>,
205        next: usize,
206    },
207}
208
209/// The decoder samples go through: the hardware's, or the native one.
210enum SampleDecoder {
211    Native(Box<NativeDecoder>),
212    /// The native H.264 or HEVC decoder reconstructing several pictures at
213    /// once on the thread pool. Pictures come out as they finish, so each is
214    /// keyed by its output id until then.
215    Threaded {
216        decoder: Box<NativeDecoder>,
217        /// Output id to display time, for pictures not finished yet.
218        in_flight: std::collections::HashMap<u64, f64>,
219    },
220    #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
221    VideoToolbox(crate::videotoolbox_decode::VideoToolboxDecoder),
222    #[cfg(all(
223        feature = "nvidia",
224        any(target_os = "linux", target_os = "windows"),
225        target_pointer_width = "64"
226    ))]
227    Nvdec(Box<crate::nvidia::NvdecDecoder>),
228    #[cfg(all(feature = "vaapi", target_os = "linux"))]
229    Vaapi(Box<crate::vaapi::VaapiDecoder>),
230}
231
232impl SampleDecoder {
233    /// The hardware's where it is wanted and takes the stream, else native.
234    /// `first` is the first coded sample, which describes a VP9 stream that
235    /// came with no `vpcC` (Matroska carries none).
236    fn open(
237        codec: TrackCodec,
238        config: &[u8],
239        width: u32,
240        height: u32,
241        hardware: bool,
242        first: Option<&[u8]>,
243        pool: &FramePool,
244    ) -> Result<Self, String> {
245        #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
246        if hardware {
247            let described = match (codec, config.is_empty(), first) {
248                (TrackCodec::Vp9, true, Some(frame)) => {
249                    vp9_record(frame, width, height).unwrap_or_default()
250                }
251                _ => config.to_vec(),
252            };
253            let track = crate::mp4_demux::VideoTrack {
254                codec,
255                width,
256                height,
257                config: described,
258                timescale: TIMESCALE,
259                samples: Vec::new(),
260            };
261            if let Ok(mut d) = crate::videotoolbox_decode::VideoToolboxDecoder::new_planar(&track) {
262                d.set_pool(pool.clone());
263                return Ok(Self::VideoToolbox(d));
264            }
265        }
266        // NVDEC, then VAAPI: both are loaded at run time and refuse when the
267        // machine has no driver.
268        #[cfg(any(
269            all(
270                feature = "nvidia",
271                any(target_os = "linux", target_os = "windows"),
272                target_pointer_width = "64"
273            ),
274            all(feature = "vaapi", target_os = "linux")
275        ))]
276        if hardware {
277            let track = crate::mp4_demux::VideoTrack {
278                codec,
279                width,
280                height,
281                config: config.to_vec(),
282                timescale: 90_000,
283                samples: Vec::new(),
284            };
285            #[cfg(all(
286                feature = "nvidia",
287                any(target_os = "linux", target_os = "windows"),
288                target_pointer_width = "64"
289            ))]
290            if Backend::Nvdec.available()
291                && let Ok(mut d) = crate::nvidia::NvdecDecoder::new(&track)
292            {
293                d.set_pool(pool.clone());
294                return Ok(Self::Nvdec(Box::new(d)));
295            }
296            #[cfg(all(feature = "vaapi", target_os = "linux"))]
297            if Backend::Vaapi.available()
298                && let Ok(mut d) = crate::vaapi::VaapiDecoder::new(&track)
299            {
300                d.set_pool(pool.clone());
301                return Ok(Self::Vaapi(Box::new(d)));
302            }
303        }
304        let _ = (hardware, first, pool);
305        let mut decoder = NativeDecoder::from_config(codec, config, width, height)?;
306        decoder.set_threads(crate::frame_threads());
307        if decoder.threaded() {
308            return Ok(Self::Threaded {
309                decoder: Box::new(decoder),
310                in_flight: Default::default(),
311            });
312        }
313        Ok(Self::Native(Box::new(decoder)))
314    }
315
316    /// The earliest display time among pictures still being reconstructed:
317    /// nothing later can go out before it has.
318    fn earliest_in_flight(&self) -> f64 {
319        match self {
320            Self::Threaded { in_flight, .. } => {
321                in_flight.values().copied().fold(f64::INFINITY, f64::min)
322            }
323            _ => f64::INFINITY,
324        }
325    }
326
327    /// Pictures that finished on the thread pool since last asked, with
328    /// their display times.
329    fn finished(&mut self) -> Vec<(f64, Picture)> {
330        let Self::Threaded { decoder, in_flight } = self else {
331            return Vec::new();
332        };
333        let mut out = Vec::new();
334        for (id, frame) in decoder.take_finished() {
335            let at = in_flight.remove(&id);
336            if let (Some(at), Some(frame)) = (at, frame) {
337                out.push((at, Picture::Planes(frame)));
338            }
339        }
340        out
341    }
342
343    /// Wait for every picture in flight, when there is nothing left to push.
344    fn wait(&mut self) -> bool {
345        match self {
346            Self::Threaded { decoder, in_flight } if !in_flight.is_empty() => {
347                decoder.wait_for_pictures();
348                true
349            }
350            _ => false,
351        }
352    }
353
354    fn backend(&self) -> Backend {
355        match self {
356            Self::Native(_) | Self::Threaded { .. } => Backend::Native,
357            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
358            Self::VideoToolbox(_) => Backend::VideoToolbox,
359            #[cfg(all(
360                feature = "nvidia",
361                any(target_os = "linux", target_os = "windows"),
362                target_pointer_width = "64"
363            ))]
364            Self::Nvdec(_) => Backend::Nvdec,
365            #[cfg(all(feature = "vaapi", target_os = "linux"))]
366            Self::Vaapi(_) => Backend::Vaapi,
367        }
368    }
369
370    /// `seconds` is when the sample shows, for a threaded decoder to key
371    /// its pictures by.
372    fn push(
373        &mut self,
374        sample: &[u8],
375        pts: i64,
376        dts: i64,
377        duration: i64,
378        seconds: f64,
379    ) -> Result<(), String> {
380        match self {
381            Self::Native(d) => d.push(sample, pts, dts, duration),
382            Self::Threaded { decoder, in_flight } => {
383                let first = decoder.next_output_id();
384                let pushed = decoder.push_nals(sample);
385                for id in first..decoder.next_output_id() {
386                    in_flight.insert(id, seconds);
387                }
388                pushed
389            }
390            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
391            Self::VideoToolbox(d) => d.push(sample, pts, dts, duration),
392            #[cfg(all(
393                feature = "nvidia",
394                any(target_os = "linux", target_os = "windows"),
395                target_pointer_width = "64"
396            ))]
397            Self::Nvdec(d) => d.push(sample, pts, dts, duration),
398            #[cfg(all(feature = "vaapi", target_os = "linux"))]
399            Self::Vaapi(d) => d.push(sample, pts, dts, duration),
400        }
401    }
402
403    /// The picture for `pts`, packed as the hardware left it where it
404    /// did.
405    fn take_frame(&mut self, pts: i64) -> Option<Picture> {
406        match self {
407            Self::Native(d) => d.take_frame(pts).map(Picture::Planes),
408            // Pictures come out through `finished`.
409            Self::Threaded { .. } => None,
410            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
411            Self::VideoToolbox(d) => d.take_packed(pts).map(Picture::Packed),
412            #[cfg(all(
413                feature = "nvidia",
414                any(target_os = "linux", target_os = "windows"),
415                target_pointer_width = "64"
416            ))]
417            Self::Nvdec(d) => d.take_packed(pts).map(Picture::Packed),
418            #[cfg(all(feature = "vaapi", target_os = "linux"))]
419            Self::Vaapi(d) => d.take_picture(pts),
420        }
421    }
422}
423
424/// A `vpcC` record for a profile 0 stream (8-bit 4:2:0, the usual VP9) from
425/// its first frame's header; `None` for the other profiles, which stay on
426/// the native decoder. Colour is left unspecified, as the frame header has it
427/// in a form the record does not.
428pub(crate) fn vp9_record(frame: &[u8], width: u32, height: u32) -> Option<Vec<u8>> {
429    let b = *frame.first()?;
430    // frame_marker (2 bits, 0b10), profile_low_bit, profile_high_bit.
431    if b >> 6 != 0b10 || (b >> 5) & 1 != 0 || (b >> 4) & 1 != 0 {
432        return None;
433    }
434    let luma = width as u64 * height as u64;
435    let level = if luma <= 2_228_224 {
436        41
437    } else if luma <= 8_912_896 {
438        51
439    } else {
440        62
441    };
442    // Version 1, no flags; profile 0; level; 8 bits, 4:2:0 colocated, video
443    // range; primaries, transfer and matrix unspecified (2); no init data.
444    Some(vec![
445        1,
446        0,
447        0,
448        0,
449        0,
450        level,
451        (8 << 4) | (1 << 1),
452        2,
453        2,
454        2,
455        0,
456        0,
457    ])
458}
459
460/// The timescale handed to a hardware session. Only `pts` round-trips
461/// through it, as a key, so any will do.
462#[cfg(all(feature = "videotoolbox", target_os = "macos"))]
463const TIMESCALE: u32 = 90_000;
464
465/// A video file, read a picture at a time in display order. See the
466/// [module docs](self).
467pub struct VideoReader {
468    source: Source,
469    width: u32,
470    height: u32,
471    duration: f64,
472    frames: usize,
473    hardware: bool,
474    /// The hardware decoders' picture buffers, given back by the caller.
475    pool: FramePool,
476}
477
478impl VideoReader {
479    /// Open a file held in memory, decoding in hardware where this build and
480    /// machine can. Fails if the container or codec is not one this crate
481    /// reads, or the stream's configuration does not parse.
482    pub fn open(data: Vec<u8>) -> Result<Self, String> {
483        Self::open_with(data.into(), true)
484    }
485
486    /// Open the file at `path`, mapped rather than read: the operating
487    /// system pages in what decoding touches as it touches it, so a film of
488    /// many gigabytes starts at once and is never all in memory. See
489    /// [`Bytes::map`].
490    pub fn open_file(path: impl AsRef<std::path::Path>) -> Result<Self, String> {
491        let path = path.as_ref();
492        let data = Bytes::map(path).map_err(|e| format!("{}: {e}", path.display()))?;
493        Self::open_with(data, true)
494    }
495
496    /// As [`open`](Self::open), holding the bytes shared rather than owned:
497    /// for a caller that reads them again, as a player decoding the
498    /// soundtrack does, without a second copy of the file.
499    pub fn open_shared(data: impl Into<Bytes>) -> Result<Self, String> {
500        Self::open_with(data.into(), true)
501    }
502
503    /// As [`open`](Self::open), with the native decoders only.
504    pub fn open_software(data: Vec<u8>) -> Result<Self, String> {
505        Self::open_with(data.into(), false)
506    }
507
508    fn open_with(data: Bytes, hardware: bool) -> Result<Self, String> {
509        if data.starts_with(&[0, 0, 1]) || data.starts_with(&[0, 0, 0, 1]) {
510            let codec = super::sniff_annexb(&data)
511                .ok_or("no H.264 SPS or HEVC VPS in the elementary stream")?;
512            let frames: Vec<_> = super::decode_annexb(codec, &data)?
513                .into_iter()
514                .map(Some)
515                .collect();
516            let (width, height) = frames
517                .first()
518                .and_then(|f| f.as_ref())
519                .map_or((0, 0), |f| (f.width, f.height));
520            let count = frames.len();
521            return Ok(Self {
522                source: Source::Decoded {
523                    data,
524                    codec,
525                    frames,
526                    next: 0,
527                },
528                width,
529                height,
530                duration: count as f64 / UNTIMED_FPS,
531                frames: count,
532                hardware: false,
533                pool: FramePool::default(),
534            });
535        }
536
537        let Demuxed {
538            codec,
539            config,
540            width,
541            height,
542            bytes,
543            samples,
544        } = demux(data)?;
545
546        let mut earliest_after = vec![f64::INFINITY; samples.len() + 1];
547        for i in (0..samples.len()).rev() {
548            earliest_after[i] = earliest_after[i + 1].min(samples[i].seconds);
549        }
550        let duration = clip_length(&samples);
551        let frames = samples.len();
552        let first = samples.first().map(|s| &bytes[s.start..s.end]);
553        let pool = FramePool::default();
554        let decoder = SampleDecoder::open(codec, &config, width, height, hardware, first, &pool)?;
555        Ok(Self {
556            source: Source::Samples {
557                bytes,
558                samples,
559                earliest_after,
560                codec,
561                config,
562                decoder,
563                next: 0,
564                waiting: VecDeque::new(),
565            },
566            width,
567            height,
568            duration,
569            frames,
570            hardware,
571            pool,
572        })
573    }
574
575    /// The pool the hardware decoders copy pictures into. Give a
576    /// [`Picture::Packed`]'s buffers back here once it has been shown
577    /// ([`recycle`](Self::recycle) does, or keep this handle on another
578    /// thread and call [`FramePool::give`]): a few buffers then serve the
579    /// whole film, instead of a fresh allocation a picture, which at 8K costs
580    /// more than the copy. Giving nothing back is fine, only slower.
581    pub fn frame_pool(&self) -> FramePool {
582        self.pool.clone()
583    }
584
585    /// Give a picture's buffers back to [`frame_pool`](Self::frame_pool).
586    pub fn recycle(&self, picture: Picture) {
587        if let Picture::Packed(frame) = picture {
588            self.pool.give(frame.y);
589            self.pool.give(frame.uv);
590        }
591    }
592
593    /// Which decoder is doing the work.
594    pub fn backend(&self) -> Backend {
595        match &self.source {
596            Source::Samples { decoder, .. } => decoder.backend(),
597            Source::Decoded { .. } => Backend::Native,
598        }
599    }
600
601    /// The picture size the container states.
602    pub fn size(&self) -> (u32, u32) {
603        (self.width, self.height)
604    }
605
606    /// How long the clip runs, in seconds: from the first picture shown to the
607    /// end of the last.
608    pub fn duration(&self) -> f64 {
609        self.duration
610    }
611
612    /// How many coded pictures the file holds.
613    pub fn frame_count(&self) -> usize {
614        self.frames
615    }
616
617    /// The next picture in display order and when it shows, in seconds from
618    /// the first; `None` at the end. A sample that fails to decode is skipped,
619    /// as a player would, unless nothing has decoded at all.
620    pub fn next_frame(&mut self) -> Result<Option<(f64, DecodedFrame)>, String> {
621        Ok(self.next_picture()?.map(|(at, p)| (at, p.into_decoded())))
622    }
623
624    /// [`next_frame`](Self::next_frame), with the picture as the decoder
625    /// left it: packed NV12 or P010 from a hardware decoder, which a GPU can
626    /// convert without a pass over it on the CPU; planes otherwise.
627    pub fn next_picture(&mut self) -> Result<Option<(f64, Picture)>, String> {
628        let (width, height) = (self.width, self.height);
629        match &mut self.source {
630            Source::Decoded { frames, next, .. } => {
631                if *next >= frames.len() {
632                    return Ok(None);
633                }
634                let at = *next;
635                *next += 1;
636                // Taken rather than cloned; `rewind` decodes the stream again.
637                Ok(frames[at]
638                    .take()
639                    .map(|f| (at as f64 / UNTIMED_FPS, Picture::Planes(f))))
640            }
641            Source::Samples {
642                bytes,
643                samples,
644                earliest_after,
645                codec,
646                config,
647                decoder,
648                next,
649                waiting,
650            } => loop {
651                for (at, frame) in decoder.finished() {
652                    let i = waiting.partition_point(|(t, _)| *t <= at);
653                    waiting.insert(i, (at, frame));
654                }
655                // The earliest waiting picture can go once nothing still to
656                // decode, or still being reconstructed, shows before it.
657                if let Some((seconds, _)) = waiting.front()
658                    && (*seconds < earliest_after[*next] || *next == samples.len())
659                    && *seconds <= decoder.earliest_in_flight()
660                {
661                    return Ok(waiting.pop_front());
662                }
663                if *next == samples.len() {
664                    if decoder.wait() {
665                        continue;
666                    }
667                    return Ok(waiting.pop_front());
668                }
669                let s = samples[*next];
670                *next += 1;
671                let mut pushed =
672                    decoder.push(&bytes[s.start..s.end], s.pts, s.dts, s.duration, s.seconds);
673                let mut frame = pushed.as_ref().ok().and_then(|_| decoder.take_frame(s.pts));
674                // A hardware decoder that takes the stream and then gives
675                // nothing for its first picture (NVDEC with VP9 below its
676                // smallest size does) is no use: the native one starts over.
677                if *next == 1 && frame.is_none() && decoder.backend() != Backend::Native {
678                    *decoder = SampleDecoder::open(
679                        *codec, config, width, height, false, None, &self.pool,
680                    )?;
681                    pushed =
682                        decoder.push(&bytes[s.start..s.end], s.pts, s.dts, s.duration, s.seconds);
683                    frame = pushed.as_ref().ok().and_then(|_| decoder.take_frame(s.pts));
684                }
685                match pushed {
686                    Ok(()) => {
687                        if let Some(frame) = frame {
688                            let at = waiting.partition_point(|(t, _)| *t <= s.seconds);
689                            waiting.insert(at, (s.seconds, frame));
690                        }
691                    }
692                    // Nothing shown yet and nothing left: say why.
693                    Err(e) if *next == samples.len() && waiting.is_empty() => return Err(e),
694                    Err(_) => {}
695                }
696            },
697        }
698    }
699
700    /// Back to the first picture.
701    pub fn rewind(&mut self) -> Result<(), String> {
702        match &mut self.source {
703            Source::Decoded {
704                data,
705                codec,
706                frames,
707                next,
708            } => {
709                if *next > 0 {
710                    // The pictures handed out were moved, not copied.
711                    *frames = super::decode_annexb(*codec, data)?
712                        .into_iter()
713                        .map(Some)
714                        .collect();
715                    *next = 0;
716                }
717            }
718            Source::Samples {
719                bytes,
720                samples,
721                codec,
722                config,
723                decoder,
724                next,
725                waiting,
726                ..
727            } => {
728                let first = samples.first().map(|s| &bytes[s.start..s.end]);
729                *decoder = SampleDecoder::open(
730                    *codec,
731                    config,
732                    self.width,
733                    self.height,
734                    self.hardware,
735                    first,
736                    &self.pool,
737                )?;
738                *next = 0;
739                waiting.clear();
740            }
741        }
742        Ok(())
743    }
744}
745
746/// The span from the first picture shown to the end of the last: the last
747/// display time plus one frame, the frame being the last sample's stated
748/// duration where it has one and the typical spacing otherwise.
749fn clip_length(samples: &[Sample]) -> f64 {
750    let mut times: Vec<f64> = samples.iter().map(|s| s.seconds).collect();
751    times.sort_by(f64::total_cmp);
752    let Some(&last) = times.last() else {
753        return 0.0;
754    };
755    let mut gaps: Vec<f64> = times
756        .windows(2)
757        .map(|w| w[1] - w[0])
758        .filter(|g| *g > 0.0)
759        .collect();
760    gaps.sort_by(f64::total_cmp);
761    let frame = gaps
762        .get(gaps.len() / 2)
763        .copied()
764        .unwrap_or(1.0 / UNTIMED_FPS);
765    last + frame
766}
767
768#[cfg(test)]
769mod tests {
770    use super::*;
771
772    /// Matroska and IVF samples are found where they lie in the file: the
773    /// demuxed bytes are the file's own, not a copy.
774    #[test]
775    fn webm_and_ivf_demux_in_place() {
776        use crate::webm::{mux_webm, WebmCodec, WebmFrame, WebmParams};
777        let packets: [&[u8]; 3] = [&[1, 2, 3, 4], &[5, 6], &[7, 8, 9]];
778        let frames: Vec<WebmFrame> = packets
779            .iter()
780            .enumerate()
781            .map(|(i, &data)| WebmFrame {
782                data,
783                alpha: None,
784                timecode: i as u64 * 40,
785                keyframe: i == 0,
786            })
787            .collect();
788        let webm = Bytes::from(mux_webm(&WebmParams {
789            width: 64,
790            height: 48,
791            codec: WebmCodec::Av1,
792            timecode_scale_ns: 1_000_000,
793            frames: &frames,
794        }));
795
796        let mut ivf = b"DKIF\0\0\x20\0AV01\x40\0\x30\0\x19\0\0\0\x01\0\0\0".to_vec();
797        ivf.resize(32, 0);
798        for (i, p) in packets.iter().enumerate() {
799            ivf.extend_from_slice(&(p.len() as u32).to_le_bytes());
800            ivf.extend_from_slice(&(i as u64).to_le_bytes());
801            ivf.extend_from_slice(p);
802        }
803        let ivf = Bytes::from(ivf);
804
805        for file in [webm, ivf] {
806            let d = demux(file.clone()).unwrap();
807            assert_eq!(d.bytes.as_ptr(), file.as_ptr(), "the file's own bytes");
808            let got: Vec<&[u8]> = d.samples.iter().map(|s| &d.bytes[s.start..s.end]).collect();
809            assert_eq!(got, packets);
810            assert!(
811                (d.samples[2].seconds - 0.08).abs() < 1e-9,
812                "{:?}",
813                d.samples
814            );
815        }
816    }
817}