Skip to main content

zvidlib_core/
codec.rs

1use crate::{
2    AudioBuffer, Codec, ColorRange, Error, ErrorKind, FrameIndex, FrameSource, Limits, PixelFormat,
3    Plane, Result, VideoDimensions, VideoFrame,
4};
5use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
6use std::future::Future;
7use std::pin::Pin;
8use std::sync::Arc;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::time::Duration;
11
12/// A non-`Send` encoder future that works on native and single-threaded WASM.
13pub type EncoderFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T>> + 'a>>;
14
15/// The kind of media carried by an encoded track.
16#[derive(Clone, Copy, Debug, Eq, PartialEq)]
17pub enum TrackKind {
18    Video,
19    Audio,
20}
21
22/// Dependency information written to the MP4 sample dependency table.
23#[derive(Clone, Copy, Debug, Eq, PartialEq)]
24pub struct SampleDependency {
25    pub is_leading: u8,
26    pub depends_on: u8,
27    pub is_depended_on: u8,
28    pub has_redundancy: u8,
29}
30
31impl SampleDependency {
32    pub const INDEPENDENT: Self = Self {
33        is_leading: 0,
34        depends_on: 2,
35        is_depended_on: 0,
36        has_redundancy: 0,
37    };
38
39    pub const DEPENDENT: Self = Self {
40        is_leading: 0,
41        depends_on: 1,
42        is_depended_on: 0,
43        has_redundancy: 0,
44    };
45
46    #[doc(hidden)]
47    pub fn to_sdtp(self) -> u8 {
48        ((self.is_leading & 3) << 6)
49            | ((self.depends_on & 3) << 4)
50            | ((self.is_depended_on & 3) << 2)
51            | (self.has_redundancy & 3)
52    }
53}
54
55/// One encoded access unit with exact decode and presentation timing.
56#[derive(Clone, Debug, Eq, PartialEq)]
57pub struct EncodedSample {
58    pub data: Vec<u8>,
59    pub dts: i64,
60    pub pts: i64,
61    pub duration: u32,
62    pub is_sync: bool,
63    pub dependency: SampleDependency,
64}
65
66/// Codec and sample-entry information needed to declare an MP4 track.
67#[derive(Clone, Debug, Eq, PartialEq)]
68pub struct EncoderConfig {
69    pub codec: Codec,
70    pub timescale: u32,
71    /// Complete codec configuration box, including its size and fourcc.
72    pub decoder_config: Vec<u8>,
73}
74
75/// Video format accepted by a video encoder.
76#[derive(Clone, Copy, Debug, Eq, PartialEq)]
77pub struct VideoEncoderFormat {
78    pub dimensions: VideoDimensions,
79    pub pixel_format: PixelFormat,
80}
81
82/// Audio format accepted by an audio encoder.
83#[derive(Clone, Copy, Debug, Eq, PartialEq)]
84pub struct AudioEncoderFormat {
85    pub sample_rate: u32,
86    pub channels: u16,
87}
88
89/// Gapless audio metadata produced when an encoder is drained.
90#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
91pub struct AudioGapless {
92    pub priming: u32,
93    pub padding: u32,
94}
95
96/// Packets and metadata emitted while draining an audio encoder.
97#[derive(Clone, Debug, Default, Eq, PartialEq)]
98pub struct AudioDrain {
99    pub samples: Vec<EncodedSample>,
100    pub gapless: AudioGapless,
101}
102
103/// Backend-neutral video encoder contract.
104pub trait VideoEncoder {
105    fn config(&self) -> &EncoderConfig;
106    fn format(&self) -> VideoEncoderFormat;
107    /// Whether this encoder runs on dedicated hardware or in software.
108    ///
109    /// A factory's [`VideoEncoderFactory::capability`] answers the same
110    /// question before an encoder exists; this is the answer for the encoder
111    /// actually created, which can differ when a backend that probed as
112    /// available fails to start and a `Prefer` factory falls back. The
113    /// default is `Software`, which is what every encoder that does not
114    /// override it is.
115    fn implementation(&self) -> CodecImplementation {
116        CodecImplementation::Software
117    }
118    /// A human-readable name for the backend doing the encoding, such as a
119    /// Media Foundation encoder's friendly name. For logs and diagnostics
120    /// only; it is not stable across platforms, drivers or releases.
121    fn backend_name(&self) -> &str {
122        "zvidlib"
123    }
124    fn encode<'a>(
125        &'a mut self,
126        index: FrameIndex,
127        frame: FrameSource<'a>,
128    ) -> EncoderFuture<'a, Vec<EncodedSample>>;
129    fn finish<'a>(&'a mut self) -> EncoderFuture<'a, Vec<EncodedSample>>;
130}
131
132/// Backend-neutral audio encoder contract.
133///
134/// # Where the audio encoders come from
135///
136/// zvidlib writes no audio codec of its own design. Each encoder behind this
137/// trait is either the platform's or a faithful port of the codec's reference
138/// encoder, so the quality a caller gets is the reference's rather than a
139/// reimplementation's:
140///
141/// * AAC-LC: `native_aac_audio_encoder_factory` delegates to the operating
142///   system's encoder, AudioToolbox on macOS and Media Foundation on Windows,
143///   and reports `HardwareUnavailable` elsewhere; the browser build encodes
144///   through `WebCodecs`. A pure-Rust AAC-LC encoder is a large subsystem in
145///   its own right -- filter bank, psychoacoustic model, quantization and rate
146///   control -- with no open reference implementation to port, and the
147///   platforms already have good ones (issue #174).
148/// * Opus: `native_opus_audio_encoder_factory` runs `opus-pure`, a pure-Rust
149///   port of libopus, on every target; the browser build can also encode
150///   through `WebCodecs`.
151/// * Vorbis: `native_vorbis_audio_encoder_factory` runs zvidlib's own port of
152///   the libvorbis 1.3.7 encoder, whose packets are bit-identical to
153///   libvorbis's for the same input and quality.
154///
155/// The concern behind issue #174 was that a mediocre encoder shipped under
156/// this crate's name is worse for callers than none, because it is harder to
157/// route around. A port of the reference encoder answers that concern where an
158/// original design could not. `crate::MediaOutput` muxes an audio track
159/// given any `AudioEncoder`, so a caller with an encoder of its own still
160/// plugs it in here.
161pub trait AudioEncoder {
162    fn config(&self) -> &EncoderConfig;
163    fn format(&self) -> AudioEncoderFormat;
164    fn encode<'a>(
165        &'a mut self,
166        index: FrameIndex,
167        buffer: AudioBuffer,
168    ) -> EncoderFuture<'a, Vec<EncodedSample>>;
169    fn finish<'a>(&'a mut self) -> EncoderFuture<'a, AudioDrain>;
170}
171
172/// A normalized codec profile that can be compared across backend implementations.
173#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
174#[non_exhaustive]
175pub enum CodecProfile {
176    UncompressedGray8,
177    HevcMain,
178    HevcMain10,
179    Av1Main,
180    Av1High,
181    Av1Professional,
182    /// VP8 has a single profile; the bitstream version each frame carries
183    /// selects its reconstruction and loop filters.
184    Vp8,
185    /// VP9 profile 0: 8-bit 4:2:0.
186    Vp9Profile0,
187    /// VP9 profile 1: 8-bit 4:2:2, 4:4:0 or 4:4:4.
188    Vp9Profile1,
189    /// VP9 profile 2: 10- or 12-bit 4:2:0.
190    Vp9Profile2,
191    /// VP9 profile 3: 10- or 12-bit 4:2:2, 4:4:0 or 4:4:4.
192    Vp9Profile3,
193    AacLowComplexity,
194    /// Opus has a single profile.
195    Opus,
196    /// Vorbis I has a single profile.
197    Vorbis,
198}
199
200/// Whether a caller permits or requires a hardware codec implementation.
201#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
202#[non_exhaustive]
203pub enum HardwarePreference {
204    Require,
205    Prefer,
206    Avoid,
207}
208
209/// The implementation class selected by a codec factory.
210#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
211#[non_exhaustive]
212pub enum CodecImplementation {
213    Software,
214    Hardware,
215}
216
217/// The result of querying a factory with a complete normalized configuration.
218#[derive(Clone, Debug, Eq, PartialEq)]
219#[non_exhaustive]
220pub enum CodecSupport {
221    Supported { implementation: CodecImplementation },
222    UnsupportedCodec,
223    UnsupportedProfile,
224    InvalidConfiguration { reason: String },
225    HardwareUnavailable,
226}
227
228impl CodecSupport {
229    pub const fn is_supported(&self) -> bool {
230        matches!(self, Self::Supported { .. })
231    }
232}
233
234/// Backend-neutral video decoder configuration.
235#[derive(Clone, Debug, Eq, PartialEq)]
236pub struct VideoDecoderConfig {
237    pub codec: Codec,
238    pub profile: CodecProfile,
239    pub coded_dimensions: VideoDimensions,
240    pub output_format: PixelFormat,
241    pub color_range: ColorRange,
242    pub hardware: HardwarePreference,
243    /// Container configuration bytes in the codec's standardized format.
244    pub configuration: Vec<u8>,
245}
246
247/// Backend-neutral video encoder configuration.
248#[derive(Clone, Debug, Eq, PartialEq)]
249pub struct VideoEncoderConfig {
250    pub codec: Codec,
251    pub profile: CodecProfile,
252    pub coded_dimensions: VideoDimensions,
253    pub input_format: PixelFormat,
254    pub color_range: ColorRange,
255    pub hardware: HardwarePreference,
256    /// Exact media-clock timescale used for emitted DTS and PTS values.
257    pub timescale: u32,
258    /// Exact duration, in `timescale` ticks, of every submitted frame.
259    pub frame_duration: u32,
260    pub configuration: Vec<u8>,
261}
262
263/// One owned compressed sample in decode order with its presentation identity.
264#[derive(Clone, Debug, Eq, PartialEq)]
265pub struct EncodedVideoSample {
266    pub presentation_index: FrameIndex,
267    pub random_access: bool,
268    pub data: Vec<u8>,
269}
270
271/// One normalized decoder output, independent of backend-private image types.
272#[derive(Clone, Debug, Eq, PartialEq)]
273pub struct DecodedVideoFrame {
274    pub presentation_index: FrameIndex,
275    pub frame: VideoFrame,
276}
277
278/// A cheap cloneable cancellation signal shared with a running codec operation.
279#[derive(Clone, Debug, Default)]
280pub struct CancellationToken {
281    canceled: Arc<AtomicBool>,
282}
283
284impl CancellationToken {
285    pub fn new() -> Self {
286        Self::default()
287    }
288
289    pub fn cancel(&self) {
290        self.canceled.store(true, Ordering::Release);
291    }
292
293    pub fn is_canceled(&self) -> bool {
294        self.canceled.load(Ordering::Acquire)
295    }
296
297    #[doc(hidden)]
298    pub fn check(&self) -> Result<()> {
299        if self.is_canceled() {
300            Err(Error::new(ErrorKind::Canceled, "codec operation canceled"))
301        } else {
302            Ok(())
303        }
304    }
305}
306
307/// A stateful video decoder that may retain reference and reorder state.
308///
309/// Requires `Send` so an [`ExactFrameReader`] (and the decoder it owns) can be moved to a
310/// background thread, which native examples rely on to decode without stalling rendering.
311pub trait VideoDecoder: Send {
312    fn submit(
313        &mut self,
314        sample: &EncodedVideoSample,
315        cancellation: &CancellationToken,
316    ) -> Result<Vec<DecodedVideoFrame>>;
317    fn drain(&mut self, cancellation: &CancellationToken) -> Result<Vec<DecodedVideoFrame>>;
318    fn reset(&mut self) -> Result<()>;
319
320    /// Whether the caller wants the pictures the next samples decode to, or is only decoding
321    /// them because a later frame references them.
322    ///
323    /// Reaching a frame in the middle of a long group of pictures means decoding every sample
324    /// before it, and on the way there nothing looks at those pictures. What they cost is not
325    /// the decoding: a hardware backend hands back NV12 and every one of these decoders then
326    /// converts a whole picture to RGBA on the CPU and allocates the frame that holds it. On
327    /// the bundled 1080p sample that conversion is 6.6 ms of the 9.6 ms a frame takes through
328    /// VideoToolbox, so a seek that skips it for the frames it passes costs a third of what it
329    /// did.
330    ///
331    /// A decoder must still *decode* a suppressed sample - the frames after it reference the
332    /// picture - and must still account for its presentation identity; what it may skip is
333    /// producing the [`DecodedVideoFrame`], which it reports by returning fewer frames than it
334    /// was given samples.
335    ///
336    /// The default implementation ignores the hint, which is always correct: a decoder that
337    /// keeps producing every frame is a decoder that is merely no faster. [`ExactFrameReader`]
338    /// tracks what it was actually handed rather than what it asked for, so a backend that
339    /// implements this and one that does not answer the same frames.
340    fn set_output_wanted(&mut self, _wanted: bool) {}
341}
342
343/// Discovers and creates video decoders without exposing backend-specific types.
344pub trait VideoDecoderFactory {
345    fn capability(&self, configuration: &VideoDecoderConfig) -> CodecSupport;
346    fn create(
347        &self,
348        configuration: &VideoDecoderConfig,
349        limits: &Limits,
350    ) -> Result<Box<dyn VideoDecoder>>;
351}
352
353/// Discovers and creates video encoders using the normalized capability model.
354pub trait VideoEncoderFactory {
355    fn capability(&self, configuration: &VideoEncoderConfig) -> CodecSupport;
356    fn create(
357        &self,
358        configuration: &VideoEncoderConfig,
359        limits: &Limits,
360    ) -> Result<Box<dyn VideoEncoder>>;
361}
362
363/// Backend-neutral audio encoder configuration.
364#[derive(Clone, Debug, Eq, PartialEq)]
365pub struct AudioEncoderConfig {
366    pub codec: Codec,
367    pub profile: CodecProfile,
368    pub sample_rate: u32,
369    pub channels: u16,
370    /// Exact media-clock timescale used for emitted DTS and PTS values.
371    pub timescale: u32,
372    pub configuration: Vec<u8>,
373}
374
375/// Discovers and creates audio encoders using the normalized capability model,
376/// the audio counterpart to [`VideoEncoderFactory`].
377///
378/// The native AAC, Opus and Vorbis factories implement it (see
379/// [`AudioEncoder`] for where each encoder comes from), and a caller's own
380/// adapter can too, so a caller that wants discoverable capability reporting
381/// is not left writing the `Result<Box<dyn AudioEncoder>>` construction by
382/// hand.
383pub trait AudioEncoderFactory {
384    fn capability(&self, configuration: &AudioEncoderConfig) -> CodecSupport;
385    fn create(
386        &self,
387        configuration: &AudioEncoderConfig,
388        limits: &Limits,
389    ) -> Result<Box<dyn AudioEncoder>>;
390}
391
392/// Reads an audio encoder's backend-private configuration: `Some(None)` for
393/// an empty one (the backend's default bit rate), `Some(Some(bits_per_second))`
394/// for four big-endian bytes naming a nonzero rate, and `None` for anything
395/// else. Zero is rejected rather than read as "no target", as the HEVC encoder
396/// does. Every native audio encoder takes this form.
397#[doc(hidden)]
398pub fn parse_audio_bit_rate(configuration: &[u8]) -> Option<Option<u32>> {
399    match configuration {
400        [] => Some(None),
401        [a, b, c, d] => match u32::from_be_bytes([*a, *b, *c, *d]) {
402            0 => None,
403            bits_per_second => Some(Some(bits_per_second)),
404        },
405        _ => None,
406    }
407}
408
409/// Observable counters for validating cache and decoder reuse behavior.
410#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
411pub struct DecodeStatistics {
412    pub samples_submitted: u64,
413    /// Of those, the ones submitted only so a later frame could reference them, with the
414    /// decoder told not to produce their pictures.
415    pub samples_skipped: u64,
416    pub cache_hits: u64,
417    pub resets: u64,
418    pub drains: u64,
419}
420
421/// Presentation-indexed exact-frame access over decode-order compressed samples.
422pub struct ExactFrameReader {
423    configuration: VideoDecoderConfig,
424    decoder: Box<dyn VideoDecoder>,
425    samples: Vec<EncodedVideoSample>,
426    decode_position_by_presentation: HashMap<FrameIndex, usize>,
427    cache: BTreeMap<FrameIndex, VideoFrame>,
428    lru: VecDeque<FrameIndex>,
429    /// Presentation frames the decoder has emitted since the last reset, which is how far
430    /// continuing the open session is assumed to reach: one of these that has since been
431    /// evicted from the cache is assumed unreachable without a reset. It is only that
432    /// assumption. A decoder that emits one of them again is published again and cached, not
433    /// refused (see [`publish`]).
434    ///
435    /// [`publish`]: Self::publish
436    published_since_reset: HashSet<FrameIndex>,
437    /// Frames whose sample was submitted while the decoder was told nothing wanted its picture,
438    /// so the decoder passed through them without producing one. Reaching one again needs the
439    /// same reset an evicted published frame does.
440    suppressed_since_reset: HashSet<FrameIndex>,
441    /// Frames whose sample has been submitted and whose picture has not come back yet, because a
442    /// reordering decoder holds one until the samples after it arrive. They are the frames a
443    /// decoder may drop the moment output stops being wanted, so turning it off writes them off
444    /// with the rest.
445    in_flight_since_reset: HashSet<FrameIndex>,
446    /// What the decoder was last told, so a walk toggles it once at the target rather than
447    /// on every sample.
448    output_wanted: bool,
449    next_decode_position: Option<usize>,
450    /// The random-access position the open decode session started from. Only frames at or
451    /// after it have been, or can still be, submitted without a reset.
452    session_start: Option<usize>,
453    /// The fast tier [`ExactFrameReader::seek`] answers from, when the caller has attached one.
454    seek_previews: Option<Arc<dyn SeekPreviewSource>>,
455    limits: Limits,
456    statistics: DecodeStatistics,
457}
458
459/// The longest a seek to any position of any track may take.
460///
461/// This is the requirement `ARCHITECTURE.md` section 3.2 states, kept here as a value so the
462/// tests that hold the library to it and the callers that budget against it quote one number.
463pub const SEEK_LATENCY_BUDGET: Duration = Duration::from_millis(50);
464
465/// Already-decoded pictures spread over a track, which a seek may answer from.
466///
467/// This is a trait rather than the concrete index because [`ExactFrameReader`] is portable and
468/// `zvidlib::previews::PreviewIndex` is not - its pass owns a thread, so it is native-only - and a
469/// reader that named the concrete type could not be built for the browser at all, which is the
470/// target that needs the tier most. An implementation must not decode and must not wait on a
471/// decoder: what it returns is a picture it already had, or nothing.
472///
473/// Issue #432 gave the browser its own implementation. What answers a seek on either target is
474/// now the same `zvidlib::previews::PreviewStore`; what differs is only what fills it, a thread in
475/// `zvidlib::previews::PreviewIndex` against an idle callback in `zvidlib::web_previews`.
476pub trait SeekPreviewSource: Send + Sync {
477    /// The kept picture closest to `frame` and the frame it is actually of, or `None` while
478    /// nothing near that position has been decoded.
479    fn nearest_at(&self, frame: FrameIndex) -> Option<(FrameIndex, VideoFrame)>;
480}
481
482/// What a constant-time [`ExactFrameReader::seek`] could answer with.
483///
484/// A seek is a question about a position on the timeline, and it has to be answered now: the
485/// exact frame at an arbitrary point of a long group of pictures is a decode of every frame
486/// before it, which on a 1080p track is seconds rather than the [`SEEK_LATENCY_BUDGET`]
487/// `ARCHITECTURE.md` section 3.2 allows. So a seek returns the best picture already decoded and
488/// the caller walks to the exact one behind it with [`ExactFrameReader::get`] if it still wants
489/// it.
490#[derive(Clone, Debug, Eq, PartialEq)]
491pub enum Seek {
492    /// The frame asked for, already decoded and cached. Nothing further is needed.
493    Exact(VideoFrame),
494    /// A picture of a different frame, the closest one the seek preview index holds.
495    Preview {
496        picture: VideoFrame,
497        /// The frame `picture` is actually of, which is within one preview stride of the frame
498        /// asked for once the pass has covered that part of the track.
499        frame: FrameIndex,
500    },
501    /// Nothing is decoded yet for anywhere near this position: no cached frame, and either no
502    /// preview index attached or an index whose pass has not produced a picture.
503    Pending,
504}
505
506/// What a caller wants out of a request, which is what decides whether the reader keeps the
507/// frames behind the one it was asked for.
508///
509/// The frames a walk passes are decoded either way - a picture cannot be decoded without its
510/// references - so this only decides which of them are converted to RGBA, which is the part of
511/// a walk that costs.
512#[derive(Clone, Copy, Debug, Eq, PartialEq)]
513enum Request {
514    /// Somewhere to be: keep the frames behind it that the cache can hold, so coming back to
515    /// one of them is free.
516    Destination,
517    /// Somewhere to pass through: keep only the frame asked for, and the reordered frames at or
518    /// after it that the next step needs anyway.
519    Step,
520}
521
522impl Request {
523    /// How far behind the target this request keeps pictures, in presentation frames.
524    fn cache_tail(self, limits: &Limits) -> u64 {
525        match self {
526            Self::Destination => u64::from(limits.max_cached_frames),
527            Self::Step => 0,
528        }
529    }
530}
531
532impl ExactFrameReader {
533    pub fn new(
534        factory: &dyn VideoDecoderFactory,
535        configuration: VideoDecoderConfig,
536        samples: Vec<EncodedVideoSample>,
537        limits: Limits,
538    ) -> Result<Self> {
539        if limits.max_cached_frames == 0 || limits.max_decode_samples_per_seek == 0 {
540            return Err(Error::new(
541                ErrorKind::ResourceLimit,
542                "exact-frame limits must permit cached frames and decode work",
543            ));
544        }
545        let capability = factory.capability(&configuration);
546        if !capability.is_supported() {
547            return Err(capability_error(capability));
548        }
549        if samples.is_empty() {
550            return Err(Error::new(
551                ErrorKind::InvalidInput,
552                "an exact-frame reader requires at least one sample",
553            ));
554        }
555        if !samples[0].random_access {
556            return Err(Error::new(
557                ErrorKind::InvalidInput,
558                "the first decode-order sample must be a random-access point",
559            ));
560        }
561        let mut positions = HashMap::with_capacity(samples.len());
562        for (position, sample) in samples.iter().enumerate() {
563            if positions
564                .insert(sample.presentation_index, position)
565                .is_some()
566            {
567                return Err(Error::new(
568                    ErrorKind::InvalidInput,
569                    "presentation frame identities must be unique",
570                ));
571            }
572        }
573        let decoder = factory.create(&configuration, &limits)?;
574        Ok(Self {
575            configuration,
576            decoder,
577            samples,
578            decode_position_by_presentation: positions,
579            cache: BTreeMap::new(),
580            lru: VecDeque::new(),
581            published_since_reset: HashSet::new(),
582            suppressed_since_reset: HashSet::new(),
583            in_flight_since_reset: HashSet::new(),
584            output_wanted: true,
585            next_decode_position: None,
586            session_start: None,
587            seek_previews: None,
588            limits,
589            statistics: DecodeStatistics::default(),
590        })
591    }
592
593    /// Attaches (or, with `None`, detaches) the preview source [`Self::seek`] answers from.
594    ///
595    /// The source is normally a `zvidlib::previews::PreviewIndex` filling itself on a decoder of its
596    /// own, and it is shared rather than owned so a reader can start answering seeks from it
597    /// while its pass is still walking the track.
598    pub fn set_seek_previews(&mut self, previews: Option<Arc<dyn SeekPreviewSource>>) {
599        self.seek_previews = previews;
600    }
601
602    /// Answers "what is at this position of the timeline" without decoding anything.
603    ///
604    /// This is the seek path, and it is bounded by `ARCHITECTURE.md` section 3.2: it takes the
605    /// same time whichever position of whatever track it is asked about, because it only ever
606    /// looks at pictures that are already decoded. It reads the presentation cache first - a
607    /// seek back to somewhere the reader has just been is exact for nothing - and then the
608    /// attached seek preview index, and it returns [`Seek::Pending`] rather than blocking when
609    /// neither holds anything.
610    ///
611    /// Reaching the exact frame is [`Self::get`], which is unbounded by construction: the frame
612    /// depends on every frame back to its random-access point and they all have to be decoded.
613    /// A caller that wants both draws what this returns and walks to the exact frame behind it.
614    pub fn seek(&mut self, presentation_index: FrameIndex) -> Seek {
615        if let Some(frame) = self.cache.get(&presentation_index).cloned() {
616            self.statistics.cache_hits = self.statistics.cache_hits.saturating_add(1);
617            self.touch(presentation_index);
618            return Seek::Exact(frame);
619        }
620        match self
621            .seek_previews
622            .as_ref()
623            .and_then(|previews| previews.nearest_at(presentation_index))
624        {
625            Some((frame, picture)) => Seek::Preview { picture, frame },
626            None => Seek::Pending,
627        }
628    }
629
630    /// Returns exactly the requested presentation frame, and the frames behind it that the
631    /// cache can hold, so a request that comes back for one of those is answered for nothing.
632    ///
633    /// This is the destination of a seek. A walk that is going to keep walking forwards should
634    /// ask for its intermediate frames with [`get_step`] instead, which converts the frame it
635    /// was asked for and nothing else.
636    ///
637    /// [`get_step`]: Self::get_step
638    pub fn get(
639        &mut self,
640        presentation_index: FrameIndex,
641        cancellation: &CancellationToken,
642    ) -> Result<VideoFrame> {
643        self.get_request(presentation_index, Request::Destination, cancellation)
644    }
645
646    /// Returns exactly the requested presentation frame and converts nothing else.
647    ///
648    /// A step is a picture on the way to somewhere, not somewhere: the caller is going to ask
649    /// for a frame further forward next, so the frames behind this one that [`get`] would keep
650    /// are converted for a request that never comes. That tail is bounded per call and so is
651    /// affordable for one seek, but a walk that publishes as it goes pays it once per published
652    /// picture, and once the stride is shorter than the tail the tails overlap and the walk
653    /// converts every picture it passes rather than the few it publishes (issue #402).
654    ///
655    /// Frames at or after the target in presentation order are still kept, exactly as [`get`]
656    /// keeps them: on a stream with B-pictures those are the reordered frames the next request
657    /// asks for, and dropping them would reset the decoder and re-walk the group of pictures.
658    ///
659    /// [`get`]: Self::get
660    pub fn get_step(
661        &mut self,
662        presentation_index: FrameIndex,
663        cancellation: &CancellationToken,
664    ) -> Result<VideoFrame> {
665        self.get_request(presentation_index, Request::Step, cancellation)
666    }
667
668    fn get_request(
669        &mut self,
670        presentation_index: FrameIndex,
671        request: Request,
672        cancellation: &CancellationToken,
673    ) -> Result<VideoFrame> {
674        cancellation.check()?;
675        if let Some(frame) = self.cache.get(&presentation_index).cloned() {
676            self.statistics.cache_hits = self.statistics.cache_hits.saturating_add(1);
677            self.touch(presentation_index);
678            return Ok(frame);
679        }
680        let target_position = *self
681            .decode_position_by_presentation
682            .get(&presentation_index)
683            .ok_or_else(|| {
684                Error::new(ErrorKind::InvalidInput, "presentation frame is not indexed")
685            })?;
686        let random_access_position = self.nearest_random_access(target_position);
687        // A decoder with output reordering (e.g. hierarchical B-frames) may need to be fed
688        // samples *past* `target_position` before the reordered frame at `target_position` is
689        // actually emitted. So `next_decode_position > target_position` does not by itself mean
690        // the frame is unreachable without a reset: it may simply still be buffered inside the
691        // decoder, pending release as more samples are submitted. The only case that truly
692        // requires a reset is when the frame was already published once and evicted from the
693        // cache, since a decoder is not expected to emit the same presentation frame twice
694        // without an intervening reset. That is an assumption about what continuing the session
695        // can reach, not a rule the decoder is held to: one that does emit the frame again is
696        // published a second time and cached rather than refused (see `publish`).
697        //
698        // A frame the decoder walked past without producing is in the same position as one that
699        // was published and evicted: the decoder will not emit it again, so only a reset can
700        // reach it.
701        //
702        // And a session that started at a later random-access point than the target's never
703        // submitted the target at all, however far it has since walked. On a track whose frames
704        // are all random-access points, a request for frame 2 opens the session at 2, and a
705        // request for frame 0 after it would otherwise walk on from 3 to the end of the track.
706        let can_reuse = self
707            .next_decode_position
708            .is_some_and(|position| position >= random_access_position)
709            && self
710                .session_start
711                .is_some_and(|start| start <= random_access_position)
712            && !self.published_since_reset.contains(&presentation_index)
713            && !self.suppressed_since_reset.contains(&presentation_index);
714        if !can_reuse {
715            self.decoder.reset()?;
716            self.statistics.resets = self.statistics.resets.saturating_add(1);
717            self.published_since_reset.clear();
718            self.suppressed_since_reset.clear();
719            self.in_flight_since_reset.clear();
720            self.next_decode_position = Some(random_access_position);
721            self.session_start = Some(random_access_position);
722        }
723
724        let mut work = 0_u32;
725        while let Some(position) = self.next_decode_position {
726            cancellation.check()?;
727            if work >= self.limits.max_decode_samples_per_seek {
728                return Err(Error::new(
729                    ErrorKind::ResourceLimit,
730                    "exact-frame request exceeded the configured decode-work limit",
731                ));
732            }
733            if position == self.samples.len() {
734                self.set_output_wanted(true);
735                self.drain_internal(cancellation)?;
736                break;
737            }
738            // Nothing looks at a picture decoded on the way to the target, and skipping the
739            // color conversion it would otherwise pay is most of what a long walk costs. Two
740            // kinds of frame are kept anyway, both because the request after this one is very
741            // likely to be for them.
742            //
743            // The first is anything at or after the target in *presentation* order. A walk
744            // arrives at its target from behind and carries on forwards, and deciding this by
745            // decode position instead discards exactly the reordered frames a stream with
746            // B-pictures asks for next: on the bundled sample every fourth request would then
747            // reset the decoder and walk the whole group of pictures again.
748            //
749            // The second is the frames immediately behind the target, as many as the cache can
750            // hold, and that one belongs to the request rather than to the reader. Walking here
751            // fills the cache with them as a side effect, and that is what makes stepping
752            // backwards a frame at a time cost nothing after a seek; skipping them would leave
753            // the cache empty behind the target and turn every backward step into another walk
754            // from the random-access point. For a destination they are a bounded charge - the
755            // last `max_cached_frames` conversions of a walk however long - against a saving
756            // that grows with the distance. For a step they are a charge with no saving behind
757            // it at all: the caller is walking forwards and will never ask for them, and paying
758            // the tail once per step is what made a 150 ms preview cadence convert 765 of the
759            // bundled sample's 768 pictures (issue #402).
760            let cache_tail = request.cache_tail(&self.limits);
761            let wanted = position >= target_position
762                || self.samples[position].presentation_index.0
763                    >= presentation_index.0.saturating_sub(cache_tail);
764            self.set_output_wanted(wanted);
765            let suppressed = !self.output_wanted;
766            let outputs = self.decoder.submit(&self.samples[position], cancellation)?;
767            self.statistics.samples_submitted = self.statistics.samples_submitted.saturating_add(1);
768            if suppressed {
769                self.statistics.samples_skipped = self.statistics.samples_skipped.saturating_add(1);
770                self.suppressed_since_reset
771                    .insert(self.samples[position].presentation_index);
772            } else {
773                self.in_flight_since_reset
774                    .insert(self.samples[position].presentation_index);
775            }
776            self.next_decode_position = Some(position + 1);
777            work += 1;
778            self.publish(outputs)?;
779            if let Some(frame) = self.cache.get(&presentation_index).cloned() {
780                self.touch(presentation_index);
781                return Ok(frame);
782            }
783        }
784        if let Some(frame) = self.cache.get(&presentation_index).cloned() {
785            self.touch(presentation_index);
786            Ok(frame)
787        } else {
788            Err(Error::new(
789                ErrorKind::MalformedMedia,
790                "decoder did not produce the requested presentation frame",
791            ))
792        }
793    }
794
795    /// Drains delayed output into the bounded presentation cache.
796    pub fn drain(&mut self, cancellation: &CancellationToken) -> Result<usize> {
797        cancellation.check()?;
798        self.set_output_wanted(true);
799        self.drain_internal(cancellation)
800    }
801
802    /// Tells the decoder whether the next samples' pictures are wanted, when that has changed.
803    ///
804    /// Turning output off writes off the frames the decoder is still holding as well as the ones
805    /// it is about to be handed. A reordering decoder releases a picture several samples after
806    /// the one that carried it, so a frame submitted while output was wanted can come out - and
807    /// be dropped - during the suppressed stretch that follows. Not writing those off is a
808    /// decoder that will never produce them and a reader that still believes it can ask, which
809    /// ends in `decoder did not produce the requested presentation frame`. A picture that does
810    /// arrive after all takes itself back off the list in [`publish`].
811    ///
812    /// [`publish`]: Self::publish
813    fn set_output_wanted(&mut self, wanted: bool) {
814        if self.output_wanted == wanted {
815            return;
816        }
817        if !wanted {
818            for frame in self.in_flight_since_reset.drain() {
819                self.suppressed_since_reset.insert(frame);
820            }
821        }
822        self.decoder.set_output_wanted(wanted);
823        self.output_wanted = wanted;
824    }
825
826    /// Clears decoder, reorder, and frame-cache state.
827    pub fn reset(&mut self) -> Result<()> {
828        self.set_output_wanted(true);
829        self.decoder.reset()?;
830        self.statistics.resets = self.statistics.resets.saturating_add(1);
831        self.next_decode_position = None;
832        self.session_start = None;
833        self.cache.clear();
834        self.lru.clear();
835        self.published_since_reset.clear();
836        self.suppressed_since_reset.clear();
837        self.in_flight_since_reset.clear();
838        Ok(())
839    }
840
841    pub const fn statistics(&self) -> DecodeStatistics {
842        self.statistics
843    }
844
845    pub fn cached_frames(&self) -> usize {
846        self.cache.len()
847    }
848
849    /// The random-access point a decode reaching `target_position` starts from.
850    ///
851    /// A random-access point presented *after* the target cannot be it, even when it comes
852    /// first in decode order: the target is then one of its leading pictures, and an open-GOP
853    /// stream's leading pictures (HEVC RASL pictures) reference pictures from before the
854    /// random-access point, so a decode that starts there cannot reconstruct them (issue #506).
855    /// The walk goes back to a random-access point the target is not leading.
856    fn nearest_random_access(&self, target_position: usize) -> usize {
857        let target = self.samples[target_position].presentation_index;
858        (0..=target_position)
859            .rev()
860            .find(|position| {
861                let sample = &self.samples[*position];
862                sample.random_access && sample.presentation_index <= target
863            })
864            .unwrap_or(0)
865    }
866
867    fn drain_internal(&mut self, cancellation: &CancellationToken) -> Result<usize> {
868        let outputs = self.decoder.drain(cancellation)?;
869        let count = outputs.len();
870        self.statistics.drains = self.statistics.drains.saturating_add(1);
871        self.next_decode_position = Some(self.samples.len());
872        self.publish(outputs)?;
873        Ok(count)
874    }
875
876    fn publish(&mut self, outputs: Vec<DecodedVideoFrame>) -> Result<()> {
877        for output in outputs {
878            // A decoder that ignores the output hint, or one that only emits a suppressed
879            // sample's picture later, hands back a frame this reader had written off. What it
880            // was handed is what counts, so it is a cached frame again rather than a reset.
881            self.suppressed_since_reset
882                .remove(&output.presentation_index);
883            self.in_flight_since_reset
884                .remove(&output.presentation_index);
885            // A repeat of a frame already published since the last reset is not an error, and
886            // was one until issue #415. Nothing here depends on a decoder emitting each
887            // presentation frame at most once per session: `published_since_reset` only decides
888            // whether a re-request can continue the open session or has to reset, and a decoder
889            // that hands the frame back anyway makes that answer conservative rather than wrong.
890            // Refusing it rejected the whole stream as `MalformedMedia` over a backend's
891            // behavior, and rejected it for a picture the reader was about to cache and could
892            // have answered the very request from. The `WebCodecs` backend already re-caches
893            // such a frame, so this is also what makes the two backends agree.
894            self.published_since_reset.insert(output.presentation_index);
895            if !self
896                .decode_position_by_presentation
897                .contains_key(&output.presentation_index)
898            {
899                return Err(Error::new(
900                    ErrorKind::MalformedMedia,
901                    "decoder produced an unindexed presentation frame",
902                ));
903            }
904            if output.frame.dimensions != self.configuration.coded_dimensions
905                || output.frame.pixel_format != self.configuration.output_format
906                || output.frame.color_range != self.configuration.color_range
907            {
908                return Err(Error::new(
909                    ErrorKind::MalformedMedia,
910                    "decoder output does not match its normalized configuration",
911                ));
912            }
913            self.insert_cache(output.presentation_index, output.frame);
914        }
915        Ok(())
916    }
917
918    fn insert_cache(&mut self, index: FrameIndex, frame: VideoFrame) {
919        self.cache.insert(index, frame);
920        self.touch(index);
921        while self.cache.len() > self.limits.max_cached_frames as usize {
922            if let Some(evicted) = self.lru.pop_front() {
923                self.cache.remove(&evicted);
924            }
925        }
926    }
927
928    fn touch(&mut self, index: FrameIndex) {
929        self.lru.retain(|candidate| *candidate != index);
930        self.lru.push_back(index);
931    }
932}
933
934fn capability_error(capability: CodecSupport) -> Error {
935    let (kind, message) = match capability {
936        CodecSupport::UnsupportedCodec => (
937            ErrorKind::Unsupported,
938            "decoder does not support the requested codec".into(),
939        ),
940        CodecSupport::UnsupportedProfile => (
941            ErrorKind::Unsupported,
942            "decoder does not support the requested profile".into(),
943        ),
944        CodecSupport::InvalidConfiguration { reason } => (ErrorKind::InvalidInput, reason),
945        CodecSupport::HardwareUnavailable => (
946            ErrorKind::Unsupported,
947            "requested hardware decoder is unavailable".into(),
948        ),
949        CodecSupport::Supported { .. } => (
950            ErrorKind::Internal,
951            "decoder capability changed unexpectedly".into(),
952        ),
953    };
954    Error::new(kind, message)
955}
956
957/// Returns the portable, software-only uncompressed Gray8 decoder backend.
958///
959/// Each packet contains one tightly packed Gray8 image. The concrete backend
960/// remains private while this factory provides an end-to-end reference path.
961pub fn uncompressed_video_decoder_factory() -> impl VideoDecoderFactory {
962    UncompressedVideoDecoderFactory
963}
964
965struct UncompressedVideoDecoderFactory;
966
967impl VideoDecoderFactory for UncompressedVideoDecoderFactory {
968    fn capability(&self, configuration: &VideoDecoderConfig) -> CodecSupport {
969        if configuration.codec != Codec::UncompressedVideo {
970            return CodecSupport::UnsupportedCodec;
971        }
972        if configuration.profile != CodecProfile::UncompressedGray8 {
973            return CodecSupport::UnsupportedProfile;
974        }
975        if configuration.output_format != PixelFormat::Gray8
976            || !configuration.configuration.is_empty()
977        {
978            return CodecSupport::InvalidConfiguration {
979                reason: "uncompressed Gray8 requires Gray8 output and no configuration bytes"
980                    .into(),
981            };
982        }
983        if configuration.hardware == HardwarePreference::Require {
984            return CodecSupport::HardwareUnavailable;
985        }
986        CodecSupport::Supported {
987            implementation: CodecImplementation::Software,
988        }
989    }
990
991    fn create(
992        &self,
993        configuration: &VideoDecoderConfig,
994        limits: &Limits,
995    ) -> Result<Box<dyn VideoDecoder>> {
996        let capability = self.capability(configuration);
997        if !capability.is_supported() {
998            return Err(capability_error(capability));
999        }
1000        if configuration.coded_dimensions.width > limits.max_width
1001            || configuration.coded_dimensions.height > limits.max_height
1002        {
1003            return Err(Error::new(
1004                ErrorKind::ResourceLimit,
1005                "decoded dimensions exceed the configured limits",
1006            ));
1007        }
1008        let required = u64::from(configuration.coded_dimensions.width)
1009            .checked_mul(u64::from(configuration.coded_dimensions.height))
1010            .ok_or_else(|| Error::new(ErrorKind::ResourceLimit, "decoded frame size overflow"))?;
1011        if required > limits.max_allocation_bytes {
1012            return Err(Error::new(
1013                ErrorKind::ResourceLimit,
1014                "decoded frame exceeds the configured allocation limit",
1015            ));
1016        }
1017        Ok(Box::new(UncompressedVideoDecoder {
1018            configuration: configuration.clone(),
1019            limits: *limits,
1020        }))
1021    }
1022}
1023
1024struct UncompressedVideoDecoder {
1025    configuration: VideoDecoderConfig,
1026    limits: Limits,
1027}
1028
1029impl VideoDecoder for UncompressedVideoDecoder {
1030    fn submit(
1031        &mut self,
1032        sample: &EncodedVideoSample,
1033        cancellation: &CancellationToken,
1034    ) -> Result<Vec<DecodedVideoFrame>> {
1035        cancellation.check()?;
1036        let width = usize::try_from(self.configuration.coded_dimensions.width)
1037            .map_err(|_| Error::new(ErrorKind::ResourceLimit, "decoded width is too large"))?;
1038        let height = usize::try_from(self.configuration.coded_dimensions.height)
1039            .map_err(|_| Error::new(ErrorKind::ResourceLimit, "decoded height is too large"))?;
1040        let length = width
1041            .checked_mul(height)
1042            .ok_or_else(|| Error::new(ErrorKind::ResourceLimit, "decoded frame size overflow"))?;
1043        if sample.data.len() != length {
1044            return Err(Error::new(
1045                ErrorKind::MalformedMedia,
1046                "uncompressed Gray8 packet size does not match coded dimensions",
1047            ));
1048        }
1049        let frame = VideoFrame::new(
1050            self.configuration.coded_dimensions,
1051            PixelFormat::Gray8,
1052            self.configuration.color_range,
1053            vec![Plane {
1054                data: sample.data.clone(),
1055                stride: width,
1056            }],
1057            &self.limits,
1058        )?;
1059        Ok(vec![DecodedVideoFrame {
1060            presentation_index: sample.presentation_index,
1061            frame,
1062        }])
1063    }
1064
1065    fn drain(&mut self, cancellation: &CancellationToken) -> Result<Vec<DecodedVideoFrame>> {
1066        cancellation.check()?;
1067        Ok(Vec::new())
1068    }
1069
1070    fn reset(&mut self) -> Result<()> {
1071        Ok(())
1072    }
1073}
1074
1075#[cfg(test)]
1076mod tests {
1077    use super::*;
1078    use std::sync::Mutex;
1079
1080    fn config() -> VideoDecoderConfig {
1081        VideoDecoderConfig {
1082            codec: Codec::UncompressedVideo,
1083            profile: CodecProfile::UncompressedGray8,
1084            coded_dimensions: VideoDimensions::new(1, 1, &Limits::default()).unwrap(),
1085            output_format: PixelFormat::Gray8,
1086            color_range: ColorRange::Full,
1087            hardware: HardwarePreference::Avoid,
1088            configuration: Vec::new(),
1089        }
1090    }
1091
1092    fn sample(index: u64, value: u8, random_access: bool) -> EncodedVideoSample {
1093        EncodedVideoSample {
1094            presentation_index: FrameIndex(index),
1095            random_access,
1096            data: vec![value],
1097        }
1098    }
1099
1100    fn value(frame: &VideoFrame) -> u8 {
1101        frame.planes[0].data[0]
1102    }
1103
1104    /// A reader over a single-group track: one random-access point at frame zero, so reaching
1105    /// any frame exactly means decoding every frame before it. This is the shape the bundled
1106    /// sample has and the shape the seek budget is hard for.
1107    fn single_group_reader(frames: u64) -> ExactFrameReader {
1108        let samples = (0..frames)
1109            .map(|index| sample(index, (index % 251) as u8, index == 0))
1110            .collect();
1111        ExactFrameReader::new(
1112            &uncompressed_video_decoder_factory(),
1113            config(),
1114            samples,
1115            Limits::default(),
1116        )
1117        .unwrap()
1118    }
1119
1120    /// A preview source holding one picture every `stride` frames, standing in for the pass a
1121    /// `previews::PreviewIndex` runs. The reader's contract is that a seek reads pictures and
1122    /// never decodes, and asserting that does not need a decoder behind the source.
1123    struct StridedPreviews {
1124        stride: u64,
1125        frames: u64,
1126    }
1127
1128    impl SeekPreviewSource for StridedPreviews {
1129        fn nearest_at(&self, frame: FrameIndex) -> Option<(FrameIndex, VideoFrame)> {
1130            if self.frames == 0 {
1131                return None;
1132            }
1133            let at = (frame.0 / self.stride * self.stride).min(self.frames - 1);
1134            let picture = VideoFrame::new(
1135                config().coded_dimensions,
1136                PixelFormat::Gray8,
1137                ColorRange::Full,
1138                vec![Plane {
1139                    data: vec![(at % 251) as u8],
1140                    stride: 1,
1141                }],
1142                &Limits::default(),
1143            )
1144            .ok()?;
1145            Some((FrameIndex(at), picture))
1146        }
1147    }
1148
1149    /// A seek with nothing decoded anywhere near it says so rather than deciding to decode: the
1150    /// caller falls through to its own exact request instead of the seek path silently becoming
1151    /// the walk it is bounded against.
1152    #[test]
1153    fn a_seek_with_nothing_decoded_is_pending_and_submits_no_samples() {
1154        let mut reader = single_group_reader(512);
1155        assert_eq!(reader.seek(FrameIndex(511)), Seek::Pending);
1156        assert_eq!(reader.statistics().samples_submitted, 0);
1157        assert_eq!(reader.statistics().resets, 0);
1158    }
1159
1160    /// A seek back to somewhere the reader has just been is the frame itself, from the cache.
1161    #[test]
1162    fn a_seek_into_the_cache_answers_exactly() {
1163        let mut reader = single_group_reader(64);
1164        let cancellation = CancellationToken::new();
1165        reader.get(FrameIndex(40), &cancellation).unwrap();
1166        let submitted = reader.statistics().samples_submitted;
1167        let Seek::Exact(frame) = reader.seek(FrameIndex(40)) else {
1168            panic!("a cached frame seeks exactly");
1169        };
1170        assert_eq!(value(&frame), 40);
1171        assert_eq!(
1172            reader.statistics().samples_submitted,
1173            submitted,
1174            "an exact seek decodes nothing"
1175        );
1176    }
1177
1178    /// The requirement in `ARCHITECTURE.md` section 3.2: a seek to any position of a track that
1179    /// is one group of pictures answers from an already-decoded picture and decodes nothing, so
1180    /// what it costs does not grow with how far along the track the position is.
1181    #[test]
1182    fn a_seek_anywhere_on_a_single_group_track_decodes_nothing() {
1183        let mut reader = single_group_reader(512);
1184        reader.set_seek_previews(Some(Arc::new(StridedPreviews {
1185            stride: 12,
1186            frames: 512,
1187        })));
1188        for target in [0_u64, 1, 255, 400, 511] {
1189            match reader.seek(FrameIndex(target)) {
1190                Seek::Preview { frame, .. } => assert!(
1191                    frame.0.abs_diff(target) < 12,
1192                    "frame {target} answered by {frame:?}, further than one stride away"
1193                ),
1194                Seek::Exact(_) => {}
1195                Seek::Pending => panic!("frame {target} had no answer"),
1196            }
1197        }
1198        assert_eq!(
1199            reader.statistics().samples_submitted,
1200            0,
1201            "seeking is not decoding"
1202        );
1203        assert_eq!(reader.statistics().resets, 0);
1204    }
1205
1206    /// The 50 ms half of the same requirement, held on every run rather than only in the
1207    /// host-specific measurement over the bundled sample: a hundred positions of a single-group
1208    /// track, the far end no slower than the near one.
1209    #[test]
1210    fn a_seek_to_any_position_answers_inside_the_latency_budget() {
1211        let frames = 768;
1212        let mut reader = single_group_reader(frames);
1213        reader.set_seek_previews(Some(Arc::new(StridedPreviews { stride: 12, frames })));
1214        let mut worst = Duration::ZERO;
1215        for step in 0..=100_u64 {
1216            let target = FrameIndex(step * (frames - 1) / 100);
1217            let started = std::time::Instant::now();
1218            let answer = reader.seek(target);
1219            worst = worst.max(started.elapsed());
1220            assert_ne!(answer, Seek::Pending, "frame {} had no answer", target.0);
1221        }
1222        assert!(
1223            worst < SEEK_LATENCY_BUDGET,
1224            "the worst seek took {worst:?}, over the {SEEK_LATENCY_BUDGET:?} budget"
1225        );
1226    }
1227
1228    /// Detaching the source puts the reader back to answering out of its own cache alone, so a
1229    /// caller that drops a preview index does not keep serving its pictures.
1230    #[test]
1231    fn detaching_the_preview_source_leaves_only_the_cache() {
1232        let mut reader = single_group_reader(64);
1233        reader.set_seek_previews(Some(Arc::new(StridedPreviews {
1234            stride: 12,
1235            frames: 64,
1236        })));
1237        assert!(matches!(reader.seek(FrameIndex(63)), Seek::Preview { .. }));
1238        reader.set_seek_previews(None);
1239        assert_eq!(reader.seek(FrameIndex(63)), Seek::Pending);
1240    }
1241
1242    /// Issue #506: frame 3 follows the random-access point at frame 4 in decode order but is
1243    /// presented before it, the shape of an open-GOP stream's leading pictures. A decoder that
1244    /// starts at frame 4 cannot reconstruct it, so the reader starts from frame 0 instead. Frame
1245    /// 5 is not leading, so it is still decoded from frame 4.
1246    #[test]
1247    fn a_leading_picture_is_decoded_from_the_random_access_point_before_it() {
1248        let samples = vec![
1249            sample(0, 10, true),
1250            sample(1, 11, false),
1251            sample(2, 12, false),
1252            sample(4, 14, true),
1253            sample(3, 13, false),
1254            sample(5, 15, false),
1255        ];
1256        let mut reader = ExactFrameReader::new(
1257            &uncompressed_video_decoder_factory(),
1258            config(),
1259            samples.clone(),
1260            Limits::default(),
1261        )
1262        .unwrap();
1263        let cancellation = CancellationToken::new();
1264        assert_eq!(
1265            value(&reader.get(FrameIndex(3), &cancellation).unwrap()),
1266            13
1267        );
1268        assert_eq!(reader.statistics().samples_submitted, 5);
1269
1270        let mut reader = ExactFrameReader::new(
1271            &uncompressed_video_decoder_factory(),
1272            config(),
1273            samples,
1274            Limits::default(),
1275        )
1276        .unwrap();
1277        assert_eq!(
1278            value(&reader.get(FrameIndex(5), &cancellation).unwrap()),
1279            15
1280        );
1281        assert_eq!(reader.statistics().samples_submitted, 3);
1282    }
1283
1284    #[test]
1285    fn capability_distinguishes_codec_profile_configuration_and_hardware() {
1286        let factory = uncompressed_video_decoder_factory();
1287        assert!(factory.capability(&config()).is_supported());
1288        let mut candidate = config();
1289        candidate.codec = Codec::Av1;
1290        assert_eq!(
1291            factory.capability(&candidate),
1292            CodecSupport::UnsupportedCodec
1293        );
1294        candidate = config();
1295        candidate.profile = CodecProfile::Av1Main;
1296        assert_eq!(
1297            factory.capability(&candidate),
1298            CodecSupport::UnsupportedProfile
1299        );
1300        candidate = config();
1301        candidate.output_format = PixelFormat::Rgba8;
1302        assert!(matches!(
1303            factory.capability(&candidate),
1304            CodecSupport::InvalidConfiguration { .. }
1305        ));
1306        candidate = config();
1307        candidate.hardware = HardwarePreference::Require;
1308        assert_eq!(
1309            factory.capability(&candidate),
1310            CodecSupport::HardwareUnavailable
1311        );
1312
1313        candidate = config();
1314        candidate.output_format = PixelFormat::Rgba8;
1315        assert_eq!(
1316            factory
1317                .create(&candidate, &Limits::default())
1318                .err()
1319                .unwrap()
1320                .kind(),
1321            ErrorKind::InvalidInput
1322        );
1323
1324        let restrictive = Limits {
1325            max_width: 0,
1326            ..Limits::default()
1327        };
1328        assert_eq!(
1329            factory
1330                .create(&config(), &restrictive)
1331                .err()
1332                .unwrap()
1333                .kind(),
1334            ErrorKind::ResourceLimit
1335        );
1336    }
1337
1338    /// Issue #504: every frame of an all-intra track is a random-access point, so a request for
1339    /// frame 2 opens its decode session at frame 2. Frame 0 was never submitted in that session,
1340    /// so reaching it needs a reset rather than a walk onwards from frame 3.
1341    #[test]
1342    fn a_request_before_the_open_session_start_resets_the_decoder() {
1343        let samples = (0..4)
1344            .map(|index| sample(index, index as u8 + 10, true))
1345            .collect();
1346        let mut reader = ExactFrameReader::new(
1347            &uncompressed_video_decoder_factory(),
1348            config(),
1349            samples,
1350            Limits {
1351                max_cached_frames: 1,
1352                ..Limits::default()
1353            },
1354        )
1355        .unwrap();
1356        let cancellation = CancellationToken::new();
1357        assert_eq!(
1358            value(&reader.get(FrameIndex(2), &cancellation).unwrap()),
1359            12
1360        );
1361        assert_eq!(
1362            value(&reader.get(FrameIndex(0), &cancellation).unwrap()),
1363            10
1364        );
1365        assert_eq!(reader.statistics().resets, 2);
1366    }
1367
1368    #[test]
1369    fn conformance_vector_reorders_exact_frames_and_reuses_bounded_state() {
1370        let samples = vec![
1371            sample(0, 10, true),
1372            sample(3, 40, false),
1373            sample(1, 20, false),
1374            sample(2, 30, false),
1375            sample(4, 50, true),
1376            sample(6, 70, false),
1377            sample(5, 60, false),
1378        ];
1379        let factory = uncompressed_video_decoder_factory();
1380        let mut reader = ExactFrameReader::new(
1381            &factory,
1382            config(),
1383            samples,
1384            Limits {
1385                max_cached_frames: 3,
1386                ..Limits::default()
1387            },
1388        )
1389        .unwrap();
1390        let cancellation = CancellationToken::new();
1391        assert_eq!(
1392            value(&reader.get(FrameIndex(1), &cancellation).unwrap()),
1393            20
1394        );
1395        let first = reader.statistics();
1396        assert_eq!(
1397            value(&reader.get(FrameIndex(2), &cancellation).unwrap()),
1398            30
1399        );
1400        assert_eq!(reader.statistics().resets, first.resets);
1401        assert_eq!(
1402            value(&reader.get(FrameIndex(3), &cancellation).unwrap()),
1403            40
1404        );
1405        assert!(reader.statistics().cache_hits > 0);
1406        assert_eq!(
1407            value(&reader.get(FrameIndex(5), &cancellation).unwrap()),
1408            60
1409        );
1410        assert_eq!(
1411            value(&reader.get(FrameIndex(0), &cancellation).unwrap()),
1412            10
1413        );
1414        assert!(reader.statistics().resets > first.resets);
1415        assert!(reader.cached_frames() <= 3);
1416    }
1417
1418    /// Records what each sample was asked to produce, so a test can see the hint the reader
1419    /// actually gave rather than only its effect.
1420    struct HintFactory {
1421        wanted: Arc<Mutex<Vec<(u64, bool)>>>,
1422    }
1423
1424    struct HintDecoder {
1425        configuration: VideoDecoderConfig,
1426        limits: Limits,
1427        wanted: Arc<Mutex<Vec<(u64, bool)>>>,
1428        output_wanted: bool,
1429    }
1430
1431    impl VideoDecoderFactory for HintFactory {
1432        fn capability(&self, _: &VideoDecoderConfig) -> CodecSupport {
1433            CodecSupport::Supported {
1434                implementation: CodecImplementation::Software,
1435            }
1436        }
1437
1438        fn create(
1439            &self,
1440            configuration: &VideoDecoderConfig,
1441            limits: &Limits,
1442        ) -> Result<Box<dyn VideoDecoder>> {
1443            Ok(Box::new(HintDecoder {
1444                configuration: configuration.clone(),
1445                limits: *limits,
1446                wanted: Arc::clone(&self.wanted),
1447                output_wanted: true,
1448            }))
1449        }
1450    }
1451
1452    impl VideoDecoder for HintDecoder {
1453        fn submit(
1454            &mut self,
1455            sample: &EncodedVideoSample,
1456            cancellation: &CancellationToken,
1457        ) -> Result<Vec<DecodedVideoFrame>> {
1458            cancellation.check()?;
1459            self.wanted
1460                .lock()
1461                .expect("hint log poisoned")
1462                .push((sample.presentation_index.0, self.output_wanted));
1463            if !self.output_wanted {
1464                return Ok(Vec::new());
1465            }
1466            Ok(vec![DecodedVideoFrame {
1467                presentation_index: sample.presentation_index,
1468                frame: VideoFrame::new(
1469                    self.configuration.coded_dimensions,
1470                    PixelFormat::Gray8,
1471                    ColorRange::Full,
1472                    vec![Plane {
1473                        data: sample.data.clone(),
1474                        stride: 1,
1475                    }],
1476                    &self.limits,
1477                )?,
1478            }])
1479        }
1480
1481        fn drain(&mut self, _: &CancellationToken) -> Result<Vec<DecodedVideoFrame>> {
1482            Ok(Vec::new())
1483        }
1484
1485        fn reset(&mut self) -> Result<()> {
1486            Ok(())
1487        }
1488
1489        fn set_output_wanted(&mut self, wanted: bool) {
1490            self.output_wanted = wanted;
1491        }
1492    }
1493
1494    /// A decoder that hands back the previous sample's picture again alongside the current
1495    /// one, as a backend that does not deduplicate its output across a session does. Both
1496    /// pictures are genuine and correctly identified; the repeat is only a frame the reader has
1497    /// already seen since its last reset.
1498    struct ReplayFactory;
1499
1500    struct ReplayDecoder {
1501        configuration: VideoDecoderConfig,
1502        limits: Limits,
1503        previous: Option<(FrameIndex, u8)>,
1504    }
1505
1506    impl ReplayDecoder {
1507        fn produce(&self, presentation_index: FrameIndex, value: u8) -> Result<DecodedVideoFrame> {
1508            Ok(DecodedVideoFrame {
1509                presentation_index,
1510                frame: VideoFrame::new(
1511                    self.configuration.coded_dimensions,
1512                    PixelFormat::Gray8,
1513                    ColorRange::Full,
1514                    vec![Plane {
1515                        data: vec![value],
1516                        stride: 1,
1517                    }],
1518                    &self.limits,
1519                )?,
1520            })
1521        }
1522    }
1523
1524    impl VideoDecoderFactory for ReplayFactory {
1525        fn capability(&self, _: &VideoDecoderConfig) -> CodecSupport {
1526            CodecSupport::Supported {
1527                implementation: CodecImplementation::Software,
1528            }
1529        }
1530
1531        fn create(
1532            &self,
1533            configuration: &VideoDecoderConfig,
1534            limits: &Limits,
1535        ) -> Result<Box<dyn VideoDecoder>> {
1536            Ok(Box::new(ReplayDecoder {
1537                configuration: configuration.clone(),
1538                limits: *limits,
1539                previous: None,
1540            }))
1541        }
1542    }
1543
1544    impl VideoDecoder for ReplayDecoder {
1545        fn submit(
1546            &mut self,
1547            sample: &EncodedVideoSample,
1548            cancellation: &CancellationToken,
1549        ) -> Result<Vec<DecodedVideoFrame>> {
1550            cancellation.check()?;
1551            let value = sample.data[0];
1552            let mut outputs = Vec::new();
1553            if let Some((index, previous)) = self.previous {
1554                outputs.push(self.produce(index, previous)?);
1555            }
1556            outputs.push(self.produce(sample.presentation_index, value)?);
1557            self.previous = Some((sample.presentation_index, value));
1558            Ok(outputs)
1559        }
1560
1561        fn drain(&mut self, _: &CancellationToken) -> Result<Vec<DecodedVideoFrame>> {
1562            Ok(Vec::new())
1563        }
1564
1565        fn reset(&mut self) -> Result<()> {
1566            self.previous = None;
1567            Ok(())
1568        }
1569    }
1570
1571    /// Issue #415: publishing a presentation frame twice within one decode session used to fail
1572    /// the whole request with `MalformedMedia`, which held the backend to a rule the reader does
1573    /// not depend on and threw away a picture it was about to cache.
1574    #[test]
1575    fn a_frame_the_decoder_emits_twice_is_cached_again_rather_than_refused() {
1576        let mut reader = ExactFrameReader::new(
1577            &ReplayFactory,
1578            config(),
1579            single_group_of_pictures(),
1580            Limits::default(),
1581        )
1582        .unwrap();
1583        let cancellation = CancellationToken::new();
1584
1585        // Reaching frame 3 submits samples 0 through 3, and every sample after the first
1586        // republishes its predecessor, so frames 0, 1, and 2 are each published twice.
1587        assert_eq!(
1588            value(&reader.get(FrameIndex(3), &cancellation).unwrap()),
1589            30
1590        );
1591        assert_eq!(
1592            value(&reader.get(FrameIndex(6), &cancellation).unwrap()),
1593            60
1594        );
1595
1596        // The repeats were cached rather than refused, so the frames behind the target are
1597        // still answered from the cache instead of costing another reset.
1598        let resets = reader.statistics().resets;
1599        let hits = reader.statistics().cache_hits;
1600        assert_eq!(
1601            value(&reader.get(FrameIndex(5), &cancellation).unwrap()),
1602            50
1603        );
1604        assert_eq!(reader.statistics().resets, resets);
1605        assert!(reader.statistics().cache_hits > hits);
1606    }
1607
1608    /// Ten frames behind one random-access point, as the bundled sample's single group of
1609    /// pictures is, with `value == index * 10`.
1610    fn single_group_of_pictures() -> Vec<EncodedVideoSample> {
1611        (0..10)
1612            .map(|index| sample(index, (index * 10) as u8, index == 0))
1613            .collect()
1614    }
1615
1616    /// A cache small enough that a walk over ten frames is longer than the tail the reader
1617    /// keeps behind its target; the default cache would hold the whole group of pictures.
1618    fn small_cache() -> Limits {
1619        Limits {
1620            max_cached_frames: 2,
1621            ..Limits::default()
1622        }
1623    }
1624
1625    #[test]
1626    fn walking_to_a_distant_frame_does_not_ask_for_the_pictures_it_passes() {
1627        let log = Arc::new(Mutex::new(Vec::new()));
1628        let factory = HintFactory {
1629            wanted: Arc::clone(&log),
1630        };
1631        let mut reader = ExactFrameReader::new(
1632            &factory,
1633            config(),
1634            single_group_of_pictures(),
1635            small_cache(),
1636        )
1637        .unwrap();
1638        let cancellation = CancellationToken::new();
1639
1640        assert_eq!(
1641            value(&reader.get(FrameIndex(8), &cancellation).unwrap()),
1642            80
1643        );
1644        assert_eq!(
1645            *log.lock().unwrap(),
1646            vec![
1647                (0, false),
1648                (1, false),
1649                (2, false),
1650                (3, false),
1651                (4, false),
1652                (5, false),
1653                (6, true),
1654                (7, true),
1655                (8, true),
1656            ],
1657            "only the target and the tail the cache can hold are produced"
1658        );
1659        let statistics = reader.statistics();
1660        assert_eq!(statistics.samples_submitted, 9);
1661        assert_eq!(statistics.samples_skipped, 6);
1662        assert_eq!(statistics.resets, 1, "the walk itself is one decode");
1663    }
1664
1665    #[test]
1666    fn a_frame_the_walk_went_past_is_decoded_again_rather_than_reported_missing() {
1667        let log = Arc::new(Mutex::new(Vec::new()));
1668        let factory = HintFactory {
1669            wanted: Arc::clone(&log),
1670        };
1671        let mut reader = ExactFrameReader::new(
1672            &factory,
1673            config(),
1674            single_group_of_pictures(),
1675            small_cache(),
1676        )
1677        .unwrap();
1678        let cancellation = CancellationToken::new();
1679        reader.get(FrameIndex(8), &cancellation).unwrap();
1680        let resets = reader.statistics().resets;
1681
1682        // Frame 4 was submitted with nothing wanting its picture, so this decoder will never
1683        // emit it again. The reader has to notice that and start over, exactly as it does for a
1684        // frame it published and then evicted.
1685        assert_eq!(
1686            value(&reader.get(FrameIndex(4), &cancellation).unwrap()),
1687            40
1688        );
1689        assert_eq!(reader.statistics().resets, resets + 1);
1690        // And having decoded it, it is a cached frame again rather than a second reset.
1691        assert_eq!(
1692            value(&reader.get(FrameIndex(4), &cancellation).unwrap()),
1693            40
1694        );
1695        assert_eq!(reader.statistics().resets, resets + 1);
1696    }
1697
1698    #[test]
1699    fn a_reordered_frame_at_or_past_the_target_is_kept_for_the_request_that_follows() {
1700        // Decode order 0, 4, 2, 1, 3: reaching frame 4 means submitting samples whose own
1701        // frames are 2, 1 and 3, which is a walk *backwards* in presentation order. Skipping by
1702        // decode position would discard them and make every following request a fresh walk.
1703        let samples = vec![
1704            sample(0, 0, true),
1705            sample(4, 40, false),
1706            sample(2, 20, false),
1707            sample(1, 10, false),
1708            sample(3, 30, false),
1709            sample(8, 80, false),
1710            sample(6, 60, false),
1711            sample(5, 50, false),
1712            sample(7, 70, false),
1713        ];
1714        let log = Arc::new(Mutex::new(Vec::new()));
1715        let factory = HintFactory {
1716            wanted: Arc::clone(&log),
1717        };
1718        let mut reader = ExactFrameReader::new(&factory, config(), samples, small_cache()).unwrap();
1719        let cancellation = CancellationToken::new();
1720
1721        assert_eq!(
1722            value(&reader.get(FrameIndex(4), &cancellation).unwrap()),
1723            40
1724        );
1725        let resets = reader.statistics().resets;
1726        for (index, expected) in [(1_u64, 10_u8), (2, 20), (3, 30)] {
1727            assert_eq!(
1728                value(&reader.get(FrameIndex(index), &cancellation).unwrap()),
1729                expected,
1730                "frame {index} is still available"
1731            );
1732        }
1733        assert_eq!(
1734            reader.statistics().resets,
1735            resets,
1736            "a frame after the target in decode order is not skipped"
1737        );
1738        assert_eq!(
1739            reader.statistics().samples_skipped,
1740            1,
1741            "only frame 0, the random-access point the walk starts from, is behind frame 4"
1742        );
1743    }
1744
1745    #[test]
1746    fn a_walk_leaves_the_frames_behind_its_target_cached_so_stepping_back_is_free() {
1747        // Walking fills the cache with the frames just before the target as a side effect, and
1748        // that is what makes the example's previous-frame key cheap. Skipping those too would
1749        // send every backward step all the way back to the random-access point.
1750        let log = Arc::new(Mutex::new(Vec::new()));
1751        let factory = HintFactory {
1752            wanted: Arc::clone(&log),
1753        };
1754        let samples: Vec<_> = (0..40)
1755            .map(|index| sample(index, index as u8, index == 0))
1756            .collect();
1757        let mut reader = ExactFrameReader::new(
1758            &factory,
1759            config(),
1760            samples,
1761            Limits {
1762                max_cached_frames: 4,
1763                ..Limits::default()
1764            },
1765        )
1766        .unwrap();
1767        let cancellation = CancellationToken::new();
1768        reader.get(FrameIndex(32), &cancellation).unwrap();
1769        let after_walk = reader.statistics();
1770        assert_eq!(after_walk.samples_skipped, 28, "frames 0 to 27 are skipped");
1771
1772        for index in [31_u64, 30, 29] {
1773            assert_eq!(
1774                value(&reader.get(FrameIndex(index), &cancellation).unwrap()),
1775                index as u8
1776            );
1777        }
1778        assert_eq!(
1779            reader.statistics().resets,
1780            after_walk.resets,
1781            "the tail the walk kept is in the cache, so stepping back decodes nothing"
1782        );
1783        assert_eq!(
1784            reader.statistics().samples_submitted,
1785            after_walk.samples_submitted
1786        );
1787    }
1788
1789    #[test]
1790    fn a_step_converts_its_own_picture_and_no_tail_behind_it() {
1791        // The charge issue #402 is about. A destination keeps the frames behind it because the
1792        // request after it is very likely to be for one of them; a step is passed through on
1793        // the way somewhere else, so those frames are converted for a request that never comes,
1794        // and a walk that steps more often than the tail is long converts everything it passes.
1795        let log = Arc::new(Mutex::new(Vec::new()));
1796        let factory = HintFactory {
1797            wanted: Arc::clone(&log),
1798        };
1799        let samples: Vec<_> = (0..40)
1800            .map(|index| sample(index, index as u8, index == 0))
1801            .collect();
1802        let mut reader = ExactFrameReader::new(
1803            &factory,
1804            config(),
1805            samples,
1806            Limits {
1807                max_cached_frames: 4,
1808                ..Limits::default()
1809            },
1810        )
1811        .unwrap();
1812        let cancellation = CancellationToken::new();
1813
1814        assert_eq!(
1815            value(&reader.get_step(FrameIndex(32), &cancellation).unwrap()),
1816            32
1817        );
1818        let after_step = reader.statistics();
1819        assert_eq!(
1820            after_step.samples_skipped, 32,
1821            "frames 0 to 31 are all passed through, tail included"
1822        );
1823        assert_eq!(
1824            reader.cached_frames(),
1825            1,
1826            "only the picture the step asked for was converted"
1827        );
1828    }
1829
1830    #[test]
1831    fn a_walk_of_steps_converts_what_it_asks_for_rather_than_what_it_passes() {
1832        // The shape of a drag's preview walk: repeated forward steps towards a far target. Each
1833        // step used to pay its own cache tail, and with a stride shorter than the tail the tails
1834        // overlapped and covered the whole span. As steps, the walk converts one picture per
1835        // step however long the span is.
1836        let log = Arc::new(Mutex::new(Vec::new()));
1837        let factory = HintFactory {
1838            wanted: Arc::clone(&log),
1839        };
1840        let samples: Vec<_> = (0..64)
1841            .map(|index| sample(index, index as u8, index == 0))
1842            .collect();
1843        let mut reader = ExactFrameReader::new(
1844            &factory,
1845            config(),
1846            samples,
1847            Limits {
1848                max_cached_frames: 8,
1849                ..Limits::default()
1850            },
1851        )
1852        .unwrap();
1853        let cancellation = CancellationToken::new();
1854
1855        // A stride of 4 against a tail of 8: every frame in the span is within some step's tail.
1856        let steps: Vec<u64> = (4..60).step_by(4).collect();
1857        for step in &steps {
1858            reader.get_step(FrameIndex(*step), &cancellation).unwrap();
1859        }
1860        let converted = reader.statistics().samples_submitted - reader.statistics().samples_skipped;
1861        assert_eq!(
1862            converted,
1863            steps.len() as u64,
1864            "one conversion per published step, not one per frame passed"
1865        );
1866        assert_eq!(
1867            reader.statistics().resets,
1868            1,
1869            "only the cold start resets; a forward walk of steps never starts over again"
1870        );
1871    }
1872
1873    #[test]
1874    fn a_step_walk_that_ends_in_a_destination_still_leaves_a_cache_tail_to_step_back_through() {
1875        // What the request split must not cost: a committed scrub is a walk of steps that ends
1876        // at the frame under the pointer, and stepping backwards from there is still expected
1877        // to come out of the cache rather than walking from the random-access point again.
1878        let log = Arc::new(Mutex::new(Vec::new()));
1879        let factory = HintFactory {
1880            wanted: Arc::clone(&log),
1881        };
1882        let samples: Vec<_> = (0..40)
1883            .map(|index| sample(index, index as u8, index == 0))
1884            .collect();
1885        let mut reader = ExactFrameReader::new(
1886            &factory,
1887            config(),
1888            samples,
1889            Limits {
1890                max_cached_frames: 4,
1891                ..Limits::default()
1892            },
1893        )
1894        .unwrap();
1895        let cancellation = CancellationToken::new();
1896
1897        for step in [8_u64, 16, 24] {
1898            reader.get_step(FrameIndex(step), &cancellation).unwrap();
1899        }
1900        reader.get(FrameIndex(32), &cancellation).unwrap();
1901        let after_walk = reader.statistics();
1902
1903        for index in [31_u64, 30, 29] {
1904            assert_eq!(
1905                value(&reader.get(FrameIndex(index), &cancellation).unwrap()),
1906                index as u8
1907            );
1908        }
1909        assert_eq!(
1910            reader.statistics().resets,
1911            after_walk.resets,
1912            "the destination kept its tail, so stepping back decodes nothing"
1913        );
1914        assert_eq!(
1915            reader.statistics().samples_submitted,
1916            after_walk.samples_submitted
1917        );
1918    }
1919
1920    /// A decoder that releases each picture one sample late, and drops whatever it is holding
1921    /// when nothing wants its output - which is what a reordering decoder does.
1922    struct LateFactory;
1923
1924    struct LateDecoder {
1925        configuration: VideoDecoderConfig,
1926        limits: Limits,
1927        held: Option<EncodedVideoSample>,
1928        output_wanted: bool,
1929    }
1930
1931    impl VideoDecoderFactory for LateFactory {
1932        fn capability(&self, _: &VideoDecoderConfig) -> CodecSupport {
1933            CodecSupport::Supported {
1934                implementation: CodecImplementation::Software,
1935            }
1936        }
1937
1938        fn create(
1939            &self,
1940            configuration: &VideoDecoderConfig,
1941            limits: &Limits,
1942        ) -> Result<Box<dyn VideoDecoder>> {
1943            Ok(Box::new(LateDecoder {
1944                configuration: configuration.clone(),
1945                limits: *limits,
1946                held: None,
1947                output_wanted: true,
1948            }))
1949        }
1950    }
1951
1952    impl LateDecoder {
1953        fn release(&self, sample: EncodedVideoSample) -> Result<Vec<DecodedVideoFrame>> {
1954            if !self.output_wanted {
1955                return Ok(Vec::new());
1956            }
1957            Ok(vec![DecodedVideoFrame {
1958                presentation_index: sample.presentation_index,
1959                frame: VideoFrame::new(
1960                    self.configuration.coded_dimensions,
1961                    PixelFormat::Gray8,
1962                    ColorRange::Full,
1963                    vec![Plane {
1964                        data: sample.data,
1965                        stride: 1,
1966                    }],
1967                    &self.limits,
1968                )?,
1969            }])
1970        }
1971    }
1972
1973    impl VideoDecoder for LateDecoder {
1974        fn submit(
1975            &mut self,
1976            sample: &EncodedVideoSample,
1977            cancellation: &CancellationToken,
1978        ) -> Result<Vec<DecodedVideoFrame>> {
1979            cancellation.check()?;
1980            match self.held.replace(sample.clone()) {
1981                Some(previous) => self.release(previous),
1982                None => Ok(Vec::new()),
1983            }
1984        }
1985
1986        fn drain(&mut self, _: &CancellationToken) -> Result<Vec<DecodedVideoFrame>> {
1987            match self.held.take() {
1988                Some(held) => self.release(held),
1989                None => Ok(Vec::new()),
1990            }
1991        }
1992
1993        fn reset(&mut self) -> Result<()> {
1994            self.held = None;
1995            Ok(())
1996        }
1997
1998        fn set_output_wanted(&mut self, wanted: bool) {
1999            self.output_wanted = wanted;
2000        }
2001    }
2002
2003    #[test]
2004    fn a_frame_still_inside_a_reordering_decoder_when_output_stops_is_written_off_too() {
2005        // The frame a decoder is holding when output stops being wanted comes out during the
2006        // suppressed stretch and is dropped, even though its own sample was submitted while
2007        // output was still wanted. A reader that does not write it off believes it can still be
2008        // asked for and reports the decoder as malformed when it never arrives.
2009        let mut reader = ExactFrameReader::new(
2010            &LateFactory,
2011            config(),
2012            single_group_of_pictures(),
2013            small_cache(),
2014        )
2015        .unwrap();
2016        let cancellation = CancellationToken::new();
2017
2018        // Reaching frame 4 means submitting frame 5's sample to release it, so the decoder is
2019        // left holding frame 5 - which it releases, and drops, on the first suppressed submit of
2020        // the request that follows.
2021        assert_eq!(
2022            value(&reader.get(FrameIndex(4), &cancellation).unwrap()),
2023            40
2024        );
2025        assert_eq!(
2026            value(&reader.get(FrameIndex(9), &cancellation).unwrap()),
2027            90
2028        );
2029        assert_eq!(
2030            value(&reader.get(FrameIndex(5), &cancellation).unwrap()),
2031            50,
2032            "the frame that was dropped in flight is decoded again, not reported missing"
2033        );
2034    }
2035
2036    #[test]
2037    fn a_decoder_that_ignores_the_hint_still_answers_every_frame() {
2038        // `set_output_wanted` is advisory: `uncompressed_video_decoder_factory` does not
2039        // implement it, and the reader has to be no less correct for that.
2040        let factory = uncompressed_video_decoder_factory();
2041        let mut reader = ExactFrameReader::new(
2042            &factory,
2043            config(),
2044            single_group_of_pictures(),
2045            small_cache(),
2046        )
2047        .unwrap();
2048        let cancellation = CancellationToken::new();
2049        reader.get(FrameIndex(8), &cancellation).unwrap();
2050        assert_eq!(reader.statistics().samples_skipped, 6);
2051        for index in 0..9_u64 {
2052            assert_eq!(
2053                value(&reader.get(FrameIndex(index), &cancellation).unwrap()),
2054                (index * 10) as u8,
2055                "frame {index} came back from the decoder that ignored the hint"
2056            );
2057        }
2058    }
2059
2060    #[test]
2061    fn malformed_packets_and_cancellation_are_reported() {
2062        let factory = uncompressed_video_decoder_factory();
2063        let mut reader = ExactFrameReader::new(
2064            &factory,
2065            config(),
2066            vec![EncodedVideoSample {
2067                presentation_index: FrameIndex(0),
2068                random_access: true,
2069                data: Vec::new(),
2070            }],
2071            Limits::default(),
2072        )
2073        .unwrap();
2074        assert_eq!(
2075            reader
2076                .get(FrameIndex(0), &CancellationToken::new())
2077                .unwrap_err()
2078                .kind(),
2079            ErrorKind::MalformedMedia
2080        );
2081        let canceled = CancellationToken::new();
2082        canceled.cancel();
2083        assert_eq!(
2084            reader.get(FrameIndex(0), &canceled).unwrap_err().kind(),
2085            ErrorKind::Canceled
2086        );
2087    }
2088
2089    #[test]
2090    fn random_access_decode_work_obeys_the_configured_bound() {
2091        let factory = uncompressed_video_decoder_factory();
2092        let mut reader = ExactFrameReader::new(
2093            &factory,
2094            config(),
2095            vec![sample(0, 1, true), sample(1, 2, false), sample(2, 3, false)],
2096            Limits {
2097                max_decode_samples_per_seek: 2,
2098                ..Limits::default()
2099            },
2100        )
2101        .unwrap();
2102        let error = reader
2103            .get(FrameIndex(2), &CancellationToken::new())
2104            .unwrap_err();
2105        assert_eq!(error.kind(), ErrorKind::ResourceLimit);
2106        assert_eq!(reader.statistics().samples_submitted, 2);
2107    }
2108
2109    struct DelayedFactory;
2110    impl VideoDecoderFactory for DelayedFactory {
2111        fn capability(&self, _: &VideoDecoderConfig) -> CodecSupport {
2112            CodecSupport::Supported {
2113                implementation: CodecImplementation::Software,
2114            }
2115        }
2116        fn create(
2117            &self,
2118            configuration: &VideoDecoderConfig,
2119            limits: &Limits,
2120        ) -> Result<Box<dyn VideoDecoder>> {
2121            Ok(Box::new(DelayedDecoder {
2122                configuration: configuration.clone(),
2123                limits: *limits,
2124                delayed: None,
2125            }))
2126        }
2127    }
2128    struct DelayedDecoder {
2129        configuration: VideoDecoderConfig,
2130        limits: Limits,
2131        delayed: Option<EncodedVideoSample>,
2132    }
2133    impl DelayedDecoder {
2134        fn decode(&self, sample: EncodedVideoSample) -> Result<DecodedVideoFrame> {
2135            Ok(DecodedVideoFrame {
2136                presentation_index: sample.presentation_index,
2137                frame: VideoFrame::new(
2138                    self.configuration.coded_dimensions,
2139                    PixelFormat::Gray8,
2140                    ColorRange::Full,
2141                    vec![Plane {
2142                        data: sample.data,
2143                        stride: 1,
2144                    }],
2145                    &self.limits,
2146                )?,
2147            })
2148        }
2149    }
2150    impl VideoDecoder for DelayedDecoder {
2151        fn submit(
2152            &mut self,
2153            sample: &EncodedVideoSample,
2154            cancellation: &CancellationToken,
2155        ) -> Result<Vec<DecodedVideoFrame>> {
2156            cancellation.check()?;
2157            let previous = self.delayed.replace(sample.clone());
2158            previous
2159                .map(|sample| self.decode(sample).map(|frame| vec![frame]))
2160                .unwrap_or(Ok(Vec::new()))
2161        }
2162        fn drain(&mut self, cancellation: &CancellationToken) -> Result<Vec<DecodedVideoFrame>> {
2163            cancellation.check()?;
2164            self.delayed
2165                .take()
2166                .map(|sample| self.decode(sample).map(|frame| vec![frame]))
2167                .unwrap_or(Ok(Vec::new()))
2168        }
2169        fn reset(&mut self) -> Result<()> {
2170            self.delayed = None;
2171            Ok(())
2172        }
2173    }
2174
2175    #[test]
2176    fn delayed_output_drains_and_reset_clears_state() {
2177        let mut reader = ExactFrameReader::new(
2178            &DelayedFactory,
2179            config(),
2180            vec![sample(0, 11, true), sample(1, 22, false)],
2181            Limits::default(),
2182        )
2183        .unwrap();
2184        let cancellation = CancellationToken::new();
2185        assert_eq!(
2186            value(&reader.get(FrameIndex(1), &cancellation).unwrap()),
2187            22
2188        );
2189        assert_eq!(reader.statistics().drains, 1);
2190        reader.reset().unwrap();
2191        assert_eq!(reader.cached_frames(), 0);
2192        assert_eq!(
2193            value(&reader.get(FrameIndex(0), &cancellation).unwrap()),
2194            11
2195        );
2196    }
2197
2198    #[test]
2199    fn sequential_playback_through_a_pipelined_decoder_does_not_reset_every_frame() {
2200        // `DelayedDecoder` only emits a frame's output on the *next* `submit` call (or on
2201        // `drain`), modeling the one-frame pipeline latency a real reordering decoder exhibits.
2202        // Even though `next_decode_position` therefore always runs one step ahead of the frame
2203        // a straightforward sequential `get(0), get(1), get(2), ...` walk is actually waiting
2204        // on, that lag must not force a full reset-and-redecode-from-keyframe on every call.
2205        let mut reader = ExactFrameReader::new(
2206            &DelayedFactory,
2207            config(),
2208            vec![
2209                sample(0, 11, true),
2210                sample(1, 22, false),
2211                sample(2, 33, false),
2212                sample(3, 44, false),
2213                sample(4, 55, false),
2214            ],
2215            Limits::default(),
2216        )
2217        .unwrap();
2218        let cancellation = CancellationToken::new();
2219
2220        assert_eq!(
2221            value(&reader.get(FrameIndex(0), &cancellation).unwrap()),
2222            11
2223        );
2224        assert_eq!(reader.statistics().resets, 1);
2225
2226        for (index, expected) in [(1, 22_u8), (2, 33), (3, 44), (4, 55)] {
2227            let frame = reader.get(FrameIndex(index), &cancellation).unwrap();
2228            assert_eq!(value(&frame), expected);
2229            assert_eq!(
2230                reader.statistics().resets,
2231                1,
2232                "sequential forward playback must not reset once decoding has started"
2233            );
2234        }
2235        assert_eq!(reader.statistics().samples_submitted, 5);
2236        assert_eq!(reader.statistics().drains, 1);
2237    }
2238
2239    struct MalformedOutputFactory;
2240
2241    impl VideoDecoderFactory for MalformedOutputFactory {
2242        fn capability(&self, _: &VideoDecoderConfig) -> CodecSupport {
2243            CodecSupport::Supported {
2244                implementation: CodecImplementation::Software,
2245            }
2246        }
2247
2248        fn create(
2249            &self,
2250            configuration: &VideoDecoderConfig,
2251            limits: &Limits,
2252        ) -> Result<Box<dyn VideoDecoder>> {
2253            let factory = UncompressedVideoDecoderFactory;
2254            factory.create(configuration, limits).map(|decoder| {
2255                Box::new(MalformedOutputDecoder {
2256                    decoder,
2257                    previous: None,
2258                }) as Box<dyn VideoDecoder>
2259            })
2260        }
2261    }
2262
2263    struct MalformedOutputDecoder {
2264        decoder: Box<dyn VideoDecoder>,
2265        previous: Option<FrameIndex>,
2266    }
2267
2268    impl VideoDecoder for MalformedOutputDecoder {
2269        fn submit(
2270            &mut self,
2271            sample: &EncodedVideoSample,
2272            cancellation: &CancellationToken,
2273        ) -> Result<Vec<DecodedVideoFrame>> {
2274            let mut output = self.decoder.submit(sample, cancellation)?;
2275            if let (Some(previous), Some(frame)) = (self.previous, output.first_mut()) {
2276                frame.presentation_index = previous;
2277            }
2278            self.previous = Some(sample.presentation_index);
2279            Ok(output)
2280        }
2281
2282        fn drain(&mut self, cancellation: &CancellationToken) -> Result<Vec<DecodedVideoFrame>> {
2283            self.decoder.drain(cancellation)
2284        }
2285
2286        fn reset(&mut self) -> Result<()> {
2287            self.previous = None;
2288            self.decoder.reset()
2289        }
2290    }
2291
2292    #[test]
2293    fn malformed_backend_output_is_rejected() {
2294        let mut reader = ExactFrameReader::new(
2295            &MalformedOutputFactory,
2296            config(),
2297            vec![sample(0, 1, true), sample(1, 2, false)],
2298            Limits::default(),
2299        )
2300        .unwrap();
2301        let error = reader
2302            .get(FrameIndex(1), &CancellationToken::new())
2303            .unwrap_err();
2304        assert_eq!(error.kind(), ErrorKind::MalformedMedia);
2305    }
2306}