re_video/player/sample_decoder.rs
1use std::collections::BTreeMap;
2
3use crate::{Chunk, Frame, Receiver, Sender, Time, VideoDataDescription};
4
5use super::{TimedDecodingError, VideoPlayerError};
6
7#[derive(Default, re_byte_size::SizeBytes)]
8pub(super) struct DecoderOutput {
9 /// Frames sorted by PTS.
10 ///
11 /// *Almost* all decoders are outputting frames in presentation timestamp order.
12 /// However, WebCodec decoders on Firefox & Safari have been observed to output frames in decode order.
13 /// (i.e. the order in which they were submitted)
14 /// Therefore, we have to be careful not to assume that an incoming frame isn't in the past even on a freshly
15 /// reset decoder.
16 /// See also <https://github.com/rerun-io/rerun/issues/7961>
17 ///
18 /// Note that this technically a bug in their respective WebCodec implementations as the spec says
19 /// (<https://www.w3.org/TR/webcodecs/#dom-videodecoder-decode>):
20 /// `VideoDecoder` requires that frames are output in the order they expect to be presented, commonly known as presentation order.
21 /// Either way, being robust against this seems like a good idea!
22 frames_by_pts: BTreeMap<Time, Frame>,
23
24 /// Set on error; reset on success.
25 #[size_bytes(ignore)]
26 error: Option<TimedDecodingError>,
27}
28
29impl DecoderOutput {
30 fn clear(&mut self) {
31 self.error = None;
32 self.frames_by_pts.clear();
33 }
34}
35
36/// Internal implementation detail of the [`super::VideoPlayer`].
37///
38/// Expected to be reset upon backwards seeking.
39#[derive(re_byte_size::SizeBytes)]
40pub struct VideoSampleDecoder {
41 debug_name: String,
42 // TODO(RR-3800): maybe we should count this
43 #[size_bytes(ignore)]
44 decoder: Box<dyn crate::AsyncDecoder>,
45
46 frame_receiver: Receiver<crate::FrameResult>,
47 decoder_output: DecoderOutput,
48
49 /// The [`Chunk::sample_idx`] of the latest submitted sample.
50 latest_sample_idx: Option<crate::SampleIndex>,
51}
52
53impl VideoSampleDecoder {
54 pub fn new(
55 debug_name: String,
56 make_decoder: impl FnOnce(
57 Sender<crate::FrameResult>,
58 ) -> crate::DecodeResult<Box<dyn crate::AsyncDecoder>>,
59 ) -> Result<Self, VideoPlayerError> {
60 re_tracing::profile_function!();
61
62 let (decoder_output_sender, frame_receiver) =
63 crate::channel(format!("{debug_name}-VideoSampleDecoder"));
64 let decoder = make_decoder(decoder_output_sender)?;
65
66 Ok(Self {
67 debug_name,
68 decoder,
69 decoder_output: DecoderOutput::default(),
70 frame_receiver,
71 latest_sample_idx: None,
72 })
73 }
74
75 /// Processes all frames received from the asynchronously running decoder.
76 fn process_decoder_output(&mut self) {
77 loop {
78 match self.frame_receiver.try_recv() {
79 Ok(frame) => {
80 match frame {
81 Ok(frame) => {
82 re_log::trace!(
83 "Decoded frame at PTS {:?}",
84 frame.info.presentation_timestamp
85 );
86 self.decoder_output
87 .frames_by_pts
88 .insert(frame.info.presentation_timestamp, frame);
89 self.decoder_output.error = None; // We successfully decoded a frame, reset the error state.
90 }
91 Err(err) => {
92 // Many of the errors we get from a decoder are recoverable.
93 // They may be very frequent, but it's still useful to see them in the debug log for troubleshooting.
94 re_log::debug!("Error during decoding of {}: {err}", self.debug_name);
95
96 let err = VideoPlayerError::Decoding(err);
97 if let Some(error) = &mut self.decoder_output.error {
98 error.latest_error = err;
99 } else {
100 self.decoder_output.error = Some(TimedDecodingError::new(err));
101 }
102 }
103 }
104 }
105
106 Err(crate::TryRecvError::Empty) => {
107 break;
108 }
109
110 Err(crate::TryRecvError::Disconnected) => {
111 self.decoder_output.error = Some(TimedDecodingError::new(
112 VideoPlayerError::DecoderUnexpectedlyExited,
113 ));
114 break;
115 }
116 }
117 }
118 }
119
120 pub fn debug_name(&self) -> &str {
121 &self.debug_name
122 }
123
124 /// Start decoding the given chunk.
125 pub fn decode(&mut self, chunk: Chunk) -> Result<(), VideoPlayerError> {
126 let sample_idx = chunk.sample_idx;
127
128 if let Some(latest_sample_idx) = self.latest_sample_idx {
129 // Some sanity checks:
130 if latest_sample_idx + 1 == sample_idx {
131 // All good!
132 } else if latest_sample_idx < sample_idx {
133 return Err(super::InsufficientSampleDataError::MissingSamples.into());
134 } else if sample_idx == latest_sample_idx {
135 return Err(super::InsufficientSampleDataError::DuplicateSampleIdx.into());
136 } else {
137 return Err(super::InsufficientSampleDataError::OutOfOrderSampleIdx.into());
138 }
139 }
140
141 self.decoder.submit_chunk(chunk)?;
142
143 self.latest_sample_idx = Some(sample_idx);
144
145 Ok(())
146 }
147
148 /// Called after submitting the last chunk.
149 ///
150 /// Should flush all pending frames.
151 pub fn end_of_video(&mut self) -> Result<(), VideoPlayerError> {
152 self.decoder.end_of_video()?;
153 self.latest_sample_idx = None;
154 Ok(())
155 }
156
157 /// Minimum number of samples the decoder requests to stay head of the currently requested sample.
158 ///
159 /// I.e. if sample N is requested, then the encoder would like to see at least all the samples from
160 /// [start of N's GOP] until [N + `min_num_samples_to_enqueue_ahead`].
161 /// Codec specific constraints regarding what samples can be decoded (samples may depend on other samples in their GOP)
162 /// still apply independently of this.
163 ///
164 /// This can be used as a workaround for decoders that are known to need additional samples to produce outputs.
165 pub fn min_num_samples_to_enqueue_ahead(&self) -> usize {
166 self.decoder.min_num_samples_to_enqueue_ahead()
167 }
168
169 pub fn max_num_samples_to_enqueue_ahead(&self) -> usize {
170 // To not fill memory up too much, only queue up a limited amount of samples.
171 //
172 // 25 here is arbitrary so far, but seems to work well with the encoder
173 // giving back frames and not waiting for a secondary keyframe.
174 self.min_num_samples_to_enqueue_ahead() + 25
175 }
176
177 /// Returns the latest decoded frame at the given PTS and drops all earlier frames than the given PTS.
178 ///
179 /// Afterwards, you can retrieve the frame that is at or after the PTS using [`Self::oldest_available_frame`]
180 /// (without a mutable reference to the decoder).
181 pub fn process_incoming_frames_and_drop_earlier_than(&mut self, pts: Time) {
182 self.process_decoder_output();
183
184 // Latest-at semantics means that if `pts` doesn't land on the exact PTS of a decode frame we have,
185 // we provide the next *older* frame.
186 let frames_by_pts = &mut self.decoder_output.frames_by_pts;
187 let latest_at_pts = frames_by_pts
188 .range(..=pts)
189 .next_back()
190 .map_or(pts, |(k, _v)| *k);
191
192 // Keep everything at or after the given PTS.
193 *frames_by_pts = frames_by_pts.split_off(&latest_at_pts);
194 }
195
196 /// Returns the latest decoded frame.
197 pub fn oldest_available_frame(&self) -> Option<&Frame> {
198 self.decoder_output
199 .frames_by_pts
200 .first_key_value()
201 .map(|(_, v)| v)
202 }
203
204 /// Reset the video decoder and discard all frames.
205 pub fn reset(&mut self, video_descr: &VideoDataDescription) -> Result<(), VideoPlayerError> {
206 self.decoder.reset(video_descr)?;
207
208 // Flush out any pending frames.
209 self.process_decoder_output();
210 self.decoder_output.clear();
211 self.latest_sample_idx = None;
212
213 Ok(())
214 }
215
216 /// Return and clear the latest error that happened during decoding.
217 pub fn take_error(&mut self) -> Option<TimedDecodingError> {
218 self.decoder_output.error.take()
219 }
220}