1use super::{Backend, DecodedFrame, NativeDecoder, Picture};
38use crate::bytes::{Bytes, FramePool};
39use crate::mp4_demux::TrackCodec;
40use std::collections::VecDeque;
41
42const UNTIMED_FPS: f64 = 30.0;
44
45#[derive(Debug, Clone, Copy)]
47pub struct CodedSample {
48 pub start: usize,
50 pub end: usize,
52 pub pts: i64,
54 pub dts: i64,
56 pub duration: i64,
58 pub seconds: f64,
60}
61
62type Sample = CodedSample;
63
64#[derive(Debug, Clone)]
70pub struct Demuxed {
71 pub codec: TrackCodec,
73 pub config: Vec<u8>,
77 pub width: u32,
79 pub height: u32,
81 pub bytes: Bytes,
84 pub samples: Vec<CodedSample>,
86}
87
88impl Demuxed {
89 pub fn sample(&self, i: usize) -> &[u8] {
91 let s = &self.samples[i];
92 &self.bytes[s.start..s.end]
93 }
94
95 pub fn duration(&self) -> f64 {
97 clip_length(&self.samples)
98 }
99}
100
101pub fn demux(data: impl Into<Bytes>) -> Result<Demuxed, String> {
104 let data: Bytes = data.into();
105 let (codec, config, width, height, bytes, samples) = if data.starts_with(b"DKIF") {
106 let (fourcc, w, h, (num, den), packets) = crate::mkv_demux::index_ivf(&data)?;
107 let codec = match &fourcc {
108 b"VP90" => TrackCodec::Vp9,
109 b"AV01" => TrackCodec::Av1,
110 other => {
111 return Err(format!(
112 "IVF codec {} is not supported",
113 String::from_utf8_lossy(other)
114 ))
115 }
116 };
117 let first = packets.iter().map(|(pts, _)| *pts).min().unwrap_or(0);
118 let tick = num as f64 / den.max(1) as f64;
119 let samples = packets
121 .iter()
122 .map(|(pts, at)| Sample {
123 start: at.start,
124 end: at.end,
125 pts: *pts,
126 dts: *pts,
127 duration: 0,
128 seconds: (pts - first) as f64 * tick,
129 })
130 .collect();
131 (codec, Vec::new(), w, h, data, samples)
132 } else if data.starts_with(&[0, 0, 1]) || data.starts_with(&[0, 0, 0, 1]) {
133 return Err("an Annex B elementary stream has no container to demux".into());
134 } else {
135 let (track, bytes) = if data.starts_with(&[0x1A, 0x45, 0xDF, 0xA3]) {
136 let (_, track, packets) = crate::ffcompat::mkv::parse_in_place(&data)?;
140 (track, packets.map_or(data, Bytes::from))
141 } else {
142 (crate::mp4_demux::demux_video(&data)?, data)
143 };
144 let first = track.samples.iter().map(|s| s.pts).min().unwrap_or(0);
145 let timescale = track.timescale.max(1) as f64;
146 let samples = track
147 .samples
148 .iter()
149 .map(|s| {
150 let start = s.offset as usize;
151 let end = start + s.size as usize;
152 if end > bytes.len() {
153 return Err("sample runs past the end of the file".to_string());
154 }
155 Ok(Sample {
156 start,
157 end,
158 pts: s.pts,
159 dts: s.dts,
160 duration: s.duration as i64,
161 seconds: (s.pts - first) as f64 / timescale,
162 })
163 })
164 .collect::<Result<Vec<_>, String>>()?;
165 (
166 track.codec,
167 track.config,
168 track.width,
169 track.height,
170 bytes,
171 samples,
172 )
173 };
174 Ok(Demuxed {
175 codec,
176 config,
177 width,
178 height,
179 bytes,
180 samples,
181 })
182}
183
184enum Source {
186 Samples {
188 bytes: Bytes,
189 samples: Vec<Sample>,
190 earliest_after: Vec<f64>,
193 codec: TrackCodec,
194 config: Vec<u8>,
195 decoder: SampleDecoder,
196 next: usize,
197 waiting: VecDeque<(f64, Picture)>,
199 },
200 Decoded {
202 data: Bytes,
203 codec: TrackCodec,
204 frames: Vec<Option<DecodedFrame>>,
205 next: usize,
206 },
207}
208
209enum SampleDecoder {
211 Native(Box<NativeDecoder>),
212 Threaded {
216 decoder: Box<NativeDecoder>,
217 in_flight: std::collections::HashMap<u64, f64>,
219 },
220 #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
221 VideoToolbox(crate::videotoolbox_decode::VideoToolboxDecoder),
222 #[cfg(all(
223 feature = "nvidia",
224 any(target_os = "linux", target_os = "windows"),
225 target_pointer_width = "64"
226 ))]
227 Nvdec(Box<crate::nvidia::NvdecDecoder>),
228 #[cfg(all(feature = "vaapi", target_os = "linux"))]
229 Vaapi(Box<crate::vaapi::VaapiDecoder>),
230}
231
232impl SampleDecoder {
233 fn open(
237 codec: TrackCodec,
238 config: &[u8],
239 width: u32,
240 height: u32,
241 hardware: bool,
242 first: Option<&[u8]>,
243 pool: &FramePool,
244 ) -> Result<Self, String> {
245 #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
246 if hardware {
247 let described = match (codec, config.is_empty(), first) {
248 (TrackCodec::Vp9, true, Some(frame)) => {
249 vp9_record(frame, width, height).unwrap_or_default()
250 }
251 _ => config.to_vec(),
252 };
253 let track = crate::mp4_demux::VideoTrack {
254 codec,
255 width,
256 height,
257 config: described,
258 timescale: TIMESCALE,
259 samples: Vec::new(),
260 };
261 if let Ok(mut d) = crate::videotoolbox_decode::VideoToolboxDecoder::new_planar(&track) {
262 d.set_pool(pool.clone());
263 return Ok(Self::VideoToolbox(d));
264 }
265 }
266 #[cfg(any(
269 all(
270 feature = "nvidia",
271 any(target_os = "linux", target_os = "windows"),
272 target_pointer_width = "64"
273 ),
274 all(feature = "vaapi", target_os = "linux")
275 ))]
276 if hardware {
277 let track = crate::mp4_demux::VideoTrack {
278 codec,
279 width,
280 height,
281 config: config.to_vec(),
282 timescale: 90_000,
283 samples: Vec::new(),
284 };
285 #[cfg(all(
286 feature = "nvidia",
287 any(target_os = "linux", target_os = "windows"),
288 target_pointer_width = "64"
289 ))]
290 if Backend::Nvdec.available()
291 && let Ok(mut d) = crate::nvidia::NvdecDecoder::new(&track)
292 {
293 d.set_pool(pool.clone());
294 return Ok(Self::Nvdec(Box::new(d)));
295 }
296 #[cfg(all(feature = "vaapi", target_os = "linux"))]
297 if Backend::Vaapi.available()
298 && let Ok(mut d) = crate::vaapi::VaapiDecoder::new(&track)
299 {
300 d.set_pool(pool.clone());
301 return Ok(Self::Vaapi(Box::new(d)));
302 }
303 }
304 let _ = (hardware, first, pool);
305 let mut decoder = NativeDecoder::from_config(codec, config, width, height)?;
306 decoder.set_threads(crate::frame_threads());
307 if decoder.threaded() {
308 return Ok(Self::Threaded {
309 decoder: Box::new(decoder),
310 in_flight: Default::default(),
311 });
312 }
313 Ok(Self::Native(Box::new(decoder)))
314 }
315
316 fn earliest_in_flight(&self) -> f64 {
319 match self {
320 Self::Threaded { in_flight, .. } => {
321 in_flight.values().copied().fold(f64::INFINITY, f64::min)
322 }
323 _ => f64::INFINITY,
324 }
325 }
326
327 fn finished(&mut self) -> Vec<(f64, Picture)> {
330 let Self::Threaded { decoder, in_flight } = self else {
331 return Vec::new();
332 };
333 let mut out = Vec::new();
334 for (id, frame) in decoder.take_finished() {
335 let at = in_flight.remove(&id);
336 if let (Some(at), Some(frame)) = (at, frame) {
337 out.push((at, Picture::Planes(frame)));
338 }
339 }
340 out
341 }
342
343 fn wait(&mut self) -> bool {
345 match self {
346 Self::Threaded { decoder, in_flight } if !in_flight.is_empty() => {
347 decoder.wait_for_pictures();
348 true
349 }
350 _ => false,
351 }
352 }
353
354 fn backend(&self) -> Backend {
355 match self {
356 Self::Native(_) | Self::Threaded { .. } => Backend::Native,
357 #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
358 Self::VideoToolbox(_) => Backend::VideoToolbox,
359 #[cfg(all(
360 feature = "nvidia",
361 any(target_os = "linux", target_os = "windows"),
362 target_pointer_width = "64"
363 ))]
364 Self::Nvdec(_) => Backend::Nvdec,
365 #[cfg(all(feature = "vaapi", target_os = "linux"))]
366 Self::Vaapi(_) => Backend::Vaapi,
367 }
368 }
369
370 fn push(
373 &mut self,
374 sample: &[u8],
375 pts: i64,
376 dts: i64,
377 duration: i64,
378 seconds: f64,
379 ) -> Result<(), String> {
380 match self {
381 Self::Native(d) => d.push(sample, pts, dts, duration),
382 Self::Threaded { decoder, in_flight } => {
383 let first = decoder.next_output_id();
384 let pushed = decoder.push_nals(sample);
385 for id in first..decoder.next_output_id() {
386 in_flight.insert(id, seconds);
387 }
388 pushed
389 }
390 #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
391 Self::VideoToolbox(d) => d.push(sample, pts, dts, duration),
392 #[cfg(all(
393 feature = "nvidia",
394 any(target_os = "linux", target_os = "windows"),
395 target_pointer_width = "64"
396 ))]
397 Self::Nvdec(d) => d.push(sample, pts, dts, duration),
398 #[cfg(all(feature = "vaapi", target_os = "linux"))]
399 Self::Vaapi(d) => d.push(sample, pts, dts, duration),
400 }
401 }
402
403 fn take_frame(&mut self, pts: i64) -> Option<Picture> {
406 match self {
407 Self::Native(d) => d.take_frame(pts).map(Picture::Planes),
408 Self::Threaded { .. } => None,
410 #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
411 Self::VideoToolbox(d) => d.take_packed(pts).map(Picture::Packed),
412 #[cfg(all(
413 feature = "nvidia",
414 any(target_os = "linux", target_os = "windows"),
415 target_pointer_width = "64"
416 ))]
417 Self::Nvdec(d) => d.take_packed(pts).map(Picture::Packed),
418 #[cfg(all(feature = "vaapi", target_os = "linux"))]
419 Self::Vaapi(d) => d.take_picture(pts),
420 }
421 }
422}
423
424pub(crate) fn vp9_record(frame: &[u8], width: u32, height: u32) -> Option<Vec<u8>> {
429 let b = *frame.first()?;
430 if b >> 6 != 0b10 || (b >> 5) & 1 != 0 || (b >> 4) & 1 != 0 {
432 return None;
433 }
434 let luma = width as u64 * height as u64;
435 let level = if luma <= 2_228_224 {
436 41
437 } else if luma <= 8_912_896 {
438 51
439 } else {
440 62
441 };
442 Some(vec![
445 1,
446 0,
447 0,
448 0,
449 0,
450 level,
451 (8 << 4) | (1 << 1),
452 2,
453 2,
454 2,
455 0,
456 0,
457 ])
458}
459
460#[cfg(all(feature = "videotoolbox", target_os = "macos"))]
463const TIMESCALE: u32 = 90_000;
464
465pub struct VideoReader {
468 source: Source,
469 width: u32,
470 height: u32,
471 duration: f64,
472 frames: usize,
473 hardware: bool,
474 pool: FramePool,
476}
477
478impl VideoReader {
479 pub fn open(data: Vec<u8>) -> Result<Self, String> {
483 Self::open_with(data.into(), true)
484 }
485
486 pub fn open_file(path: impl AsRef<std::path::Path>) -> Result<Self, String> {
491 let path = path.as_ref();
492 let data = Bytes::map(path).map_err(|e| format!("{}: {e}", path.display()))?;
493 Self::open_with(data, true)
494 }
495
496 pub fn open_shared(data: impl Into<Bytes>) -> Result<Self, String> {
500 Self::open_with(data.into(), true)
501 }
502
503 pub fn open_software(data: Vec<u8>) -> Result<Self, String> {
505 Self::open_with(data.into(), false)
506 }
507
508 fn open_with(data: Bytes, hardware: bool) -> Result<Self, String> {
509 if data.starts_with(&[0, 0, 1]) || data.starts_with(&[0, 0, 0, 1]) {
510 let codec = super::sniff_annexb(&data)
511 .ok_or("no H.264 SPS or HEVC VPS in the elementary stream")?;
512 let frames: Vec<_> = super::decode_annexb(codec, &data)?
513 .into_iter()
514 .map(Some)
515 .collect();
516 let (width, height) = frames
517 .first()
518 .and_then(|f| f.as_ref())
519 .map_or((0, 0), |f| (f.width, f.height));
520 let count = frames.len();
521 return Ok(Self {
522 source: Source::Decoded {
523 data,
524 codec,
525 frames,
526 next: 0,
527 },
528 width,
529 height,
530 duration: count as f64 / UNTIMED_FPS,
531 frames: count,
532 hardware: false,
533 pool: FramePool::default(),
534 });
535 }
536
537 let Demuxed {
538 codec,
539 config,
540 width,
541 height,
542 bytes,
543 samples,
544 } = demux(data)?;
545
546 let mut earliest_after = vec![f64::INFINITY; samples.len() + 1];
547 for i in (0..samples.len()).rev() {
548 earliest_after[i] = earliest_after[i + 1].min(samples[i].seconds);
549 }
550 let duration = clip_length(&samples);
551 let frames = samples.len();
552 let first = samples.first().map(|s| &bytes[s.start..s.end]);
553 let pool = FramePool::default();
554 let decoder = SampleDecoder::open(codec, &config, width, height, hardware, first, &pool)?;
555 Ok(Self {
556 source: Source::Samples {
557 bytes,
558 samples,
559 earliest_after,
560 codec,
561 config,
562 decoder,
563 next: 0,
564 waiting: VecDeque::new(),
565 },
566 width,
567 height,
568 duration,
569 frames,
570 hardware,
571 pool,
572 })
573 }
574
575 pub fn frame_pool(&self) -> FramePool {
582 self.pool.clone()
583 }
584
585 pub fn recycle(&self, picture: Picture) {
587 if let Picture::Packed(frame) = picture {
588 self.pool.give(frame.y);
589 self.pool.give(frame.uv);
590 }
591 }
592
593 pub fn backend(&self) -> Backend {
595 match &self.source {
596 Source::Samples { decoder, .. } => decoder.backend(),
597 Source::Decoded { .. } => Backend::Native,
598 }
599 }
600
601 pub fn size(&self) -> (u32, u32) {
603 (self.width, self.height)
604 }
605
606 pub fn duration(&self) -> f64 {
609 self.duration
610 }
611
612 pub fn frame_count(&self) -> usize {
614 self.frames
615 }
616
617 pub fn next_frame(&mut self) -> Result<Option<(f64, DecodedFrame)>, String> {
621 Ok(self.next_picture()?.map(|(at, p)| (at, p.into_decoded())))
622 }
623
624 pub fn next_picture(&mut self) -> Result<Option<(f64, Picture)>, String> {
628 let (width, height) = (self.width, self.height);
629 match &mut self.source {
630 Source::Decoded { frames, next, .. } => {
631 if *next >= frames.len() {
632 return Ok(None);
633 }
634 let at = *next;
635 *next += 1;
636 Ok(frames[at]
638 .take()
639 .map(|f| (at as f64 / UNTIMED_FPS, Picture::Planes(f))))
640 }
641 Source::Samples {
642 bytes,
643 samples,
644 earliest_after,
645 codec,
646 config,
647 decoder,
648 next,
649 waiting,
650 } => loop {
651 for (at, frame) in decoder.finished() {
652 let i = waiting.partition_point(|(t, _)| *t <= at);
653 waiting.insert(i, (at, frame));
654 }
655 if let Some((seconds, _)) = waiting.front()
658 && (*seconds < earliest_after[*next] || *next == samples.len())
659 && *seconds <= decoder.earliest_in_flight()
660 {
661 return Ok(waiting.pop_front());
662 }
663 if *next == samples.len() {
664 if decoder.wait() {
665 continue;
666 }
667 return Ok(waiting.pop_front());
668 }
669 let s = samples[*next];
670 *next += 1;
671 let mut pushed =
672 decoder.push(&bytes[s.start..s.end], s.pts, s.dts, s.duration, s.seconds);
673 let mut frame = pushed.as_ref().ok().and_then(|_| decoder.take_frame(s.pts));
674 if *next == 1 && frame.is_none() && decoder.backend() != Backend::Native {
678 *decoder = SampleDecoder::open(
679 *codec, config, width, height, false, None, &self.pool,
680 )?;
681 pushed =
682 decoder.push(&bytes[s.start..s.end], s.pts, s.dts, s.duration, s.seconds);
683 frame = pushed.as_ref().ok().and_then(|_| decoder.take_frame(s.pts));
684 }
685 match pushed {
686 Ok(()) => {
687 if let Some(frame) = frame {
688 let at = waiting.partition_point(|(t, _)| *t <= s.seconds);
689 waiting.insert(at, (s.seconds, frame));
690 }
691 }
692 Err(e) if *next == samples.len() && waiting.is_empty() => return Err(e),
694 Err(_) => {}
695 }
696 },
697 }
698 }
699
700 pub fn rewind(&mut self) -> Result<(), String> {
702 match &mut self.source {
703 Source::Decoded {
704 data,
705 codec,
706 frames,
707 next,
708 } => {
709 if *next > 0 {
710 *frames = super::decode_annexb(*codec, data)?
712 .into_iter()
713 .map(Some)
714 .collect();
715 *next = 0;
716 }
717 }
718 Source::Samples {
719 bytes,
720 samples,
721 codec,
722 config,
723 decoder,
724 next,
725 waiting,
726 ..
727 } => {
728 let first = samples.first().map(|s| &bytes[s.start..s.end]);
729 *decoder = SampleDecoder::open(
730 *codec,
731 config,
732 self.width,
733 self.height,
734 self.hardware,
735 first,
736 &self.pool,
737 )?;
738 *next = 0;
739 waiting.clear();
740 }
741 }
742 Ok(())
743 }
744}
745
746fn clip_length(samples: &[Sample]) -> f64 {
750 let mut times: Vec<f64> = samples.iter().map(|s| s.seconds).collect();
751 times.sort_by(f64::total_cmp);
752 let Some(&last) = times.last() else {
753 return 0.0;
754 };
755 let mut gaps: Vec<f64> = times
756 .windows(2)
757 .map(|w| w[1] - w[0])
758 .filter(|g| *g > 0.0)
759 .collect();
760 gaps.sort_by(f64::total_cmp);
761 let frame = gaps
762 .get(gaps.len() / 2)
763 .copied()
764 .unwrap_or(1.0 / UNTIMED_FPS);
765 last + frame
766}
767
768#[cfg(test)]
769mod tests {
770 use super::*;
771
772 #[test]
775 fn webm_and_ivf_demux_in_place() {
776 use crate::webm::{mux_webm, WebmCodec, WebmFrame, WebmParams};
777 let packets: [&[u8]; 3] = [&[1, 2, 3, 4], &[5, 6], &[7, 8, 9]];
778 let frames: Vec<WebmFrame> = packets
779 .iter()
780 .enumerate()
781 .map(|(i, &data)| WebmFrame {
782 data,
783 alpha: None,
784 timecode: i as u64 * 40,
785 keyframe: i == 0,
786 })
787 .collect();
788 let webm = Bytes::from(mux_webm(&WebmParams {
789 width: 64,
790 height: 48,
791 codec: WebmCodec::Av1,
792 timecode_scale_ns: 1_000_000,
793 frames: &frames,
794 }));
795
796 let mut ivf = b"DKIF\0\0\x20\0AV01\x40\0\x30\0\x19\0\0\0\x01\0\0\0".to_vec();
797 ivf.resize(32, 0);
798 for (i, p) in packets.iter().enumerate() {
799 ivf.extend_from_slice(&(p.len() as u32).to_le_bytes());
800 ivf.extend_from_slice(&(i as u64).to_le_bytes());
801 ivf.extend_from_slice(p);
802 }
803 let ivf = Bytes::from(ivf);
804
805 for file in [webm, ivf] {
806 let d = demux(file.clone()).unwrap();
807 assert_eq!(d.bytes.as_ptr(), file.as_ptr(), "the file's own bytes");
808 let got: Vec<&[u8]> = d.samples.iter().map(|s| &d.bytes[s.start..s.end]).collect();
809 assert_eq!(got, packets);
810 assert!(
811 (d.samples[2].seconds - 0.08).abs() < 1e-9,
812 "{:?}",
813 d.samples
814 );
815 }
816 }
817}