1#![allow(clippy::needless_range_loop)]
53
54#[cfg(feature = "decode")]
55pub mod av1;
56pub mod bits;
57#[cfg(feature = "decode")]
58pub mod cabac;
59pub mod h264;
60pub mod hevc;
61#[cfg(feature = "decode")]
62pub(crate) mod jobs;
63pub mod picture;
64#[cfg(feature = "decode")]
65pub mod platform;
66#[cfg(feature = "decode")]
67pub mod reader;
68#[cfg(feature = "decode")]
69pub mod vp9;
70pub mod vp9_packet;
71
72#[cfg(feature = "decode")]
73pub use h264::H264Decoder;
74#[cfg(feature = "decode")]
75pub use hevc::HevcDecoder;
76#[cfg(feature = "decode")]
77pub use picture::{ChromaFormat, ColorInfo, DecodedFrame, Matrix, PackedFrame, Picture, Plane};
78#[cfg(feature = "decode")]
79pub use platform::{Backend, Decoder};
80#[cfg(feature = "decode")]
81pub use reader::{demux, CodedSample, Demuxed, VideoReader};
82#[cfg(feature = "decode")]
83pub mod webcodecs;
84
85#[cfg(feature = "decode")]
86use crate::mp4_demux::{TrackCodec, VideoTrack};
87
88#[cfg(feature = "decode")]
89enum Codec {
90 H264(Box<H264Decoder>),
91 Hevc(Box<HevcDecoder>),
92 Vp9(Box<vp9::Vp9Decoder>),
93 Av1(Box<av1::Av1Decoder>),
94}
95
96#[cfg(feature = "decode")]
97impl Codec {
98 fn decode_nal(&mut self, nal: &[u8]) -> Result<(), String> {
99 match self {
100 Codec::H264(d) => d.decode_nal(nal),
101 Codec::Hevc(d) => d.decode_nal(nal),
102 Codec::Vp9(_) => Err("VP9 has no NAL units".into()),
103 Codec::Av1(_) => Err("AV1 has no NAL units".into()),
104 }
105 }
106
107 fn flush(&mut self) -> Vec<DecodedFrame> {
110 match self {
111 Codec::H264(d) => {
112 d.finish_picture();
113 d.take_pictures().into_iter().map(|p| p.frame).collect()
114 }
115 Codec::Hevc(d) => {
116 d.finish_picture();
117 d.take_pictures().into_iter().map(|p| p.frame).collect()
118 }
119 Codec::Vp9(_) | Codec::Av1(_) => Vec::new(),
120 }
121 }
122}
123
124#[cfg(feature = "decode")]
125pub(crate) fn split_avcc(cfg: &[u8]) -> Result<(Vec<&[u8]>, usize), String> {
127 if cfg.len() < 7 {
128 return Err("avcC too short".into());
129 }
130 let nal_len = (cfg[4] & 3) as usize + 1;
131 let mut sets = Vec::new();
132 let mut at = 5;
133 for pass in 0..2 {
134 let count = if pass == 0 {
135 (cfg[at] & 31) as usize
136 } else {
137 cfg.get(at).copied().unwrap_or(0) as usize
138 };
139 at += 1;
140 for _ in 0..count {
141 if at + 2 > cfg.len() {
142 return Err("avcC ended inside a parameter set".into());
143 }
144 let len = u16::from_be_bytes([cfg[at], cfg[at + 1]]) as usize;
145 at += 2;
146 if at + len > cfg.len() {
147 return Err("avcC parameter set runs past the box".into());
148 }
149 sets.push(&cfg[at..at + len]);
150 at += len;
151 }
152 if at >= cfg.len() {
153 break;
154 }
155 }
156 Ok((sets, nal_len))
157}
158
159#[cfg(feature = "decode")]
160pub(crate) fn split_hvcc(cfg: &[u8]) -> Result<(Vec<&[u8]>, usize), String> {
162 if cfg.len() < 23 {
163 return Err("hvcC too short".into());
164 }
165 let nal_len = (cfg[21] & 3) as usize + 1;
166 let arrays = cfg[22] as usize;
167 let mut sets = Vec::new();
168 let mut at = 23;
169 for _ in 0..arrays {
170 if at + 3 > cfg.len() {
171 return Err("hvcC ended inside an array header".into());
172 }
173 let count = u16::from_be_bytes([cfg[at + 1], cfg[at + 2]]) as usize;
174 at += 3;
175 for _ in 0..count {
176 if at + 2 > cfg.len() {
177 return Err("hvcC ended inside a parameter set".into());
178 }
179 let len = u16::from_be_bytes([cfg[at], cfg[at + 1]]) as usize;
180 at += 2;
181 if at + len > cfg.len() {
182 return Err("hvcC parameter set runs past the box".into());
183 }
184 sets.push(&cfg[at..at + len]);
185 at += len;
186 }
187 }
188 Ok((sets, nal_len))
189}
190
191#[cfg(feature = "decode")]
192pub struct NativeDecoder {
195 codec: Codec,
196 nal_len: usize,
197 width: u32,
198 height: u32,
199 ready: Vec<(i64, DecodedFrame)>,
200}
201
202#[cfg(feature = "decode")]
203impl NativeDecoder {
204 pub fn new(track: &VideoTrack) -> Result<Self, String> {
206 Self::from_config(track.codec, &track.config, track.width, track.height)
207 }
208
209 pub fn from_config(
215 codec: TrackCodec,
216 config: &[u8],
217 width: u32,
218 height: u32,
219 ) -> Result<Self, String> {
220 let (mut codec, (sets, nal_len)) = match codec {
221 TrackCodec::H264 => (Codec::H264(Box::default()), split_avcc(config)?),
222 TrackCodec::Hevc => (Codec::Hevc(Box::default()), split_hvcc(config)?),
223 TrackCodec::Vp9 => (Codec::Vp9(Box::default()), (Vec::new(), 0)),
224 TrackCodec::Av1 => (Codec::Av1(Box::default()), (Vec::new(), 0)),
225 };
226 for nal in sets {
227 codec.decode_nal(nal)?;
228 }
229 Ok(NativeDecoder {
230 codec,
231 nal_len,
232 width,
233 height,
234 ready: Vec::new(),
235 })
236 }
237
238 pub fn size(&self) -> (u32, u32) {
240 (self.width, self.height)
241 }
242
243 pub fn push(
248 &mut self,
249 sample: &[u8],
250 pts: i64,
251 _dts: i64,
252 _duration: i64,
253 ) -> Result<(), String> {
254 if let Codec::Av1(d) = &mut self.codec {
255 for (i, f) in d.decode(sample)?.into_iter().enumerate() {
257 self.ready.push((pts + i as i64, f.to_decoded()));
258 }
259 return Ok(());
260 }
261 if let Codec::Vp9(d) = &mut self.codec {
262 for (i, f) in d.decode(sample)?.into_iter().enumerate() {
264 self.ready.push((pts + i as i64, f.to_decoded()?));
265 }
266 return Ok(());
267 }
268 for nal in bits::split_length_prefixed(sample, self.nal_len)? {
269 self.codec.decode_nal(nal)?;
270 }
271 for (i, frame) in self.codec.flush().into_iter().enumerate() {
272 self.ready.push((pts + i as i64, frame));
275 }
276 Ok(())
277 }
278
279 pub fn take(&mut self, pts: i64) -> Option<Vec<u8>> {
283 self.take_frame(pts).map(|f| f.to_bgra())
284 }
285
286 pub fn take_frame(&mut self, pts: i64) -> Option<DecodedFrame> {
288 let hit = self.ready.iter().position(|(p, _)| *p == pts)?;
289 let (_, frame) = self.ready.remove(hit);
290 self.ready.retain(|(p, _)| *p > pts);
291 Some(frame)
292 }
293
294 fn set_threads(&mut self, threads: usize) {
297 match &mut self.codec {
298 Codec::H264(d) => d.set_threads(threads),
299 Codec::Hevc(d) => d.set_threads(threads),
300 _ => {}
301 }
302 }
303
304 fn threaded(&self) -> bool {
305 match &self.codec {
306 Codec::H264(d) => d.threaded(),
307 Codec::Hevc(d) => d.threaded(),
308 _ => false,
309 }
310 }
311
312 fn next_output_id(&self) -> u64 {
313 match &self.codec {
314 Codec::H264(d) => d.next_output_id(),
315 Codec::Hevc(d) => d.next_output_id(),
316 _ => 0,
317 }
318 }
319
320 fn push_nals(&mut self, sample: &[u8]) -> Result<(), String> {
322 for nal in bits::split_length_prefixed(sample, self.nal_len)? {
323 self.codec.decode_nal(nal)?;
324 }
325 match &mut self.codec {
326 Codec::H264(d) => d.finish_picture(),
327 Codec::Hevc(d) => d.finish_picture(),
328 _ => {}
329 }
330 Ok(())
331 }
332
333 fn take_finished(&mut self) -> Vec<(u64, Option<DecodedFrame>)> {
334 match &mut self.codec {
335 Codec::H264(d) => d
336 .take_finished()
337 .into_iter()
338 .map(|(id, p)| (id, p.map(|p| p.frame)))
339 .collect(),
340 Codec::Hevc(d) => d
341 .take_finished()
342 .into_iter()
343 .map(|(id, p)| (id, p.map(|p| p.frame)))
344 .collect(),
345 _ => Vec::new(),
346 }
347 }
348
349 fn wait_for_pictures(&mut self) {
350 match &mut self.codec {
351 Codec::H264(d) => d.wait_for_pictures(),
352 Codec::Hevc(d) => d.wait_for_pictures(),
353 _ => {}
354 }
355 }
356
357 fn take_thread_error(&mut self) -> Option<String> {
358 match &mut self.codec {
359 Codec::H264(d) => d.take_thread_error(),
360 Codec::Hevc(d) => d.take_thread_error(),
361 _ => None,
362 }
363 }
364
365 pub fn queued(&self) -> usize {
367 self.ready.len()
368 }
369}
370
371#[cfg(feature = "decode")]
372pub fn decode_annexb(codec: TrackCodec, stream: &[u8]) -> Result<Vec<DecodedFrame>, String> {
375 let mut pics: Vec<(bool, i32, DecodedFrame)> = Vec::new();
377 match codec {
378 TrackCodec::H264 => {
379 let mut d = H264Decoder::new();
380 d.set_threads(crate::frame_threads());
381 for nal in bits::split_annexb(stream) {
382 d.decode_nal(nal)?;
383 }
384 d.finish_picture();
385 d.wait_for_pictures();
386 d.take_thread_error().map_or(Ok(()), Err)?;
387 pics.extend(
388 d.take_pictures()
389 .into_iter()
390 .map(|p| (p.idr, p.poc, p.frame)),
391 );
392 }
393 TrackCodec::Vp9 => return Err("VP9 is not an Annex B stream".into()),
394 TrackCodec::Av1 => return Err("AV1 is not an Annex B H.26x stream".into()),
395 TrackCodec::Hevc => {
396 let mut d = HevcDecoder::new();
399 d.set_threads(crate::frame_threads());
400 for nal in bits::split_annexb(stream) {
401 d.decode_nal(nal)?;
402 }
403 d.flush_output();
404 d.take_thread_error().map_or(Ok(()), Err)?;
405 let mut by_id: std::collections::HashMap<u64, DecodedFrame> = d
406 .take_pictures()
407 .into_iter()
408 .map(|p| (p.id, p.frame))
409 .collect();
410 return Ok(d
411 .take_output_order()
412 .into_iter()
413 .filter_map(|id| by_id.remove(&id))
414 .collect());
415 }
416 }
417 let mut period = 0u32;
418 let mut keyed: Vec<((u32, i32), DecodedFrame)> = Vec::with_capacity(pics.len());
419 for (i, (starts, poc, frame)) in pics.into_iter().enumerate() {
420 if starts && i > 0 {
421 period += 1;
422 }
423 keyed.push(((period, poc), frame));
424 }
425 keyed.sort_by_key(|(k, _)| *k);
426 Ok(keyed.into_iter().map(|(_, f)| f).collect())
427}
428
429#[cfg(feature = "decode")]
435pub fn decode_video_in_order(
436 data: &[u8],
437 mut each: impl FnMut(Option<f64>, DecodedFrame) -> Result<bool, String>,
438) -> Result<usize, String> {
439 const MAX_REORDER: usize = 16;
441 const STOPPED: &str = "\0stopped";
442 let mut held: Vec<(f64, DecodedFrame)> = Vec::new();
443 let mut emit = |t: Option<f64>, f: DecodedFrame| match each(t, f) {
444 Ok(true) => Ok(()),
445 Ok(false) => Err(STOPPED.to_string()),
446 Err(e) => Err(e),
447 };
448 let earliest = |held: &[(f64, DecodedFrame)]| {
449 (0..held.len())
450 .min_by(|&a, &b| held[a].0.total_cmp(&held[b].0))
451 .unwrap()
452 };
453 let mut decoded = decode_video(data, |t, f| {
454 let Some(t) = t else { return emit(None, f) };
455 held.push((t, f));
456 if held.len() > MAX_REORDER {
457 let (t, f) = held.remove(earliest(&held));
458 emit(Some(t), f)?;
459 }
460 Ok(())
461 });
462 while decoded.is_ok() && !held.is_empty() {
463 let (t, f) = held.remove(earliest(&held));
464 if let Err(e) = emit(Some(t), f) {
465 decoded = Err(e);
466 }
467 }
468 match decoded {
469 Err(e) if e == STOPPED => Ok(0),
470 r => r,
471 }
472}
473
474#[cfg(feature = "decode")]
476fn decode_av1_units(
477 units: &[&[u8]],
478 time: impl Fn(usize) -> f64,
479 mut each: impl FnMut(Option<f64>, DecodedFrame) -> Result<(), String>,
480) -> Result<usize, String> {
481 av1::Av1Decoder::new().decode_each(units, crate::frame_threads(), |i, f| {
482 each(Some(time(i)), f.to_decoded())
483 })
484}
485
486#[cfg(feature = "decode")]
489fn decode_samples<'a>(
490 dec: &mut NativeDecoder,
491 samples: impl Iterator<Item = Result<(&'a [u8], i64, i64, i64, f64), String>>,
492 mut each: impl FnMut(Option<f64>, DecodedFrame) -> Result<(), String>,
493) -> Result<usize, String> {
494 let (mut shown, mut skipped, mut first_error) = (0usize, 0usize, None);
495 for s in samples {
496 let (bytes, pts, dts, duration, time) = s?;
497 match dec.push(bytes, pts, dts, duration) {
498 Ok(()) => {
499 if let Some(f) = dec.take_frame(pts) {
500 shown += 1;
501 each(Some(time), f)?;
502 }
503 }
504 Err(e) => {
505 skipped += 1;
506 first_error.get_or_insert(e);
507 }
508 }
509 }
510 match first_error {
511 Some(e) if shown == 0 => Err(e),
512 _ => Ok(skipped),
513 }
514}
515
516#[cfg(feature = "decode")]
517pub fn decode_video(
526 data: &[u8],
527 mut each: impl FnMut(Option<f64>, DecodedFrame) -> Result<(), String>,
528) -> Result<usize, String> {
529 if data.starts_with(b"DKIF") {
530 let (fourcc, w, h, (num, den), packets) = crate::mkv_demux::parse_ivf(data)?;
531 let codec = match &fourcc {
532 b"VP90" => TrackCodec::Vp9,
533 b"AV01" => TrackCodec::Av1,
534 other => {
535 return Err(format!(
536 "IVF codec {} is not supported",
537 String::from_utf8_lossy(other)
538 ))
539 }
540 };
541 let first = packets.iter().map(|(pts, _)| *pts).min().unwrap_or(0);
542 let time = |pts: i64| (pts - first) as f64 * num as f64 / den.max(1) as f64;
543 if codec == TrackCodec::Av1 {
544 let units: Vec<&[u8]> = packets.iter().map(|(_, p)| &p[..]).collect();
545 return decode_av1_units(&units, |i| time(packets[i].0), each);
546 }
547 let mut dec = NativeDecoder::from_config(codec, &[], w, h)?;
548 return decode_samples(
549 &mut dec,
550 packets
551 .iter()
552 .map(|(pts, p)| Ok((&p[..], *pts, *pts, 0, time(*pts)))),
553 each,
554 );
555 }
556 if data.starts_with(&[0, 0, 1]) || data.starts_with(&[0, 0, 0, 1]) {
557 let codec =
558 sniff_annexb(data).ok_or("no H.264 SPS or HEVC VPS in the elementary stream")?;
559 for f in decode_annexb(codec, data)? {
560 each(None, f)?;
561 }
562 return Ok(0);
563 }
564 let (track, packets) = if data.starts_with(&[0x1A, 0x45, 0xDF, 0xA3]) {
565 let (_, track, packets) = crate::ffcompat::mkv::parse(data)?;
568 (track, std::borrow::Cow::Owned(packets))
569 } else {
570 (
571 crate::mp4_demux::demux_video(data)?,
572 std::borrow::Cow::Borrowed(data),
573 )
574 };
575 let first = track.samples.iter().map(|s| s.pts).min().unwrap_or(0);
576 let timescale = track.timescale.max(1) as f64;
577 if track.codec == TrackCodec::Av1 {
578 let units = track
579 .samples
580 .iter()
581 .map(|s| {
582 packets
583 .get(s.offset as usize..s.offset as usize + s.size as usize)
584 .ok_or("sample runs past the end of the file".to_string())
585 })
586 .collect::<Result<Vec<_>, _>>()?;
587 return decode_av1_units(
588 &units,
589 |i| (track.samples[i].pts - first) as f64 / timescale,
590 each,
591 );
592 }
593 let mut dec = NativeDecoder::new(&track)?;
594 dec.set_threads(crate::frame_threads());
595 let samples = track.samples.iter().map(|s| {
596 let bytes = packets
597 .get(s.offset as usize..s.offset as usize + s.size as usize)
598 .ok_or("sample runs past the end of the file".to_string())?;
599 Ok((
600 bytes,
601 s.pts,
602 s.dts,
603 s.duration as i64,
604 (s.pts - first) as f64 / timescale,
605 ))
606 });
607 if dec.threaded() {
608 return decode_samples_threaded(&mut dec, samples, each);
609 }
610 decode_samples(&mut dec, samples, each)
611}
612
613#[cfg(feature = "decode")]
616fn decode_samples_threaded<'a>(
617 dec: &mut NativeDecoder,
618 samples: impl Iterator<Item = Result<(&'a [u8], i64, i64, i64, f64), String>>,
619 mut each: impl FnMut(Option<f64>, DecodedFrame) -> Result<(), String>,
620) -> Result<usize, String> {
621 let mut times = std::collections::HashMap::new();
622 let (mut shown, mut skipped, mut first_error) = (0usize, 0usize, None);
623 let mut emit = |dec: &mut NativeDecoder,
624 times: &mut std::collections::HashMap<u64, f64>,
625 skipped: &mut usize|
626 -> Result<(), String> {
627 for (id, frame) in dec.take_finished() {
628 let time = times.remove(&id);
629 match frame {
630 Some(f) => {
631 shown += 1;
632 each(time, f)?;
633 }
634 None => *skipped += 1,
635 }
636 }
637 Ok(())
638 };
639 for s in samples {
640 let (bytes, _, _, _, time) = s?;
641 let first = dec.next_output_id();
642 if let Err(e) = dec.push_nals(bytes) {
643 skipped += 1;
644 first_error.get_or_insert(e);
645 }
646 for id in first..dec.next_output_id() {
647 times.insert(id, time);
648 }
649 emit(dec, &mut times, &mut skipped)?;
650 }
651 dec.wait_for_pictures();
652 emit(dec, &mut times, &mut skipped)?;
653 let error = first_error.or_else(|| dec.take_thread_error());
654 match error {
655 Some(e) if shown == 0 => Err(e),
656 _ => Ok(skipped),
657 }
658}
659
660#[cfg(feature = "decode")]
664fn sniff_annexb(stream: &[u8]) -> Option<TrackCodec> {
665 bits::split_annexb(stream)
666 .into_iter()
667 .take(64)
668 .find_map(|nal| match nal {
669 [0x40, 0x01, ..] => Some(TrackCodec::Hevc),
670 [b, ..] if b & 0x9f == 0x07 => Some(TrackCodec::H264),
671 _ => None,
672 })
673}