Skip to main content

threers_codec/decode/hevc/
mod.rs

1//! A from-scratch HEVC / H.265 decoder.
2//!
3//! Main, Main 10 and Main Still Picture streams (and the 4:2:0 subset of
4//! Main 12): every coding tool those profiles allow — AMP, tiles, wavefront
5//! parallel processing, dependent slice segments, PCM, transform skip,
6//! transquant bypass, sign data hiding, scaling lists, weighted
7//! prediction, long-term references, TMVP, deblocking and SAO — and Main
8//! 4:2:2 10 (and its 8-bit subset): 4:2:2 chroma without the range
9//! extension coding tools. Not supported, and refused with an error: 4:4:4,
10//! the range extension tools, and anything in a layer above the base
11//! (multi-layer streams decode their base layer — Apple's alpha layer is
12//! skipped).
13//!
14//! Known gap: with 16×16 CTBs ffmpeg's interleaved per-CTB deblocking/SAO
15//! schedule lets SAO read chroma a horizontal deblock has not reached yet,
16//! so its output differs from the spec order used here by a few samples
17//! (~1 in 15 000); 32×32 and 64×64 CTBs match it exactly.
18//!
19//! Feed it NAL units with [`HevcDecoder::decode_nal`]; finished pictures
20//! come out of [`HevcDecoder::take_pictures`].
21
22#[cfg(feature = "decode")]
23pub mod cabac_init;
24#[cfg(feature = "decode")]
25pub mod ctu;
26#[cfg(feature = "decode")]
27pub mod filters;
28#[cfg(feature = "decode")]
29pub mod inter;
30#[cfg(feature = "decode")]
31pub mod intra;
32#[cfg(feature = "decode")]
33pub mod mv;
34// Parameter sets parse without the decoder: the MP4 muxer reads them.
35#[cfg_attr(not(feature = "decode"), allow(dead_code))]
36pub mod params;
37#[cfg(feature = "decode")]
38pub mod pic;
39// Parameter sets parse without the decoder: the MP4 muxer reads them.
40#[cfg_attr(not(feature = "decode"), allow(dead_code))]
41pub mod tables;
42#[cfg(feature = "decode")]
43pub mod transform;
44
45#[cfg(feature = "decode")]
46use std::collections::BTreeMap;
47#[cfg(feature = "decode")]
48use std::sync::{Arc, Mutex};
49
50#[cfg(feature = "decode")]
51use self::ctu::{Carry, SliceDec};
52#[cfg(feature = "decode")]
53use self::params::{Pps, SliceHeader, SliceType, Sps};
54#[cfg(feature = "decode")]
55use self::pic::{Layout, MvField, PicState, SliceInfo};
56#[cfg(feature = "decode")]
57use super::bits::unescape;
58#[cfg(feature = "decode")]
59use super::jobs::{caught, Jobs, Slot};
60#[cfg(feature = "decode")]
61use super::picture::{ChromaFormat, ColorInfo, DecodedFrame, Plane};
62
63/// Decoding errors are messages naming what was malformed or unsupported.
64#[cfg(feature = "decode")]
65pub type Result<T> = std::result::Result<T, String>;
66
67#[cfg(feature = "decode")]
68/// A decoded picture and where it goes in display order.
69#[derive(Debug, Clone)]
70pub struct HevcPicture {
71    /// Decode-order number, which [`HevcDecoder::take_output_order`] refers to.
72    pub id: u64,
73    /// The picture, cropped to its conformance window.
74    pub frame: DecodedFrame,
75    /// `PicOrderCntVal`.
76    pub poc: i32,
77    /// Starts a new coded video sequence (an IRAP picture after which POC
78    /// ordering restarts).
79    pub new_sequence: bool,
80}
81
82#[cfg(feature = "decode")]
83/// The motion a picture leaves for later ones to use as collocated.
84pub struct Motion {
85    /// Motion of each 4×4 block, row-major.
86    pub mv: Vec<MvField>,
87    /// Width of the picture in 4×4 blocks (the row stride of `mv`).
88    pub w4: usize,
89    /// Index into `slices` of the slice each CTB belongs to.
90    pub ctb_slice: Vec<u16>,
91    /// What collocated prediction needs from each slice of the picture.
92    pub slices: Vec<SliceInfo>,
93    /// log2 of the CTB size.
94    pub log2_ctb: usize,
95    /// Width of the picture in CTBs.
96    pub ctb_w: usize,
97}
98
99/// A finished picture, as later ones read it.
100#[cfg(feature = "decode")]
101pub(crate) struct RefData {
102    planes: Arc<[Plane; 3]>,
103    motion: Arc<Motion>,
104}
105
106/// A picture in the decoded picture buffer.
107#[cfg(feature = "decode")]
108#[derive(Clone)]
109pub struct RefPic {
110    /// `PicOrderCntVal`.
111    pub poc: i32,
112    /// Marked "used for long-term reference".
113    pub long_term: bool,
114    /// The reconstructed samples, before cropping.
115    pub planes: Arc<[Plane; 3]>,
116    /// The motion it leaves for temporal motion vector prediction.
117    pub motion: Arc<Motion>,
118    /// Width and height in luma samples.
119    size: (usize, usize),
120    /// With threads, where `planes` and `motion` will come from: until a job
121    /// resolves it, both are empty.
122    pending: Option<Arc<Slot<RefData>>>,
123}
124
125/// One entry of a reference picture list.
126#[cfg(feature = "decode")]
127#[derive(Clone, Copy, Debug)]
128pub struct RefEntry {
129    /// Index of the picture in the decoded picture buffer.
130    pub dpb: usize,
131    /// Its `PicOrderCntVal`.
132    pub poc: i32,
133    /// Whether it is a long-term reference.
134    pub long_term: bool,
135}
136
137#[cfg(feature = "decode")]
138struct Current {
139    pic: PicState,
140    sps: Sps,
141    pps: Pps,
142    carry: Carry,
143    output: bool,
144    new_sequence: bool,
145    st_before: Vec<usize>,
146    st_after: Vec<usize>,
147    lt_curr: Vec<usize>,
148    /// With threads, the slice segments for its job to reconstruct.
149    work: Vec<SliceWork>,
150}
151
152/// A slice segment whose headers are read and whose data is not.
153#[cfg(feature = "decode")]
154struct SliceWork {
155    rbsp: Vec<u8>,
156    sh: SliceHeader,
157    refs: [Vec<RefEntry>; 2],
158    slice_idx: u16,
159    slice_addr: u32,
160}
161
162/// The state shared with picture jobs.
163#[cfg(feature = "decode")]
164struct Threads {
165    jobs: Arc<Jobs>,
166    /// Finished output pictures by id; `None` for one that failed.
167    finished: Arc<Mutex<BTreeMap<u64, Option<HevcPicture>>>>,
168    /// The first failure in decode order.
169    error: Arc<Mutex<Option<(u64, String)>>>,
170    next_take: u64,
171    next_job: u64,
172}
173
174/// A hardware back end for [`HevcDecoder`]: the decoder parses, derives
175/// POCs and reference picture sets, builds the lists and orders output; the
176/// accelerator reconstructs. See
177/// [`H264Accel`](crate::decode::h264::H264Accel) for the shape.
178#[cfg(feature = "decode")]
179pub trait HevcAccel {
180    /// Decode one slice segment of the current picture.
181    fn slice(&mut self, s: &AccelSlice<'_>) -> Result<()>;
182    /// The current picture has had all its slice segments.
183    fn end_picture(&mut self) -> Result<()>;
184    /// The samples of the picture with POC `poc`, cropped as `sps` says.
185    fn frame(&mut self, poc: i32, sps: &Sps) -> Result<DecodedFrame>;
186}
187
188/// Everything an [`HevcAccel`] is told about one slice segment.
189#[cfg(feature = "decode")]
190pub struct AccelSlice<'a> {
191    /// The NAL unit as coded: 2-byte header included, emulation prevention in.
192    pub nal: &'a [u8],
193    /// `nal_unit_type`.
194    pub nal_type: u32,
195    /// The active sequence parameter set.
196    pub sps: &'a Sps,
197    /// The active picture parameter set.
198    pub pps: &'a Pps,
199    /// The parsed slice segment header.
200    pub sh: &'a SliceHeader,
201    /// The first segment of a new picture.
202    pub first: bool,
203    /// `PicOrderCntVal` of the picture.
204    pub poc: i32,
205    /// The reference pictures; the current one is not among them.
206    pub dpb: &'a [RefPic],
207    /// `RefPicSetStCurrBefore`, `RefPicSetStCurrAfter`, `RefPicSetLtCurr`
208    /// as indices into `dpb`.
209    pub st_before: &'a [usize],
210    /// See `st_before`.
211    pub st_after: &'a [usize],
212    /// See `st_before`.
213    pub lt_curr: &'a [usize],
214    /// `RefPicList0` and `RefPicList1`.
215    pub refs: &'a [Vec<RefEntry>; 2],
216}
217
218/// An HEVC decoder fed one NAL unit at a time ([`decode_nal`](Self::decode_nal)).
219///
220/// Most callers want [`NativeDecoder`](crate::decode::NativeDecoder), which
221/// unpacks MP4 samples and orders the output; this is the layer below it.
222#[cfg(feature = "decode")]
223pub struct HevcDecoder {
224    spss: Vec<Option<Sps>>,
225    ppss: Vec<Option<Pps>>,
226    dpb: Vec<RefPic>,
227    cur: Option<Current>,
228    /// The last independent slice segment header of the current picture.
229    last_header: Option<SliceHeader>,
230    prev_tid0_poc: i32,
231    /// The next IRAP picture starts with `NoRaslOutputFlag` set.
232    at_start: bool,
233    /// RASL pictures of the current IRAP cannot be decoded.
234    skip_rasl: bool,
235    /// The picture now arriving is being skipped (a RASL picture).
236    skipping: bool,
237    layout: Option<(usize, usize, Arc<Layout>)>,
238    done: Vec<HevcPicture>,
239    /// The DPB output process (§C.5.2): pictures decoded but not yet output,
240    /// as `(id, POC, PicLatencyCount)`.
241    pending: Vec<(u64, i32, u32)>,
242    /// Ids of pictures in the order the output process emits them. Pictures
243    /// it discards (`no_output_of_prior_pics`) never appear.
244    output: Vec<u64>,
245    next_id: u64,
246    /// Whether any picture has been decoded — the first needs no flush.
247    decoded_any: bool,
248    accel: Option<Box<dyn HevcAccel>>,
249    accel_error: Option<String>,
250    accel_started: bool,
251    threads: Option<Threads>,
252}
253
254#[cfg(feature = "decode")]
255impl Default for HevcDecoder {
256    fn default() -> Self {
257        Self::new()
258    }
259}
260
261#[cfg(feature = "decode")]
262impl HevcDecoder {
263    /// A decoder with no parameter sets yet.
264    pub fn new() -> Self {
265        HevcDecoder {
266            spss: vec![None; 16],
267            ppss: vec![None; 64],
268            dpb: Vec::new(),
269            cur: None,
270            last_header: None,
271            prev_tid0_poc: 0,
272            at_start: true,
273            skip_rasl: false,
274            skipping: false,
275            layout: None,
276            done: Vec::new(),
277            pending: Vec::new(),
278            output: Vec::new(),
279            next_id: 0,
280            decoded_any: false,
281            accel: None,
282            accel_error: None,
283            accel_started: false,
284            threads: None,
285        }
286    }
287
288    /// Reconstruct up to `threads` pictures at once, each as soon as its
289    /// references are finished. Headers stay in order on the calling thread,
290    /// so [`decode_nal`](Self::decode_nal) can block when too many are
291    /// waiting, and must not be called from the thread pool itself: this does
292    /// nothing there, with one thread, or with an accelerator.
293    ///
294    /// [`take_pictures`](Self::take_pictures) then gives pictures once they
295    /// are finished, leaving out any that failed; decoding carries on past
296    /// those, and [`take_thread_error`](Self::take_thread_error) has the first.
297    pub fn set_threads(&mut self, threads: usize) {
298        if threads > 1 && self.accel.is_none() && crate::pool_can_serve_caller() {
299            self.threads = Some(Threads {
300                jobs: Jobs::new(threads * 2),
301                finished: Arc::default(),
302                error: Arc::default(),
303                next_take: 0,
304                next_job: 0,
305            });
306        }
307    }
308
309    /// The first error from a picture job, in decode order, once
310    /// [`flush_output`](Self::flush_output) has waited for them all.
311    pub fn take_thread_error(&mut self) -> Option<String> {
312        let t = self.threads.as_ref()?;
313        t.error.lock().unwrap().take().map(|e| e.1)
314    }
315
316    /// Hand reconstruction to `accel` (see [`HevcAccel`]).
317    pub fn set_accel(&mut self, accel: Box<dyn HevcAccel>) {
318        self.accel = Some(accel);
319    }
320
321    /// The first error the accelerator reported from
322    /// [`finish_picture`](Self::finish_picture), which cannot return one.
323    pub fn take_accel_error(&mut self) -> Option<String> {
324        self.accel_error.take()
325    }
326
327    /// Decode one NAL unit (without start code or length prefix).
328    pub fn decode_nal(&mut self, nal: &[u8]) -> Result<()> {
329        if nal.len() < 2 {
330            return Ok(());
331        }
332        let nal_type = ((nal[0] >> 1) & 63) as u32;
333        let layer = (((nal[0] & 1) << 5) | (nal[1] >> 3)) as u32;
334        let tid = (nal[1] & 7).saturating_sub(1) as u32;
335        if layer > 0 {
336            return Ok(());
337        }
338        match nal_type {
339            0..=9 | 16..=21 => {
340                let rbsp = unescape(&nal[2..]);
341                self.decode_slice(nal, &rbsp, nal_type, tid)
342            }
343            33 => {
344                let sps = Sps::parse(&unescape(&nal[2..]))?;
345                let id = sps.id as usize;
346                self.spss[id] = Some(sps);
347                // A re-sent set may change under the same id.
348                self.layout = None;
349                Ok(())
350            }
351            34 => {
352                let pps = Pps::parse(&unescape(&nal[2..]), &self.spss)?;
353                let id = pps.id as usize;
354                self.ppss[id] = Some(pps);
355                // Encoders re-send a PPS under the same id with a new tile
356                // grid; the cached layout is only good for the old one.
357                self.layout = None;
358                Ok(())
359            }
360            35 => {
361                self.finish_picture();
362                Ok(())
363            }
364            36 | 37 => {
365                self.finish_picture();
366                self.at_start = true;
367                Ok(())
368            }
369            _ => Ok(()),
370        }
371    }
372
373    /// Pictures finished so far, in decode order.
374    /// Ids of pictures (see [`HevcPicture::id`]) in display order, as the
375    /// DPB output process of §C.5.2 emits them so far. Pictures still in the
376    /// DPB come out with [`flush_output`](Self::flush_output); pictures the
377    /// stream asks to be discarded never do.
378    pub fn take_output_order(&mut self) -> Vec<u64> {
379        std::mem::take(&mut self.output)
380    }
381
382    /// End of stream: output everything still waiting, in POC order.
383    pub fn flush_output(&mut self) {
384        self.finish_picture();
385        while self.bump() {}
386        if let Some(t) = &self.threads {
387            t.jobs.wait();
388        }
389    }
390
391    /// Output the waiting picture with the smallest POC ("bumping").
392    fn bump(&mut self) -> bool {
393        let Some(i) = (0..self.pending.len()).min_by_key(|&i| self.pending[i].1) else {
394            return false;
395        };
396        let (id, _, _) = self.pending.remove(i);
397        self.output.push(id);
398        true
399    }
400
401    /// Bump while the reorder or latency limits are exceeded (and, before a
402    /// picture is decoded, while the DPB is full).
403    fn bump_while_over(&mut self, sps: &Sps, dpb_full: bool) {
404        loop {
405            let reorder = self.pending.len() > sps.max_num_reorder as usize;
406            let latency = sps.max_latency_increase_plus1 != 0 && {
407                let max = sps.max_num_reorder + sps.max_latency_increase_plus1 - 1;
408                self.pending.iter().any(|e| e.2 >= max)
409            };
410            let full = dpb_full && {
411                let refs = self
412                    .dpb
413                    .iter()
414                    .filter(|p| !self.pending.iter().any(|e| e.1 == p.poc))
415                    .count();
416                refs + self.pending.len() >= sps.max_dec_pic_buffering as usize
417            };
418            if !(reorder || latency || full) || !self.bump() {
419                return;
420            }
421        }
422    }
423
424    /// The pictures finished since the last call, in decode order.
425    pub fn take_pictures(&mut self) -> Vec<HevcPicture> {
426        let finished = self.take_finished().into_iter().filter_map(|p| p.1);
427        let finished: Vec<_> = finished.collect();
428        self.done.extend(finished);
429        std::mem::take(&mut self.done)
430    }
431
432    /// With threads: output pictures finished so far by id, in order, `None`
433    /// for one that failed.
434    pub(crate) fn take_finished(&mut self) -> Vec<(u64, Option<HevcPicture>)> {
435        let mut out = Vec::new();
436        if let Some(t) = &mut self.threads {
437            let mut finished = t.finished.lock().unwrap();
438            while let Some(picture) = finished.remove(&t.next_take) {
439                out.push((t.next_take, picture));
440                t.next_take += 1;
441            }
442        }
443        out
444    }
445
446    /// Wait for every picture job to finish.
447    pub(crate) fn wait_for_pictures(&self) {
448        if let Some(t) = &self.threads {
449            t.jobs.wait();
450        }
451    }
452
453    /// Whether [`set_threads`](Self::set_threads) took effect.
454    pub(crate) fn threaded(&self) -> bool {
455        self.threads.is_some()
456    }
457
458    /// The id the next output picture will have.
459    pub(crate) fn next_output_id(&self) -> u64 {
460        self.next_id
461    }
462
463    fn decode_slice(&mut self, nal: &[u8], rbsp: &[u8], nal_type: u32, tid: u32) -> Result<()> {
464        let first = rbsp.first().is_some_and(|b| b & 0x80 != 0);
465        if first {
466            self.finish_picture();
467            self.skipping = (nal_type == 8 || nal_type == 9) && self.skip_rasl;
468            self.last_header = None;
469        }
470        if self.skipping {
471            return Ok(());
472        }
473        if !first && self.cur.is_none() {
474            // The picture's first slice was lost; nothing to attach to.
475            return Ok(());
476        }
477        let prev = self.last_header.as_ref();
478        let sh = SliceHeader::parse(rbsp, nal_type, &self.spss, &self.ppss, prev)?;
479        if first {
480            self.start_picture(&sh, nal_type, tid)?;
481        }
482        if !sh.dependent {
483            self.last_header = Some(sh.clone());
484        }
485        let refs = self.build_lists(&sh)?;
486        if let Some(accel) = self.accel.as_mut() {
487            let cur = self.cur.as_ref().unwrap();
488            let slice = AccelSlice {
489                nal,
490                nal_type,
491                sps: &cur.sps,
492                pps: &cur.pps,
493                sh: &sh,
494                first: !self.accel_started,
495                poc: cur.pic.poc,
496                dpb: &self.dpb,
497                st_before: &cur.st_before,
498                st_after: &cur.st_after,
499                lt_curr: &cur.lt_curr,
500                refs: &refs,
501            };
502            self.accel_started = true;
503            return accel.slice(&slice);
504        }
505        let cur = self.cur.as_mut().unwrap();
506        // SliceAddrRs: a dependent segment belongs to the slice before it.
507        let slice_addr = if sh.dependent {
508            cur.pic.slices.last().map_or(0, |s| s.addr_rs)
509        } else {
510            sh.segment_address
511        };
512        let slice_idx = cur.pic.slices.len() as u16;
513        cur.pic.slices.push(SliceInfo {
514            addr_rs: slice_addr,
515            deblocking_disabled: sh.deblocking_disabled,
516            beta_offset: sh.beta_offset,
517            tc_offset: sh.tc_offset,
518            lf_across_slices: sh.loop_filter_across_slices,
519            cb_qp_offset: sh.cb_qp_offset,
520            cr_qp_offset: sh.cr_qp_offset,
521            refs: [
522                refs[0].iter().map(|e| (e.poc, e.long_term)).collect(),
523                refs[1].iter().map(|e| (e.poc, e.long_term)).collect(),
524            ],
525        });
526        if self.threads.is_some() {
527            cur.work.push(SliceWork {
528                rbsp: rbsp.to_vec(),
529                sh,
530                refs,
531                slice_idx,
532                slice_addr,
533            });
534            return Ok(());
535        }
536        let mut dec = SliceDec::new(
537            &cur.sps,
538            &cur.pps,
539            &sh,
540            &mut cur.pic,
541            &self.dpb,
542            &refs,
543            &mut cur.carry,
544            slice_idx,
545            slice_addr,
546        );
547        dec.decode(rbsp)
548    }
549
550    fn start_picture(&mut self, sh: &SliceHeader, nal_type: u32, tid: u32) -> Result<()> {
551        let pps = self.ppss[sh.pps_id as usize].clone().ok_or("missing PPS")?;
552        let sps = self.spss[pps.sps_id as usize]
553            .clone()
554            .ok_or("missing SPS")?;
555        let layout = match &self.layout {
556            Some((s, p, l))
557                if *s == pps.sps_id as usize
558                    && *p == pps.id as usize
559                    && l.ctb_w == sps.width_ctbs() as usize
560                    && l.ctb_h == sps.height_ctbs() as usize =>
561            {
562                l.clone()
563            }
564            _ => {
565                let l = Arc::new(Layout::new(&sps, &pps));
566                self.layout = Some((pps.sps_id as usize, pps.id as usize, l.clone()));
567                l
568            }
569        };
570        if let Some(p) = self.dpb.first()
571            && p.size != (sps.width as usize, sps.height as usize)
572        {
573            self.dpb.clear();
574        }
575        let irap = (16..=23).contains(&nal_type);
576        let idr = nal_type == 19 || nal_type == 20;
577        let no_rasl_output = irap && (idr || (16..=18).contains(&nal_type) || self.at_start);
578        if irap {
579            self.skip_rasl = no_rasl_output && !idr;
580            self.at_start = false;
581        }
582        // POC (§8.3.1).
583        let max_lsb = 1i32 << sps.log2_max_poc_lsb;
584        let lsb = sh.poc_lsb as i32;
585        let msb = if irap && no_rasl_output {
586            0
587        } else {
588            let prev_lsb = self.prev_tid0_poc & (max_lsb - 1);
589            let prev_msb = self.prev_tid0_poc - prev_lsb;
590            if lsb < prev_lsb && prev_lsb - lsb >= max_lsb / 2 {
591                prev_msb + max_lsb
592            } else if lsb > prev_lsb && lsb - prev_lsb > max_lsb / 2 {
593                prev_msb - max_lsb
594            } else {
595                prev_msb
596            }
597        };
598        let poc = msb + lsb;
599        let sub_layer_non_ref = nal_type < 16 && nal_type.is_multiple_of(2);
600        if tid == 0 && !(6..=9).contains(&nal_type) && !sub_layer_non_ref {
601            self.prev_tid0_poc = poc;
602        }
603
604        // Reference picture set (§8.3.2).
605        let mut st_before = Vec::new();
606        let mut st_after = Vec::new();
607        let mut lt_curr = Vec::new();
608        if idr || (irap && no_rasl_output) {
609            self.dpb.clear();
610        } else {
611            let mut keep = vec![false; self.dpb.len()];
612            let mut lt_mark = vec![false; self.dpb.len()];
613            let mut missing: Vec<(i32, bool, u8)> = Vec::new();
614            for &(v, msb_present, used) in &sh.lt {
615                let full = if msb_present {
616                    v + poc - (poc & (max_lsb - 1))
617                } else {
618                    v
619                };
620                let found = self.dpb.iter().position(|p| {
621                    if msb_present {
622                        p.poc == full
623                    } else {
624                        p.poc & (max_lsb - 1) == v
625                    }
626                });
627                match found {
628                    Some(i) => {
629                        keep[i] = true;
630                        lt_mark[i] = true;
631                        if used {
632                            lt_curr.push(i);
633                        }
634                    }
635                    None if used => missing.push((full, true, 2)),
636                    None => {}
637                }
638            }
639            for (list, entries) in [(0u8, &sh.st_rps.neg), (1u8, &sh.st_rps.pos)] {
640                for &(d, used) in entries {
641                    let target = poc + d;
642                    let found = self
643                        .dpb
644                        .iter()
645                        .enumerate()
646                        .position(|(i, p)| !lt_mark[i] && !p.long_term && p.poc == target);
647                    match found {
648                        Some(i) => {
649                            keep[i] = true;
650                            if used {
651                                if list == 0 {
652                                    st_before.push(i);
653                                } else {
654                                    st_after.push(i);
655                                }
656                            }
657                        }
658                        None if used => missing.push((target, false, list)),
659                        None => {}
660                    }
661                }
662            }
663            // Drop everything the set does not mention, renumbering the
664            // indices collected above.
665            let mut remap = vec![usize::MAX; self.dpb.len()];
666            let mut kept = Vec::with_capacity(self.dpb.len());
667            for (i, p) in std::mem::take(&mut self.dpb).into_iter().enumerate() {
668                if keep[i] {
669                    remap[i] = kept.len();
670                    let mut p = p;
671                    if lt_mark[i] {
672                        p.long_term = true;
673                    }
674                    kept.push(p);
675                }
676            }
677            self.dpb = kept;
678            for v in [&mut st_before, &mut st_after, &mut lt_curr] {
679                for i in v.iter_mut() {
680                    *i = remap[*i];
681                }
682            }
683            // References the stream needs and does not have (lost, or a
684            // broken stream): stand in a mid-grey picture so decoding goes on.
685            for (target, long_term, list) in missing {
686                let i = self.dpb.len();
687                self.dpb
688                    .push(Self::placeholder(&sps, &layout, target, long_term));
689                match list {
690                    0 => st_before.push(i),
691                    1 => st_after.push(i),
692                    _ => lt_curr.push(i),
693                }
694            }
695        }
696
697        // Output of prior pictures (§C.5.2.2).
698        if irap && no_rasl_output && self.decoded_any {
699            // A CRA here always discards; otherwise the stream says.
700            if nal_type == 21 || sh.no_output_of_prior_pics {
701                self.pending.clear();
702            } else {
703                while self.bump() {}
704            }
705        } else {
706            self.bump_while_over(&sps, true);
707        }
708        self.decoded_any = true;
709
710        let mut pic = if self.accel.is_some() || self.threads.is_some() {
711            PicState::sized(&sps, layout)
712        } else {
713            PicState::new(&sps, layout)
714        };
715        pic.poc = poc;
716        self.cur = Some(Current {
717            pic,
718            sps,
719            pps,
720            carry: Carry::default(),
721            output: sh.pic_output && !((8..=9).contains(&nal_type) && self.skip_rasl),
722            new_sequence: irap && no_rasl_output,
723            st_before,
724            st_after,
725            lt_curr,
726            work: Vec::new(),
727        });
728        Ok(())
729    }
730
731    fn placeholder(sps: &Sps, layout: &Layout, poc: i32, long_term: bool) -> RefPic {
732        let (w, h) = (sps.width as usize, sps.height as usize);
733        let planes = [
734            Plane::new(w, h, 1 << (sps.bit_depth_y - 1)),
735            Plane::new(w.div_ceil(2), h.div_ceil(2), 1 << (sps.bit_depth_c - 1)),
736            Plane::new(w.div_ceil(2), h.div_ceil(2), 1 << (sps.bit_depth_c - 1)),
737        ];
738        RefPic {
739            poc,
740            long_term,
741            planes: Arc::new(planes),
742            motion: Arc::new(Motion {
743                mv: vec![MvField::INTRA; layout.w4 * layout.h4],
744                w4: layout.w4,
745                ctb_slice: vec![u16::MAX; layout.ctb_w * layout.ctb_h],
746                slices: Vec::new(),
747                log2_ctb: layout.log2_ctb as usize,
748                ctb_w: layout.ctb_w,
749            }),
750            size: (w, h),
751            pending: None,
752        }
753    }
754
755    fn build_lists(&self, sh: &SliceHeader) -> Result<[Vec<RefEntry>; 2]> {
756        let mut lists: [Vec<RefEntry>; 2] = [Vec::new(), Vec::new()];
757        if sh.slice_type == SliceType::I {
758            return Ok(lists);
759        }
760        let cur = self.cur.as_ref().unwrap();
761        let total = cur.st_before.len() + cur.st_after.len() + cur.lt_curr.len();
762        if total == 0 {
763            return Err("inter slice with no reference pictures".into());
764        }
765        let entry = |i: usize| RefEntry {
766            dpb: i,
767            poc: self.dpb[i].poc,
768            long_term: self.dpb[i].long_term,
769        };
770        for l in 0..(1 + (sh.slice_type == SliceType::B) as usize) {
771            let n = sh.num_ref_idx[l] as usize;
772            let order: Vec<usize> = if l == 0 {
773                cur.st_before
774                    .iter()
775                    .chain(&cur.st_after)
776                    .chain(&cur.lt_curr)
777                    .copied()
778                    .collect()
779            } else {
780                cur.st_after
781                    .iter()
782                    .chain(&cur.st_before)
783                    .chain(&cur.lt_curr)
784                    .copied()
785                    .collect()
786            };
787            let len = n.max(total);
788            let temp: Vec<usize> = (0..len).map(|k| order[k % total]).collect();
789            lists[l] = (0..n)
790                .map(|i| {
791                    let k = match &sh.list_entry[l] {
792                        Some(e) => e[i] as usize,
793                        None => i,
794                    };
795                    temp.get(k)
796                        .copied()
797                        .map(entry)
798                        .ok_or("list_entry out of range")
799                })
800                .collect::<std::result::Result<_, _>>()?;
801        }
802        Ok(lists)
803    }
804
805    /// Finish the picture in progress, if any: loop filters, then keep it as
806    /// a reference and queue it for output.
807    pub fn finish_picture(&mut self) {
808        let Some(mut cur) = self.cur.take() else {
809            return;
810        };
811        if self.threads.is_some() {
812            self.submit_picture(cur);
813            return;
814        }
815        if let Some(accel) = self.accel.as_mut() {
816            if std::mem::take(&mut self.accel_started)
817                && let Err(e) = accel.end_picture()
818            {
819                self.accel_error.get_or_insert(e);
820            }
821        } else {
822            filters::deblock(&mut cur.pic, &cur.sps, &cur.pps);
823            if cur.sps.sao {
824                filters::sao(&mut cur.pic, &cur.sps, &cur.pps);
825            }
826        }
827        let sps = &cur.sps;
828        let pic = cur.pic;
829        if cur.output {
830            let (fmt, crop, color) = output_format(sps);
831            let frame = match self.accel.as_mut() {
832                Some(a) => match a.frame(pic.poc, sps) {
833                    Ok(f) => f,
834                    Err(e) => {
835                        // The accelerator's error is reported; the picture
836                        // still takes its place in output order, mid-grey.
837                        self.accel_error.get_or_insert(e);
838                        let grey =
839                            |p: &Plane, bd: u32| Plane::new(p.width, p.height, 1 << (bd - 1));
840                        let planes = [
841                            grey(&pic.planes[0], sps.bit_depth_y),
842                            grey(&pic.planes[1], sps.bit_depth_c),
843                            grey(&pic.planes[2], sps.bit_depth_c),
844                        ];
845                        DecodedFrame::from_planes(&planes, fmt, sps.bit_depth_y as u8, crop, color)
846                    }
847                },
848                None => {
849                    DecodedFrame::from_planes(&pic.planes, fmt, sps.bit_depth_y as u8, crop, color)
850                }
851            };
852            let id = self.next_id;
853            self.next_id += 1;
854            self.done.push(HevcPicture {
855                id,
856                frame,
857                poc: pic.poc,
858                new_sequence: cur.new_sequence,
859            });
860            // §C.5.2.3: everything waiting ages by one; then "additional
861            // bumping".
862            for e in &mut self.pending {
863                e.2 += 1;
864            }
865            self.pending.push((id, pic.poc, 0));
866            self.bump_while_over(sps, false);
867        }
868        let layout = pic.layout.clone();
869        let size = (sps.width as usize, sps.height as usize);
870        let cap = (sps.max_dec_pic_buffering as usize).max(1) + 16;
871        let poc = pic.poc;
872        let data = ref_data(pic, &layout);
873        self.dpb.push(RefPic {
874            poc,
875            long_term: false,
876            planes: data.planes,
877            motion: data.motion,
878            size,
879            pending: None,
880        });
881        self.limit_dpb(cap);
882    }
883
884    /// A broken stream that never lets go of references must not grow the
885    /// store without bound.
886    fn limit_dpb(&mut self, cap: usize) {
887        while self.dpb.len() > cap {
888            self.dpb.remove(0);
889        }
890    }
891
892    /// Everything [`finish_picture`](Self::finish_picture) does that needs
893    /// only headers; a job does the rest.
894    fn submit_picture(&mut self, cur: Current) {
895        let poc = cur.pic.poc;
896        let out = cur.output.then(|| {
897            let id = self.next_id;
898            self.next_id += 1;
899            for e in &mut self.pending {
900                e.2 += 1;
901            }
902            self.pending.push((id, poc, 0));
903            self.bump_while_over(&cur.sps, false);
904            id
905        });
906        let dpb = self.dpb.clone();
907        // Only what this picture can read: the buffer also holds pictures kept
908        // for later ones, and waiting on those would serialise siblings.
909        let deps: Vec<_> = [&cur.st_before, &cur.st_after, &cur.lt_curr]
910            .into_iter()
911            .flatten()
912            .filter_map(|&i| dpb.get(i)?.pending.clone())
913            .collect();
914        let slot = Slot::new();
915        let size = (cur.sps.width as usize, cur.sps.height as usize);
916        let cap = (cur.sps.max_dec_pic_buffering as usize).max(1) + 16;
917        let layout = cur.pic.layout.clone();
918        let empty = Self::empty_ref(poc, size, slot.clone(), &layout);
919        self.dpb.push(empty);
920        self.limit_dpb(cap);
921
922        let t = self.threads.as_mut().unwrap();
923        let order = t.next_job;
924        t.next_job += 1;
925        let (finished, error) = (t.finished.clone(), t.error.clone());
926        let new_sequence = cur.new_sequence;
927        let sps = cur.sps.clone();
928        t.jobs.clone().submit(&deps, move || {
929            let result = caught(|| reconstruct(cur, dpb));
930            let (data, frame) = match result {
931                Ok((data, frame)) => (data, frame),
932                Err(e) => {
933                    let mut first = error.lock().unwrap();
934                    if first.as_ref().is_none_or(|f| order < f.0) {
935                        *first = Some((order, e));
936                    }
937                    let grey = Self::placeholder(&sps, &layout, poc, false);
938                    (
939                        RefData {
940                            planes: grey.planes,
941                            motion: grey.motion,
942                        },
943                        None,
944                    )
945                }
946            };
947            if let Some(id) = out {
948                let picture = frame.map(|frame| HevcPicture {
949                    id,
950                    frame,
951                    poc,
952                    new_sequence,
953                });
954                finished.lock().unwrap().insert(id, picture);
955            }
956            slot.finish(data);
957        });
958    }
959
960    fn empty_ref(
961        poc: i32,
962        size: (usize, usize),
963        slot: Arc<Slot<RefData>>,
964        layout: &Layout,
965    ) -> RefPic {
966        let none = || Plane::new(0, 0, 0);
967        RefPic {
968            poc,
969            long_term: false,
970            planes: Arc::new([none(), none(), none()]),
971            motion: Arc::new(Motion {
972                mv: Vec::new(),
973                w4: layout.w4,
974                ctb_slice: Vec::new(),
975                slices: Vec::new(),
976                log2_ctb: layout.log2_ctb as usize,
977                ctb_w: layout.ctb_w,
978            }),
979            size,
980            pending: Some(slot),
981        }
982    }
983}
984
985/// How `sps` says a picture leaves the decoder.
986#[cfg(feature = "decode")]
987fn output_format(sps: &Sps) -> (ChromaFormat, [usize; 4], ColorInfo) {
988    let color = ColorInfo::from_vui(
989        sps.matrix_coefficients,
990        sps.full_range,
991        sps.height - sps.crop[2] - sps.crop[3],
992    );
993    let fmt = match sps.chroma_format_idc {
994        3 => ChromaFormat::Yuv444,
995        2 => ChromaFormat::Yuv422,
996        _ => ChromaFormat::Yuv420,
997    };
998    (fmt, sps.crop.map(|v| v as usize), color)
999}
1000
1001/// What a finished picture leaves in the decoded picture buffer.
1002#[cfg(feature = "decode")]
1003fn ref_data(pic: PicState, layout: &Layout) -> RefData {
1004    RefData {
1005        planes: Arc::new(pic.planes),
1006        motion: Arc::new(Motion {
1007            mv: pic.mv,
1008            w4: layout.w4,
1009            ctb_slice: pic.ctb_slice,
1010            slices: pic.slices,
1011            log2_ctb: layout.log2_ctb as usize,
1012            ctb_w: layout.ctb_w,
1013        }),
1014    }
1015}
1016
1017/// A picture job: its slices, the loop filters, and what comes out. `dpb` is
1018/// the buffer as this picture saw it, its references all finished by now.
1019#[cfg(feature = "decode")]
1020fn reconstruct(mut cur: Current, mut dpb: Vec<RefPic>) -> Result<(RefData, Option<DecodedFrame>)> {
1021    for p in &mut dpb {
1022        if let Some(data) = p.pending.as_ref().and_then(|s| s.get()) {
1023            p.planes = data.planes.clone();
1024            p.motion = data.motion.clone();
1025        }
1026    }
1027    // The buffers come here rather than in the header walk, which they would
1028    // otherwise hold up.
1029    let mut pic = PicState::new(&cur.sps, cur.pic.layout.clone());
1030    pic.poc = cur.pic.poc;
1031    pic.slices = std::mem::take(&mut cur.pic.slices);
1032    cur.pic = pic;
1033    for w in std::mem::take(&mut cur.work) {
1034        let mut dec = SliceDec::new(
1035            &cur.sps,
1036            &cur.pps,
1037            &w.sh,
1038            &mut cur.pic,
1039            &dpb,
1040            &w.refs,
1041            &mut cur.carry,
1042            w.slice_idx,
1043            w.slice_addr,
1044        );
1045        dec.decode(&w.rbsp)?;
1046    }
1047    filters::deblock(&mut cur.pic, &cur.sps, &cur.pps);
1048    if cur.sps.sao {
1049        filters::sao(&mut cur.pic, &cur.sps, &cur.pps);
1050    }
1051    let frame = cur.output.then(|| {
1052        let (fmt, crop, color) = output_format(&cur.sps);
1053        DecodedFrame::from_planes(&cur.pic.planes, fmt, cur.sps.bit_depth_y as u8, crop, color)
1054    });
1055    let layout = cur.pic.layout.clone();
1056    Ok((ref_data(cur.pic, &layout), frame))
1057}