Skip to main content

threers_codec/decode/hevc/
ctu.rs

1//! HEVC slice segment data: CTU parsing and reconstruction (§7.3.8, §8.4,
2//! §8.6). Motion vector derivation is in `mv`, loop filters in `filters`.
3
4use super::cabac_init as ci;
5use super::inter::{self, Weighting};
6use super::intra;
7use super::params::{Pps, SliceHeader, SliceType, Sps};
8use super::pic::{flag, MvField, PicState, Sao};
9use super::tables::{chroma_qp, CTX_IDX_MAP, SCAN4, SCAN_SB1, SCAN_SB2, SCAN_SB4, SCAN_SB8};
10use super::transform;
11use super::{RefEntry, RefPic};
12use crate::decode::bits::BitReader;
13use crate::decode::cabac::{Cabac, Ctx};
14
15pub type Result<T> = std::result::Result<T, String>;
16
17/// CABAC state carried between the slice segments of one picture.
18#[derive(Default)]
19pub struct Carry {
20    /// Entropy state saved after the second CTB of the last WPP row.
21    pub wpp: Option<EntropyState>,
22    /// Entropy state at the end of the previous slice segment.
23    pub ds: Option<EntropyState>,
24    /// `QpY` of the last coding unit — `qPY_PREV` for a dependent segment.
25    pub qp_prev: i32,
26}
27
28/// What a parsed `residual_coding()` asks of reconstruction.
29struct ResidualInfo {
30    transform_skip: bool,
31    /// Residual DPCM direction: `Some(true)` vertical, `Some(false)` horizontal.
32    rdpcm: Option<bool>,
33}
34
35/// What WPP and dependent slice segments carry over: the contexts, and the
36/// persistent Rice statistics (`StatCoeff`) that are synchronised with them.
37#[derive(Clone)]
38pub struct EntropyState {
39    pub ctx: Vec<Ctx>,
40    pub stat_coeff: [u32; 4],
41}
42
43#[derive(Clone, Copy, Debug, PartialEq, Eq)]
44pub enum PartMode {
45    P2Nx2N,
46    P2NxN,
47    PNx2N,
48    PNxN,
49    P2NxnU,
50    P2NxnD,
51    PnLx2N,
52    PnRx2N,
53}
54
55impl PartMode {
56    /// Prediction blocks as `(x, y, w, h)` within a CU of size `n`.
57    pub fn parts(self, n: usize) -> Vec<(usize, usize, usize, usize)> {
58        let (h2, q) = (n / 2, n / 4);
59        match self {
60            PartMode::P2Nx2N => vec![(0, 0, n, n)],
61            PartMode::P2NxN => vec![(0, 0, n, h2), (0, h2, n, h2)],
62            PartMode::PNx2N => vec![(0, 0, h2, n), (h2, 0, h2, n)],
63            PartMode::PNxN => vec![
64                (0, 0, h2, h2),
65                (h2, 0, h2, h2),
66                (0, h2, h2, h2),
67                (h2, h2, h2, h2),
68            ],
69            PartMode::P2NxnU => vec![(0, 0, n, q), (0, q, n, n - q)],
70            PartMode::P2NxnD => vec![(0, 0, n, n - q), (0, n - q, n, q)],
71            PartMode::PnLx2N => vec![(0, 0, q, n), (q, 0, n - q, n)],
72            PartMode::PnRx2N => vec![(0, 0, n - q, n), (n - q, 0, q, n)],
73        }
74    }
75}
76
77pub struct SliceDec<'a> {
78    pub sps: &'a Sps,
79    pub pps: &'a Pps,
80    pub sh: &'a SliceHeader,
81    pub pic: &'a mut PicState,
82    pub dpb: &'a [RefPic],
83    pub refs: &'a [Vec<RefEntry>; 2],
84    pub carry: &'a mut Carry,
85    pub slice_idx: u16,
86    /// `SliceAddrRs`: raster address of the slice's first CTB.
87    pub slice_addr: u32,
88    pub slice_qp: i32,
89    pub ctx: Vec<Ctx>,
90    pub ctb_rs: usize,
91    qp_y: i32,
92    qp_pred: i32,
93    last_qp: i32,
94    first_qg: bool,
95    cu_qp_delta_coded: bool,
96    cu_qp_delta: i32,
97    /// Whether the current chroma QP offset group has coded its choice, and
98    /// the (Cb, Cr) offsets that choice selected (`CuQpOffsetCb/Cr`).
99    cu_chroma_qp_offset_coded: bool,
100    cu_chroma_qp_offset: [i32; 2],
101    cu_bypass: bool,
102    cu_intra: bool,
103    cu_depth: u32,
104    intra_split: bool,
105    max_trafo_depth: u32,
106    /// Chroma intra modes: one per prediction block in a 4:4:4 NxN coding
107    /// unit, otherwise all four the same. `cu_origin` and `cu_pb` place them.
108    ipm_c: [u32; 4],
109    cu_origin: (usize, usize),
110    cu_pb: usize,
111    /// Whether each block's chroma mode was coded as "same as luma" (4),
112    /// which cross-component prediction requires of intra blocks.
113    ipm_c_dm: [bool; 4],
114    /// `StatCoeff`: persistent Rice adaptation statistics (§9.3.3.11).
115    stat_coeff: [u32; 4],
116    /// The current transform unit's luma residual, for cross-component
117    /// prediction.
118    res_y: Vec<i32>,
119    pub part_mode: PartMode,
120    merge_2nx2n: bool,
121    /// Index into `dpb` of the collocated picture.
122    pub col: Option<usize>,
123    pub no_backward_pred: bool,
124    coeff: Vec<i32>,
125    pub mc: [Vec<i16>; 2],
126    mc_scratch: inter::Scratch,
127}
128
129impl<'a> SliceDec<'a> {
130    #[allow(clippy::too_many_arguments)]
131    pub fn new(
132        sps: &'a Sps,
133        pps: &'a Pps,
134        sh: &'a SliceHeader,
135        pic: &'a mut PicState,
136        dpb: &'a [RefPic],
137        refs: &'a [Vec<RefEntry>; 2],
138        carry: &'a mut Carry,
139        slice_idx: u16,
140        slice_addr: u32,
141    ) -> Self {
142        let slice_qp = pps.init_qp + sh.qp_delta;
143        let col = if sh.temporal_mvp && sh.slice_type != SliceType::I {
144            let l = if sh.collocated_from_l0 { 0 } else { 1 };
145            refs[l].get(sh.collocated_ref_idx as usize).map(|e| e.dpb)
146        } else {
147            None
148        };
149        let poc = pic.poc;
150        let no_backward_pred = refs.iter().all(|l| l.iter().all(|e| e.poc <= poc));
151        SliceDec {
152            sps,
153            pps,
154            sh,
155            pic,
156            dpb,
157            refs,
158            carry,
159            slice_idx,
160            slice_addr,
161            slice_qp,
162            ctx: Vec::new(),
163            ctb_rs: 0,
164            qp_y: slice_qp,
165            qp_pred: slice_qp,
166            last_qp: slice_qp,
167            first_qg: true,
168            cu_qp_delta_coded: false,
169            cu_qp_delta: 0,
170            cu_chroma_qp_offset_coded: false,
171            cu_chroma_qp_offset: [0, 0],
172            cu_bypass: false,
173            cu_intra: false,
174            cu_depth: 0,
175            intra_split: false,
176            max_trafo_depth: 0,
177            ipm_c: [0; 4],
178            cu_origin: (0, 0),
179            cu_pb: 0,
180            ipm_c_dm: [false; 4],
181            stat_coeff: [0; 4],
182            res_y: vec![0; 32 * 32],
183            part_mode: PartMode::P2Nx2N,
184            merge_2nx2n: false,
185            col,
186            no_backward_pred,
187            coeff: vec![0; 32 * 32],
188            mc: [vec![0; 64 * 64], vec![0; 64 * 64]],
189            mc_scratch: inter::Scratch::default(),
190        }
191    }
192
193    fn init_contexts(&mut self) {
194        let init_type = match (self.sh.slice_type, self.sh.cabac_init_flag) {
195            (SliceType::I, _) => 0,
196            (SliceType::P, false) | (SliceType::B, true) => 1,
197            _ => 2,
198        };
199        self.ctx = ci::INIT[init_type]
200            .iter()
201            .map(|&v| Ctx::hevc(v, self.slice_qp))
202            .collect();
203        self.stat_coeff = [0; 4];
204    }
205
206    fn entropy(&self) -> EntropyState {
207        EntropyState {
208            ctx: self.ctx.clone(),
209            stat_coeff: self.stat_coeff,
210        }
211    }
212
213    fn load_entropy(&mut self, e: &EntropyState) {
214        self.ctx.clone_from(&e.ctx);
215        self.stat_coeff = e.stat_coeff;
216    }
217
218    #[inline(always)]
219    fn bin(&mut self, c: &mut Cabac, idx: usize) -> u32 {
220        c.decision(&mut self.ctx[idx])
221    }
222
223    /// Z-scan availability (§6.4.1) of luma location `(xn, yn)` from
224    /// `(xc, yc)`.
225    #[inline]
226    pub fn avail(&self, xc: isize, yc: isize, xn: isize, yn: isize) -> bool {
227        if xn < 0 || yn < 0 || xn >= self.pic.width as isize || yn >= self.pic.height as isize {
228            return false;
229        }
230        let l = &self.pic.layout;
231        let zn = l.z[(yn as usize >> 2) * l.w4 + (xn as usize >> 2)];
232        let zc = l.z[(yc as usize >> 2) * l.w4 + (xc as usize >> 2)];
233        if zn > zc {
234            return false;
235        }
236        let ctb_n = self.pic.ctb_rs(xn as usize, yn as usize);
237        let ctb_c = self.pic.ctb_rs(xc as usize, yc as usize);
238        let sn = self.pic.ctb_slice[ctb_n];
239        if sn == u16::MAX || self.pic.slices[sn as usize].addr_rs != self.slice_addr {
240            return false;
241        }
242        l.tile[ctb_n] == l.tile[ctb_c]
243    }
244
245    /// Decode the slice segment's CTUs.
246    pub fn decode(&mut self, rbsp: &[u8]) -> Result<()> {
247        let layout = self.pic.layout.clone();
248        let total = layout.ctb_w * layout.ctb_h;
249        let log2 = self.sps.log2_ctb;
250        let mut ts = layout.rs2ts[self.sh.segment_address as usize] as usize;
251        let first_ts = ts;
252        if self.sh.data_offset > rbsp.len() {
253            return Err("slice data starts past the NAL".into());
254        }
255        let mut c = Cabac::new(rbsp, self.sh.data_offset);
256        loop {
257            let rs = layout.ts2rs[ts] as usize;
258            self.ctb_rs = rs;
259            let (cx, cy) = (rs % layout.ctb_w, rs / layout.ctb_w);
260            let tile_start = layout.tile_start(ts);
261            let row_start = self.pps.entropy_sync && layout.tile_col_start(cx as u32) == cx as u32;
262            // The CTB is part of this slice before anything asks.
263            self.pic.ctb_slice[rs] = self.slice_idx;
264            if tile_start {
265                self.init_contexts();
266                self.first_qg = true;
267            } else if row_start {
268                let (x0, y0) = ((cx << log2) as isize, (cy << log2) as isize);
269                let t = self.avail(x0, y0, x0 + (1 << log2), y0 - (1 << log2));
270                match (&self.carry.wpp, t) {
271                    (Some(saved), true) => self.load_entropy(&saved.clone()),
272                    _ => self.init_contexts(),
273                }
274                self.first_qg = true;
275            } else if ts == first_ts {
276                if self.sh.dependent {
277                    match &self.carry.ds {
278                        Some(saved) => self.load_entropy(&saved.clone()),
279                        None => self.init_contexts(),
280                    }
281                    self.last_qp = self.carry.qp_prev;
282                    self.first_qg = false;
283                } else {
284                    self.init_contexts();
285                    self.first_qg = true;
286                }
287            }
288            self.sao_syntax(&mut c, cx, cy)?;
289            self.coding_quadtree(&mut c, cx << log2, cy << log2, log2, 0)?;
290            let end = c.terminate();
291            self.pic.decoded_ctbs += 1;
292            if self.pps.entropy_sync
293                && (rs % layout.ctb_w == 1 || (rs > 1 && layout.tile[rs] != layout.tile[rs - 2]))
294            {
295                self.carry.wpp = Some(self.entropy());
296            }
297            self.carry.qp_prev = self.last_qp;
298            if c.overrun() {
299                return Err("CABAC slice data overrun".into());
300            }
301            ts += 1;
302            if end == 1 {
303                self.carry.ds = Some(self.entropy());
304                break;
305            }
306            if ts >= total {
307                return Err("slice runs past the end of the picture".into());
308            }
309            let rs_next = layout.ts2rs[ts] as usize;
310            if (self.pps.tiles && layout.tile_start(ts))
311                || (self.pps.entropy_sync
312                    && (rs_next.is_multiple_of(layout.ctb_w)
313                        || layout.tile[rs_next] != layout.tile[rs_next - 1]))
314            {
315                if c.terminate() != 1 {
316                    return Err("end_of_subset_one_bit missing".into());
317                }
318                c.restart(c.aligned_byte_pos());
319            }
320        }
321        Ok(())
322    }
323
324    // ------------------------------------------------------------------
325    // SAO syntax (§7.3.8.3)
326    // ------------------------------------------------------------------
327
328    fn sao_syntax(&mut self, c: &mut Cabac, cx: usize, cy: usize) -> Result<()> {
329        let rs = self.ctb_rs;
330        if !self.sh.sao_luma && !self.sh.sao_chroma {
331            self.pic.sao[rs] = [Sao::default(); 3];
332            return Ok(());
333        }
334        let l = self.pic.layout.clone();
335        let w = l.ctb_w;
336        let mut merge_left = false;
337        let mut merge_up = false;
338        if cx > 0 && rs > self.slice_addr as usize && l.tile[rs] == l.tile[rs - 1] {
339            merge_left = self.bin(c, ci::SAO_MERGE) == 1;
340        }
341        if cy > 0
342            && !merge_left
343            && rs - w >= self.slice_addr as usize
344            && l.tile[rs] == l.tile[rs - w]
345        {
346            merge_up = self.bin(c, ci::SAO_MERGE) == 1;
347        }
348        if merge_left {
349            self.pic.sao[rs] = self.pic.sao[rs - 1];
350            return Ok(());
351        }
352        if merge_up {
353            self.pic.sao[rs] = self.pic.sao[rs - w];
354            return Ok(());
355        }
356        let mut out = [Sao::default(); 3];
357        for cidx in 0..3 {
358            if (cidx == 0 && !self.sh.sao_luma) || (cidx > 0 && !self.sh.sao_chroma) {
359                continue;
360            }
361            let bd = if cidx == 0 {
362                self.sps.bit_depth_y
363            } else {
364                self.sps.bit_depth_c
365            };
366            let kind = if cidx == 2 {
367                out[1].kind
368            } else if self.bin(c, ci::SAO_TYPE) == 0 {
369                0
370            } else if c.bypass() == 0 {
371                1
372            } else {
373                2
374            };
375            out[cidx].kind = kind;
376            if kind == 0 {
377                continue;
378            }
379            // Offsets are not scaled up above 10 bits: version 1 did, the
380            // range extensions replaced that with `log2_sao_offset_scale`
381            // (0 unless signalled, and a non-zero one is refused).
382            let cmax = (1 << (bd.min(10) - 5)) - 1;
383            let mut abs = [0i16; 4];
384            for a in abs.iter_mut() {
385                let mut v = 0;
386                while v < cmax && c.bypass() == 1 {
387                    v += 1;
388                }
389                *a = v as i16;
390            }
391            if kind == 1 {
392                for a in abs.iter_mut() {
393                    if *a != 0 && c.bypass() == 1 {
394                        *a = -*a;
395                    }
396                }
397                out[cidx].band_or_class = c.bypass_bits(5) as u8;
398                out[cidx].offsets[..4].copy_from_slice(&abs[..4]);
399            } else {
400                out[cidx].band_or_class = if cidx == 2 {
401                    out[1].band_or_class
402                } else {
403                    c.bypass_bits(2) as u8
404                };
405                out[cidx].offsets = [abs[0], abs[1], -abs[2], -abs[3]];
406            }
407            let scale = self.pps.log2_sao_offset_scale[(cidx > 0) as usize];
408            out[cidx].offsets = out[cidx].offsets.map(|o| o << scale);
409        }
410        self.pic.sao[rs] = out;
411        Ok(())
412    }
413
414    // ------------------------------------------------------------------
415    // Coding quadtree and coding unit
416    // ------------------------------------------------------------------
417
418    fn start_qg(&mut self, x: usize, y: usize) {
419        self.cu_qp_delta_coded = false;
420        self.cu_qp_delta = 0;
421        let prev = if self.first_qg {
422            self.slice_qp
423        } else {
424            self.last_qp
425        };
426        self.first_qg = false;
427        let ctb = self.ctb_rs;
428        let nb = |s: &Self, xn: isize, yn: isize| -> i32 {
429            if s.avail(x as isize, y as isize, xn, yn)
430                && s.pic.ctb_rs(xn as usize, yn as usize) == ctb
431            {
432                s.pic.qp[s.pic.idx4(xn as usize, yn as usize)] as i32
433            } else {
434                prev
435            }
436        };
437        let a = nb(self, x as isize - 1, y as isize);
438        let b = nb(self, x as isize, y as isize - 1);
439        self.qp_pred = (a + b + 1) >> 1;
440    }
441
442    fn update_qp_y(&mut self) {
443        let off = 6 * (self.sps.bit_depth_y as i32 - 8);
444        self.qp_y = ((self.qp_pred + self.cu_qp_delta + 52 + 2 * off) % (52 + off)) - off;
445    }
446
447    fn coding_quadtree(
448        &mut self,
449        c: &mut Cabac,
450        x0: usize,
451        y0: usize,
452        log2: u32,
453        depth: u32,
454    ) -> Result<()> {
455        let n = 1usize << log2;
456        let (w, h) = (self.pic.width, self.pic.height);
457        let split = if x0 + n <= w && y0 + n <= h && log2 > self.sps.log2_min_cb {
458            let cond = |s: &Self, xn: isize, yn: isize| {
459                (s.avail(x0 as isize, y0 as isize, xn, yn)
460                    && s.pic.depth[s.pic.idx4(xn as usize, yn as usize)] as u32 > depth)
461                    as usize
462            };
463            let inc =
464                cond(self, x0 as isize - 1, y0 as isize) + cond(self, x0 as isize, y0 as isize - 1);
465            self.bin(c, ci::SPLIT_CU + inc) == 1
466        } else {
467            log2 > self.sps.log2_min_cb
468        };
469        // A quantization group is the quadtree node of the minimum delta-QP
470        // size, or a coding unit larger than that. Larger nodes reset the
471        // delta too, but only the group itself predicts a QP — starting one
472        // at every level would spend the "first group in the row" on a
473        // node that is not a group.
474        let qg = self
475            .sps
476            .log2_ctb
477            .saturating_sub(self.pps.diff_cu_qp_delta_depth);
478        if self.pps.cu_qp_delta && (log2 == qg || (log2 > qg && !split)) {
479            self.start_qg(x0, y0);
480        }
481        // Chroma QP offset groups have no prediction, so every node at or
482        // above their size simply starts a new one.
483        if self.sh.cu_chroma_qp_offset
484            && log2
485                >= self
486                    .sps
487                    .log2_ctb
488                    .saturating_sub(self.pps.diff_cu_chroma_qp_offset_depth)
489        {
490            self.cu_chroma_qp_offset_coded = false;
491        }
492        if split {
493            let h2 = n / 2;
494            for (dx, dy) in [(0, 0), (h2, 0), (0, h2), (h2, h2)] {
495                if x0 + dx < w && y0 + dy < h {
496                    self.coding_quadtree(c, x0 + dx, y0 + dy, log2 - 1, depth + 1)?;
497                }
498            }
499            Ok(())
500        } else {
501            self.coding_unit(c, x0, y0, log2, depth)
502        }
503    }
504
505    /// Set a per-4×4 array over a block, clipped to the picture.
506    fn fill4<T: Copy>(
507        arr: &mut [T],
508        w4: usize,
509        x: usize,
510        y: usize,
511        w: usize,
512        h: usize,
513        pw: usize,
514        ph: usize,
515        v: T,
516    ) {
517        let x1 = (x + w).min(pw);
518        let y1 = (y + h).min(ph);
519        for yy in (y..y1).step_by(4) {
520            for xx in (x..x1).step_by(4) {
521                arr[(yy >> 2) * w4 + (xx >> 2)] = v;
522            }
523        }
524    }
525
526    fn set_flags(&mut self, x: usize, y: usize, w: usize, h: usize, set: u8, clear: u8) {
527        let (w4, pw, ph) = (self.pic.layout.w4, self.pic.width, self.pic.height);
528        for yy in (y..(y + h).min(ph)).step_by(4) {
529            for xx in (x..(x + w).min(pw)).step_by(4) {
530                let f = &mut self.pic.flags[(yy >> 2) * w4 + (xx >> 2)];
531                *f = (*f & !clear) | set;
532            }
533        }
534    }
535
536    /// Mark the left and top edges of a block for deblocking.
537    fn mark_edges(&mut self, x: usize, y: usize, w: usize, h: usize, pu: bool) {
538        let (w4, pw, ph) = (self.pic.layout.w4, self.pic.width, self.pic.height);
539        for yy in (y..(y + h).min(ph)).step_by(4) {
540            let i = (yy >> 2) * w4 + (x >> 2);
541            if pu {
542                self.pic.pu_edge[i] |= 1;
543            } else {
544                self.pic.flags[i] |= flag::EDGE_V;
545            }
546        }
547        for xx in (x..(x + w).min(pw)).step_by(4) {
548            let i = (y >> 2) * w4 + (xx >> 2);
549            if pu {
550                self.pic.pu_edge[i] |= 2;
551            } else {
552                self.pic.flags[i] |= flag::EDGE_H;
553            }
554        }
555    }
556
557    fn coding_unit(
558        &mut self,
559        c: &mut Cabac,
560        x0: usize,
561        y0: usize,
562        log2: u32,
563        depth: u32,
564    ) -> Result<()> {
565        let n = 1usize << log2;
566        let (w4, pw, ph) = (self.pic.layout.w4, self.pic.width, self.pic.height);
567        self.cu_depth = depth;
568        self.cu_bypass = self.pps.transquant_bypass && self.bin(c, ci::TRANSQUANT_BYPASS) == 1;
569        let skip = if self.sh.slice_type != SliceType::I {
570            let cond = |s: &Self, xn: isize, yn: isize| {
571                (s.avail(x0 as isize, y0 as isize, xn, yn)
572                    && s.pic.flags[s.pic.idx4(xn as usize, yn as usize)] & flag::SKIP != 0)
573                    as usize
574            };
575            let inc =
576                cond(self, x0 as isize - 1, y0 as isize) + cond(self, x0 as isize, y0 as isize - 1);
577            self.bin(c, ci::SKIP + inc) == 1
578        } else {
579            false
580        };
581        Self::fill4(&mut self.pic.depth, w4, x0, y0, n, n, pw, ph, depth as u8);
582        self.set_flags(x0, y0, n, n, if skip { flag::SKIP } else { 0 }, 0xFF);
583        if self.cu_bypass {
584            self.set_flags(x0, y0, n, n, flag::NO_FILTER, 0);
585        }
586        if !self.pps.cu_qp_delta {
587            self.qp_y = self.slice_qp;
588        } else {
589            self.update_qp_y();
590        }
591        // The CU boundary is a transform-block boundary whatever follows.
592        self.mark_edges(x0, y0, n, n, false);
593        let mut pcm = false;
594        if skip {
595            self.cu_intra = false;
596            self.part_mode = PartMode::P2Nx2N;
597            Self::fill4(&mut self.pic.ipm, w4, x0, y0, n, n, pw, ph, 1u8);
598            self.prediction_unit(c, x0, y0, n, x0, y0, n, n, 0, true)?;
599        } else {
600            self.cu_intra = self.sh.slice_type == SliceType::I || self.bin(c, ci::PRED_MODE) == 1;
601            self.part_mode = if !self.cu_intra || log2 == self.sps.log2_min_cb {
602                self.decode_part_mode(c, log2)
603            } else {
604                PartMode::P2Nx2N
605            };
606            self.intra_split = self.cu_intra && self.part_mode == PartMode::PNxN;
607            if self.cu_intra {
608                self.set_flags(x0, y0, n, n, flag::INTRA, 0);
609                Self::fill4(&mut self.pic.mv, w4, x0, y0, n, n, pw, ph, MvField::INTRA);
610                if self.part_mode == PartMode::P2Nx2N
611                    && self.sps.pcm
612                    && log2 >= self.sps.log2_min_pcm
613                    && log2 <= self.sps.log2_max_pcm
614                {
615                    pcm = c.terminate() == 1;
616                }
617                if pcm {
618                    self.pcm_samples(c, x0, y0, n)?;
619                } else {
620                    self.intra_modes(c, x0, y0, n)?;
621                }
622            } else {
623                Self::fill4(&mut self.pic.ipm, w4, x0, y0, n, n, pw, ph, 1u8);
624                self.merge_2nx2n = false;
625                for (i, (px, py, w, h)) in self.part_mode.parts(n).into_iter().enumerate() {
626                    if (px, py) != (0, 0) {
627                        self.mark_edges(x0 + px, y0 + py, w, h, true);
628                    }
629                    self.prediction_unit(c, x0, y0, n, x0 + px, y0 + py, w, h, i, false)?;
630                }
631            }
632            if !pcm {
633                let root_cbf = if !self.cu_intra
634                    && !(self.part_mode == PartMode::P2Nx2N && self.merge_2nx2n)
635                {
636                    self.bin(c, ci::ROOT_CBF) == 1
637                } else {
638                    true
639                };
640                if root_cbf {
641                    self.max_trafo_depth = if self.cu_intra {
642                        self.sps.max_th_depth_intra + self.intra_split as u32
643                    } else {
644                        self.sps.max_th_depth_inter
645                    };
646                    self.transform_tree(c, x0, y0, x0, y0, log2, 0, 0, [false; 2], [false; 2])?;
647                }
648            }
649        }
650        let qp = self.qp_y as i8;
651        Self::fill4(&mut self.pic.qp, w4, x0, y0, n, n, pw, ph, qp);
652        self.last_qp = self.qp_y;
653        Ok(())
654    }
655
656    fn decode_part_mode(&mut self, c: &mut Cabac, log2: u32) -> PartMode {
657        if self.bin(c, ci::PART_MODE) == 1 {
658            return PartMode::P2Nx2N;
659        }
660        if self.cu_intra {
661            return PartMode::PNxN;
662        }
663        if log2 == self.sps.log2_min_cb {
664            if self.bin(c, ci::PART_MODE + 1) == 1 {
665                return PartMode::P2NxN;
666            }
667            if log2 == 3 {
668                return PartMode::PNx2N;
669            }
670            if self.bin(c, ci::PART_MODE + 2) == 1 {
671                return PartMode::PNx2N;
672            }
673            return PartMode::PNxN;
674        }
675        if !self.sps.amp {
676            return if self.bin(c, ci::PART_MODE + 1) == 1 {
677                PartMode::P2NxN
678            } else {
679                PartMode::PNx2N
680            };
681        }
682        if self.bin(c, ci::PART_MODE + 1) == 1 {
683            if self.bin(c, ci::PART_MODE + 3) == 1 {
684                PartMode::P2NxN
685            } else if c.bypass() == 1 {
686                PartMode::P2NxnD
687            } else {
688                PartMode::P2NxnU
689            }
690        } else if self.bin(c, ci::PART_MODE + 3) == 1 {
691            PartMode::PNx2N
692        } else if c.bypass() == 1 {
693            PartMode::PnRx2N
694        } else {
695            PartMode::PnLx2N
696        }
697    }
698
699    fn pcm_samples(&mut self, c: &mut Cabac, x0: usize, y0: usize, n: usize) -> Result<()> {
700        let mut r = BitReader::new(c.data());
701        r.set_pos(c.aligned_byte_pos() * 8);
702        let (bdy, bdc) = (self.sps.bit_depth_y, self.sps.bit_depth_c);
703        let (py, pc) = (self.sps.pcm_bit_depth_y, self.sps.pcm_bit_depth_c);
704        for y in 0..n {
705            for x in 0..n {
706                let v = (r.u(py) << (bdy - py)) as u16;
707                if x0 + x < self.pic.width && y0 + y < self.pic.height {
708                    self.pic.planes[0].set(x0 + x, y0 + y, v);
709                }
710            }
711        }
712        let (hs, vs) = self.sps.chroma_shift();
713        for cidx in 1..3 {
714            for y in 0..n >> vs {
715                for x in 0..n >> hs {
716                    let v = (r.u(pc) << (bdc - pc)) as u16;
717                    let p = &mut self.pic.planes[cidx];
718                    if (x0 >> hs) + x < p.width && (y0 >> vs) + y < p.height {
719                        p.set((x0 >> hs) + x, (y0 >> vs) + y, v);
720                    }
721                }
722            }
723        }
724        if r.overrun() {
725            return Err("PCM samples run past the slice".into());
726        }
727        c.restart(r.byte_pos());
728        let f = flag::PCM
729            | if self.sps.pcm_loop_filter_disabled {
730                flag::NO_FILTER
731            } else {
732                0
733            };
734        self.set_flags(x0, y0, n, n, f, 0);
735        Ok(())
736    }
737
738    // ------------------------------------------------------------------
739    // Intra modes (§8.4.2, §8.4.3)
740    // ------------------------------------------------------------------
741
742    fn intra_modes(&mut self, c: &mut Cabac, x0: usize, y0: usize, n: usize) -> Result<()> {
743        let nparts = if self.intra_split { 4 } else { 1 };
744        let pb = if self.intra_split { n / 2 } else { n };
745        let mut prev = [false; 4];
746        for p in prev.iter_mut().take(nparts) {
747            *p = self.bin(c, ci::PREV_INTRA) == 1;
748        }
749        let (w4, pw, ph) = (self.pic.layout.w4, self.pic.width, self.pic.height);
750        let mut luma = [0u32; 4];
751        for k in 0..nparts {
752            let (xp, yp) = (x0 + (k % 2) * pb, y0 + (k / 2) * pb);
753            let (mpm_idx, rem) = if prev[k] {
754                let mut i = 0;
755                while i < 2 && c.bypass() == 1 {
756                    i += 1;
757                }
758                (Some(i), 0)
759            } else {
760                (None, c.bypass_bits(5))
761            };
762            let cand = self.mpm_candidates(xp, yp);
763            let mode = match mpm_idx {
764                Some(i) => cand[i],
765                None => {
766                    let mut s = cand;
767                    s.sort_unstable();
768                    let mut m = rem;
769                    for v in s {
770                        if m >= v {
771                            m += 1;
772                        }
773                    }
774                    m
775                }
776            };
777            Self::fill4(&mut self.pic.ipm, w4, xp, yp, pb, pb, pw, ph, mode as u8);
778            luma[k] = mode;
779        }
780        // 4:4:4 codes a chroma mode for each prediction block; otherwise one,
781        // derived from the first block's luma mode.
782        let nchroma = if self.sps.chroma_format_idc == 3 {
783            nparts
784        } else {
785            1
786        };
787        for k in 0..nchroma {
788            let cm = if self.bin(c, ci::CHROMA_MODE) == 0 {
789                4
790            } else {
791                c.bypass_bits(2)
792            };
793            self.ipm_c_dm[k] = cm == 4;
794            let m = if cm == 4 {
795                luma[k]
796            } else {
797                let m = [0, 26, 10, 1][cm as usize];
798                if m == luma[k] {
799                    34
800                } else {
801                    m
802                }
803            };
804            self.ipm_c[k] = m;
805        }
806        if nchroma == 1 {
807            self.ipm_c = [self.ipm_c[0]; 4];
808            self.ipm_c_dm = [self.ipm_c_dm[0]; 4];
809        }
810        if self.sps.chroma_format_idc == 2 {
811            // Table 8-3: the 4:2:2 chroma mode (ffmpeg `tab_mode_idx`).
812            const MODE422: [u8; 35] = [
813                0, 1, 2, 2, 2, 2, 3, 5, 7, 8, 10, 12, 13, 15, 17, 18, 19, 20, 21, 22, 23, 23, 24,
814                24, 25, 25, 26, 27, 27, 28, 28, 29, 29, 30, 31,
815            ];
816            self.ipm_c = self.ipm_c.map(|m| MODE422[m as usize] as u32);
817        }
818        self.cu_origin = (x0, y0);
819        self.cu_pb = pb;
820        Ok(())
821    }
822
823    /// The chroma intra mode at luma position `(x, y)` of the current CU.
824    fn chroma_mode(&self, x: usize, y: usize) -> u32 {
825        self.ipm_c[self.chroma_block(x, y)]
826    }
827
828    fn chroma_block(&self, x: usize, y: usize) -> usize {
829        let (x0, y0) = self.cu_origin;
830        2 * (y >= y0 + self.cu_pb) as usize + (x >= x0 + self.cu_pb) as usize
831    }
832
833    fn mpm_candidates(&self, xp: usize, yp: usize) -> [u32; 3] {
834        let cand_of = |xn: isize, yn: isize, is_b: bool| -> u32 {
835            if !self.avail(xp as isize, yp as isize, xn, yn) {
836                return 1;
837            }
838            let i = self.pic.idx4(xn as usize, yn as usize);
839            let f = self.pic.flags[i];
840            if f & flag::INTRA == 0 || f & flag::PCM != 0 {
841                return 1;
842            }
843            if is_b && (yn as usize) < ((yp >> self.sps.log2_ctb) << self.sps.log2_ctb) {
844                return 1;
845            }
846            self.pic.ipm[i] as u32
847        };
848        let a = cand_of(xp as isize - 1, yp as isize, false);
849        let b = cand_of(xp as isize, yp as isize - 1, true);
850        if a == b {
851            if a < 2 {
852                [0, 1, 26]
853            } else {
854                [a, 2 + ((a + 29) % 32), 2 + ((a - 2 + 1) % 32)]
855            }
856        } else {
857            let third = if a != 0 && b != 0 {
858                0
859            } else if a != 1 && b != 1 {
860                1
861            } else {
862                26
863            };
864            [a, b, third]
865        }
866    }
867
868    // ------------------------------------------------------------------
869    // Prediction units (§7.3.8.6)
870    // ------------------------------------------------------------------
871
872    #[allow(clippy::too_many_arguments)]
873    fn prediction_unit(
874        &mut self,
875        c: &mut Cabac,
876        xcb: usize,
877        ycb: usize,
878        ncb: usize,
879        xpb: usize,
880        ypb: usize,
881        w: usize,
882        h: usize,
883        part: usize,
884        skip: bool,
885    ) -> Result<()> {
886        let merge = skip || self.bin(c, ci::MERGE_FLAG) == 1;
887        let mvf = if merge {
888            let mut idx = 0;
889            if self.sh.max_merge_cand > 1 && self.bin(c, ci::MERGE_IDX) == 1 {
890                idx = 1;
891                while idx < self.sh.max_merge_cand as usize - 1 && c.bypass() == 1 {
892                    idx += 1;
893                }
894            }
895            if part == 0 && self.part_mode == PartMode::P2Nx2N {
896                self.merge_2nx2n = true;
897            }
898            self.merge_motion(xcb, ycb, ncb, xpb, ypb, w, h, part, idx)
899        } else {
900            let pred = if self.sh.slice_type == SliceType::B {
901                if w + h != 12 && self.bin(c, ci::INTER_PRED_IDC + self.cu_depth as usize) == 1 {
902                    3
903                } else if self.bin(c, ci::INTER_PRED_IDC + 4) == 1 {
904                    2
905                } else {
906                    1
907                }
908            } else {
909                1
910            };
911            let mut ref_idx = [-1i8; 2];
912            let mut mvd = [[0i32; 2]; 2];
913            let mut mvp_flag = [0usize; 2];
914            for l in 0..2 {
915                if pred & (1 << l) == 0 {
916                    continue;
917                }
918                let num = self.sh.num_ref_idx[l] as usize;
919                let mut r = 0;
920                if num > 1 {
921                    while r < num - 1 {
922                        let b = if r < 2 {
923                            self.bin(c, ci::REF_IDX + r)
924                        } else {
925                            c.bypass()
926                        };
927                        if b == 0 {
928                            break;
929                        }
930                        r += 1;
931                    }
932                }
933                ref_idx[l] = r as i8;
934                if l == 1 && self.sh.mvd_l1_zero && pred == 3 {
935                    mvd[1] = [0, 0];
936                } else {
937                    mvd[l] = self.mvd_coding(c)?;
938                }
939                mvp_flag[l] = self.bin(c, ci::MVP_FLAG) as usize;
940            }
941            let mut m = MvField::INTRA;
942            for l in 0..2 {
943                if ref_idx[l] < 0 {
944                    continue;
945                }
946                if ref_idx[l] as usize >= self.refs[l].len() {
947                    return Err(format!("ref_idx_l{l} {} past the list", ref_idx[l]));
948                }
949                let mvp = self.amvp(
950                    xcb,
951                    ycb,
952                    ncb,
953                    xpb,
954                    ypb,
955                    w,
956                    h,
957                    part,
958                    l,
959                    ref_idx[l],
960                    mvp_flag[l],
961                );
962                m.ref_idx[l] = ref_idx[l];
963                m.mv[l] = [
964                    (mvp[0] as i32 + mvd[l][0]) as i16,
965                    (mvp[1] as i32 + mvd[l][1]) as i16,
966                ];
967            }
968            m
969        };
970        let (w4, pw, ph) = (self.pic.layout.w4, self.pic.width, self.pic.height);
971        Self::fill4(&mut self.pic.mv, w4, xpb, ypb, w, h, pw, ph, mvf);
972        self.motion_compensate(xpb, ypb, w, h, &mvf)
973    }
974
975    fn mvd_coding(&mut self, c: &mut Cabac) -> Result<[i32; 2]> {
976        let g0 = [self.bin(c, ci::MVD_GT0), self.bin(c, ci::MVD_GT0)];
977        let mut g1 = [0; 2];
978        for k in 0..2 {
979            if g0[k] == 1 {
980                g1[k] = self.bin(c, ci::MVD_GT1);
981            }
982        }
983        let mut out = [0i32; 2];
984        for k in 0..2 {
985            if g0[k] == 0 {
986                continue;
987            }
988            let mut abs = 1i32;
989            if g1[k] == 1 {
990                let mut kk = 1;
991                let mut v = 0i32;
992                while c.bypass() == 1 {
993                    v += 1 << kk;
994                    kk += 1;
995                    if kk > 30 {
996                        return Err("abs_mvd_minus2 too long".into());
997                    }
998                }
999                v += c.bypass_bits(kk) as i32;
1000                abs = v + 2;
1001            }
1002            out[k] = if c.bypass() == 1 { -abs } else { abs };
1003        }
1004        Ok(out)
1005    }
1006
1007    /// Inter prediction of one PU from its motion (§8.5.3.3).
1008    pub fn motion_compensate(
1009        &mut self,
1010        x: usize,
1011        y: usize,
1012        w: usize,
1013        h: usize,
1014        m: &MvField,
1015    ) -> Result<()> {
1016        let weights = self.sh.weights.as_ref();
1017        let (hs, vs) = self.sps.chroma_shift();
1018        for cidx in 0..3 {
1019            let (bx, by, bw, bh) = if cidx == 0 {
1020                (x, y, w, h)
1021            } else {
1022                (x >> hs, y >> vs, w >> hs, h >> vs)
1023            };
1024            let bd = if cidx == 0 {
1025                self.sps.bit_depth_y
1026            } else {
1027                self.sps.bit_depth_c
1028            };
1029            let mut used = [false; 2];
1030            for l in 0..2 {
1031                if m.ref_idx[l] < 0 {
1032                    continue;
1033                }
1034                let e = self.refs[l]
1035                    .get(m.ref_idx[l] as usize)
1036                    .ok_or("reference index past the list")?;
1037                let p = &self.dpb[e.dpb].planes[cidx];
1038                let mv = m.mv[l];
1039                let (xi, yi, xf, yf) = if cidx == 0 {
1040                    (
1041                        bx as isize + (mv[0] >> 2) as isize,
1042                        by as isize + (mv[1] >> 2) as isize,
1043                        (mv[0] & 3) as usize,
1044                        (mv[1] & 3) as usize,
1045                    )
1046                } else {
1047                    // Chroma in eighths of a chroma sample: a subsampled
1048                    // axis takes the vector as is; a full one doubles it
1049                    // (ffmpeg `_mx = mx << 1`).
1050                    let axis = |v: i16, sub: usize| -> (isize, usize) {
1051                        if sub == 1 {
1052                            ((v >> 3) as isize, (v & 7) as usize)
1053                        } else {
1054                            ((v >> 2) as isize, ((v & 3) << 1) as usize)
1055                        }
1056                    };
1057                    let ((dx, xf), (dy, yf)) = (axis(mv[0], hs), axis(mv[1], vs));
1058                    (bx as isize + dx, by as isize + dy, xf, yf)
1059                };
1060                inter::interp(
1061                    p,
1062                    xi,
1063                    yi,
1064                    xf,
1065                    yf,
1066                    bw,
1067                    bh,
1068                    bd,
1069                    cidx == 0,
1070                    &mut self.mc[l],
1071                    &mut self.mc_scratch,
1072                );
1073                used[l] = true;
1074            }
1075            let wt = match weights {
1076                Some(wt) => {
1077                    let get = |l: usize| -> (i32, i32) {
1078                        if m.ref_idx[l] < 0 {
1079                            (0, 0)
1080                        } else {
1081                            wt.w[l][m.ref_idx[l] as usize][cidx]
1082                        }
1083                    };
1084                    let (w0, o0) = get(0);
1085                    let (w1, o1) = get(1);
1086                    let d = if cidx == 0 {
1087                        wt.luma_log2_denom
1088                    } else {
1089                        wt.chroma_log2_denom
1090                    };
1091                    Weighting::Explicit(w0, o0, w1, o1, d)
1092                }
1093                None => Weighting::Default,
1094            };
1095            let (m0, m1) = self.mc.split_at(1);
1096            inter::put(
1097                &mut self.pic.planes[cidx],
1098                bx,
1099                by,
1100                bw,
1101                bh,
1102                used[0].then_some(&m0[0][..]),
1103                used[1].then_some(&m1[0][..]),
1104                wt,
1105                bd,
1106            );
1107        }
1108        Ok(())
1109    }
1110
1111    // ------------------------------------------------------------------
1112    // Transform tree (§7.3.8.8)
1113    // ------------------------------------------------------------------
1114
1115    #[allow(clippy::too_many_arguments)]
1116    fn transform_tree(
1117        &mut self,
1118        c: &mut Cabac,
1119        x0: usize,
1120        y0: usize,
1121        xb: usize,
1122        yb: usize,
1123        log2: u32,
1124        depth: u32,
1125        blk: usize,
1126        parent_cb: [bool; 2],
1127        parent_cr: [bool; 2],
1128    ) -> Result<()> {
1129        let inter_split = self.sps.max_th_depth_inter == 0
1130            && !self.cu_intra
1131            && self.part_mode != PartMode::P2Nx2N
1132            && depth == 0;
1133        let split = if log2 <= self.sps.log2_max_tb
1134            && log2 > self.sps.log2_min_tb
1135            && depth < self.max_trafo_depth
1136            && !(self.intra_split && depth == 0)
1137        {
1138            self.bin(c, ci::SPLIT_TRANSFORM + 5 - log2 as usize) == 1
1139        } else {
1140            log2 > self.sps.log2_max_tb || (self.intra_split && depth == 0) || inter_split
1141        };
1142        // ffmpeg hls_transform_tree: flags start from the parent's, the
1143        // second (lower 4:2:2 half) only re-read at leaves and log2 == 3.
1144        let (mut cb, mut cr) = (parent_cb, parent_cr);
1145        if log2 > 2 || self.sps.chroma_format_idc == 3 {
1146            let two = self.sps.chroma_format_idc == 2 && (!split || log2 == 3);
1147            if depth == 0 || parent_cb[0] {
1148                cb[0] = self.bin(c, ci::CBF_CHROMA + depth as usize) == 1;
1149                if two {
1150                    cb[1] = self.bin(c, ci::CBF_CHROMA + depth as usize) == 1;
1151                }
1152            } else {
1153                cb = [false, false];
1154            }
1155            if depth == 0 || parent_cr[0] {
1156                cr[0] = self.bin(c, ci::CBF_CHROMA + depth as usize) == 1;
1157                if two {
1158                    cr[1] = self.bin(c, ci::CBF_CHROMA + depth as usize) == 1;
1159                }
1160            } else {
1161                cr = [false, false];
1162            }
1163        }
1164        if split {
1165            let h = 1usize << (log2 - 1);
1166            for (i, (dx, dy)) in [(0, 0), (h, 0), (0, h), (h, h)].into_iter().enumerate() {
1167                self.transform_tree(c, x0 + dx, y0 + dy, x0, y0, log2 - 1, depth + 1, i, cb, cr)?;
1168            }
1169            return Ok(());
1170        }
1171        let any_c = cb[0] || cr[0] || cb[1] || cr[1];
1172        let cbf_luma = if self.cu_intra || depth != 0 || any_c {
1173            self.bin(c, ci::CBF_LUMA + (depth == 0) as usize) == 1
1174        } else {
1175            true
1176        };
1177        self.transform_unit(c, x0, y0, xb, yb, log2, blk, cbf_luma, cb, cr)
1178    }
1179
1180    #[allow(clippy::too_many_arguments)]
1181    fn transform_unit(
1182        &mut self,
1183        c: &mut Cabac,
1184        x0: usize,
1185        y0: usize,
1186        xb: usize,
1187        yb: usize,
1188        log2: u32,
1189        blk: usize,
1190        cbf_luma: bool,
1191        cb: [bool; 2],
1192        cr: [bool; 2],
1193    ) -> Result<()> {
1194        let n = 1usize << log2;
1195        self.mark_edges(x0, y0, n, n, false);
1196        let c422 = self.sps.chroma_format_idc == 2;
1197        let any_c = cb[0] || cr[0] || (c422 && (cb[1] || cr[1]));
1198        if (cbf_luma || any_c) && self.pps.cu_qp_delta && !self.cu_qp_delta_coded {
1199            let mut v = 0i32;
1200            if self.bin(c, ci::CU_QP_DELTA) == 1 {
1201                v = 1;
1202                while v < 5 && self.bin(c, ci::CU_QP_DELTA + 1) == 1 {
1203                    v += 1;
1204                }
1205                if v == 5 {
1206                    let mut k = 0;
1207                    let mut s = 0i32;
1208                    while c.bypass() == 1 {
1209                        s += 1 << k;
1210                        k += 1;
1211                        if k > 30 {
1212                            return Err("cu_qp_delta_abs too long".into());
1213                        }
1214                    }
1215                    s += c.bypass_bits(k) as i32;
1216                    v += s;
1217                }
1218                if c.bypass() == 1 {
1219                    v = -v;
1220                }
1221            }
1222            let off = 6 * (self.sps.bit_depth_y as i32 - 8);
1223            if v < -(26 + off / 2) || v > 25 + off / 2 {
1224                return Err("CuQpDeltaVal out of range".into());
1225            }
1226            self.cu_qp_delta_coded = true;
1227            self.cu_qp_delta = v;
1228            self.update_qp_y();
1229        }
1230        if self.sh.cu_chroma_qp_offset
1231            && any_c
1232            && !self.cu_bypass
1233            && !self.cu_chroma_qp_offset_coded
1234        {
1235            let list = &self.pps.chroma_qp_offset_list;
1236            self.cu_chroma_qp_offset = if self.bin(c, ci::CU_CHROMA_QP_OFFSET_FLAG) == 1 {
1237                // Truncated unary, every bin on the one context.
1238                let mut idx = 0;
1239                while idx + 1 < list.len() && self.bin(c, ci::CU_CHROMA_QP_OFFSET_IDX) == 1 {
1240                    idx += 1;
1241                }
1242                list[idx]
1243            } else {
1244                [0, 0]
1245            };
1246            self.cu_chroma_qp_offset_coded = true;
1247        }
1248        if self.cu_intra {
1249            let mode = self.pic.ipm[self.pic.idx4(x0, y0)] as u32;
1250            self.intra_predict(0, x0, y0, n, mode);
1251        }
1252        if cbf_luma {
1253            self.set_flags(x0, y0, n, n, flag::CBF, 0);
1254            self.residual(c, x0, y0, log2, 0, 0)?;
1255        }
1256        // 4:2:2 chroma TUs are N × 2N, coded as two stacked N × N blocks.
1257        let halves = if c422 { 2 } else { 1 };
1258        let (hs, vs) = self.sps.chroma_shift();
1259        if log2 > 2 || self.sps.chroma_format_idc == 3 {
1260            // 4:4:4 chroma is the luma block's size, down to 4 × 4.
1261            let (nc, log2c) = (n >> hs, log2 - hs as u32);
1262            let mode = self.chroma_mode(x0, y0);
1263            // Cross-component prediction (4:4:4 only): each chroma residual
1264            // adds a scaled copy of this block's luma residual.
1265            let cross_pf = self.pps.cross_component_prediction
1266                && cbf_luma
1267                && (!self.cu_intra || self.ipm_c_dm[self.chroma_block(x0, y0)]);
1268            for (cidx, cbf) in [(1, cb), (2, cr)] {
1269                let res_scale = if cross_pf {
1270                    self.cross_component_pred(c, cidx - 1)
1271                } else {
1272                    0
1273                };
1274                for i in 0..halves {
1275                    let (xc, yc) = (x0 >> hs, (y0 >> vs) + i * nc);
1276                    if self.cu_intra {
1277                        self.intra_predict(cidx, xc, yc, nc, mode);
1278                    }
1279                    if cbf[i] {
1280                        self.residual(c, xc, yc, log2c, cidx, res_scale)?;
1281                    } else if res_scale != 0 {
1282                        self.add_cross_residual(xc, yc, nc, cidx, res_scale);
1283                    }
1284                }
1285            }
1286        } else if blk == 3 {
1287            let mode = self.chroma_mode(xb, yb);
1288            for (cidx, cbf) in [(1, cb), (2, cr)] {
1289                for i in 0..halves {
1290                    let (xc, yc) = (xb >> hs, (yb >> vs) + i * 4);
1291                    if self.cu_intra {
1292                        self.intra_predict(cidx, xc, yc, 4, mode);
1293                    }
1294                    if cbf[i] {
1295                        self.residual(c, xc, yc, 2, cidx, 0)?;
1296                    }
1297                }
1298            }
1299        }
1300        Ok(())
1301    }
1302
1303    // ------------------------------------------------------------------
1304    // Intra prediction driver
1305    // ------------------------------------------------------------------
1306
1307    /// Predict the `n × n` block at component location `(x, y)`.
1308    fn intra_predict(&mut self, cidx: usize, x: usize, y: usize, n: usize, mode: u32) {
1309        let (hs, vs) = if cidx == 0 {
1310            (0, 0)
1311        } else {
1312            self.sps.chroma_shift()
1313        };
1314        let (sx, sy) = (1usize << hs, 1usize << vs);
1315        let (xl, yl) = ((x * sx) as isize, (y * sy) as isize);
1316        let bd = if cidx == 0 {
1317            self.sps.bit_depth_y
1318        } else {
1319            self.sps.bit_depth_c
1320        };
1321        let len = 4 * n + 1;
1322        let mut arr = [0i32; 129];
1323        let mut av = [false; 129];
1324        let p = &self.pic.planes[cidx];
1325        let constrained = self.pps.constrained_intra_pred;
1326        let check = |xn: isize, yn: isize| -> bool {
1327            // In component samples; availability is decided in luma.
1328            let (lx, ly) = (xn * sx as isize, yn * sy as isize);
1329            if !self.avail(xl, yl, lx, ly) {
1330                return false;
1331            }
1332            !constrained
1333                || self.pic.flags[self.pic.idx4(lx as usize, ly as usize)] & flag::INTRA != 0
1334        };
1335        let (xi, yi) = (x as isize, y as isize);
1336        let c2 = 2 * n;
1337        // Availability changes at most every 4 luma samples; test in steps
1338        // (component samples: 4 / sy down the left, 4 / sx along the top).
1339        let (step_l, step_t) = (4 / sy, 4 / sx);
1340        let mut k = 0;
1341        while k < c2 {
1342            // Left column, y = k .. k + step_l.
1343            let a = check(xi - 1, yi + k as isize);
1344            for j in k..(k + step_l).min(c2) {
1345                let idx = c2 - 1 - j;
1346                av[idx] = a;
1347                if a {
1348                    arr[idx] = p.at(x - 1, y + j) as i32;
1349                }
1350            }
1351            k += step_l;
1352        }
1353        let mut k = 0;
1354        while k < c2 {
1355            // Top row, x = k .. k + step_t.
1356            let a = check(xi + k as isize, yi - 1);
1357            for j in k..(k + step_t).min(c2) {
1358                let idx = c2 + 1 + j;
1359                av[idx] = a;
1360                if a {
1361                    arr[idx] = p.at(x + j, y - 1) as i32;
1362                }
1363            }
1364            k += step_t;
1365        }
1366        if check(xi - 1, yi - 1) {
1367            av[c2] = true;
1368            arr[c2] = p.at(x - 1, y - 1) as i32;
1369        }
1370        intra::substitute(&mut arr[..len], &av[..len], bd);
1371        if !self.sps.intra_smoothing_disabled {
1372            let filtered = cidx == 0 || self.sps.chroma_format_idc == 3;
1373            let strong = cidx == 0 && self.sps.strong_intra_smoothing;
1374            intra::filter(&mut arr[..len], n, mode, filtered, strong, bd);
1375        }
1376        let mut out = [0i32; 32 * 32];
1377        // Lossless blocks under implicit RDPCM lose the horizontal and
1378        // vertical edge filters (HM `enableEdgeFilters`); DC keeps its own.
1379        let no_edges = self.sps.implicit_rdpcm && self.cu_bypass && mode != 1;
1380        intra::predict(&mut out, &arr[..len], n, mode, cidx == 0 && !no_edges, bd);
1381        let p = &mut self.pic.planes[cidx];
1382        for yy in 0..n {
1383            for xx in 0..n {
1384                if x + xx < p.width && y + yy < p.height {
1385                    p.set(x + xx, y + yy, out[yy * n + xx] as u16);
1386                }
1387            }
1388        }
1389    }
1390
1391    // ------------------------------------------------------------------
1392    // Residual coding (§7.3.8.11) and reconstruction
1393    // ------------------------------------------------------------------
1394
1395    /// `cross_component_prediction()` (§7.3.8.12): `ResScaleVal` for chroma
1396    /// component `c` (0 Cb, 1 Cr).
1397    fn cross_component_pred(&mut self, c: &mut Cabac, comp: usize) -> i32 {
1398        let mut v = 0;
1399        while v < 4 && self.bin(c, ci::LOG2_RES_SCALE_ABS + 4 * comp + v) == 1 {
1400            v += 1;
1401        }
1402        if v == 0 {
1403            return 0;
1404        }
1405        let sign = self.bin(c, ci::RES_SCALE_SIGN + comp) as i32;
1406        (1 << (v - 1)) * (1 - 2 * sign)
1407    }
1408
1409    /// The cross-component term alone, for a chroma block with no residual
1410    /// of its own: (ResScaleVal × luma residual at chroma depth) >> 3.
1411    fn add_cross_residual(&mut self, x0: usize, y0: usize, n: usize, cidx: usize, res_scale: i32) {
1412        let mut coeff = std::mem::take(&mut self.coeff);
1413        coeff[..n * n].fill(0);
1414        self.cross_term(&mut coeff[..n * n], res_scale);
1415        self.add_residual(x0, y0, n, cidx, &coeff);
1416        self.coeff = coeff;
1417    }
1418
1419    fn cross_term(&self, r: &mut [i32], res_scale: i32) {
1420        let (bdy, bdc) = (self.sps.bit_depth_y, self.sps.bit_depth_c);
1421        for (v, &y) in r.iter_mut().zip(&self.res_y) {
1422            *v += (res_scale * ((y << bdc) >> bdy)) >> 3;
1423        }
1424    }
1425
1426    fn add_residual(&mut self, x0: usize, y0: usize, n: usize, cidx: usize, coeff: &[i32]) {
1427        let bd = if cidx == 0 {
1428            self.sps.bit_depth_y
1429        } else {
1430            self.sps.bit_depth_c
1431        };
1432        let max = (1i32 << bd) - 1;
1433        let p = &mut self.pic.planes[cidx];
1434        for y in 0..n {
1435            if y0 + y >= p.height {
1436                break;
1437            }
1438            for x in 0..n {
1439                if x0 + x >= p.width {
1440                    break;
1441                }
1442                let i = (y0 + y) * p.width + x0 + x;
1443                p.data[i] = (p.data[i] as i32 + coeff[y * n + x]).clamp(0, max) as u16;
1444            }
1445        }
1446    }
1447
1448    fn residual(
1449        &mut self,
1450        c: &mut Cabac,
1451        x0: usize,
1452        y0: usize,
1453        log2: u32,
1454        cidx: usize,
1455        res_scale: i32,
1456    ) -> Result<()> {
1457        let n = 1usize << log2;
1458        let mut coeff = std::mem::take(&mut self.coeff);
1459        coeff[..n * n].fill(0);
1460        let rc = self.residual_coding(c, x0, y0, log2, cidx, &mut coeff)?;
1461        let ts = rc.transform_skip;
1462        // Untransformed 4×4 intra residuals are rotated by 180° (HM
1463        // `isNonTransformedResidualRotated`: transform skip and lossless).
1464        let rotate =
1465            self.sps.transform_skip_rotation && n == 4 && self.cu_intra && (ts || self.cu_bypass);
1466        let bd = if cidx == 0 {
1467            self.sps.bit_depth_y
1468        } else {
1469            self.sps.bit_depth_c
1470        };
1471        if !self.cu_bypass {
1472            let qp = if cidx == 0 {
1473                self.qp_y + 6 * (self.sps.bit_depth_y as i32 - 8)
1474            } else {
1475                let off_c = 6 * (self.sps.bit_depth_c as i32 - 8);
1476                let o = if cidx == 1 {
1477                    self.pps.cb_qp_offset + self.sh.cb_qp_offset
1478                } else {
1479                    self.pps.cr_qp_offset + self.sh.cr_qp_offset
1480                } + self.cu_chroma_qp_offset[cidx - 1];
1481                let qpi = (self.qp_y + o).clamp(-off_c, 57);
1482                (if self.sps.chroma_format_idc == 1 {
1483                    chroma_qp(qpi)
1484                } else {
1485                    qpi.min(51)
1486                }) + off_c
1487            };
1488            let m = match (&self.pps.scaling, &self.sps.scaling) {
1489                _ if ts && n > 4 => None,
1490                (Some(s), _) | (None, Some(s)) => {
1491                    Some(&s.m[log2 as usize - 2][cidx + if self.cu_intra { 0 } else { 3 }][..])
1492                }
1493                _ => None,
1494            };
1495            // `scaling_list_enabled_flag` gates PPS lists too.
1496            let m = if self.sps.scaling.is_some() { m } else { None };
1497            transform::dequant(&mut coeff, n, qp, bd, m);
1498            if ts {
1499                if rotate {
1500                    coeff[..16].reverse();
1501                }
1502                transform::transform_skip(&mut coeff, n, bd);
1503            } else {
1504                let dst = self.cu_intra && n == 4 && cidx == 0;
1505                transform::inverse(&mut coeff, n, dst, bd);
1506            }
1507        } else if rotate {
1508            coeff[..16].reverse();
1509        }
1510        if let Some(vertical) = rc.rdpcm {
1511            transform::rdpcm(&mut coeff, n, vertical);
1512        }
1513        if cidx == 0 && self.pps.cross_component_prediction {
1514            self.res_y[..n * n].copy_from_slice(&coeff[..n * n]);
1515        }
1516        if res_scale != 0 {
1517            self.cross_term(&mut coeff[..n * n], res_scale);
1518        }
1519        self.add_residual(x0, y0, n, cidx, &coeff);
1520        self.coeff = coeff;
1521        Ok(())
1522    }
1523
1524    /// Parse one `residual_coding()` into `out` (raster, `n × n`).
1525    fn residual_coding(
1526        &mut self,
1527        c: &mut Cabac,
1528        x0: usize,
1529        y0: usize,
1530        log2: u32,
1531        cidx: usize,
1532        out: &mut [i32],
1533    ) -> Result<ResidualInfo> {
1534        let n = 1usize << log2;
1535        let chroma = cidx > 0;
1536        let ts = self.pps.transform_skip
1537            && !self.cu_bypass
1538            && log2 <= self.pps.log2_max_transform_skip
1539            && self.bin(c, ci::TRANSFORM_SKIP + chroma as usize) == 1;
1540        let untransformed = ts || self.cu_bypass;
1541        // Residual DPCM: signalled for inter blocks, implied for intra ones
1542        // predicted straight across (mode 10 horizontal, 26 vertical).
1543        let rdpcm = if !self.cu_intra && self.sps.explicit_rdpcm && untransformed {
1544            if self.bin(c, ci::EXPLICIT_RDPCM + chroma as usize) == 1 {
1545                Some(self.bin(c, ci::EXPLICIT_RDPCM_DIR + chroma as usize) == 1)
1546            } else {
1547                None
1548            }
1549        } else if self.cu_intra && self.sps.implicit_rdpcm && untransformed {
1550            // Chroma coordinates; luma ones only matter in 4:4:4.
1551            let m = if chroma {
1552                self.chroma_mode(x0, y0)
1553            } else {
1554                self.pic.ipm[self.pic.idx4(x0, y0)] as u32
1555            };
1556            match m {
1557                10 => Some(false),
1558                26 => Some(true),
1559                _ => None,
1560            }
1561        } else {
1562            None
1563        };
1564        // Last significant position.
1565        let (off, shift) = if !chroma {
1566            (3 * (log2 - 2) + ((log2 - 1) >> 2), (log2 + 1) >> 2)
1567        } else {
1568            (15, log2 - 2)
1569        };
1570        let max = (log2 << 1) - 1;
1571        let mut xp = 0;
1572        while xp < max && self.bin(c, ci::LAST_X + (off + (xp >> shift)) as usize) == 1 {
1573            xp += 1;
1574        }
1575        let mut yp = 0;
1576        while yp < max && self.bin(c, ci::LAST_Y + (off + (yp >> shift)) as usize) == 1 {
1577            yp += 1;
1578        }
1579        let suffix = |c: &mut Cabac, p: u32| -> u32 {
1580            if p > 3 {
1581                let nb = (p >> 1) - 1;
1582                (1 << nb) * (2 + (p & 1)) + c.bypass_bits(nb)
1583            } else {
1584                p
1585            }
1586        };
1587        let mut lx = suffix(c, xp) as usize;
1588        let mut ly = suffix(c, yp) as usize;
1589        if lx >= n || ly >= n {
1590            return Err("last significant coefficient outside the block".into());
1591        }
1592        // Scan order.
1593        let c444 = self.sps.chroma_format_idc == 3;
1594        let scan_idx = if self.cu_intra && (log2 == 2 || (log2 == 3 && (cidx == 0 || c444))) {
1595            let m = if cidx == 0 {
1596                self.pic.ipm[self.pic.idx4(x0, y0)] as u32
1597            } else {
1598                // Chroma coordinates; luma ones only matter in 4:4:4.
1599                self.chroma_mode(x0, y0)
1600            };
1601            if (6..=14).contains(&m) {
1602                2
1603            } else if (22..=30).contains(&m) {
1604                1
1605            } else {
1606                0
1607            }
1608        } else {
1609            0
1610        };
1611        if scan_idx == 2 {
1612            std::mem::swap(&mut lx, &mut ly);
1613        }
1614        let sbw = 1usize << (log2 - 2);
1615        let sb_scan: &[(u8, u8)] = match log2 {
1616            2 => &SCAN_SB1[scan_idx],
1617            3 => &SCAN_SB2[scan_idx],
1618            4 => &SCAN_SB4,
1619            _ => &SCAN_SB8,
1620        };
1621        let scan = &SCAN4[scan_idx];
1622        let last_sb = sb_scan
1623            .iter()
1624            .position(|&(x, y)| x as usize == lx >> 2 && y as usize == ly >> 2)
1625            .ok_or("last sub-block not in scan")?;
1626        let last_pos = scan
1627            .iter()
1628            .position(|&(x, y)| x as usize == lx & 3 && y as usize == ly & 3)
1629            .unwrap();
1630        let mut csbf = [0u8; 64];
1631        let mut prev_c1 = 1u32;
1632        let sdh = self.pps.sign_data_hiding && !self.cu_bypass && rdpcm.is_none();
1633        // One significance context for untransformed blocks (RExt).
1634        let ts_ctx = self.sps.transform_skip_context && untransformed;
1635        let persistent = self.sps.persistent_rice_adaptation;
1636        let sb_type = 2 * (!chroma) as usize + untransformed as usize;
1637        for i in (0..=last_sb).rev() {
1638            let (xs, ys) = (sb_scan[i].0 as usize, sb_scan[i].1 as usize);
1639            let right = if xs + 1 < sbw {
1640                csbf[ys * 8 + xs + 1]
1641            } else {
1642                0
1643            };
1644            let below = if ys + 1 < sbw {
1645                csbf[(ys + 1) * 8 + xs]
1646            } else {
1647                0
1648            };
1649            let mut infer_dc = false;
1650            let coded = if i < last_sb && i > 0 {
1651                let inc = (right | below).min(1) as usize + if chroma { 2 } else { 0 };
1652                infer_dc = true;
1653                self.bin(c, ci::SIG_GROUP + inc) as u8
1654            } else {
1655                1
1656            };
1657            csbf[ys * 8 + xs] = coded;
1658            let mut sig = [0u8; 16];
1659            let mut nsig = 0;
1660            let start: isize = if i == last_sb {
1661                sig[0] = last_pos as u8;
1662                nsig = 1;
1663                last_pos as isize - 1
1664            } else {
1665                15
1666            };
1667            if coded == 1 {
1668                let prev_csbf = right | (below << 1);
1669                let mut k = start;
1670                while k >= 0 {
1671                    let (xp, yp) = (scan[k as usize].0 as usize, scan[k as usize].1 as usize);
1672                    let (xc, yc) = (xs * 4 + xp, ys * 4 + yp);
1673                    if k > 0 || !infer_dc {
1674                        let sc = if ts_ctx {
1675                            if chroma {
1676                                16
1677                            } else {
1678                                42
1679                            }
1680                        } else if log2 == 2 {
1681                            CTX_IDX_MAP[(yc << 2) + xc] as usize
1682                        } else if xc + yc == 0 {
1683                            0
1684                        } else {
1685                            let mut s = match prev_csbf {
1686                                0 => {
1687                                    if xp + yp == 0 {
1688                                        2
1689                                    } else if xp + yp < 3 {
1690                                        1
1691                                    } else {
1692                                        0
1693                                    }
1694                                }
1695                                1 => {
1696                                    if yp == 0 {
1697                                        2
1698                                    } else if yp == 1 {
1699                                        1
1700                                    } else {
1701                                        0
1702                                    }
1703                                }
1704                                2 => {
1705                                    if xp == 0 {
1706                                        2
1707                                    } else if xp == 1 {
1708                                        1
1709                                    } else {
1710                                        0
1711                                    }
1712                                }
1713                                _ => 2,
1714                            };
1715                            if !chroma {
1716                                if xs + ys > 0 {
1717                                    s += 3;
1718                                }
1719                                s += if log2 == 3 {
1720                                    if scan_idx == 0 {
1721                                        9
1722                                    } else {
1723                                        15
1724                                    }
1725                                } else {
1726                                    21
1727                                };
1728                            } else {
1729                                s += if log2 == 3 { 9 } else { 12 };
1730                            }
1731                            s
1732                        };
1733                        let inc = if chroma { 27 + sc } else { sc };
1734                        if self.bin(c, ci::SIG + inc) == 1 {
1735                            sig[nsig] = k as u8;
1736                            nsig += 1;
1737                            infer_dc = false;
1738                        }
1739                    } else {
1740                        // DC of a coded sub-block nothing else in was coded.
1741                        sig[nsig] = 0;
1742                        nsig += 1;
1743                    }
1744                    k -= 1;
1745                }
1746            }
1747            if nsig == 0 {
1748                continue;
1749            }
1750            // Levels.
1751            let mut ctx_set = if i == 0 || chroma { 0 } else { 2 };
1752            if prev_c1 == 0 {
1753                ctx_set += 1;
1754            }
1755            let mut c1 = 1u32;
1756            let mut g1 = [0u8; 16];
1757            let mut first_g1: Option<usize> = None;
1758            for k in 0..nsig.min(8) {
1759                let inc = ctx_set * 4 + c1 as usize + if chroma { 16 } else { 0 };
1760                let f = self.bin(c, ci::GT1 + inc);
1761                g1[k] = f as u8;
1762                if f == 1 {
1763                    c1 = 0;
1764                    if first_g1.is_none() {
1765                        first_g1 = Some(k);
1766                    }
1767                } else if c1 > 0 && c1 < 3 {
1768                    c1 += 1;
1769                }
1770            }
1771            prev_c1 = c1;
1772            let mut g2 = 0;
1773            if first_g1.is_some() {
1774                g2 = self.bin(c, ci::GT2 + ctx_set + if chroma { 4 } else { 0 });
1775            }
1776            let hidden = sdh && (sig[0] as i32 - sig[nsig - 1] as i32) > 3;
1777            let nsign = if hidden { nsig - 1 } else { nsig };
1778            let signs = c.bypass_bits(nsign as u32) << (32 - nsign as u32).min(31);
1779            let signs = if nsign == 0 { 0 } else { signs };
1780            let mut rice = if persistent {
1781                self.stat_coeff[sb_type] / 4
1782            } else {
1783                0
1784            };
1785            let mut rice_updated = false;
1786            let mut sum_abs = 0i32;
1787            for k in 0..nsig {
1788                let base = 1 + g1[k] as i32 + if Some(k) == first_g1 { g2 as i32 } else { 0 };
1789                let thresh = if k < 8 {
1790                    if Some(k) == first_g1 {
1791                        3
1792                    } else {
1793                        2
1794                    }
1795                } else {
1796                    1
1797                };
1798                let mut abs = base;
1799                if base == thresh {
1800                    let mut prefix = 0u32;
1801                    while prefix < 32 && c.bypass() == 1 {
1802                        prefix += 1;
1803                    }
1804                    let rem = if prefix <= 3 {
1805                        ((prefix << rice) + c.bypass_bits(rice)) as i32
1806                    } else {
1807                        let nb = prefix - 3 + rice;
1808                        if nb > 31 {
1809                            return Err("coeff_abs_level_remaining too long".into());
1810                        }
1811                        ((((1u32 << (prefix - 3)) + 2) << rice) + c.bypass_bits(nb)) as i32
1812                    };
1813                    // Wrapping: only damaged data overflows, and `dequant`
1814                    // clamps the coefficient to 16 bits further on.
1815                    abs = base.wrapping_add(rem);
1816                    if abs > 3 * (1 << rice) {
1817                        rice = if persistent {
1818                            rice + 1
1819                        } else {
1820                            (rice + 1).min(4)
1821                        };
1822                    }
1823                    if persistent && !rice_updated {
1824                        // The sub-block's first remainder moves StatCoeff.
1825                        let st = &mut self.stat_coeff[sb_type];
1826                        let init = *st / 4;
1827                        if rem >= 3 << init {
1828                            *st += 1;
1829                        } else if 2 * rem < 1 << init && *st > 0 {
1830                            *st -= 1;
1831                        }
1832                        rice_updated = true;
1833                    }
1834                }
1835                // Only its parity is read, which wrapping keeps.
1836                sum_abs = i32::wrapping_add(sum_abs, abs);
1837                let neg = if k < nsign {
1838                    (signs >> (31 - k as u32)) & 1 == 1
1839                } else {
1840                    // Hidden sign: parity of the sub-block's sum.
1841                    sum_abs & 1 == 1
1842                };
1843                let pos = sig[k] as usize;
1844                let (xc, yc) = (xs * 4 + scan[pos].0 as usize, ys * 4 + scan[pos].1 as usize);
1845                out[yc * n + xc] = if neg { abs.wrapping_neg() } else { abs };
1846            }
1847        }
1848        Ok(ResidualInfo {
1849            transform_skip: ts,
1850            rdpcm,
1851        })
1852    }
1853}