cros_codecs/codec/h264/
picture.rs

1// Copyright 2022 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use std::cell::RefCell;
6use std::ops::Deref;
7use std::rc::Rc;
8use std::rc::Weak;
9
10use log::debug;
11
12use crate::codec::h264::parser::MaxLongTermFrameIdx;
13use crate::codec::h264::parser::RefPicMarking;
14use crate::codec::h264::parser::Slice;
15use crate::codec::h264::parser::SliceType;
16use crate::codec::h264::parser::Sps;
17use crate::Resolution;
18
19#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
20pub enum Field {
21    #[default]
22    Frame,
23    Top,
24    Bottom,
25}
26
27impl Field {
28    /// Returns the field of opposite parity.
29    pub fn opposite(&self) -> Self {
30        match *self {
31            Field::Frame => Field::Frame,
32            Field::Top => Field::Bottom,
33            Field::Bottom => Field::Top,
34        }
35    }
36}
37
38#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
39pub enum Reference {
40    #[default]
41    None,
42    ShortTerm,
43    LongTerm,
44}
45
46#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
47pub enum IsIdr {
48    #[default]
49    No,
50    Yes {
51        idr_pic_id: u16,
52    },
53}
54
55/// The rank of a field, i.e. whether it is the first or second one to be parsed from the stream.
56/// This is unrelated to the `Field` type, as the first field can be either `Top` or `Bottom`.
57#[derive(Default, Debug)]
58pub enum FieldRank {
59    /// Frame has a single field.
60    #[default]
61    Single,
62    /// Frame is interlaced, and this is the first field (with a reference to the second one).
63    First(Weak<RefCell<PictureData>>),
64    /// Frame is interlaced, and this is the second field (with a reference to the first one).
65    Second(Rc<RefCell<PictureData>>),
66}
67
68#[derive(Default)]
69pub struct PictureData {
70    pub pic_order_cnt_type: u8,
71    pub top_field_order_cnt: i32,
72    pub bottom_field_order_cnt: i32,
73    pub pic_order_cnt: i32,
74    pub pic_order_cnt_msb: i32,
75    pub pic_order_cnt_lsb: i32,
76    pub delta_pic_order_cnt_bottom: i32,
77    pub delta_pic_order_cnt0: i32,
78    pub delta_pic_order_cnt1: i32,
79
80    pub pic_num: i32,
81    pub long_term_pic_num: u32,
82    pub frame_num: u32,
83    pub frame_num_offset: u32,
84    pub frame_num_wrap: i32,
85    pub long_term_frame_idx: u32,
86
87    pub coded_resolution: Resolution,
88    pub display_resolution: Resolution,
89
90    pub type_: SliceType,
91    pub nal_ref_idc: u8,
92    pub is_idr: IsIdr,
93    reference: Reference,
94    pub ref_pic_list_modification_flag_l0: i32,
95    pub abs_diff_pic_num_minus1: i32,
96
97    // Does memory management op 5 needs to be executed after this
98    // picture has finished decoding?
99    pub has_mmco_5: bool,
100
101    // Created by the decoding process for gaps in frame_num.
102    // Not for decode or output.
103    pub nonexisting: bool,
104
105    pub field: Field,
106
107    // Values from slice_hdr to be used during reference marking and
108    // memory management after finishing this picture.
109    pub ref_pic_marking: RefPicMarking,
110
111    field_rank: FieldRank,
112
113    pub timestamp: u64,
114}
115
116/// A `PictureData` within a `Rc<RefCell>` which field rank is guaranteed to be correct.
117///
118/// The field rank of `PictureData` is only final after both fields have been constructed - namely,
119/// the first field can only point to the second one after the latter is available as a Rc. Methods
120/// [`PictureData::into_rc`] and [`PictureData::split_frame`] take care of this, and is this only
121/// producer of this type, ensuring all instances are correct.
122#[derive(Default, Debug, Clone)]
123pub struct RcPictureData {
124    pic: Rc<RefCell<PictureData>>,
125}
126
127impl Deref for RcPictureData {
128    type Target = Rc<RefCell<PictureData>>;
129
130    fn deref(&self) -> &Self::Target {
131        &self.pic
132    }
133}
134
135impl PictureData {
136    pub fn new_non_existing(frame_num: u32, timestamp: u64) -> Self {
137        PictureData {
138            frame_num,
139            nonexisting: true,
140            nal_ref_idc: 1,
141            field: Field::Frame,
142            pic_num: frame_num as i32,
143            reference: Reference::ShortTerm,
144            timestamp,
145            ..Default::default()
146        }
147    }
148
149    /// Create a new picture from a `slice`, `sps`, and `timestamp`.
150    ///
151    /// `first_field` is set if this picture is the second field of a frame.
152    pub fn new_from_slice(
153        slice: &Slice,
154        sps: &Sps,
155        timestamp: u64,
156        first_field: Option<&RcPictureData>,
157    ) -> Self {
158        let hdr = &slice.header;
159        let nalu_hdr = &slice.nalu.header;
160
161        let is_idr = if nalu_hdr.idr_pic_flag {
162            IsIdr::Yes { idr_pic_id: hdr.idr_pic_id }
163        } else {
164            IsIdr::No
165        };
166
167        let field = if hdr.field_pic_flag {
168            if hdr.bottom_field_flag {
169                Field::Bottom
170            } else {
171                Field::Top
172            }
173        } else {
174            Field::Frame
175        };
176
177        let reference = if nalu_hdr.ref_idc != 0 { Reference::ShortTerm } else { Reference::None };
178
179        let pic_num = if !hdr.field_pic_flag { hdr.frame_num } else { 2 * hdr.frame_num + 1 };
180
181        let (
182            pic_order_cnt_lsb,
183            delta_pic_order_cnt_bottom,
184            delta_pic_order_cnt0,
185            delta_pic_order_cnt1,
186        ) = match sps.pic_order_cnt_type {
187            0 => (
188                hdr.pic_order_cnt_lsb,
189                hdr.delta_pic_order_cnt_bottom,
190                Default::default(),
191                Default::default(),
192            ),
193            1 => (
194                Default::default(),
195                Default::default(),
196                hdr.delta_pic_order_cnt[0],
197                hdr.delta_pic_order_cnt[1],
198            ),
199            _ => (Default::default(), Default::default(), Default::default(), Default::default()),
200        };
201
202        let coded_resolution = Resolution::from((sps.width(), sps.height()));
203
204        let visible_rect = sps.visible_rectangle();
205
206        let display_resolution = Resolution {
207            width: visible_rect.max.x - visible_rect.min.x,
208            height: visible_rect.max.y - visible_rect.min.y,
209        };
210
211        let mut pic = PictureData {
212            pic_order_cnt_type: sps.pic_order_cnt_type,
213            pic_order_cnt_lsb: i32::from(pic_order_cnt_lsb),
214            delta_pic_order_cnt_bottom,
215            delta_pic_order_cnt0,
216            delta_pic_order_cnt1,
217            pic_num: i32::from(pic_num),
218            frame_num: u32::from(hdr.frame_num),
219            nal_ref_idc: nalu_hdr.ref_idc,
220            is_idr,
221            reference,
222            field,
223            ref_pic_marking: hdr.dec_ref_pic_marking.clone(),
224            coded_resolution,
225            display_resolution,
226            timestamp,
227            ..Default::default()
228        };
229
230        if let Some(first_field) = first_field {
231            pic.set_first_field_to(first_field);
232        }
233
234        pic
235    }
236
237    /// Whether the current picture is a reference, either ShortTerm or LongTerm.
238    pub fn is_ref(&self) -> bool {
239        !matches!(self.reference, Reference::None)
240    }
241
242    /// Whether this picture is a second field.
243    pub fn is_second_field(&self) -> bool {
244        matches!(self.field_rank, FieldRank::Second(..))
245    }
246
247    /// Returns the field rank of this picture, including a reference to its other field.
248    pub fn field_rank(&self) -> &FieldRank {
249        &self.field_rank
250    }
251
252    /// Returns a reference to the picture's Reference
253    pub fn reference(&self) -> &Reference {
254        &self.reference
255    }
256
257    /// Mark the picture as a reference picture.
258    pub fn set_reference(&mut self, reference: Reference, apply_to_other_field: bool) {
259        log::debug!("Set reference of {:#?} to {:?}", self, reference);
260        self.reference = reference;
261
262        if apply_to_other_field {
263            if let Some(other_field) = self.other_field() {
264                log::debug!(
265                    "other_field: Set reference of {:#?} to {:?}",
266                    &other_field.borrow(),
267                    reference
268                );
269                other_field.borrow_mut().reference = reference;
270            }
271        }
272    }
273
274    /// Get a reference to the picture's other field, if there is any
275    /// and its reference is still valid.
276    pub fn other_field(&self) -> Option<Rc<RefCell<PictureData>>> {
277        match &self.field_rank {
278            FieldRank::Single => None,
279            FieldRank::First(other_field) => other_field.upgrade(),
280            FieldRank::Second(other_field) => Some(other_field.clone()),
281        }
282    }
283
284    /// Set this picture's second field.
285    fn set_second_field_to(&mut self, other_field: &Rc<RefCell<Self>>) {
286        self.field_rank = FieldRank::First(Rc::downgrade(other_field));
287    }
288
289    /// Whether the current picture is the second field of a complementary ref pair.
290    pub fn is_second_field_of_complementary_ref_pair(&self) -> bool {
291        self.is_ref()
292            && matches!(self.field_rank(), FieldRank::Second(first_field) if first_field.borrow().is_ref())
293    }
294
295    /// Set this picture's first field.
296    fn set_first_field_to(&mut self, other_field: &Rc<RefCell<Self>>) {
297        self.field_rank = FieldRank::Second(other_field.clone());
298    }
299
300    pub fn pic_num_f(&self, max_pic_num: i32) -> i32 {
301        if !matches!(self.reference(), Reference::LongTerm) {
302            self.pic_num
303        } else {
304            max_pic_num
305        }
306    }
307
308    pub fn long_term_pic_num_f(&self, max_long_term_frame_idx: MaxLongTermFrameIdx) -> u32 {
309        if matches!(self.reference(), Reference::LongTerm) {
310            self.long_term_pic_num
311        } else {
312            2 * max_long_term_frame_idx.to_value_plus1()
313        }
314    }
315
316    /// Consume this picture and return a Rc'd version.
317    ///
318    /// If the picture was a second field, adjust the field of the first field to point to this
319    /// one.
320    pub fn into_rc(self) -> RcPictureData {
321        let self_rc = Rc::new(RefCell::new(self));
322
323        if let FieldRank::Second(first_field) = self_rc.borrow().field_rank() {
324            first_field.borrow_mut().set_second_field_to(&self_rc);
325        }
326
327        RcPictureData { pic: self_rc }
328    }
329
330    /// Split a frame into two complementary fields that reference one another.
331    pub fn split_frame(mut self) -> (RcPictureData, RcPictureData) {
332        assert!(matches!(self.field, Field::Frame));
333        assert!(matches!(self.field_rank, FieldRank::Single));
334
335        debug!(
336            "Splitting picture (frame_num, POC) ({:?}, {:?})",
337            self.frame_num, self.pic_order_cnt
338        );
339
340        let second_pic_order_cnt = if self.top_field_order_cnt < self.bottom_field_order_cnt {
341            self.field = Field::Top;
342            self.pic_order_cnt = self.top_field_order_cnt;
343
344            self.bottom_field_order_cnt
345        } else {
346            self.field = Field::Bottom;
347            self.pic_order_cnt = self.bottom_field_order_cnt;
348
349            self.top_field_order_cnt
350        };
351
352        let second_field = PictureData {
353            top_field_order_cnt: self.top_field_order_cnt,
354            bottom_field_order_cnt: self.bottom_field_order_cnt,
355            frame_num: self.frame_num,
356            reference: self.reference,
357            nonexisting: self.nonexisting,
358            pic_order_cnt: second_pic_order_cnt,
359            field: self.field.opposite(),
360            ..Default::default()
361        };
362
363        debug!(
364            "Split into picture (frame_num, POC) ({:?}, {:?}), field: {:?}",
365            self.frame_num, self.pic_order_cnt, self.field
366        );
367        debug!(
368            "Split into picture (frame_num, POC) ({:?}, {:?}), field {:?}",
369            second_field.frame_num, second_field.pic_order_cnt, second_field.field
370        );
371
372        let first_field = Rc::new(RefCell::new(self));
373        let second_field = Rc::new(RefCell::new(second_field));
374
375        first_field.borrow_mut().set_second_field_to(&second_field);
376        second_field.borrow_mut().set_first_field_to(&first_field);
377
378        (RcPictureData { pic: first_field }, RcPictureData { pic: second_field })
379    }
380}
381
382impl std::fmt::Debug for PictureData {
383    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
384        f.debug_struct("PictureData")
385            .field("pic_order_cnt_type", &self.pic_order_cnt_type)
386            .field("top_field_order_cnt", &self.top_field_order_cnt)
387            .field("bottom_field_order_cnt", &self.bottom_field_order_cnt)
388            .field("pic_order_cnt", &self.pic_order_cnt)
389            .field("pic_order_cnt_msb", &self.pic_order_cnt_msb)
390            .field("pic_order_cnt_lsb", &self.pic_order_cnt_lsb)
391            .field("delta_pic_order_cnt_bottom", &self.delta_pic_order_cnt_bottom)
392            .field("delta_pic_order_cnt0", &self.delta_pic_order_cnt0)
393            .field("delta_pic_order_cnt1", &self.delta_pic_order_cnt1)
394            .field("pic_num", &self.pic_num)
395            .field("long_term_pic_num", &self.long_term_pic_num)
396            .field("frame_num", &self.frame_num)
397            .field("frame_num_offset", &self.frame_num_offset)
398            .field("frame_num_wrap", &self.frame_num_wrap)
399            .field("long_term_frame_idx", &self.long_term_frame_idx)
400            .field("coded_resolution", &self.coded_resolution)
401            .field("display_resolution", &self.display_resolution)
402            .field("type_", &self.type_)
403            .field("nal_ref_idc", &self.nal_ref_idc)
404            .field("is_idr", &self.is_idr)
405            .field("reference", &self.reference)
406            .field("ref_pic_list_modification_flag_l0", &self.ref_pic_list_modification_flag_l0)
407            .field("abs_diff_pic_num_minus1", &self.abs_diff_pic_num_minus1)
408            .field("has_mmco_5", &self.has_mmco_5)
409            .field("nonexisting", &self.nonexisting)
410            .field("field", &self.field)
411            .field("ref_pic_marking", &self.ref_pic_marking)
412            .field("field_rank", &self.field_rank)
413            .finish()
414    }
415}
cros_codecs/codec/h264/picture.rs

cros_codecs/codec/h264/
picture.rs
