cros_codecs/decoder/stateless/

av1.rs

// Copyright 2023 The ChromiumOS Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use std::os::fd::AsFd;
use std::os::fd::BorrowedFd;
use std::rc::Rc;

use anyhow::anyhow;

use crate::codec::av1::parser::FrameHeaderObu;
use crate::codec::av1::parser::FrameObu;
use crate::codec::av1::parser::FrameType;
use crate::codec::av1::parser::ObuType;
use crate::codec::av1::parser::ParsedObu;
use crate::codec::av1::parser::Parser;
use crate::codec::av1::parser::SequenceHeaderObu;
use crate::codec::av1::parser::TileGroupObu;
use crate::codec::av1::parser::NUM_REF_FRAMES;
use crate::decoder::stateless::DecodeError;
use crate::decoder::stateless::DecodingState;
use crate::decoder::stateless::NewPictureResult;
use crate::decoder::stateless::StatelessBackendResult;
use crate::decoder::stateless::StatelessCodec;
use crate::decoder::stateless::StatelessDecoder;
use crate::decoder::stateless::StatelessDecoderBackend;
use crate::decoder::stateless::StatelessDecoderBackendPicture;
use crate::decoder::stateless::StatelessVideoDecoder;
use crate::decoder::stateless::TryFormat;
use crate::decoder::BlockingMode;
use crate::decoder::DecodedHandle;
use crate::decoder::PoolLayer;
use crate::Resolution;

#[cfg(test)]
mod dummy;
#[cfg(feature = "vaapi")]
mod vaapi;

/// Stateless backend methods specific to AV1.
pub trait StatelessAV1DecoderBackend:
    StatelessDecoderBackend + StatelessDecoderBackendPicture<Av1>
{
    /// Called when a new Sequence Header OBU is parsed. The
    /// `highest_spatial_layer` argument refers to the maximum layer selected by
    /// the client through `set_operating_point()` and the scalability
    /// information present in the stream, if any.
    fn new_sequence(
        &mut self,
        sequence: &Rc<SequenceHeaderObu>,
        highest_spatial_layer: Option<u32>,
    ) -> StatelessBackendResult<()>;

    /// Called when the decoder determines that a new picture was found. The backend allocates all
    /// the resources it needs to process that picture.
    fn new_picture(
        &mut self,
        hdr: &FrameHeaderObu,
        timestamp: u64,
        highest_spatial_layer: Option<u32>,
    ) -> NewPictureResult<Self::Picture>;

    /// Called to set the global parameters of a picture.
    fn begin_picture(
        &mut self,
        picture: &mut Self::Picture,
        sequence: &SequenceHeaderObu,
        hdr: &FrameHeaderObu,
        reference_frames: &[Option<Self::Handle>; NUM_REF_FRAMES],
    ) -> StatelessBackendResult<()>;

    /// Called to dispatch a decode operation to the backend.
    #[allow(clippy::too_many_arguments)]
    fn decode_tile_group(
        &mut self,
        picture: &mut Self::Picture,
        tile_group: TileGroupObu,
    ) -> StatelessBackendResult<()>;

    /// Called when the decoder wants the backend to finish the decoding
    /// operations for `picture`. At this point, `decode_tile` has been called
    /// for all tiles.
    fn submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult<Self::Handle>;
}

/// State of the picture being currently decoded.
///
/// Stored between calls to [`StatelessDecoder::decode_tile_group`] that belong to the same
/// picture.
enum CurrentPicState<H: DecodedHandle, P> {
    /// A regular frame
    RegularFrame {
        /// Data for the current picture as extracted from the stream.
        header: FrameHeaderObu,
        /// Backend-specific data for that picture.
        backend_picture: P,
    },

    /// A frame that has 'show_existing_frame' set.
    ShowExistingFrame {
        /// Data for the current picture as extracted from the stream.
        header: FrameHeaderObu,
        /// The handle of the reference frame that this frame points to.
        handle: H,
    },
}

pub struct AV1DecoderState<H: DecodedHandle, P> {
    /// AV1 bitstream parser.
    parser: Parser,

    /// The reference frames in use.
    reference_frames: [Option<H>; NUM_REF_FRAMES],

    /// Keeps track of the last values seen for negotiation purposes.
    sequence: Option<Rc<SequenceHeaderObu>>,

    /// The picture currently being decoded. We need to preserve it between
    /// calls to `decode` because multiple tiles will be processed in different
    /// calls to `decode`.
    current_pic: Option<CurrentPicState<H, P>>,

    /// Keep track of the number of frames we've processed for logging purposes.
    frame_count: u32,

    /// For SVC streams, we only want to output the highest layer possible given
    /// the choice of operating point.
    highest_spatial_layer: Option<u32>,
}

impl<H, P> Default for AV1DecoderState<H, P>
where
    H: DecodedHandle,
{
    fn default() -> Self {
        Self {
            parser: Default::default(),
            reference_frames: Default::default(),
            sequence: Default::default(),
            current_pic: Default::default(),
            frame_count: Default::default(),
            highest_spatial_layer: Default::default(),
        }
    }
}

/// [`StatelessCodec`] structure to use in order to create a AV1 stateless decoder.
///
/// # Accepted input
///
/// the VP9 specification requires the last byte of the chunk to contain the superframe marker.
/// Thus, a decoder using this codec processes exactly one encoded chunk per call to
/// [`StatelessDecoder::decode`], and always returns the size of the passed input if successful.
pub struct Av1;

impl StatelessCodec for Av1 {
    type FormatInfo = Rc<SequenceHeaderObu>;
    type DecoderState<H: DecodedHandle, P> = AV1DecoderState<H, P>;
}

impl<B> StatelessDecoder<Av1, B>
where
    B: StatelessAV1DecoderBackend,
    B::Handle: Clone,
{
    fn decode_frame_header(
        &mut self,
        frame_header: FrameHeaderObu,
        timestamp: u64,
    ) -> Result<(), DecodeError> {
        log::debug!(
            "Processing frame {} with timestamp {}",
            self.codec.frame_count,
            timestamp
        );

        if frame_header.show_existing_frame {
            let ref_frame = self.codec.reference_frames
                [frame_header.frame_to_show_map_idx as usize]
                .as_ref()
                .ok_or(anyhow!("Broken stream: no reference picture to display"))?;
            self.codec.current_pic = Some(CurrentPicState::ShowExistingFrame {
                header: frame_header,
                handle: ref_frame.clone(),
            });
        } else if let Some(sequence) = &self.codec.sequence {
            let mut backend_picture = self.backend.new_picture(
                &frame_header,
                timestamp,
                self.codec.highest_spatial_layer,
            )?;

            self.backend.begin_picture(
                &mut backend_picture,
                sequence,
                &frame_header,
                &self.codec.reference_frames,
            )?;

            self.codec.current_pic = Some(CurrentPicState::RegularFrame {
                header: frame_header.clone(),
                backend_picture,
            });
        } else {
            log::warn!("invalid stream: frame header received while no valid sequence ongoing");
        }

        Ok(())
    }

    fn decode_tile_group(&mut self, tile_group: TileGroupObu) -> anyhow::Result<()> {
        let picture = match self.codec.current_pic.as_mut() {
            Some(CurrentPicState::RegularFrame {
                backend_picture, ..
            }) => backend_picture,
            Some(CurrentPicState::ShowExistingFrame { .. }) => {
                return Err(anyhow!("Broken stream: cannot decode a tile group for a frame with show_existing_frame set"));
            }
            None => {
                return Err(anyhow!(
                "Broken stream: cannot decode a tile group without first decoding a frame header"
            ))
            }
        };

        self.backend.decode_tile_group(picture, tile_group)?;
        Ok(())
    }

    fn decode_frame(&mut self, frame: FrameObu, timestamp: u64) -> Result<(), DecodeError> {
        let FrameObu { header, tile_group } = frame;
        self.decode_frame_header(header, timestamp)?;
        self.decode_tile_group(tile_group)?;
        Ok(())
    }

    fn submit_frame(&mut self, timestamp: u64) -> anyhow::Result<()> {
        log::debug!(
            "Finishing frame {} with timestamp: {}",
            self.codec.frame_count,
            timestamp
        );

        let picture = self.codec.current_pic.take();

        let (handle, header) = match picture {
            Some(CurrentPicState::RegularFrame {
                header,
                backend_picture,
            }) => {
                let handle = self.backend.submit_picture(backend_picture)?;

                if self.blocking_mode == BlockingMode::Blocking {
                    handle.sync()?;
                }
                (handle, header)
            }
            Some(CurrentPicState::ShowExistingFrame { header, handle }) => (handle, header),
            None => return Err(anyhow!("Broken stream: no picture to submit")),
        };

        let update_refs = if header.show_existing_frame {
            header.frame_type == FrameType::KeyFrame
        } else {
            true
        };

        if update_refs {
            let mut refresh_frame_flags = header.refresh_frame_flags;

            #[allow(clippy::needless_range_loop)]
            for i in 0..NUM_REF_FRAMES {
                if (refresh_frame_flags & 1) == 1 {
                    log::debug!(
                        "Replacing reference frame {} to new timestamp {} on frame count: {}",
                        i,
                        timestamp,
                        self.codec.frame_count
                    );
                    self.codec.reference_frames[i] = Some(handle.clone());
                }

                refresh_frame_flags >>= 1;
            }
        }

        let show_existing_frame = header.show_existing_frame;
        if header.show_frame || show_existing_frame {
            match self.codec.highest_spatial_layer {
                None => self.ready_queue.push(handle),
                Some(highest_spatial_layer) => {
                    if header.obu_header.spatial_id >= highest_spatial_layer {
                        self.ready_queue.push(handle);
                    } else {
                        log::debug!(
                            "Dropping frame with spatial_id {}",
                            header.obu_header.spatial_id
                        );
                    }
                }
            }
        }

        self.codec.parser.ref_frame_update(&header)?;
        self.codec.frame_count += 1;
        Ok(())
    }
}

impl<B> StatelessVideoDecoder for StatelessDecoder<Av1, B>
where
    B: StatelessAV1DecoderBackend + TryFormat<Av1>,
    B::Handle: Clone + 'static,
{
    type Handle = B::Handle;
    type FramePool = B::FramePool;

    /// Decode an AV1 stream.
    ///
    /// `bitstream` should initially be submitted as a whole temporal unit, however a call to this
    /// method will only consume a single OBU. The caller must be careful to check the return value
    /// and resubmit the remainder if the whole bitstream has not been consumed.
    fn decode(&mut self, timestamp: u64, bitstream: &[u8]) -> Result<usize, DecodeError> {
        let obu = match self.codec.parser.parse_obu(bitstream)? {
            ParsedObu::Process(obu) => obu,
            // This OBU should be dropped.
            ParsedObu::Drop(length) => return Ok(length as usize),
        };
        let obu_length = obu.data.len();

        let is_decode_op = matches!(
            obu.header.obu_type,
            ObuType::Frame | ObuType::FrameHeader | ObuType::TileGroup
        );

        if is_decode_op {
            match self.decoding_state {
                /* we want to be here */
                DecodingState::Decoding => (),

                /* otherwise... */
                DecodingState::AwaitingStreamInfo => {
                    /* Skip input until we get information from the stream. */
                    return Ok(obu_length);
                }
                /* Ask the client to confirm the format before we can process this. */
                DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents),
                DecodingState::Reset => {
                    let mut parser = self.codec.parser.clone();

                    let is_key_frame = match obu.header.obu_type {
                        ObuType::Frame => {
                            let frame = parser.parse_frame_obu(obu.clone())?;
                            frame.header.frame_type == FrameType::KeyFrame
                        }
                        ObuType::FrameHeader => {
                            let fh = parser.parse_frame_header_obu(&obu)?;
                            fh.frame_type == FrameType::KeyFrame
                        }
                        _ => false,
                    };

                    /* we can only resume from key frames */
                    if !is_key_frame {
                        return Ok(obu_length);
                    } else {
                        self.decoding_state = DecodingState::Decoding;
                    }
                }
            }
        }

        /* We are in `Decoding` state if we reached here */

        match obu.header.obu_type {
            ObuType::SequenceHeader => {
                let sequence = self.codec.parser.parse_sequence_header_obu(&obu)?;
                let sequence_differs = match &self.codec.sequence {
                    Some(old_sequence) => **old_sequence != *sequence,
                    None => true,
                };

                if matches!(self.decoding_state, DecodingState::AwaitingStreamInfo)
                    || sequence_differs
                {
                    if self.codec.current_pic.is_some() {
                        return Err(DecodeError::DecoderError(anyhow!(
                                "broken stream: a picture is being decoded while a new sequence header is encountered"
                            )));
                    }

                    /* make sure we sync *before* we clear any state in the backend */
                    for f in &mut self.ready_queue.queue {
                        /* TODO: this fixes av1-1-b8-03-sizeup on Intel
                         * gen12, but we apparently do not do the same in
                         * VP9. How is it that we do not get similar crashes there?
                         *
                         * TODO: syncing before calling new_sequence() in VP9 may fix some tests
                         */
                        f.sync()?;
                    }

                    log::debug!(
                        "found new sequence, resolution: {:?}, profile: {:?}, bit depth: {:?}",
                        Resolution::from((
                            sequence.max_frame_width_minus_1 as u32 + 1,
                            sequence.max_frame_height_minus_1 as u32 + 1
                        )),
                        sequence.seq_profile,
                        sequence.bit_depth
                    );
                    /* there is nothing to drain, much like vp8 and vp9 */
                    self.codec.highest_spatial_layer = self.codec.parser.highest_operating_point();
                    self.backend
                        .new_sequence(&sequence, self.codec.highest_spatial_layer)?;
                    self.await_format_change(sequence);
                }
            }
            ObuType::TemporalDelimiter => self.codec.parser.parse_temporal_delimiter_obu(&obu)?,
            ObuType::FrameHeader => {
                if self.codec.current_pic.is_some() {
                    /* submit this frame immediately, as we need to update the
                     * DPB and the reference info state *before* processing the
                     * next frame */
                    self.submit_frame(timestamp)?;
                }
                let frame_header = self.codec.parser.parse_frame_header_obu(&obu)?;
                self.decode_frame_header(frame_header, timestamp)?;
            }
            ObuType::TileGroup => {
                let tile_group = self.codec.parser.parse_tile_group_obu(obu)?;
                self.decode_tile_group(tile_group)?;
            }
            ObuType::Frame => {
                let frame = self.codec.parser.parse_frame_obu(obu)?;
                self.decode_frame(frame, timestamp)?;
                /* submit this frame immediately, as we need to update the
                 * DPB and the reference info state *before* processing the
                 * next frame */
                self.submit_frame(timestamp)?;
            }
            ObuType::TileList => {
                return Err(DecodeError::DecoderError(anyhow!(
                    "large tile scale mode is not supported"
                )));
            }
            other => {
                log::debug!("skipping OBU of type {:?}", other);
            }
        }

        /* Submit the last frame if we have reached the end of the temporal unit. */
        if bitstream.len() == obu_length && self.codec.current_pic.is_some() {
            self.submit_frame(timestamp)?;
        }

        Ok(obu_length)
    }

    fn flush(&mut self) -> Result<(), super::DecodeError> {
        // Note: all the submitted frames are already in the ready queue.
        self.codec.reference_frames = Default::default();
        self.decoding_state = DecodingState::Reset;

        Ok(())
    }

    fn frame_pool(&mut self, layer: PoolLayer) -> Vec<&mut B::FramePool> {
        self.backend.frame_pool(layer)
    }

    fn stream_info(&self) -> Option<&crate::decoder::StreamInfo> {
        self.backend.stream_info()
    }

    fn next_event(&mut self) -> Option<crate::decoder::DecoderEvent<B::Handle>> {
        self.query_next_event(|decoder, sequence| {
            decoder.codec.sequence = Some(Rc::clone(sequence));
        })
    }

    fn poll_fd(&self) -> BorrowedFd {
        self.epoll_fd.0.as_fd()
    }
}

#[cfg(test)]
pub mod tests {
    use crate::decoder::stateless::av1::Av1;
    use crate::decoder::stateless::tests::test_decode_stream;
    use crate::decoder::stateless::tests::TestStream;
    use crate::decoder::stateless::StatelessDecoder;
    use crate::decoder::BlockingMode;
    use crate::utils::simple_playback_loop;
    use crate::utils::simple_playback_loop_owned_frames;
    use crate::utils::IvfIterator;
    use crate::DecodedFormat;

    /// Run `test` using the dummy decoder, in both blocking and non-blocking modes.
    fn test_decoder_dummy(test: &TestStream, blocking_mode: BlockingMode) {
        let decoder = StatelessDecoder::<Av1, _>::new_dummy(blocking_mode).unwrap();

        test_decode_stream(
            |d, s, f| {
                simple_playback_loop(
                    d,
                    IvfIterator::new(s),
                    f,
                    &mut simple_playback_loop_owned_frames,
                    DecodedFormat::NV12,
                    blocking_mode,
                )
            },
            decoder,
            test,
            false,
            false,
        );
    }

    /// Same as Chromium's test-25fps.av1.ivf
    pub const DECODE_TEST_25FPS: TestStream = TestStream {
        stream: include_bytes!("../../codec/av1/test_data/test-25fps.ivf.av1"),
        crcs: include_str!("../../codec/av1/test_data/test-25fps.ivf.av1.crc"),
    };

    #[test]
    fn test_25fps_block() {
        test_decoder_dummy(&DECODE_TEST_25FPS, BlockingMode::Blocking);
    }

    #[test]
    fn test_25fps_nonblock() {
        test_decoder_dummy(&DECODE_TEST_25FPS, BlockingMode::NonBlocking);
    }
}