use crate::context::MAX_TX_SIZE;
use crate::cpu_features::CpuFeatureLevel;
use crate::predict::{
rust, IntraEdgeFilterParameters, PredictionMode, PredictionVariant,
};
use crate::tiling::PlaneRegionMut;
use crate::transform::TxSize;
use crate::util::Aligned;
use crate::Pixel;
use v_frame::pixel::PixelType;
macro_rules! decl_angular_ipred_fn {
($($f:ident),+) => {
extern {
$(
fn $f(
dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
width: libc::c_int, height: libc::c_int, angle: libc::c_int,
);
)*
}
};
}
decl_angular_ipred_fn! {
rav1e_ipred_dc_8bpc_avx2,
rav1e_ipred_dc_8bpc_ssse3,
rav1e_ipred_dc_128_8bpc_avx2,
rav1e_ipred_dc_128_8bpc_ssse3,
rav1e_ipred_dc_left_8bpc_avx2,
rav1e_ipred_dc_left_8bpc_ssse3,
rav1e_ipred_dc_top_8bpc_avx2,
rav1e_ipred_dc_top_8bpc_ssse3,
rav1e_ipred_v_8bpc_avx2,
rav1e_ipred_v_8bpc_ssse3,
rav1e_ipred_h_8bpc_avx2,
rav1e_ipred_h_8bpc_ssse3,
rav1e_ipred_z1_8bpc_avx2,
rav1e_ipred_z3_8bpc_avx2,
rav1e_ipred_smooth_8bpc_avx2,
rav1e_ipred_smooth_8bpc_ssse3,
rav1e_ipred_smooth_v_8bpc_avx2,
rav1e_ipred_smooth_v_8bpc_ssse3,
rav1e_ipred_smooth_h_8bpc_avx2,
rav1e_ipred_smooth_h_8bpc_ssse3,
rav1e_ipred_paeth_8bpc_avx2,
rav1e_ipred_paeth_8bpc_ssse3
}
macro_rules! decl_angular_ipred_hbd_fn {
($($f:ident),+) => {
extern {
$(
fn $f(
dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
width: libc::c_int, height: libc::c_int, angle: libc::c_int,
max_width: libc::c_int, max_height: libc::c_int,
bit_depth_max: libc::c_int,
);
)*
}
};
}
decl_angular_ipred_hbd_fn! {
rav1e_ipred_dc_16bpc_avx2,
rav1e_ipred_dc_128_16bpc_avx2,
rav1e_ipred_dc_left_16bpc_avx2,
rav1e_ipred_dc_top_16bpc_avx2,
rav1e_ipred_v_16bpc_avx2,
rav1e_ipred_h_16bpc_avx2,
rav1e_ipred_z1_16bpc_avx2,
rav1e_ipred_z3_16bpc_avx2,
rav1e_ipred_smooth_16bpc_avx2,
rav1e_ipred_smooth_v_16bpc_avx2,
rav1e_ipred_smooth_h_16bpc_avx2,
rav1e_ipred_paeth_16bpc_avx2
}
extern {
fn rav1e_ipred_z2_8bpc_avx2(
dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
width: libc::c_int, height: libc::c_int, angle: libc::c_int,
dx: libc::c_int, dy: libc::c_int,
);
fn rav1e_ipred_z2_16bpc_avx2(
dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
width: libc::c_int, height: libc::c_int, angle: libc::c_int,
dx: libc::c_int, dy: libc::c_int, bit_depth_max: libc::c_int,
);
}
macro_rules! decl_cfl_pred_fn {
($($f:ident),+) => {
extern {
$(
fn $f(
dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
width: libc::c_int, height: libc::c_int, ac: *const i16,
alpha: libc::c_int,
);
)*
}
};
}
decl_cfl_pred_fn! {
rav1e_ipred_cfl_8bpc_avx2,
rav1e_ipred_cfl_8bpc_ssse3,
rav1e_ipred_cfl_128_8bpc_avx2,
rav1e_ipred_cfl_128_8bpc_ssse3,
rav1e_ipred_cfl_left_8bpc_avx2,
rav1e_ipred_cfl_left_8bpc_ssse3,
rav1e_ipred_cfl_top_8bpc_avx2,
rav1e_ipred_cfl_top_8bpc_ssse3
}
macro_rules! decl_cfl_pred_hbd_fn {
($($f:ident),+) => {
extern {
$(
fn $f(
dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
width: libc::c_int, height: libc::c_int, ac: *const i16,
alpha: libc::c_int, bit_depth_max: libc::c_int,
);
)*
}
};
}
decl_cfl_pred_hbd_fn! {
rav1e_ipred_cfl_16bpc_avx2,
rav1e_ipred_cfl_128_16bpc_avx2,
rav1e_ipred_cfl_left_16bpc_avx2,
rav1e_ipred_cfl_top_16bpc_avx2
}
#[inline(always)]
pub fn dispatch_predict_intra<T: Pixel>(
mode: PredictionMode, variant: PredictionVariant,
dst: &mut PlaneRegionMut<'_, T>, tx_size: TxSize, bit_depth: usize,
ac: &[i16], angle: isize, ief_params: Option<IntraEdgeFilterParameters>,
edge_buf: &Aligned<[T; 4 * MAX_TX_SIZE + 1]>, cpu: CpuFeatureLevel,
) {
let call_rust = |dst: &mut PlaneRegionMut<'_, T>| {
rust::dispatch_predict_intra(
mode, variant, dst, tx_size, bit_depth, ac, angle, ief_params, edge_buf,
cpu,
);
};
unsafe {
let stride = T::to_asm_stride(dst.plane_cfg.stride) as libc::ptrdiff_t;
let w = tx_size.width() as libc::c_int;
let h = tx_size.height() as libc::c_int;
let angle = angle as libc::c_int;
match T::type_enum() {
PixelType::U8 if cpu >= CpuFeatureLevel::SSSE3 => {
let dst_ptr = dst.data_ptr_mut() as *mut _;
let edge_ptr =
edge_buf.data.as_ptr().offset(2 * MAX_TX_SIZE as isize) as *const _;
if cpu >= CpuFeatureLevel::AVX2 {
match mode {
PredictionMode::DC_PRED => {
(match variant {
PredictionVariant::NONE => rav1e_ipred_dc_128_8bpc_avx2,
PredictionVariant::LEFT => rav1e_ipred_dc_left_8bpc_avx2,
PredictionVariant::TOP => rav1e_ipred_dc_top_8bpc_avx2,
PredictionVariant::BOTH => rav1e_ipred_dc_8bpc_avx2,
})(dst_ptr, stride, edge_ptr, w, h, angle);
}
PredictionMode::V_PRED if angle == 90 => {
rav1e_ipred_v_8bpc_avx2(dst_ptr, stride, edge_ptr, w, h, angle);
}
PredictionMode::H_PRED if angle == 180 => {
rav1e_ipred_h_8bpc_avx2(dst_ptr, stride, edge_ptr, w, h, angle);
}
PredictionMode::V_PRED
| PredictionMode::H_PRED
| PredictionMode::D45_PRED
| PredictionMode::D135_PRED
| PredictionMode::D113_PRED
| PredictionMode::D157_PRED
| PredictionMode::D203_PRED
| PredictionMode::D67_PRED => {
let (enable_ief, ief_smooth_filter) =
if let Some(params) = ief_params {
(
true as libc::c_int,
params.use_smooth_filter() as libc::c_int,
)
} else {
(false as libc::c_int, false as libc::c_int)
};
let angle_arg =
angle | (enable_ief << 10) | (ief_smooth_filter << 9);
let (bw, bh) = (
((dst.plane_cfg.width + 7) >> 3) << 3,
((dst.plane_cfg.height + 7) >> 3) << 3,
);
let (dx, dy) = (
(bw as isize - dst.rect().x) as libc::c_int,
(bh as isize - dst.rect().y) as libc::c_int,
);
if angle <= 90 {
rav1e_ipred_z1_8bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle_arg,
);
} else if angle < 180 {
rav1e_ipred_z2_8bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle_arg, dx, dy,
);
} else {
rav1e_ipred_z3_8bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle_arg,
);
}
}
PredictionMode::SMOOTH_PRED => {
rav1e_ipred_smooth_8bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle,
);
}
PredictionMode::SMOOTH_V_PRED => {
rav1e_ipred_smooth_v_8bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle,
);
}
PredictionMode::SMOOTH_H_PRED => {
rav1e_ipred_smooth_h_8bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle,
);
}
PredictionMode::PAETH_PRED => {
rav1e_ipred_paeth_8bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle,
);
}
PredictionMode::UV_CFL_PRED => {
let ac_ptr = ac.as_ptr() as *const _;
(match variant {
PredictionVariant::NONE => rav1e_ipred_cfl_128_8bpc_avx2,
PredictionVariant::LEFT => rav1e_ipred_cfl_left_8bpc_avx2,
PredictionVariant::TOP => rav1e_ipred_cfl_top_8bpc_avx2,
PredictionVariant::BOTH => rav1e_ipred_cfl_8bpc_avx2,
})(dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle);
}
_ => call_rust(dst),
}
} else if cpu >= CpuFeatureLevel::SSSE3 {
match mode {
PredictionMode::DC_PRED => {
(match variant {
PredictionVariant::NONE => rav1e_ipred_dc_128_8bpc_ssse3,
PredictionVariant::LEFT => rav1e_ipred_dc_left_8bpc_ssse3,
PredictionVariant::TOP => rav1e_ipred_dc_top_8bpc_ssse3,
PredictionVariant::BOTH => rav1e_ipred_dc_8bpc_ssse3,
})(dst_ptr, stride, edge_ptr, w, h, angle);
}
PredictionMode::V_PRED if angle == 90 => {
rav1e_ipred_v_8bpc_ssse3(dst_ptr, stride, edge_ptr, w, h, angle);
}
PredictionMode::H_PRED if angle == 180 => {
rav1e_ipred_h_8bpc_ssse3(dst_ptr, stride, edge_ptr, w, h, angle);
}
PredictionMode::SMOOTH_PRED => {
rav1e_ipred_smooth_8bpc_ssse3(
dst_ptr, stride, edge_ptr, w, h, angle,
);
}
PredictionMode::SMOOTH_V_PRED => {
rav1e_ipred_smooth_v_8bpc_ssse3(
dst_ptr, stride, edge_ptr, w, h, angle,
);
}
PredictionMode::SMOOTH_H_PRED => {
rav1e_ipred_smooth_h_8bpc_ssse3(
dst_ptr, stride, edge_ptr, w, h, angle,
);
}
PredictionMode::PAETH_PRED => {
rav1e_ipred_paeth_8bpc_ssse3(
dst_ptr, stride, edge_ptr, w, h, angle,
);
}
PredictionMode::UV_CFL_PRED => {
let ac_ptr = ac.as_ptr() as *const _;
(match variant {
PredictionVariant::NONE => rav1e_ipred_cfl_128_8bpc_ssse3,
PredictionVariant::LEFT => rav1e_ipred_cfl_left_8bpc_ssse3,
PredictionVariant::TOP => rav1e_ipred_cfl_top_8bpc_ssse3,
PredictionVariant::BOTH => rav1e_ipred_cfl_8bpc_ssse3,
})(dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle);
}
_ => call_rust(dst),
}
}
}
PixelType::U16 if cpu >= CpuFeatureLevel::AVX2 && bit_depth > 8 => {
let dst_ptr = dst.data_ptr_mut() as *mut _;
let edge_ptr =
edge_buf.data.as_ptr().offset(2 * MAX_TX_SIZE as isize) as *const _;
let bd_max = (1 << bit_depth) - 1;
match mode {
PredictionMode::DC_PRED => {
(match variant {
PredictionVariant::NONE => rav1e_ipred_dc_128_16bpc_avx2,
PredictionVariant::LEFT => rav1e_ipred_dc_left_16bpc_avx2,
PredictionVariant::TOP => rav1e_ipred_dc_top_16bpc_avx2,
PredictionVariant::BOTH => rav1e_ipred_dc_16bpc_avx2,
})(
dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max
);
}
PredictionMode::V_PRED if angle == 90 => {
rav1e_ipred_v_16bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
);
}
PredictionMode::H_PRED if angle == 180 => {
rav1e_ipred_h_16bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
);
}
PredictionMode::V_PRED
| PredictionMode::H_PRED
| PredictionMode::D45_PRED
| PredictionMode::D135_PRED
| PredictionMode::D113_PRED
| PredictionMode::D157_PRED
| PredictionMode::D203_PRED
| PredictionMode::D67_PRED => {
let (enable_ief, ief_smooth_filter) = if let Some(params) =
ief_params
{
(true as libc::c_int, params.use_smooth_filter() as libc::c_int)
} else {
(false as libc::c_int, false as libc::c_int)
};
let angle_arg =
angle | (enable_ief << 10) | (ief_smooth_filter << 9);
let (bw, bh) = (
((dst.plane_cfg.width + 7) >> 3) << 3,
((dst.plane_cfg.height + 7) >> 3) << 3,
);
let (dx, dy) = (
(bw as isize - dst.rect().x) as libc::c_int,
(bh as isize - dst.rect().y) as libc::c_int,
);
if angle <= 90 {
rav1e_ipred_z1_16bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle_arg, 0, 0, bd_max,
);
} else if angle < 180 {
rav1e_ipred_z2_16bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle_arg, dx, dy, bd_max,
);
} else {
rav1e_ipred_z3_16bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle_arg, 0, 0, bd_max,
);
}
}
PredictionMode::SMOOTH_PRED => {
rav1e_ipred_smooth_16bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
);
}
PredictionMode::SMOOTH_V_PRED => {
rav1e_ipred_smooth_v_16bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
);
}
PredictionMode::SMOOTH_H_PRED => {
rav1e_ipred_smooth_h_16bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
);
}
PredictionMode::PAETH_PRED => {
rav1e_ipred_paeth_16bpc_avx2(
dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
);
}
PredictionMode::UV_CFL_PRED => {
let ac_ptr = ac.as_ptr() as *const _;
(match variant {
PredictionVariant::NONE => rav1e_ipred_cfl_128_16bpc_avx2,
PredictionVariant::LEFT => rav1e_ipred_cfl_left_16bpc_avx2,
PredictionVariant::TOP => rav1e_ipred_cfl_top_16bpc_avx2,
PredictionVariant::BOTH => rav1e_ipred_cfl_16bpc_avx2,
})(
dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle, bd_max
);
}
_ => call_rust(dst),
}
}
_ => call_rust(dst),
}
}
}