use super::{load_u16, *};
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
const fn depack_pn<const BITS: u32, const LOW_PACKED: bool>(value: u16) -> u16 {
if LOW_PACKED {
value & ((1u16 << BITS) - 1)
} else {
value >> (16 - BITS)
}
}
#[cfg(all(any(feature = "std", feature = "alloc"), feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_2to1_center_h_p0xx<const BITS: u32, const LOW_PACKED: bool>(
uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(uv_half.len() >= width, "uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let shift = 16 - BITS;
let load = |c: usize, comp: usize| -> u32 {
let raw = uv_half[2 * c + comp];
let host = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
depack_pn::<BITS, LOW_PACKED>(host) as u32
};
let store = |logical: u32| -> u16 {
let v = if LOW_PACKED {
logical as u16
} else {
(logical as u16) << shift
};
if big_endian { v.to_be() } else { v.to_le() }
};
let half = width / 2;
for j in 0..half {
let lj = j.saturating_sub(1);
let rj = if j + 1 < half { j + 1 } else { j };
let (ul, um, ur) = (load(lj, 0), load(j, 0), load(rj, 0));
let (vl, vm, vr) = (load(lj, 1), load(j, 1), load(rj, 1));
uv_full[2 * (2 * j)] = store((ul + 3 * um + 2) >> 2);
uv_full[2 * (2 * j) + 1] = store((vl + 3 * vm + 2) >> 2);
uv_full[2 * (2 * j + 1)] = store((3 * um + ur + 2) >> 2);
uv_full[2 * (2 * j + 1) + 1] = store((3 * vm + vr + 2) >> 2);
}
}
#[cfg(all(any(feature = "std", feature = "alloc"), feature = "yuv-planar"))]
#[cfg_attr(
not(all(
feature = "yuv-semi-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_2to1_center_h_p0xx_pair<const BITS: u32, const LOW_PACKED: bool>(
uv_half: &[u16],
uv_full: &mut [u16],
j: usize,
half: usize,
big_endian: bool,
) {
let shift = 16 - BITS;
let load = |c: usize, comp: usize| -> u32 {
let raw = uv_half[2 * c + comp];
let host = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
depack_pn::<BITS, LOW_PACKED>(host) as u32
};
let store = |logical: u32| -> u16 {
let v = if LOW_PACKED {
logical as u16
} else {
(logical as u16) << shift
};
if big_endian { v.to_be() } else { v.to_le() }
};
let lj = j.saturating_sub(1);
let rj = if j + 1 < half { j + 1 } else { j };
let (ul, um, ur) = (load(lj, 0), load(j, 0), load(rj, 0));
let (vl, vm, vr) = (load(lj, 1), load(j, 1), load(rj, 1));
uv_full[2 * (2 * j)] = store((ul + 3 * um + 2) >> 2);
uv_full[2 * (2 * j) + 1] = store((vl + 3 * vm + 2) >> 2);
uv_full[2 * (2 * j + 1)] = store((3 * um + ur + 2) >> 2);
uv_full[2 * (2 * j + 1) + 1] = store((3 * vm + vr + 2) >> 2);
}
#[cfg(all(any(feature = "std", feature = "alloc"), feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottom_even_h_p0xx<const BITS: u32>(
prev_uv_half: &[u16],
cur_uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(prev_uv_half.len() >= width, "prev_uv_half row too short");
debug_assert!(cur_uv_half.len() >= width, "cur_uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let shift = 16 - BITS;
let load = |row: &[u16], c: usize, comp: usize| -> u32 {
let raw = row[2 * c + comp];
(if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
} >> shift) as u32
};
let store = |logical: u32| -> u16 {
let v = (logical as u16) << shift;
if big_endian { v.to_be() } else { v.to_le() }
};
let vblend = |c: usize, comp: usize| -> u32 {
(load(prev_uv_half, c, comp) + load(cur_uv_half, c, comp) + 1) >> 1
};
let half = width / 2;
for j in 0..half {
let lj = j.saturating_sub(1);
let rj = if j + 1 < half { j + 1 } else { j };
let (ul, um, ur) = (vblend(lj, 0), vblend(j, 0), vblend(rj, 0));
let (vl, vm, vr) = (vblend(lj, 1), vblend(j, 1), vblend(rj, 1));
uv_full[2 * (2 * j)] = store((ul + 3 * um + 2) >> 2);
uv_full[2 * (2 * j) + 1] = store((vl + 3 * vm + 2) >> 2);
uv_full[2 * (2 * j + 1)] = store((3 * um + ur + 2) >> 2);
uv_full[2 * (2 * j + 1) + 1] = store((3 * vm + vr + 2) >> 2);
}
}
#[cfg(all(any(feature = "std", feature = "alloc"), feature = "yuv-planar"))]
#[cfg_attr(
not(all(
feature = "yuv-semi-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottom_even_h_p0xx_pair<const BITS: u32>(
prev_uv_half: &[u16],
cur_uv_half: &[u16],
uv_full: &mut [u16],
j: usize,
half: usize,
big_endian: bool,
) {
let shift = 16 - BITS;
let load = |row: &[u16], c: usize, comp: usize| -> u32 {
let raw = row[2 * c + comp];
(if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
} >> shift) as u32
};
let store = |logical: u32| -> u16 {
let v = (logical as u16) << shift;
if big_endian { v.to_be() } else { v.to_le() }
};
let vblend = |c: usize, comp: usize| -> u32 {
(load(prev_uv_half, c, comp) + load(cur_uv_half, c, comp) + 1) >> 1
};
let lj = j.saturating_sub(1);
let rj = if j + 1 < half { j + 1 } else { j };
let (ul, um, ur) = (vblend(lj, 0), vblend(j, 0), vblend(rj, 0));
let (vl, vm, vr) = (vblend(lj, 1), vblend(j, 1), vblend(rj, 1));
uv_full[2 * (2 * j)] = store((ul + 3 * um + 2) >> 2);
uv_full[2 * (2 * j) + 1] = store((vl + 3 * vm + 2) >> 2);
uv_full[2 * (2 * j + 1)] = store((3 * um + ur + 2) >> 2);
uv_full[2 * (2 * j + 1) + 1] = store((3 * vm + vr + 2) >> 2);
}
#[cfg(all(any(feature = "std", feature = "alloc"), feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_2to1_cosited_h_p0xx<const BITS: u32, const LOW_PACKED: bool>(
uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(uv_half.len() >= width, "uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let shift = 16 - BITS;
let load = |c: usize, comp: usize| -> u32 {
let raw = uv_half[2 * c + comp];
let host = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
depack_pn::<BITS, LOW_PACKED>(host) as u32
};
let store = |logical: u32| -> u16 {
let v = if LOW_PACKED {
logical as u16
} else {
(logical as u16) << shift
};
if big_endian { v.to_be() } else { v.to_le() }
};
let half = width / 2;
for j in 0..half {
let u = store(load(j, 0));
let v = store(load(j, 1));
uv_full[2 * (2 * j)] = u;
uv_full[2 * (2 * j) + 1] = v;
uv_full[2 * (2 * j + 1)] = u;
uv_full[2 * (2 * j + 1) + 1] = v;
}
}
#[cfg(all(any(feature = "std", feature = "alloc"), feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottomleft_even_h_p0xx<const BITS: u32>(
prev_uv_half: &[u16],
cur_uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(prev_uv_half.len() >= width, "prev_uv_half row too short");
debug_assert!(cur_uv_half.len() >= width, "cur_uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let shift = 16 - BITS;
let load = |row: &[u16], c: usize, comp: usize| -> u32 {
let raw = row[2 * c + comp];
(if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
} >> shift) as u32
};
let store = |logical: u32| -> u16 {
let v = (logical as u16) << shift;
if big_endian { v.to_be() } else { v.to_le() }
};
let vblend = |c: usize, comp: usize| -> u32 {
(load(prev_uv_half, c, comp) + load(cur_uv_half, c, comp) + 1) >> 1
};
let half = width / 2;
for j in 0..half {
let u = store(vblend(j, 0));
let v = store(vblend(j, 1));
uv_full[2 * (2 * j)] = u;
uv_full[2 * (2 * j) + 1] = v;
uv_full[2 * (2 * j + 1)] = u;
uv_full[2 * (2 * j + 1) + 1] = v;
}
}
#[cfg(all(any(feature = "std", feature = "alloc"), feature = "yuv-planar"))]
#[cfg_attr(
not(all(
feature = "yuv-semi-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottomleft_even_h_p0xx_pair<const BITS: u32>(
prev_uv_half: &[u16],
cur_uv_half: &[u16],
uv_full: &mut [u16],
j: usize,
big_endian: bool,
) {
let shift = 16 - BITS;
let load = |row: &[u16], c: usize, comp: usize| -> u32 {
let raw = row[2 * c + comp];
(if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
} >> shift) as u32
};
let store = |logical: u32| -> u16 {
let v = (logical as u16) << shift;
if big_endian { v.to_be() } else { v.to_le() }
};
let vblend =
|comp: usize| -> u32 { (load(prev_uv_half, j, comp) + load(cur_uv_half, j, comp) + 1) >> 1 };
let u = store(vblend(0));
let v = store(vblend(1));
uv_full[2 * (2 * j)] = u;
uv_full[2 * (2 * j) + 1] = v;
uv_full[2 * (2 * j + 1)] = u;
uv_full[2 * (2 * j + 1) + 1] = v;
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_to_rgb_row<const BITS: u32, const BE: bool>(
y: &[u16],
uv_half: &[u16],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_to_rgb_or_rgba_row::<BITS, false, BE, false>(y, uv_half, rgb_out, width, matrix, full_range);
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_to_rgba_row<const BITS: u32, const BE: bool>(
y: &[u16],
uv_half: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_to_rgb_or_rgba_row::<BITS, true, BE, false>(y, uv_half, rgba_out, width, matrix, full_range);
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn nv20_to_rgb_row<const BE: bool>(
y: &[u16],
uv_half: &[u16],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_to_rgb_or_rgba_row::<10, false, BE, true>(y, uv_half, rgb_out, width, matrix, full_range);
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn nv20_to_rgba_row<const BE: bool>(
y: &[u16],
uv_half: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_to_rgb_or_rgba_row::<10, true, BE, true>(y, uv_half, rgba_out, width, matrix, full_range);
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_to_rgb_or_rgba_row<
const BITS: u32,
const ALPHA: bool,
const BE: bool,
const LOW_PACKED: bool,
>(
y: &[u16],
uv_half: &[u16],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
const { assert!(BITS == 10 || BITS == 12) };
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert_eq!(width & 1, 0, "semi-planar high-bit requires even width");
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(uv_half.len() >= width, "uv row too short");
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<BITS, 8>(full_range);
let bias = chroma_bias::<BITS>();
let mut x = 0;
while x < width {
let c_idx = x / 2;
let u_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_half[c_idx * 2]));
let v_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_half[c_idx * 2 + 1]));
let u_d = q15_scale(u_sample as i32 - bias, c_scale);
let v_d = q15_scale(v_sample as i32 - bias, c_scale);
let r_chroma = q15_chroma(coeffs.r_u(), u_d, coeffs.r_v(), v_d);
let g_chroma = q15_chroma(coeffs.g_u(), u_d, coeffs.g_v(), v_d);
let b_chroma = q15_chroma(coeffs.b_u(), u_d, coeffs.b_v(), v_d);
let y0 = q15_scale(
depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(y[x])) as i32 - y_off,
y_scale,
);
out[x * bpp] = clamp_u8(y0 + r_chroma);
out[x * bpp + 1] = clamp_u8(y0 + g_chroma);
out[x * bpp + 2] = clamp_u8(y0 + b_chroma);
if ALPHA {
out[x * bpp + 3] = 0xFF;
}
let y1 = q15_scale(
depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(y[x + 1])) as i32 - y_off,
y_scale,
);
out[(x + 1) * bpp] = clamp_u8(y1 + r_chroma);
out[(x + 1) * bpp + 1] = clamp_u8(y1 + g_chroma);
out[(x + 1) * bpp + 2] = clamp_u8(y1 + b_chroma);
if ALPHA {
out[(x + 1) * bpp + 3] = 0xFF;
}
x += 2;
}
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_to_rgb_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
uv_half: &[u16],
rgb_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_to_rgb_or_rgba_u16_row::<BITS, false, BE, false>(
y, uv_half, rgb_out, width, matrix, full_range,
);
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_to_rgba_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
uv_half: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_to_rgb_or_rgba_u16_row::<BITS, true, BE, false>(
y, uv_half, rgba_out, width, matrix, full_range,
);
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn nv20_to_rgb_u16_row<const BE: bool>(
y: &[u16],
uv_half: &[u16],
rgb_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_to_rgb_or_rgba_u16_row::<10, false, BE, true>(y, uv_half, rgb_out, width, matrix, full_range);
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn nv20_to_rgba_u16_row<const BE: bool>(
y: &[u16],
uv_half: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_to_rgb_or_rgba_u16_row::<10, true, BE, true>(y, uv_half, rgba_out, width, matrix, full_range);
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_to_rgb_or_rgba_u16_row<
const BITS: u32,
const ALPHA: bool,
const BE: bool,
const LOW_PACKED: bool,
>(
y: &[u16],
uv_half: &[u16],
out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
const { assert!(BITS == 10 || BITS == 12) };
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert_eq!(width & 1, 0, "semi-planar high-bit requires even width");
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(uv_half.len() >= width, "uv row too short");
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<BITS, BITS>(full_range);
let bias = chroma_bias::<BITS>();
let out_max: i32 = (1i32 << BITS) - 1;
let alpha_max: u16 = out_max as u16;
let mut x = 0;
while x < width {
let c_idx = x / 2;
let u_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_half[c_idx * 2]));
let v_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_half[c_idx * 2 + 1]));
let u_d = q15_scale(u_sample as i32 - bias, c_scale);
let v_d = q15_scale(v_sample as i32 - bias, c_scale);
let r_chroma = q15_chroma(coeffs.r_u(), u_d, coeffs.r_v(), v_d);
let g_chroma = q15_chroma(coeffs.g_u(), u_d, coeffs.g_v(), v_d);
let b_chroma = q15_chroma(coeffs.b_u(), u_d, coeffs.b_v(), v_d);
let y0 = q15_scale(
depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(y[x])) as i32 - y_off,
y_scale,
);
out[x * bpp] = (y0 + r_chroma).clamp(0, out_max) as u16;
out[x * bpp + 1] = (y0 + g_chroma).clamp(0, out_max) as u16;
out[x * bpp + 2] = (y0 + b_chroma).clamp(0, out_max) as u16;
if ALPHA {
out[x * bpp + 3] = alpha_max;
}
let y1 = q15_scale(
depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(y[x + 1])) as i32 - y_off,
y_scale,
);
out[(x + 1) * bpp] = (y1 + r_chroma).clamp(0, out_max) as u16;
out[(x + 1) * bpp + 1] = (y1 + g_chroma).clamp(0, out_max) as u16;
out[(x + 1) * bpp + 2] = (y1 + b_chroma).clamp(0, out_max) as u16;
if ALPHA {
out[(x + 1) * bpp + 3] = alpha_max;
}
x += 2;
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_to_rgb_row<const BITS: u32, const BE: bool>(
y: &[u16],
uv_full: &[u16],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_444_to_rgb_or_rgba_row::<BITS, false, BE, false>(
y, uv_full, rgb_out, width, matrix, full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_to_rgba_row<const BITS: u32, const BE: bool>(
y: &[u16],
uv_full: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_444_to_rgb_or_rgba_row::<BITS, true, BE, false>(
y, uv_full, rgba_out, width, matrix, full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_to_rgb_or_rgba_row<
const BITS: u32,
const ALPHA: bool,
const BE: bool,
const LOW_PACKED: bool,
>(
y: &[u16],
uv_full: &[u16],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
const { assert!(BITS == 10 || BITS == 12) };
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<BITS, 8>(full_range);
let bias = chroma_bias::<BITS>();
for x in 0..width {
let u_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_full[x * 2]));
let v_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_full[x * 2 + 1]));
let u_d = q15_scale(u_sample as i32 - bias, c_scale);
let v_d = q15_scale(v_sample as i32 - bias, c_scale);
let r_chroma = q15_chroma(coeffs.r_u(), u_d, coeffs.r_v(), v_d);
let g_chroma = q15_chroma(coeffs.g_u(), u_d, coeffs.g_v(), v_d);
let b_chroma = q15_chroma(coeffs.b_u(), u_d, coeffs.b_v(), v_d);
let y0 = q15_scale(
depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(y[x])) as i32 - y_off,
y_scale,
);
out[x * bpp] = clamp_u8(y0 + r_chroma);
out[x * bpp + 1] = clamp_u8(y0 + g_chroma);
out[x * bpp + 2] = clamp_u8(y0 + b_chroma);
if ALPHA {
out[x * bpp + 3] = 0xFF;
}
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_to_rgb_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
uv_full: &[u16],
rgb_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_444_to_rgb_or_rgba_u16_row::<BITS, false, BE, false>(
y, uv_full, rgb_out, width, matrix, full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_to_rgba_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
uv_full: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_444_to_rgb_or_rgba_u16_row::<BITS, true, BE, false>(
y, uv_full, rgba_out, width, matrix, full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_to_rgb_or_rgba_u16_row<
const BITS: u32,
const ALPHA: bool,
const BE: bool,
const LOW_PACKED: bool,
>(
y: &[u16],
uv_full: &[u16],
out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
const { assert!(BITS == 10 || BITS == 12) };
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<BITS, BITS>(full_range);
let bias = chroma_bias::<BITS>();
let out_max: i32 = (1i32 << BITS) - 1;
let alpha_max: u16 = out_max as u16;
for x in 0..width {
let u_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_full[x * 2]));
let v_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_full[x * 2 + 1]));
let u_d = q15_scale(u_sample as i32 - bias, c_scale);
let v_d = q15_scale(v_sample as i32 - bias, c_scale);
let r_chroma = q15_chroma(coeffs.r_u(), u_d, coeffs.r_v(), v_d);
let g_chroma = q15_chroma(coeffs.g_u(), u_d, coeffs.g_v(), v_d);
let b_chroma = q15_chroma(coeffs.b_u(), u_d, coeffs.b_v(), v_d);
let y0 = q15_scale(
depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(y[x])) as i32 - y_off,
y_scale,
);
out[x * bpp] = (y0 + r_chroma).clamp(0, out_max) as u16;
out[x * bpp + 1] = (y0 + g_chroma).clamp(0, out_max) as u16;
out[x * bpp + 2] = (y0 + b_chroma).clamp(0, out_max) as u16;
if ALPHA {
out[x * bpp + 3] = alpha_max;
}
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_16_to_rgb_row<const BE: bool>(
y: &[u16],
uv_full: &[u16],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_444_16_to_rgb_or_rgba_row::<false, BE>(y, uv_full, rgb_out, width, matrix, full_range);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_16_to_rgba_row<const BE: bool>(
y: &[u16],
uv_full: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_444_16_to_rgb_or_rgba_row::<true, BE>(y, uv_full, rgba_out, width, matrix, full_range);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_16_to_rgb_or_rgba_row<const ALPHA: bool, const BE: bool>(
y: &[u16],
uv_full: &[u16],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<16, 8>(full_range);
let bias = chroma_bias::<16>();
for x in 0..width {
let u_sample = load_u16::<BE>(uv_full[x * 2]);
let v_sample = load_u16::<BE>(uv_full[x * 2 + 1]);
let u_d = q15_scale(u_sample as i32 - bias, c_scale);
let v_d = q15_scale(v_sample as i32 - bias, c_scale);
let r_chroma = q15_chroma(coeffs.r_u(), u_d, coeffs.r_v(), v_d);
let g_chroma = q15_chroma(coeffs.g_u(), u_d, coeffs.g_v(), v_d);
let b_chroma = q15_chroma(coeffs.b_u(), u_d, coeffs.b_v(), v_d);
let y0 = q15_scale(load_u16::<BE>(y[x]) as i32 - y_off, y_scale);
out[x * bpp] = clamp_u8(y0 + r_chroma);
out[x * bpp + 1] = clamp_u8(y0 + g_chroma);
out[x * bpp + 2] = clamp_u8(y0 + b_chroma);
if ALPHA {
out[x * bpp + 3] = 0xFF;
}
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_16_to_rgb_u16_row<const BE: bool>(
y: &[u16],
uv_full: &[u16],
rgb_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_444_16_to_rgb_or_rgba_u16_row::<false, BE>(y, uv_full, rgb_out, width, matrix, full_range);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_16_to_rgba_u16_row<const BE: bool>(
y: &[u16],
uv_full: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_444_16_to_rgb_or_rgba_u16_row::<true, BE>(y, uv_full, rgba_out, width, matrix, full_range);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_444_16_to_rgb_or_rgba_u16_row<const ALPHA: bool, const BE: bool>(
y: &[u16],
uv_full: &[u16],
out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<16, 16>(full_range);
let bias = chroma_bias::<16>();
let out_max: i32 = 0xFFFF;
for x in 0..width {
let u_sample = load_u16::<BE>(uv_full[x * 2]);
let v_sample = load_u16::<BE>(uv_full[x * 2 + 1]);
let u_d = q15_scale(u_sample as i32 - bias, c_scale);
let v_d = q15_scale(v_sample as i32 - bias, c_scale);
let r_chroma = q15_chroma64(coeffs.r_u(), u_d, coeffs.r_v(), v_d);
let g_chroma = q15_chroma64(coeffs.g_u(), u_d, coeffs.g_v(), v_d);
let b_chroma = q15_chroma64(coeffs.b_u(), u_d, coeffs.b_v(), v_d);
let y0 = q15_scale64(load_u16::<BE>(y[x]) as i32 - y_off, y_scale);
out[x * bpp] = (y0 + r_chroma).clamp(0, out_max) as u16;
out[x * bpp + 1] = (y0 + g_chroma).clamp(0, out_max) as u16;
out[x * bpp + 2] = (y0 + b_chroma).clamp(0, out_max) as u16;
if ALPHA {
out[x * bpp + 3] = 0xFFFF;
}
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn p_n_444_to_hsv_row<const BITS: u32, const BE: bool, const LOW_PACKED: bool>(
y: &[u16],
uv_full: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
const { assert!(BITS == 10 || BITS == 12) };
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<BITS, 8>(full_range);
let bias = chroma_bias::<BITS>();
for x in 0..width {
let u_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_full[x * 2]));
let v_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_full[x * 2 + 1]));
let u_d = q15_scale(u_sample as i32 - bias, c_scale);
let v_d = q15_scale(v_sample as i32 - bias, c_scale);
let r_chroma = q15_chroma(coeffs.r_u(), u_d, coeffs.r_v(), v_d);
let g_chroma = q15_chroma(coeffs.g_u(), u_d, coeffs.g_v(), v_d);
let b_chroma = q15_chroma(coeffs.b_u(), u_d, coeffs.b_v(), v_d);
let y0 = q15_scale(
depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(y[x])) as i32 - y_off,
y_scale,
);
let (h, s, v) = rgb_to_hsv_pixel(
clamp_u8(y0 + r_chroma) as i32,
clamp_u8(y0 + g_chroma) as i32,
clamp_u8(y0 + b_chroma) as i32,
);
h_out[x] = h;
s_out[x] = s;
v_out[x] = v;
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn p_n_444_16_to_hsv_row<const BE: bool>(
y: &[u16],
uv_full: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<16, 8>(full_range);
let bias = chroma_bias::<16>();
for x in 0..width {
let u_sample = load_u16::<BE>(uv_full[x * 2]);
let v_sample = load_u16::<BE>(uv_full[x * 2 + 1]);
let u_d = q15_scale(u_sample as i32 - bias, c_scale);
let v_d = q15_scale(v_sample as i32 - bias, c_scale);
let r_chroma = q15_chroma(coeffs.r_u(), u_d, coeffs.r_v(), v_d);
let g_chroma = q15_chroma(coeffs.g_u(), u_d, coeffs.g_v(), v_d);
let b_chroma = q15_chroma(coeffs.b_u(), u_d, coeffs.b_v(), v_d);
let y0 = q15_scale(load_u16::<BE>(y[x]) as i32 - y_off, y_scale);
let (h, s, v) = rgb_to_hsv_pixel(
clamp_u8(y0 + r_chroma) as i32,
clamp_u8(y0 + g_chroma) as i32,
clamp_u8(y0 + b_chroma) as i32,
);
h_out[x] = h;
s_out[x] = s;
v_out[x] = v;
}
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn p_n_to_hsv_row<const BITS: u32, const BE: bool, const LOW_PACKED: bool>(
y: &[u16],
uv_half: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
const { assert!(BITS == 10 || BITS == 12) };
debug_assert_eq!(width & 1, 0, "semi-planar high-bit requires even width");
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(uv_half.len() >= width, "uv row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<BITS, 8>(full_range);
let bias = chroma_bias::<BITS>();
let mut x = 0;
while x < width {
let c_idx = x / 2;
let u_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_half[c_idx * 2]));
let v_sample = depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(uv_half[c_idx * 2 + 1]));
let u_d = q15_scale(u_sample as i32 - bias, c_scale);
let v_d = q15_scale(v_sample as i32 - bias, c_scale);
let r_chroma = q15_chroma(coeffs.r_u(), u_d, coeffs.r_v(), v_d);
let g_chroma = q15_chroma(coeffs.g_u(), u_d, coeffs.g_v(), v_d);
let b_chroma = q15_chroma(coeffs.b_u(), u_d, coeffs.b_v(), v_d);
let y0 = q15_scale(
depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(y[x])) as i32 - y_off,
y_scale,
);
let (h0, s0, v0) = rgb_to_hsv_pixel(
clamp_u8(y0 + r_chroma) as i32,
clamp_u8(y0 + g_chroma) as i32,
clamp_u8(y0 + b_chroma) as i32,
);
h_out[x] = h0;
s_out[x] = s0;
v_out[x] = v0;
let y1 = q15_scale(
depack_pn::<BITS, LOW_PACKED>(load_u16::<BE>(y[x + 1])) as i32 - y_off,
y_scale,
);
let (h1, s1, v1) = rgb_to_hsv_pixel(
clamp_u8(y1 + r_chroma) as i32,
clamp_u8(y1 + g_chroma) as i32,
clamp_u8(y1 + b_chroma) as i32,
);
h_out[x + 1] = h1;
s_out[x + 1] = s1;
v_out[x + 1] = v1;
x += 2;
}
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn nv20_to_hsv_row<const BE: bool>(
y: &[u16],
uv_half: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
p_n_to_hsv_row::<10, BE, true>(y, uv_half, h_out, s_out, v_out, width, matrix, full_range);
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn p_n_to_luma_row<const BITS: u32, const BE: bool>(
y: &[u16],
luma_out: &mut [u8],
width: usize,
) {
const { assert!(BITS == 10 || BITS == 12 || BITS == 16) };
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(luma_out.len() >= width, "luma_out row too short");
for (d, &s) in luma_out[..width].iter_mut().zip(y[..width].iter()) {
*d = (load_u16::<BE>(s) >> 8) as u8;
}
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn nv20_to_luma_row<const BE: bool>(y: &[u16], luma_out: &mut [u8], width: usize) {
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(luma_out.len() >= width, "luma_out row too short");
for (d, &s) in luma_out[..width].iter_mut().zip(y[..width].iter()) {
*d = (depack_pn::<10, true>(load_u16::<BE>(s)) >> 2) as u8;
}
}