use super::{
GeometryOverflow, HsvFrameMut, MixedSinkerError, deinterleave_y_high_bit_masked,
frozen_outputs_check, planar_8bit::native_preflight_core_check_only,
};
use crate::{
KernelMatrix,
resample::{AreaStream, PlanGeometry, ResampleError, ResamplePlan, try_box, try_zeroed},
row::{
expand_rgb_to_rgba_row, expand_rgb_u16_to_rgba_u16_row, rgb_to_hsv_row,
yuv444p9_to_rgb_row_endian, yuv444p9_to_rgb_u16_row_endian, yuv444p10_to_rgb_row_endian,
yuv444p10_to_rgb_u16_row_endian, yuv444p12_to_rgb_row_endian, yuv444p12_to_rgb_u16_row_endian,
yuv444p14_to_rgb_row_endian, yuv444p14_to_rgb_u16_row_endian, yuv444p16_to_rgb_row_endian,
yuv444p16_to_rgb_u16_row_endian,
},
};
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
std::thread_local! {
static FORCE_PLANAR_HB_NATIVE_ALLOC_FAILURE: core::cell::Cell<Option<usize>> =
const { core::cell::Cell::new(None) };
}
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
pub(crate) fn arm_planar_hb_native_alloc_failure() {
FORCE_PLANAR_HB_NATIVE_ALLOC_FAILURE.with(|f| f.set(Some(0)));
}
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
pub(crate) fn arm_planar_hb_native_alloc_failure_after(skip: usize) {
FORCE_PLANAR_HB_NATIVE_ALLOC_FAILURE.with(|f| f.set(Some(skip)));
}
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
std::thread_local! {
static FORCE_PLANAR_HB_NATIVE_CHROMA_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(
test,
feature = "std",
feature = "yuv-planar",
any(
feature = "rgb",
feature = "v210",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-semi-planar"
)
))]
pub(crate) fn arm_planar_hb_native_chroma_failure() {
FORCE_PLANAR_HB_NATIVE_CHROMA_FAILURE.with(|f| f.set(true));
}
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
std::thread_local! {
static FORCE_PLANAR_HB_NATIVE_RGB_U16_SCRATCH_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
pub(crate) fn arm_planar_hb_native_rgb_u16_scratch_failure() {
FORCE_PLANAR_HB_NATIVE_RGB_U16_SCRATCH_FAILURE.with(|f| f.set(true));
}
pub(crate) struct NativePlanarYuvU16 {
y: AreaStream<u16>,
y_src: std::vec::Vec<u16>,
y_stage: std::vec::Vec<u16>,
chroma: Option<NativePlanarChromaU16>,
chroma_w: usize,
chroma_vsub: usize,
staged: [[bool; 2]; 3],
next_emit: usize,
chroma_centered: bool,
chroma_bottom: bool,
chroma_top: bool,
}
struct NativePlanarChromaU16 {
u: AreaStream<u16>,
v: AreaStream<u16>,
u_src: std::vec::Vec<u16>,
v_src: std::vec::Vec<u16>,
u_stage: std::vec::Vec<u16>,
v_stage: std::vec::Vec<u16>,
}
impl NativePlanarYuvU16 {
fn new(
plan: &ResamplePlan,
build_chroma_plan: impl FnOnce() -> Result<ResamplePlan, ResampleError>,
chroma_vsub: usize,
chroma_w: usize,
w: usize,
h: usize,
need_color: bool,
) -> Result<Self, ResampleError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter());
}
let y = AreaStream::new(plan.h(), plan.v(), w, h, 1)?;
let alloc =
|_| ResampleError::AllocationFailed(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()));
let stage_len = plan.out_w().checked_mul(2).ok_or_else(|| {
ResampleError::Overflow(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()))
})?;
let mut chroma_centered = false;
let mut chroma_bottom = false;
let mut chroma_top = false;
let chroma = if need_color {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if FORCE_PLANAR_HB_NATIVE_CHROMA_FAILURE.with(|f| f.take()) {
return Err(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)));
}
let chroma_plan = build_chroma_plan()?;
chroma_centered = chroma_plan.has_chroma_phase();
chroma_bottom = chroma_plan.has_chroma_v_phase();
chroma_top = chroma_plan.has_chroma_v_top();
Some(NativePlanarChromaU16 {
u: AreaStream::new(
chroma_plan.h(),
chroma_plan.v(),
chroma_plan.src_w(),
chroma_plan.src_h(),
1,
)?,
v: AreaStream::new(
chroma_plan.h(),
chroma_plan.v(),
chroma_plan.src_w(),
chroma_plan.src_h(),
1,
)?,
u_src: std::vec::Vec::new(),
v_src: std::vec::Vec::new(),
u_stage: try_zeroed(stage_len).map_err(alloc)?,
v_stage: try_zeroed(stage_len).map_err(alloc)?,
})
} else {
None
};
Ok(Self {
y,
y_src: std::vec::Vec::new(),
y_stage: try_zeroed(stage_len).map_err(alloc)?,
chroma,
chroma_w,
chroma_vsub,
staged: [[false; 2]; 3],
next_emit: 0,
chroma_centered,
chroma_bottom,
chroma_top,
})
}
pub(crate) fn reset(&mut self) {
self.y.reset();
if let Some(chroma) = self.chroma.as_mut() {
chroma.u.reset();
chroma.v.reset();
}
self.staged = [[false; 2]; 3];
self.next_emit = 0;
}
pub(crate) fn next_y(&self) -> usize {
self.y.next_y()
}
pub(crate) fn chroma_phase_centered(&self) -> Option<bool> {
self.chroma.as_ref().map(|_| self.chroma_centered)
}
pub(crate) fn chroma_bottom(&self) -> Option<bool> {
self.chroma.as_ref().map(|_| self.chroma_bottom)
}
pub(crate) fn chroma_top(&self) -> Option<bool> {
self.chroma.as_ref().map(|_| self.chroma_top)
}
fn check_sequence(&self, idx: usize) -> Result<(), MixedSinkerError> {
if self.y.next_y() != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(self.y.next_y(), idx),
)));
}
if let Some(chroma) = self.chroma.as_ref() {
let chroma_expected = idx.div_ceil(self.chroma_vsub);
for stream in [&chroma.u, &chroma.v] {
if stream.next_y() != chroma_expected {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(
stream.next_y().saturating_mul(self.chroma_vsub),
idx,
),
)));
}
}
}
Ok(())
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn native_planar_hb_preflight_check_only(
join: &Option<std::boxed::Box<NativePlanarYuvU16>>,
resample_outputs: &Option<super::FrozenOutputs>,
rgb: &Option<&mut [u8]>,
rgba: &Option<&mut [u8]>,
rgb_u16: &Option<&mut [u16]>,
rgba_u16: &Option<&mut [u16]>,
luma: &Option<&mut [u8]>,
luma_u16: &Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
idx: usize,
need_luma: bool,
need_color: bool,
) -> Result<bool, MixedSinkerError> {
native_preflight_core_check_only(
join.as_ref().map_or(0, |j| j.next_y()),
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn grow_src_scratch(
scratch: &mut std::vec::Vec<u16>,
len: usize,
w: usize,
h: usize,
plan: &ResamplePlan,
) -> Result<(), MixedSinkerError> {
if scratch.len() < len {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if let Some(remaining) = FORCE_PLANAR_HB_NATIVE_ALLOC_FAILURE.with(|f| f.get()) {
if remaining == 0 {
FORCE_PLANAR_HB_NATIVE_ALLOC_FAILURE.with(|f| f.set(None));
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
)));
}
FORCE_PLANAR_HB_NATIVE_ALLOC_FAILURE.with(|f| f.set(Some(remaining - 1)));
}
scratch
.try_reserve_exact(len - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?;
scratch.resize(len, 0);
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_planar16_process_native<const BITS: u32, const BE: bool>(
plan: &ResamplePlan,
native: &mut Option<std::boxed::Box<NativePlanarYuvU16>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
rgb_scratch_u16: &mut std::vec::Vec<u16>,
y_row: &[u16],
u_row: &[u16],
v_row: &[u16],
matrix: KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
chroma_vsub: usize,
chroma_w: usize,
build_chroma_plan: impl FnOnce() -> Result<ResamplePlan, ResampleError>,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
const {
assert!(
BITS > 8 && BITS <= 16,
"BITS must be in (8, 16] for high-bit non-4:2:0 planar YUV"
)
};
let ow = plan.out_w();
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color_u8 = rgb.is_some() || rgba.is_some() || hsv.is_some();
let need_color_u16 = rgb_u16.is_some() || rgba_u16.is_some();
let need_color = need_color_u8 || need_color_u16;
if !native_planar_hb_preflight_check_only(
native,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
let rebuild = native.is_none()
|| native
.as_ref()
.is_some_and(|join| join.chroma.is_some() != need_color);
let mut new_join = if rebuild {
let join = NativePlanarYuvU16::new(
plan,
build_chroma_plan,
chroma_vsub,
chroma_w,
w,
h,
need_color,
)?;
Some(try_box(join).map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?)
} else {
None
};
if need_color_u8 {
let row_bytes =
ow.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
ow,
plan.out_h(),
3,
)))?;
if rgb_scratch.len() < row_bytes {
rgb_scratch
.try_reserve_exact(row_bytes - rgb_scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?;
rgb_scratch.resize(row_bytes, 0);
}
}
if need_color_u16 {
let row_elems =
ow.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
ow,
plan.out_h(),
3,
)))?;
if rgb_scratch_u16.len() < row_elems {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if FORCE_PLANAR_HB_NATIVE_RGB_U16_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
)));
}
rgb_scratch_u16
.try_reserve_exact(row_elems - rgb_scratch_u16.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?;
rgb_scratch_u16.resize(row_elems, 0);
}
}
let feed_chroma;
{
let target: &mut NativePlanarYuvU16 = match new_join.as_deref_mut() {
Some(join) => join,
None => native.as_deref_mut().expect("reuse implies a cached join"),
};
grow_src_scratch(&mut target.y_src, w, w, h, plan)?;
feed_chroma = target.chroma.is_some() && idx.is_multiple_of(target.chroma_vsub);
if feed_chroma {
let cw = target.chroma_w;
let chroma = target.chroma.as_mut().expect("feed_chroma implies Some");
grow_src_scratch(&mut chroma.u_src, cw, w, h, plan)?;
grow_src_scratch(&mut chroma.v_src, cw, w, h, plan)?;
}
}
if let Some(boxed) = new_join {
*native = Some(boxed);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let join = native.as_mut().expect("built or reused in the preflight");
join.check_sequence(idx)?;
deinterleave_y_high_bit_masked::<BITS, BE>(y_row, &mut join.y_src, w);
if feed_chroma {
let cw = join.chroma_w;
let chroma = join.chroma.as_mut().expect("feed_chroma implies Some");
deinterleave_y_high_bit_masked::<BITS, BE>(u_row, &mut chroma.u_src, cw);
deinterleave_y_high_bit_masked::<BITS, BE>(v_row, &mut chroma.v_src, cw);
}
let NativePlanarYuvU16 {
y,
y_src,
y_stage,
chroma,
chroma_w,
chroma_vsub,
staged,
next_emit,
chroma_centered: _,
chroma_bottom: _,
chroma_top: _,
} = &mut **join;
y.feed_row(idx, &y_src[..w], use_simd, |oy, out_row| {
let slot = oy & 1;
y_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[0][slot] = true;
})?;
if let Some(c) = chroma.as_mut()
&& idx.is_multiple_of(*chroma_vsub)
{
let cidx = idx / *chroma_vsub;
let cw = *chroma_w;
let NativePlanarChromaU16 {
u,
v,
u_src,
v_src,
u_stage,
v_stage,
} = c;
u.feed_row(cidx, &u_src[..cw], use_simd, |oy, out_row| {
let slot = oy & 1;
u_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[1][slot] = true;
})?;
v.feed_row(cidx, &v_src[..cw], use_simd, |oy, out_row| {
let slot = oy & 1;
v_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[2][slot] = true;
})?;
}
while *next_emit < plan.out_h() {
let slot = *next_emit & 1;
let chroma_ready = match chroma.as_ref() {
Some(_) => staged[1][slot] && staged[2][slot],
None => true,
};
if !(staged[0][slot] && chroma_ready) {
break;
}
let oy = *next_emit;
let y_out = &y_stage[slot * ow..slot * ow + ow];
if need_luma {
let native_max: u16 = ((1u32 << BITS) - 1) as u16;
if let Some(buf) = luma.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(y_out) {
*dst = (src.min(native_max) >> (BITS - 8)) as u8;
}
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(y_out) {
*dst = src.min(native_max);
}
}
}
if let Some(c) = chroma.as_ref() {
let u_out = &c.u_stage[slot * ow..slot * ow + ow];
let v_out = &c.v_stage[slot * ow..slot * ow + ow];
if need_color_u16 {
let out_rgb = &mut rgb_scratch_u16[..ow * 3];
emit_rgb_u16::<BITS>(
y_out, u_out, v_out, out_rgb, ow, matrix, full_range, use_simd,
);
if let Some(buf) = rgb_u16.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_rgb);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
expand_rgb_u16_to_rgba_u16_row::<BITS>(
out_rgb,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
);
}
}
if need_color_u8 {
let out_rgb = &mut rgb_scratch[..ow * 3];
emit_rgb_u8::<BITS>(
y_out, u_out, v_out, out_rgb, ow, matrix, full_range, use_simd,
);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_rgb);
}
if let Some(hsv) = hsv.as_mut() {
let (hp, sp, vp) = hsv.hsv();
rgb_to_hsv_row(
out_rgb,
&mut hp[oy * ow..(oy + 1) * ow],
&mut sp[oy * ow..(oy + 1) * ow],
&mut vp[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
expand_rgb_to_rgba_row(out_rgb, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
}
}
staged[0][slot] = false;
staged[1][slot] = false;
staged[2][slot] = false;
*next_emit += 1;
}
Ok(())
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
fn emit_rgb_u16<const BITS: u32>(
y: &[u16],
u: &[u16],
v: &[u16],
rgb_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
match BITS {
9 => yuv444p9_to_rgb_u16_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
10 => yuv444p10_to_rgb_u16_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
12 => yuv444p12_to_rgb_u16_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
14 => yuv444p14_to_rgb_u16_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
16 => yuv444p16_to_rgb_u16_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
_ => unreachable!("BITS pinned to 9/10/12/14/16 by the call sites"),
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
fn emit_rgb_u8<const BITS: u32>(
y: &[u16],
u: &[u16],
v: &[u16],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
match BITS {
9 => yuv444p9_to_rgb_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
10 => yuv444p10_to_rgb_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
12 => yuv444p12_to_rgb_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
14 => yuv444p14_to_rgb_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
16 => yuv444p16_to_rgb_row_endian(
y,
u,
v,
rgb_out,
width,
matrix,
full_range,
use_simd,
HOST_NATIVE_BE,
),
_ => unreachable!("BITS pinned to 9/10/12/14/16 by the call sites"),
}
}