use super::super::{
GeometryOverflow, InsufficientBuffer, MixedSinker, MixedSinkerError, RowIndexOutOfRange,
RowShapeMismatch, RowSlice, WidthAlignment, check_dimensions_match,
packed_yuv422_triple_filter_resample, packed_yuv422_triple_resample, reset_high_bit_yuv_streams,
rgb_row_buf_or_scratch, rgba_plane_row_slice, rgba_u16_plane_row_slice,
};
use crate::{PixelSink, row::*, source::*};
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
use super::super::{
ChromaSitingChanged, FrozenOutputs, HsvFrameMut, NativePlanarYuvU16, NativeRouteChanged,
chroma_422_center_sited_h, native_planar_hb_preflight_check_only,
planar_8bit::YUV422P_CENTERED_H_PHASE,
resample_preflight_check_only,
subsampled_4_2_0_high_bit::{
reserve_pn_chroma_full_u16, upsample_pn_chroma_center_h, upsample_pn_chroma_center_h_low_packed,
},
yuv_planar16_process_native,
};
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
use crate::{
KernelMatrix,
resample::{
AveragingDomain, InsertionContext, InsertionPoint, PlanGeometry, ResampleError, ResamplePlan,
select_insertion_point,
},
};
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
#[cfg(all(
test,
feature = "std",
feature = "yuv-semi-planar",
feature = "yuv-planar"
))]
std::thread_local! {
static FORCE_P2XX_ALLOC_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(
test,
feature = "std",
feature = "yuv-semi-planar",
feature = "yuv-planar"
))]
pub(crate) fn arm_p2xx_alloc_failure() {
FORCE_P2XX_ALLOC_FAILURE.with(|f| f.set(true));
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn grow_depack_scratch(
scratch: &mut std::vec::Vec<u16>,
len: usize,
fail: bool,
w: usize,
h: usize,
plan: &ResamplePlan,
) -> Result<(), MixedSinkerError> {
let _ = fail;
if scratch.len() < len {
#[cfg(all(
test,
feature = "std",
feature = "yuv-semi-planar",
feature = "yuv-planar"
))]
if fail && FORCE_P2XX_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
)));
}
scratch
.try_reserve_exact(len - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?;
scratch.resize(len, 0);
}
Ok(())
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
#[allow(clippy::too_many_arguments)]
fn p2xx_process_native<const BITS: u32, const BE: bool, const LOW_PACKED: bool>(
plan: &ResamplePlan,
native_planar_u16: &mut Option<std::boxed::Box<NativePlanarYuvU16>>,
y_scratch: &mut std::vec::Vec<u16>,
u_scratch: &mut std::vec::Vec<u16>,
v_scratch: &mut std::vec::Vec<u16>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
rgb_scratch_u16: &mut std::vec::Vec<u16>,
y_row: &[u16],
uv_half: &[u16],
chroma_h_phase: f64,
matrix: KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
const {
assert!(
BITS > 8 && BITS <= 16,
"BITS must be in (8, 16] for high-bit semi-planar 4:2:2 P-format"
)
};
let need_luma = luma.is_some();
let need_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
let cw = w / 2;
if !native_planar_hb_preflight_check_only(
native_planar_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&None,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
grow_depack_scratch(y_scratch, w, true, w, h, plan)?;
if need_color {
grow_depack_scratch(u_scratch, cw, false, w, h, plan)?;
grow_depack_scratch(v_scratch, cw, false, w, h, plan)?;
}
let depack = |logical: u16| -> u16 {
if LOW_PACKED {
logical & ((1u16 << BITS) - 1)
} else {
logical >> (16 - BITS)
}
};
for (d, &s) in y_scratch[..w].iter_mut().zip(y_row.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = depack(logical);
}
if need_color {
for (i, pair) in uv_half.as_chunks::<2>().0.iter().enumerate() {
let u = if BE {
u16::from_be(pair[0])
} else {
u16::from_le(pair[0])
};
let v = if BE {
u16::from_be(pair[1])
} else {
u16::from_le(pair[1])
};
u_scratch[i] = depack(u);
v_scratch[i] = depack(v);
}
}
let (u_plane, v_plane): (&[u16], &[u16]) = if need_color {
(&u_scratch[..cw], &v_scratch[..cw])
} else {
(&[], &[])
};
yuv_planar16_process_native::<BITS, HOST_NATIVE_BE>(
plan,
native_planar_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
rgb_scratch,
rgb_scratch_u16,
&y_scratch[..w],
u_plane,
v_plane,
matrix,
full_range,
idx,
w,
h,
1,
cw,
|| ResamplePlan::area_chroma_422(cw, h, plan.out_w(), plan.out_h(), chroma_h_phase, 0.0),
use_simd,
)
}
impl<'a, R, const BE: bool> MixedSinker<'a, P210<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> P210Sink<BE> for MixedSinker<'_, P210<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, P210<BE>, R> {
type Input<'r> = P210Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
reset_high_bit_yuv_streams(self);
Ok(())
}
fn process(&mut self, row: P210Row<'_>) -> Result<(), Self::Error> {
const BITS: u32 = 10;
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y10,
idx,
w,
row.y().len(),
)));
}
if row.uv_half().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UvHalf10,
idx,
w,
row.uv_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
#[cfg(feature = "yuv-planar")]
let chroma_location = self.chroma_location.clone();
let Self {
rgb,
rgb_u16,
rgba,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
luma_scratch_u16,
rgb_stream,
rgb_stream_u16,
luma_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
luma_filter_stream_u16,
resample_outputs,
plan,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
native,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
native_planar_u16,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_y_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_u_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_v_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
frozen_native_route,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
chroma_full_u16,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
frozen_chroma_centered,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let (y, uv_half) = (row.y(), row.uv_half());
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let center_sited = chroma_422_center_sited_h(&chroma_location);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let want_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let need_output = luma.is_some()
|| rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| hsv.is_some();
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_filter() {
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
{
plan.ensure_single_kernel_filter()?;
if center_sited && want_color {
let expected = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
let uv_full =
upsample_pn_chroma_center_h::<BITS>(chroma_full_u16, uv_half, w, BE, use_simd);
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| {
p410_to_rgb_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
p410_to_rgb_u16_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
}
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| {
p010_to_rgb_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
p010_to_rgb_u16_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let take_native = matches!(
select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: cfg!(all(feature = "yuv-semi-planar", feature = "yuv-planar")),
with_native: *native,
area_plan: true,
},
),
InsertionPoint::NativeCodes
);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != take_native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if take_native {
let stale_native = idx == 0
&& native_planar_u16.as_ref().is_some_and(|join| {
join.chroma_phase_centered() == Some(!center_sited) && join.next_y() == 0
});
let prev_native = if stale_native {
native_planar_u16.take()
} else {
None
};
let native_result = p2xx_process_native::<BITS, BE, false>(
plan,
native_planar_u16,
p0xx_y_half,
p0xx_u_half,
p0xx_v_half,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
y,
uv_half,
chroma_h_phase,
matrix,
full_range,
idx,
w,
h,
use_simd,
);
if stale_native && native_result.is_err() {
*native_planar_u16 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if center_sited && want_color {
let expected = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
let uv_full =
upsample_pn_chroma_center_h::<BITS>(chroma_full_u16, uv_half, w, BE, use_simd);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| {
p410_to_rgb_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
p410_to_rgb_u16_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
)?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| p010_to_rgb_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE),
|scratch| {
p010_to_rgb_u16_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
)?;
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
let need_output =
luma.is_some() || want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16;
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
#[cfg(feature = "yuv-planar")]
let center_sited = chroma_422_center_sited_h(&chroma_location);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
#[cfg(feature = "yuv-planar")]
let need_centered_chroma =
center_sited && (want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16);
#[cfg(feature = "yuv-planar")]
if need_centered_chroma {
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
#[cfg(feature = "yuv-planar")]
let centered: Option<&[u16]> = if need_centered_chroma {
Some(upsample_pn_chroma_center_h::<BITS>(
chroma_full_u16,
row.uv_half(),
w,
BE,
use_simd,
))
} else {
None
};
#[cfg(not(feature = "yuv-planar"))]
let centered: Option<&[u16]> = None;
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
if let Some(luma) = luma.as_deref_mut() {
let dst = &mut luma[one_plane_start..one_plane_end];
for (d, &s) in dst.iter_mut().zip(row.y().iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = (logical >> 8) as u8;
}
}
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
if let Some(uv_full) = centered {
p410_to_rgba_u16_row_endian(
row.y(),
uv_full,
rgba_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p010_to_rgba_u16_row_endian(
row.y(),
row.uv_half(),
rgba_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end =
one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
if let Some(uv_full) = centered {
p410_to_rgb_u16_row_endian(
row.y(),
uv_full,
rgb_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p010_to_rgb_u16_row_endian(
row.y(),
row.uv_half(),
rgb_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if let Some(uv_full) = centered {
p410_to_hsv_row_endian(
row.y(),
uv_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p010_to_hsv_row_endian(
row.y(),
row.uv_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if let Some(uv_full) = centered {
p410_to_rgba_row_endian(
row.y(),
uv_full,
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p010_to_rgba_row_endian(
row.y(),
row.uv_half(),
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if let Some(uv_full) = centered {
p410_to_rgb_row_endian(
row.y(),
uv_full,
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p010_to_rgb_row_endian(
row.y(),
row.uv_half(),
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R, const BE: bool> MixedSinker<'a, P212<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> P212Sink<BE> for MixedSinker<'_, P212<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, P212<BE>, R> {
type Input<'r> = P212Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
reset_high_bit_yuv_streams(self);
Ok(())
}
fn process(&mut self, row: P212Row<'_>) -> Result<(), Self::Error> {
const BITS: u32 = 12;
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y12,
idx,
w,
row.y().len(),
)));
}
if row.uv_half().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UvHalf12,
idx,
w,
row.uv_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
#[cfg(feature = "yuv-planar")]
let chroma_location = self.chroma_location.clone();
let Self {
rgb,
rgb_u16,
rgba,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
luma_scratch_u16,
rgb_stream,
rgb_stream_u16,
luma_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
luma_filter_stream_u16,
resample_outputs,
plan,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
native,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
native_planar_u16,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_y_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_u_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_v_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
frozen_native_route,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
chroma_full_u16,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
frozen_chroma_centered,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let (y, uv_half) = (row.y(), row.uv_half());
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let center_sited = chroma_422_center_sited_h(&chroma_location);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let want_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let need_output = luma.is_some()
|| rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| hsv.is_some();
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_filter() {
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
{
plan.ensure_single_kernel_filter()?;
if center_sited && want_color {
let expected = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
let uv_full =
upsample_pn_chroma_center_h::<BITS>(chroma_full_u16, uv_half, w, BE, use_simd);
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| {
p412_to_rgb_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
p412_to_rgb_u16_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
}
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| {
p012_to_rgb_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
p012_to_rgb_u16_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let take_native = matches!(
select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: cfg!(all(feature = "yuv-semi-planar", feature = "yuv-planar")),
with_native: *native,
area_plan: true,
},
),
InsertionPoint::NativeCodes
);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != take_native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if take_native {
let stale_native = idx == 0
&& native_planar_u16.as_ref().is_some_and(|join| {
join.chroma_phase_centered() == Some(!center_sited) && join.next_y() == 0
});
let prev_native = if stale_native {
native_planar_u16.take()
} else {
None
};
let native_result = p2xx_process_native::<BITS, BE, false>(
plan,
native_planar_u16,
p0xx_y_half,
p0xx_u_half,
p0xx_v_half,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
y,
uv_half,
chroma_h_phase,
matrix,
full_range,
idx,
w,
h,
use_simd,
);
if stale_native && native_result.is_err() {
*native_planar_u16 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if center_sited && want_color {
let expected = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
let uv_full =
upsample_pn_chroma_center_h::<BITS>(chroma_full_u16, uv_half, w, BE, use_simd);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| {
p412_to_rgb_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
p412_to_rgb_u16_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
)?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| p012_to_rgb_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE),
|scratch| {
p012_to_rgb_u16_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
)?;
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
let need_output =
luma.is_some() || want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16;
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
#[cfg(feature = "yuv-planar")]
let center_sited = chroma_422_center_sited_h(&chroma_location);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
#[cfg(feature = "yuv-planar")]
let need_centered_chroma =
center_sited && (want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16);
#[cfg(feature = "yuv-planar")]
if need_centered_chroma {
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
#[cfg(feature = "yuv-planar")]
let centered: Option<&[u16]> = if need_centered_chroma {
Some(upsample_pn_chroma_center_h::<BITS>(
chroma_full_u16,
row.uv_half(),
w,
BE,
use_simd,
))
} else {
None
};
#[cfg(not(feature = "yuv-planar"))]
let centered: Option<&[u16]> = None;
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
if let Some(luma) = luma.as_deref_mut() {
let dst = &mut luma[one_plane_start..one_plane_end];
for (d, &s) in dst.iter_mut().zip(row.y().iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = (logical >> 8) as u8;
}
}
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
if let Some(uv_full) = centered {
p412_to_rgba_u16_row_endian(
row.y(),
uv_full,
rgba_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p012_to_rgba_u16_row_endian(
row.y(),
row.uv_half(),
rgba_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end =
one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
if let Some(uv_full) = centered {
p412_to_rgb_u16_row_endian(
row.y(),
uv_full,
rgb_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p012_to_rgb_u16_row_endian(
row.y(),
row.uv_half(),
rgb_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if let Some(uv_full) = centered {
p412_to_hsv_row_endian(
row.y(),
uv_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p012_to_hsv_row_endian(
row.y(),
row.uv_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if let Some(uv_full) = centered {
p412_to_rgba_row_endian(
row.y(),
uv_full,
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p012_to_rgba_row_endian(
row.y(),
row.uv_half(),
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if let Some(uv_full) = centered {
p412_to_rgb_row_endian(
row.y(),
uv_full,
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p012_to_rgb_row_endian(
row.y(),
row.uv_half(),
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R, const BE: bool> MixedSinker<'a, P216<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> P216Sink<BE> for MixedSinker<'_, P216<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, P216<BE>, R> {
type Input<'r> = P216Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
reset_high_bit_yuv_streams(self);
Ok(())
}
fn process(&mut self, row: P216Row<'_>) -> Result<(), Self::Error> {
const BITS: u32 = 16;
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y16,
idx,
w,
row.y().len(),
)));
}
if row.uv_half().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UvHalf16,
idx,
w,
row.uv_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
#[cfg(feature = "yuv-planar")]
let chroma_location = self.chroma_location.clone();
let Self {
rgb,
rgb_u16,
rgba,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
luma_scratch_u16,
rgb_stream,
rgb_stream_u16,
luma_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
luma_filter_stream_u16,
resample_outputs,
plan,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
native,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
native_planar_u16,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_y_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_u_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_v_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
frozen_native_route,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
chroma_full_u16,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
frozen_chroma_centered,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let (y, uv_half) = (row.y(), row.uv_half());
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let center_sited = chroma_422_center_sited_h(&chroma_location);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let want_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let need_output = luma.is_some()
|| rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| hsv.is_some();
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_filter() {
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
{
plan.ensure_single_kernel_filter()?;
if center_sited && want_color {
let expected = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
let uv_full =
upsample_pn_chroma_center_h::<BITS>(chroma_full_u16, uv_half, w, BE, use_simd);
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| {
p416_to_rgb_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
p416_to_rgb_u16_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
}
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| {
p016_to_rgb_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
p016_to_rgb_u16_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let take_native = matches!(
select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: cfg!(all(feature = "yuv-semi-planar", feature = "yuv-planar")),
with_native: *native,
area_plan: true,
},
),
InsertionPoint::NativeCodes
);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != take_native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if take_native {
let stale_native = idx == 0
&& native_planar_u16.as_ref().is_some_and(|join| {
join.chroma_phase_centered() == Some(!center_sited) && join.next_y() == 0
});
let prev_native = if stale_native {
native_planar_u16.take()
} else {
None
};
let native_result = p2xx_process_native::<BITS, BE, false>(
plan,
native_planar_u16,
p0xx_y_half,
p0xx_u_half,
p0xx_v_half,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
y,
uv_half,
chroma_h_phase,
matrix,
full_range,
idx,
w,
h,
use_simd,
);
if stale_native && native_result.is_err() {
*native_planar_u16 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if center_sited && want_color {
let expected = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
let uv_full =
upsample_pn_chroma_center_h::<BITS>(chroma_full_u16, uv_half, w, BE, use_simd);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| {
p416_to_rgb_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
p416_to_rgb_u16_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
)?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical >> (16 - BITS);
}
},
|scratch| p016_to_rgb_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE),
|scratch| {
p016_to_rgb_u16_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
)?;
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
let need_output =
luma.is_some() || want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16;
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
#[cfg(feature = "yuv-planar")]
let center_sited = chroma_422_center_sited_h(&chroma_location);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
#[cfg(feature = "yuv-planar")]
let need_centered_chroma =
center_sited && (want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16);
#[cfg(feature = "yuv-planar")]
if need_centered_chroma {
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
#[cfg(feature = "yuv-planar")]
let centered: Option<&[u16]> = if need_centered_chroma {
Some(upsample_pn_chroma_center_h::<BITS>(
chroma_full_u16,
row.uv_half(),
w,
BE,
use_simd,
))
} else {
None
};
#[cfg(not(feature = "yuv-planar"))]
let centered: Option<&[u16]> = None;
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
if let Some(luma) = luma.as_deref_mut() {
let dst = &mut luma[one_plane_start..one_plane_end];
for (d, &s) in dst.iter_mut().zip(row.y().iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = (logical >> 8) as u8;
}
}
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
if let Some(uv_full) = centered {
p416_to_rgba_u16_row_endian(
row.y(),
uv_full,
rgba_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p016_to_rgba_u16_row_endian(
row.y(),
row.uv_half(),
rgba_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end =
one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
if let Some(uv_full) = centered {
p416_to_rgb_u16_row_endian(
row.y(),
uv_full,
rgb_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p016_to_rgb_u16_row_endian(
row.y(),
row.uv_half(),
rgb_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if let Some(uv_full) = centered {
p416_to_hsv_row_endian(
row.y(),
uv_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p016_to_hsv_row_endian(
row.y(),
row.uv_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if let Some(uv_full) = centered {
p416_to_rgba_row_endian(
row.y(),
uv_full,
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p016_to_rgba_row_endian(
row.y(),
row.uv_half(),
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if let Some(uv_full) = centered {
p416_to_rgb_row_endian(
row.y(),
uv_full,
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
p016_to_rgb_row_endian(
row.y(),
row.uv_half(),
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R, const BE: bool> MixedSinker<'a, Nv20<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> Nv20Sink<BE> for MixedSinker<'_, Nv20<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, Nv20<BE>, R> {
type Input<'r> = Nv20Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
reset_high_bit_yuv_streams(self);
Ok(())
}
fn process(&mut self, row: Nv20Row<'_>) -> Result<(), Self::Error> {
const BITS: u32 = 10;
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y10,
idx,
w,
row.y().len(),
)));
}
if row.uv().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UvHalf10,
idx,
w,
row.uv().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
#[cfg(feature = "yuv-planar")]
let chroma_location = self.chroma_location.clone();
let Self {
rgb,
rgb_u16,
rgba,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
luma_scratch_u16,
rgb_stream,
rgb_stream_u16,
luma_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
luma_filter_stream_u16,
resample_outputs,
plan,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
native,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
native_planar_u16,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_y_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_u_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_v_half,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
frozen_native_route,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
chroma_full_u16,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
frozen_chroma_centered,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let (y, uv_half) = (row.y(), row.uv());
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let center_sited = chroma_422_center_sited_h(&chroma_location);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let want_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let need_output = luma.is_some()
|| rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| hsv.is_some();
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_filter() {
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
{
plan.ensure_single_kernel_filter()?;
if center_sited && want_color {
let expected = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
let uv_full = upsample_pn_chroma_center_h_low_packed::<BITS>(
chroma_full_u16,
uv_half,
w,
BE,
use_simd,
);
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical & ((1u16 << BITS) - 1);
}
},
|scratch| {
nv20_444_to_rgb_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
nv20_444_to_rgb_u16_row_endian(
y, uv_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
}
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical & ((1u16 << BITS) - 1);
}
},
|scratch| {
nv20_to_rgb_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
nv20_to_rgb_u16_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let take_native = matches!(
select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: cfg!(all(feature = "yuv-semi-planar", feature = "yuv-planar")),
with_native: *native,
area_plan: true,
},
),
InsertionPoint::NativeCodes
);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != take_native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if take_native {
let stale_native = idx == 0
&& native_planar_u16.as_ref().is_some_and(|join| {
join.chroma_phase_centered() == Some(!center_sited) && join.next_y() == 0
});
let prev_native = if stale_native {
native_planar_u16.take()
} else {
None
};
let native_result = p2xx_process_native::<BITS, BE, true>(
plan,
native_planar_u16,
p0xx_y_half,
p0xx_u_half,
p0xx_v_half,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
y,
uv_half,
chroma_h_phase,
matrix,
full_range,
idx,
w,
h,
use_simd,
);
if stale_native && native_result.is_err() {
*native_planar_u16 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if center_sited && want_color {
let expected = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
let uv_full =
upsample_pn_chroma_center_h_low_packed::<BITS>(chroma_full_u16, uv_half, w, BE, use_simd);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical & ((1u16 << BITS) - 1);
}
},
|scratch| {
nv20_444_to_rgb_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
|scratch| {
nv20_444_to_rgb_u16_row_endian(y, uv_full, scratch, w, matrix, full_range, use_simd, BE)
},
)?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| {
for (dst, &s) in scratch[..w].iter_mut().zip(y.iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*dst = logical & ((1u16 << BITS) - 1);
}
},
|scratch| nv20_to_rgb_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE),
|scratch| {
nv20_to_rgb_u16_row_endian(y, uv_half, scratch, w, matrix, full_range, use_simd, BE)
},
)?;
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
let need_output =
luma.is_some() || want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16;
#[cfg(feature = "yuv-planar")]
let center_sited = chroma_422_center_sited_h(&chroma_location);
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
#[cfg(feature = "yuv-planar")]
let need_centered_chroma =
center_sited && (want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16);
#[cfg(feature = "yuv-planar")]
if need_centered_chroma {
reserve_pn_chroma_full_u16(chroma_full_u16, w, h)?;
}
if want_hsv && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
#[cfg(feature = "yuv-planar")]
let centered: Option<&[u16]> = if need_centered_chroma {
Some(upsample_pn_chroma_center_h_low_packed::<BITS>(
chroma_full_u16,
row.uv(),
w,
BE,
use_simd,
))
} else {
None
};
#[cfg(not(feature = "yuv-planar"))]
let centered: Option<&[u16]> = None;
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
if let Some(luma) = luma.as_deref_mut() {
let dst = &mut luma[one_plane_start..one_plane_end];
for (d, &s) in dst.iter_mut().zip(row.y().iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = ((logical & ((1u16 << BITS) - 1)) >> (BITS - 8)) as u8;
}
}
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
if let Some(uv_full) = centered {
nv20_444_to_rgba_u16_row_endian(
row.y(),
uv_full,
rgba_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
nv20_to_rgba_u16_row_endian(
row.y(),
row.uv(),
rgba_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end =
one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
if let Some(uv_full) = centered {
nv20_444_to_rgb_u16_row_endian(
row.y(),
uv_full,
rgb_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
nv20_to_rgb_u16_row_endian(
row.y(),
row.uv(),
rgb_u16_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let need_rgb_kernel = want_rgb || want_hsv;
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if let Some(uv_full) = centered {
nv20_444_to_rgba_row_endian(
row.y(),
uv_full,
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
nv20_to_rgba_row_endian(
row.y(),
row.uv(),
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if let Some(uv_full) = centered {
nv20_444_to_rgb_row_endian(
row.y(),
uv_full,
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
} else {
nv20_to_rgb_row_endian(
row.y(),
row.uv(),
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
BE,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}