use super::{
super::{
ChromaSitingChanged, FrozenOutputs, GeometryOverflow, HsvFrameMut, InsufficientBuffer,
MixedSinker, MixedSinkerError, NativeRouteChanged, RowIndexOutOfRange, RowShapeMismatch,
RowSlice, WidthAlignment, check_dimensions_match, chroma_420_bottom_sited_v,
chroma_420_center_sited_h, chroma_420_top_sited_v, deinterleave_y_high_bit_masked,
packed_yuv422_triple_filter_resample, packed_yuv422_triple_resample,
planar_8bit::YUV422P_CENTERED_H_PHASE, resample_preflight_check_only,
reset_high_bit_yuv_streams, rgb_row_buf_or_scratch, rgba_plane_row_slice,
rgba_u16_plane_row_slice,
},
yuv420p16_process_native,
};
use crate::{
PixelSink,
resample::{
AveragingDomain, InsertionContext, InsertionPoint, PlanGeometry, ResampleError, ResamplePlan,
select_insertion_point,
},
row::*,
source::*,
};
const YUV420P_HIGH_BIT_NATIVE_ELIGIBLE: bool = true;
pub(crate) fn reserve_420_chroma_full_u16(
chroma_full: &mut std::vec::Vec<u16>,
width: usize,
height: usize,
) -> Result<(), MixedSinkerError> {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if super::super::FORCE_CHROMA_FULL_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(width, height, width, height),
)));
}
let needed = width
.checked_mul(2)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width, height, 2,
)))?;
if chroma_full.len() < needed {
chroma_full
.try_reserve_exact(needed - chroma_full.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
width, height, width, height,
)))
})?;
chroma_full.resize(needed, 0);
}
Ok(())
}
pub(crate) fn upsample_420_chroma_center_h_u16<'s, const BITS: u32>(
chroma_full: &'s mut [u16],
u_half: &[u16],
v_half: &[u16],
width: usize,
big_endian: bool,
use_simd: bool,
) -> (&'s [u16], &'s [u16]) {
debug_assert!(
chroma_full.len() >= 2 * width,
"chroma_full must be reserved via reserve_420_chroma_full_u16 first"
);
let (u_full, v_full) = chroma_full[..2 * width].split_at_mut(width);
crate::row::chroma_upsample_2to1_center_h_u16_row::<BITS>(
u_half, u_full, width, big_endian, use_simd,
);
crate::row::chroma_upsample_2to1_center_h_u16_row::<BITS>(
v_half, v_full, width, big_endian, use_simd,
);
(u_full, v_full)
}
pub(crate) fn upsample_420_chroma_cosited_h_u16<'s, const BITS: u32>(
chroma_full: &'s mut [u16],
u_half: &[u16],
v_half: &[u16],
width: usize,
big_endian: bool,
) -> (&'s [u16], &'s [u16]) {
debug_assert!(
chroma_full.len() >= 2 * width,
"chroma_full must be reserved via reserve_420_chroma_full_u16 first"
);
let (u_full, v_full) = chroma_full[..2 * width].split_at_mut(width);
crate::row::scalar::chroma_upsample_2to1_cosited_h_u16::<BITS>(u_half, u_full, width, big_endian);
crate::row::scalar::chroma_upsample_2to1_cosited_h_u16::<BITS>(v_half, v_full, width, big_endian);
(u_full, v_full)
}
pub(crate) fn reserve_420_chroma_prev_u16(
chroma_prev: &mut std::vec::Vec<u16>,
width: usize,
height: usize,
) -> Result<(), MixedSinkerError> {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if super::super::FORCE_CHROMA_PREV_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(width, height, width, height),
)));
}
if chroma_prev.len() < width {
chroma_prev
.try_reserve_exact(width - chroma_prev.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
width, height, width, height,
)))
})?;
chroma_prev.resize(width, 0);
}
Ok(())
}
#[inline]
pub(crate) fn stage_420_chroma_prev_u16(
chroma_prev: &mut [u16],
chroma_prev_row: &mut Option<usize>,
u_half: &[u16],
v_half: &[u16],
idx: usize,
width: usize,
) {
debug_assert!(
chroma_prev.len() >= width,
"chroma_prev must be reserved via reserve_420_chroma_prev_u16 first"
);
let half = width / 2;
chroma_prev[..half].copy_from_slice(&u_half[..half]);
chroma_prev[half..width].copy_from_slice(&v_half[..half]);
*chroma_prev_row = Some(idx / 2);
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn upsample_420_chroma_sited_u16<'s, const BITS: u32>(
chroma_full: &'s mut [u16],
chroma_prev: &mut [u16],
chroma_prev_row: &mut Option<usize>,
u_half: &[u16],
v_half: &[u16],
idx: usize,
bottom_v: bool,
center_h: bool,
stage: bool,
width: usize,
big_endian: bool,
use_simd: bool,
) -> (&'s [u16], &'s [u16]) {
debug_assert!(
chroma_full.len() >= 2 * width,
"chroma_full must be reserved via reserve_420_chroma_full_u16 first"
);
let half = width / 2;
let chroma_row = idx / 2;
let do_vblend =
bottom_v && idx & 1 == 0 && chroma_row > 0 && *chroma_prev_row == Some(chroma_row - 1);
let result = if do_vblend {
debug_assert!(
chroma_prev.len() >= width,
"chroma_prev must be reserved via reserve_420_chroma_prev_u16 first"
);
let (u_full, v_full) = chroma_full[..2 * width].split_at_mut(width);
let (u_prev, v_prev) = chroma_prev[..width].split_at(half);
if center_h {
crate::row::chroma_upsample_420_bottom_even_h_u16_row::<BITS>(
u_prev, u_half, u_full, width, big_endian, use_simd,
);
crate::row::chroma_upsample_420_bottom_even_h_u16_row::<BITS>(
v_prev, v_half, v_full, width, big_endian, use_simd,
);
} else {
crate::row::chroma_upsample_420_bottomleft_even_h_u16_row::<BITS>(
u_prev, u_half, u_full, width, big_endian, use_simd,
);
crate::row::chroma_upsample_420_bottomleft_even_h_u16_row::<BITS>(
v_prev, v_half, v_full, width, big_endian, use_simd,
);
}
(&*u_full, &*v_full)
} else if center_h {
upsample_420_chroma_center_h_u16::<BITS>(
chroma_full,
u_half,
v_half,
width,
big_endian,
use_simd,
)
} else {
upsample_420_chroma_cosited_h_u16::<BITS>(chroma_full, u_half, v_half, width, big_endian)
};
if bottom_v && stage {
stage_420_chroma_prev_u16(chroma_prev, chroma_prev_row, u_half, v_half, idx, width);
}
result
}
#[cfg(feature = "yuv-planar")]
macro_rules! by_bits {
($bits:expr; $k9:path, $k10:path, $k12:path, $k14:path, $k16:path; $($arg:expr),* $(,)?) => {
match $bits {
9 => $k9($($arg),*),
10 => $k10($($arg),*),
12 => $k12($($arg),*),
14 => $k14($($arg),*),
16 => $k16($($arg),*),
_ => unreachable!("BITS pinned to 9/10/12/14/16 by the call sites"),
}
};
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn emit_rgb_u8_wire<const BITS: u32>(
y: &[u16],
u: &[u16],
v: &[u16],
rgb_out: &mut [u8],
width: usize,
matrix: crate::KernelMatrix,
full_range: bool,
use_simd: bool,
big_endian: bool,
) {
by_bits!(
BITS;
yuv444p9_to_rgb_row_endian, yuv444p10_to_rgb_row_endian, yuv444p12_to_rgb_row_endian,
yuv444p14_to_rgb_row_endian, yuv444p16_to_rgb_row_endian;
y, u, v, rgb_out, width, matrix, full_range, use_simd, big_endian
);
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
fn emit_rgba_u8_wire<const BITS: u32>(
y: &[u16],
u: &[u16],
v: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: crate::KernelMatrix,
full_range: bool,
use_simd: bool,
big_endian: bool,
) {
by_bits!(
BITS;
yuv444p9_to_rgba_row_endian, yuv444p10_to_rgba_row_endian, yuv444p12_to_rgba_row_endian,
yuv444p14_to_rgba_row_endian, yuv444p16_to_rgba_row_endian;
y, u, v, rgba_out, width, matrix, full_range, use_simd, big_endian
);
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn emit_rgb_u16_wire<const BITS: u32>(
y: &[u16],
u: &[u16],
v: &[u16],
rgb_out: &mut [u16],
width: usize,
matrix: crate::KernelMatrix,
full_range: bool,
use_simd: bool,
big_endian: bool,
) {
by_bits!(
BITS;
yuv444p9_to_rgb_u16_row_endian, yuv444p10_to_rgb_u16_row_endian,
yuv444p12_to_rgb_u16_row_endian, yuv444p14_to_rgb_u16_row_endian,
yuv444p16_to_rgb_u16_row_endian;
y, u, v, rgb_out, width, matrix, full_range, use_simd, big_endian
);
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
fn emit_rgba_u16_wire<const BITS: u32>(
y: &[u16],
u: &[u16],
v: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: crate::KernelMatrix,
full_range: bool,
use_simd: bool,
big_endian: bool,
) {
by_bits!(
BITS;
yuv444p9_to_rgba_u16_row_endian, yuv444p10_to_rgba_u16_row_endian,
yuv444p12_to_rgba_u16_row_endian, yuv444p14_to_rgba_u16_row_endian,
yuv444p16_to_rgba_u16_row_endian;
y, u, v, rgba_out, width, matrix, full_range, use_simd, big_endian
);
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
fn emit_hsv_u8_wire<const BITS: u32>(
y: &[u16],
u: &[u16],
v: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: crate::KernelMatrix,
full_range: bool,
use_simd: bool,
big_endian: bool,
) {
by_bits!(
BITS;
yuv444p9_to_hsv_row_endian, yuv444p10_to_hsv_row_endian, yuv444p12_to_hsv_row_endian,
yuv444p14_to_hsv_row_endian, yuv444p16_to_hsv_row_endian;
y, u, v, h_out, s_out, v_out, width, matrix, full_range, use_simd, big_endian
);
}
#[cfg(feature = "yuv-planar")]
pub(crate) fn reserve_420_chroma_top_y_u16(
buf: &mut std::vec::Vec<u16>,
width: usize,
height: usize,
) -> Result<(), MixedSinkerError> {
if buf.len() < width {
buf.try_reserve_exact(width - buf.len()).map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
width, height, width, height,
)))
})?;
buf.resize(width, 0);
}
Ok(())
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv444p_top_identity_color_row<const BITS: u32, const BE: bool>(
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y: &[u16],
u_full: &[u16],
v_full: &[u16],
one_plane_start: usize,
one_plane_end: usize,
w: usize,
h: usize,
matrix: crate::KernelMatrix,
full_range: bool,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
emit_rgba_u16_wire::<BITS>(
y,
u_full,
v_full,
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end = one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
emit_rgb_u16_wire::<BITS>(
y,
u_full,
v_full,
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (hh, ss, vv) = hsv.hsv();
emit_hsv_u8_wire::<BITS>(
y,
u_full,
v_full,
&mut hh[one_plane_start..one_plane_end],
&mut ss[one_plane_start..one_plane_end],
&mut vv[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
emit_rgba_u8_wire::<BITS>(
y, u_full, v_full, rgba_row, w, matrix, full_range, use_simd, BE,
);
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
emit_rgb_u8_wire::<BITS>(
y, u_full, v_full, rgb_row, w, matrix, full_range, use_simd, BE,
);
if let Some(hsv) = hsv.as_mut() {
let (hh, ss, vv) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut hh[one_plane_start..one_plane_end],
&mut ss[one_plane_start..one_plane_end],
&mut vv[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
fn yuv420p_top_reconstruct_area<const BITS: u32, const BE: bool>(
luma_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
rgb_stream: &mut Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
rgb_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
hsv: &mut Option<HsvFrameMut<'_>>,
luma_scratch_u16: &mut std::vec::Vec<u16>,
rgb_scratch: &mut std::vec::Vec<u8>,
rgb_scratch_u16: &mut std::vec::Vec<u16>,
chroma_full_u16: &mut std::vec::Vec<u16>,
chroma_prev_u16: &mut std::vec::Vec<u16>,
chroma_prev_row: &mut Option<usize>,
chroma_top_pending: &mut Option<(usize, crate::KernelMatrix, bool)>,
chroma_top_y_u16: &mut std::vec::Vec<u16>,
y_row: &[u16],
u_half: &[u16],
v_half: &[u16],
w: usize,
h: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
center_sited: bool,
) -> Result<core::ops::ControlFlow<()>, MixedSinkerError> {
let cursor = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
let expected_source = cursor + usize::from(chroma_top_pending.is_some());
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected_source),
idx,
)? {
return Ok(core::ops::ControlFlow::Break(()));
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
reserve_420_chroma_top_y_u16(chroma_top_y_u16, w, h)?;
let is_last = idx + 1 == h;
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
true,
center_sited,
false,
w,
BE,
use_simd,
);
let held_y: &[u16] = &chroma_top_y_u16[..w];
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
p_idx,
use_simd,
p_matrix,
p_full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(held_y, scratch, w),
|scratch| {
emit_rgb_u8_wire::<BITS>(
held_y,
u_full,
v_full,
scratch,
w,
p_matrix,
p_full_range,
use_simd,
BE,
)
},
|scratch| {
emit_rgb_u16_wire::<BITS>(
held_y,
u_full,
v_full,
scratch,
w,
p_matrix,
p_full_range,
use_simd,
BE,
)
},
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y_row, scratch, w),
|scratch| {
emit_rgb_u8_wire::<BITS>(
y_row, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
emit_rgb_u16_wire::<BITS>(
y_row, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
} else if !is_last {
chroma_top_y_u16[..w].copy_from_slice(&y_row[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y_row, scratch, w),
|scratch| {
emit_rgb_u8_wire::<BITS>(
y_row, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
emit_rgb_u16_wire::<BITS>(
y_row, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
}
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
Ok(core::ops::ControlFlow::Continue(()))
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
fn yuv420p_top_reconstruct_filter<const BITS: u32, const BE: bool>(
luma_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
rgb_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
rgb_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
hsv: &mut Option<HsvFrameMut<'_>>,
luma_scratch_u16: &mut std::vec::Vec<u16>,
rgb_scratch: &mut std::vec::Vec<u8>,
rgb_scratch_u16: &mut std::vec::Vec<u16>,
chroma_full_u16: &mut std::vec::Vec<u16>,
chroma_prev_u16: &mut std::vec::Vec<u16>,
chroma_prev_row: &mut Option<usize>,
chroma_top_pending: &mut Option<(usize, crate::KernelMatrix, bool)>,
chroma_top_y_u16: &mut std::vec::Vec<u16>,
y_row: &[u16],
u_half: &[u16],
v_half: &[u16],
w: usize,
h: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
center_sited: bool,
) -> Result<core::ops::ControlFlow<()>, MixedSinkerError> {
let cursor = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
let expected_source = cursor + usize::from(chroma_top_pending.is_some());
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected_source),
idx,
)? {
return Ok(core::ops::ControlFlow::Break(()));
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
reserve_420_chroma_top_y_u16(chroma_top_y_u16, w, h)?;
let is_last = idx + 1 == h;
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
true,
center_sited,
false,
w,
BE,
use_simd,
);
let held_y: &[u16] = &chroma_top_y_u16[..w];
packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
p_idx,
use_simd,
p_matrix,
p_full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(held_y, scratch, w),
|scratch| {
emit_rgb_u8_wire::<BITS>(
held_y,
u_full,
v_full,
scratch,
w,
p_matrix,
p_full_range,
use_simd,
BE,
)
},
|scratch| {
emit_rgb_u16_wire::<BITS>(
held_y,
u_full,
v_full,
scratch,
w,
p_matrix,
p_full_range,
use_simd,
BE,
)
},
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y_row, scratch, w),
|scratch| {
emit_rgb_u8_wire::<BITS>(
y_row, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
emit_rgb_u16_wire::<BITS>(
y_row, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
} else if !is_last {
chroma_top_y_u16[..w].copy_from_slice(&y_row[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y_row, scratch, w),
|scratch| {
emit_rgb_u8_wire::<BITS>(
y_row, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
emit_rgb_u16_wire::<BITS>(
y_row, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
}
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
Ok(core::ops::ControlFlow::Continue(()))
}
impl<'a, R, const BE: bool> MixedSinker<'a, Yuv420p9<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> Yuv420p9Sink<BE> for MixedSinker<'_, Yuv420p9<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, Yuv420p9<BE>, R> {
type Input<'r> = Yuv420p9Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
reset_high_bit_yuv_streams(self);
Ok(())
}
#[allow(clippy::too_many_lines)]
fn process(&mut self, row: Yuv420p9Row<'_>) -> Result<(), Self::Error> {
const BITS: u32 = 9;
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y9,
idx,
w,
row.y().len(),
)));
}
if row.u_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UHalf9,
idx,
w / 2,
row.u_half().len(),
)));
}
if row.v_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VHalf9,
idx,
w / 2,
row.v_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let chroma_location = self.chroma_location.clone();
let Self {
rgb,
rgb_u16,
rgba,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
luma_scratch_u16,
rgb_stream,
rgb_stream_u16,
luma_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
luma_filter_stream_u16,
resample_outputs,
plan,
native,
native_420_u16,
frozen_native_route,
frozen_chroma_centered,
frozen_chroma_bottom_v,
frozen_chroma_top_v,
chroma_top_pending,
chroma_top_y_u16,
chroma_prev_u16,
chroma_prev_row,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let (y, u_half, v_half) = (row.y(), row.u_half(), row.v_half());
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
let chroma_v_phase = if bottom_v { 1.0 } else { 0.0 };
let top_v = chroma_420_top_sited_v(&chroma_location);
let need_output = luma.is_some()
|| rgb.is_some()
|| rgba.is_some()
|| hsv.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some();
let want_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_filter() {
plan.ensure_single_kernel_filter()?;
if top_v && want_color {
match yuv420p_top_reconstruct_filter::<BITS, BE>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p9_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p9_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p9_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p9_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != *native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
let insertion = select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: YUV420P_HIGH_BIT_NATIVE_ELIGIBLE,
with_native: *native,
area_plan: true,
},
);
match insertion {
InsertionPoint::NativeCodes => {
let stale_native = idx == 0
&& native_420_u16.as_ref().is_some_and(|join| {
(join.chroma_phase_centered() == Some(!center_sited)
|| join.chroma_bottom() == Some(!bottom_v)
|| join.chroma_top() == Some(!top_v))
&& join.next_y() == 0
});
let prev_native = if stale_native {
native_420_u16.take()
} else {
None
};
let native_result = yuv420p16_process_native::<BITS, BE>(
plan,
native_420_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
y,
u_half,
v_half,
matrix,
full_range,
idx,
w,
h,
|| {
ResamplePlan::area_chroma_420(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
chroma_v_phase,
top_v,
)
},
use_simd,
);
if stale_native && native_result.is_err() {
*native_420_u16 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::EncodedOutput => {
if top_v && want_color {
match yuv420p_top_reconstruct_area::<BITS, BE>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p9_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p9_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
if bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
} else {
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p9_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p9_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::LinearLight => {
unreachable!("encoded domain never selects the linear-light splice")
}
}
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
let want_color = want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16;
let need_output = want_color || luma.is_some();
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let top_v = chroma_420_top_sited_v(&chroma_location);
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
let need_centered_chroma = (center_sited || bottom_v) && want_color && !top_v;
if need_centered_chroma || (top_v && want_color) {
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
}
if top_v && want_color {
reserve_420_chroma_top_y_u16(chroma_top_y_u16, w, h)?;
}
if bottom_v || top_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if (bottom_v || top_v) && !want_color {
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
}
let centered = if need_centered_chroma {
Some(upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
true,
w,
BE,
use_simd,
))
} else {
None
};
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
let matrix = row.matrix();
let full_range = row.full_range();
if let Some(luma) = luma.as_deref_mut() {
let dst = &mut luma[one_plane_start..one_plane_end];
for (d, &s) in dst.iter_mut().zip(row.y().iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = (logical >> (BITS - 8)) as u8;
}
}
if top_v && want_color {
let is_last = idx + 1 == h;
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
true,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
&chroma_top_y_u16[..w],
u_full,
v_full,
p_idx * w,
p_idx * w + w,
w,
h,
p_matrix,
p_full_range,
use_simd,
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
} else if !is_last {
chroma_top_y_u16[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
}
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
return Ok(());
}
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p9_to_rgba_u16_row_endian(
row.y(),
u_full,
v_full,
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p9_to_rgba_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end =
one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
if let Some((u_full, v_full)) = centered {
yuv444p9_to_rgb_u16_row_endian(
row.y(),
u_full,
v_full,
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p9_to_rgb_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if let Some((u_full, v_full)) = centered {
yuv444p9_to_hsv_row_endian(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p9_to_hsv_row_endian(
row.y(),
row.u_half(),
row.v_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p9_to_rgba_row_endian(
row.y(),
u_full,
v_full,
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p9_to_rgba_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if let Some((u_full, v_full)) = centered {
yuv444p9_to_rgb_row_endian(
row.y(),
u_full,
v_full,
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p9_to_rgb_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R, const BE: bool> MixedSinker<'a, Yuv420p10<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected_elements = self.frame_elems(3)?;
if buf.len() < expected_elements {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected_elements, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> Yuv420p10Sink<BE> for MixedSinker<'_, Yuv420p10<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, Yuv420p10<BE>, R> {
type Input<'r> = Yuv420p10Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
reset_high_bit_yuv_streams(self);
Ok(())
}
#[allow(clippy::too_many_lines)]
fn process(&mut self, row: Yuv420p10Row<'_>) -> Result<(), Self::Error> {
const BITS: u32 = 10;
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y10,
idx,
w,
row.y().len(),
)));
}
if row.u_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UHalf10,
idx,
w / 2,
row.u_half().len(),
)));
}
if row.v_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VHalf10,
idx,
w / 2,
row.v_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let chroma_location = self.chroma_location.clone();
let Self {
rgb,
rgb_u16,
rgba,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
luma_scratch_u16,
rgb_stream,
rgb_stream_u16,
luma_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
luma_filter_stream_u16,
resample_outputs,
plan,
native,
native_420_u16,
frozen_native_route,
frozen_chroma_centered,
frozen_chroma_bottom_v,
frozen_chroma_top_v,
chroma_top_pending,
chroma_top_y_u16,
chroma_prev_u16,
chroma_prev_row,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let (y, u_half, v_half) = (row.y(), row.u_half(), row.v_half());
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
let chroma_v_phase = if bottom_v { 1.0 } else { 0.0 };
let top_v = chroma_420_top_sited_v(&chroma_location);
let need_output = luma.is_some()
|| rgb.is_some()
|| rgba.is_some()
|| hsv.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some();
let want_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_filter() {
plan.ensure_single_kernel_filter()?;
if top_v && want_color {
match yuv420p_top_reconstruct_filter::<BITS, BE>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p10_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p10_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p10_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p10_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != *native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
let insertion = select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: YUV420P_HIGH_BIT_NATIVE_ELIGIBLE,
with_native: *native,
area_plan: true,
},
);
match insertion {
InsertionPoint::NativeCodes => {
let stale_native = idx == 0
&& native_420_u16.as_ref().is_some_and(|join| {
(join.chroma_phase_centered() == Some(!center_sited)
|| join.chroma_bottom() == Some(!bottom_v)
|| join.chroma_top() == Some(!top_v))
&& join.next_y() == 0
});
let prev_native = if stale_native {
native_420_u16.take()
} else {
None
};
let native_result = yuv420p16_process_native::<BITS, BE>(
plan,
native_420_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
y,
u_half,
v_half,
matrix,
full_range,
idx,
w,
h,
|| {
ResamplePlan::area_chroma_420(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
chroma_v_phase,
top_v,
)
},
use_simd,
);
if stale_native && native_result.is_err() {
*native_420_u16 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::EncodedOutput => {
if top_v && want_color {
match yuv420p_top_reconstruct_area::<BITS, BE>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p10_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p10_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
if bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
} else {
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p10_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p10_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::LinearLight => {
unreachable!("encoded domain never selects the linear-light splice")
}
}
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
let want_color = want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16;
let need_output = want_color || luma.is_some();
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let top_v = chroma_420_top_sited_v(&chroma_location);
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
let need_centered_chroma = (center_sited || bottom_v) && want_color && !top_v;
if need_centered_chroma || (top_v && want_color) {
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
}
if top_v && want_color {
reserve_420_chroma_top_y_u16(chroma_top_y_u16, w, h)?;
}
if bottom_v || top_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if (bottom_v || top_v) && !want_color {
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
}
let centered = if need_centered_chroma {
Some(upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
true,
w,
BE,
use_simd,
))
} else {
None
};
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
let matrix = row.matrix();
let full_range = row.full_range();
if let Some(luma) = luma.as_deref_mut() {
let dst = &mut luma[one_plane_start..one_plane_end];
for (d, &s) in dst.iter_mut().zip(row.y().iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = (logical >> (BITS - 8)) as u8;
}
}
if top_v && want_color {
let is_last = idx + 1 == h;
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
true,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
&chroma_top_y_u16[..w],
u_full,
v_full,
p_idx * w,
p_idx * w + w,
w,
h,
p_matrix,
p_full_range,
use_simd,
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
} else if !is_last {
chroma_top_y_u16[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
}
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
return Ok(());
}
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p10_to_rgba_u16_row_endian(
row.y(),
u_full,
v_full,
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p10_to_rgba_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end =
one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
if let Some((u_full, v_full)) = centered {
yuv444p10_to_rgb_u16_row_endian(
row.y(),
u_full,
v_full,
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p10_to_rgb_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if let Some((u_full, v_full)) = centered {
yuv444p10_to_hsv_row_endian(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p10_to_hsv_row_endian(
row.y(),
row.u_half(),
row.v_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p10_to_rgba_row_endian(
row.y(),
u_full,
v_full,
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p10_to_rgba_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if let Some((u_full, v_full)) = centered {
yuv444p10_to_rgb_row_endian(
row.y(),
u_full,
v_full,
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p10_to_rgb_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R, const BE: bool> MixedSinker<'a, Yuv420p12<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected_elements = self.frame_elems(3)?;
if buf.len() < expected_elements {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected_elements, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> Yuv420p12Sink<BE> for MixedSinker<'_, Yuv420p12<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, Yuv420p12<BE>, R> {
type Input<'r> = Yuv420p12Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
reset_high_bit_yuv_streams(self);
Ok(())
}
#[allow(clippy::too_many_lines)]
fn process(&mut self, row: Yuv420p12Row<'_>) -> Result<(), Self::Error> {
const BITS: u32 = 12;
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y12,
idx,
w,
row.y().len(),
)));
}
if row.u_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UHalf12,
idx,
w / 2,
row.u_half().len(),
)));
}
if row.v_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VHalf12,
idx,
w / 2,
row.v_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let chroma_location = self.chroma_location.clone();
let Self {
rgb,
rgb_u16,
rgba,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
luma_scratch_u16,
rgb_stream,
rgb_stream_u16,
luma_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
luma_filter_stream_u16,
resample_outputs,
plan,
native,
native_420_u16,
frozen_native_route,
frozen_chroma_centered,
frozen_chroma_bottom_v,
frozen_chroma_top_v,
chroma_top_pending,
chroma_top_y_u16,
chroma_prev_u16,
chroma_prev_row,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let (y, u_half, v_half) = (row.y(), row.u_half(), row.v_half());
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
let chroma_v_phase = if bottom_v { 1.0 } else { 0.0 };
let top_v = chroma_420_top_sited_v(&chroma_location);
let need_output = luma.is_some()
|| rgb.is_some()
|| rgba.is_some()
|| hsv.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some();
let want_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_filter() {
plan.ensure_single_kernel_filter()?;
if top_v && want_color {
match yuv420p_top_reconstruct_filter::<BITS, BE>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p12_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p12_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p12_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p12_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != *native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
let insertion = select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: YUV420P_HIGH_BIT_NATIVE_ELIGIBLE,
with_native: *native,
area_plan: true,
},
);
match insertion {
InsertionPoint::NativeCodes => {
let stale_native = idx == 0
&& native_420_u16.as_ref().is_some_and(|join| {
(join.chroma_phase_centered() == Some(!center_sited)
|| join.chroma_bottom() == Some(!bottom_v)
|| join.chroma_top() == Some(!top_v))
&& join.next_y() == 0
});
let prev_native = if stale_native {
native_420_u16.take()
} else {
None
};
let native_result = yuv420p16_process_native::<BITS, BE>(
plan,
native_420_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
y,
u_half,
v_half,
matrix,
full_range,
idx,
w,
h,
|| {
ResamplePlan::area_chroma_420(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
chroma_v_phase,
top_v,
)
},
use_simd,
);
if stale_native && native_result.is_err() {
*native_420_u16 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::EncodedOutput => {
if top_v && want_color {
match yuv420p_top_reconstruct_area::<BITS, BE>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p12_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p12_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
if bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
} else {
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p12_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p12_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::LinearLight => {
unreachable!("encoded domain never selects the linear-light splice")
}
}
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
let want_color = want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16;
let need_output = want_color || luma.is_some();
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let top_v = chroma_420_top_sited_v(&chroma_location);
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
let need_centered_chroma = (center_sited || bottom_v) && want_color && !top_v;
if need_centered_chroma || (top_v && want_color) {
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
}
if top_v && want_color {
reserve_420_chroma_top_y_u16(chroma_top_y_u16, w, h)?;
}
if bottom_v || top_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if (bottom_v || top_v) && !want_color {
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
}
let centered = if need_centered_chroma {
Some(upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
true,
w,
BE,
use_simd,
))
} else {
None
};
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
let matrix = row.matrix();
let full_range = row.full_range();
if let Some(luma) = luma.as_deref_mut() {
let dst = &mut luma[one_plane_start..one_plane_end];
for (d, &s) in dst.iter_mut().zip(row.y().iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = (logical >> (BITS - 8)) as u8;
}
}
if top_v && want_color {
let is_last = idx + 1 == h;
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
true,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
&chroma_top_y_u16[..w],
u_full,
v_full,
p_idx * w,
p_idx * w + w,
w,
h,
p_matrix,
p_full_range,
use_simd,
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
} else if !is_last {
chroma_top_y_u16[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
}
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
return Ok(());
}
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p12_to_rgba_u16_row_endian(
row.y(),
u_full,
v_full,
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p12_to_rgba_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end =
one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
if let Some((u_full, v_full)) = centered {
yuv444p12_to_rgb_u16_row_endian(
row.y(),
u_full,
v_full,
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p12_to_rgb_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if let Some((u_full, v_full)) = centered {
yuv444p12_to_hsv_row_endian(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p12_to_hsv_row_endian(
row.y(),
row.u_half(),
row.v_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p12_to_rgba_row_endian(
row.y(),
u_full,
v_full,
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p12_to_rgba_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if let Some((u_full, v_full)) = centered {
yuv444p12_to_rgb_row_endian(
row.y(),
u_full,
v_full,
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p12_to_rgb_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R, const BE: bool> MixedSinker<'a, Yuv420p14<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected_elements = self.frame_elems(3)?;
if buf.len() < expected_elements {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected_elements, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> Yuv420p14Sink<BE> for MixedSinker<'_, Yuv420p14<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, Yuv420p14<BE>, R> {
type Input<'r> = Yuv420p14Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
reset_high_bit_yuv_streams(self);
Ok(())
}
#[allow(clippy::too_many_lines)]
fn process(&mut self, row: Yuv420p14Row<'_>) -> Result<(), Self::Error> {
const BITS: u32 = 14;
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y14,
idx,
w,
row.y().len(),
)));
}
if row.u_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UHalf14,
idx,
w / 2,
row.u_half().len(),
)));
}
if row.v_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VHalf14,
idx,
w / 2,
row.v_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let chroma_location = self.chroma_location.clone();
let Self {
rgb,
rgb_u16,
rgba,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
luma_scratch_u16,
rgb_stream,
rgb_stream_u16,
luma_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
luma_filter_stream_u16,
resample_outputs,
plan,
native,
native_420_u16,
frozen_native_route,
frozen_chroma_centered,
frozen_chroma_bottom_v,
frozen_chroma_top_v,
chroma_top_pending,
chroma_top_y_u16,
chroma_prev_u16,
chroma_prev_row,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let (y, u_half, v_half) = (row.y(), row.u_half(), row.v_half());
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
let chroma_v_phase = if bottom_v { 1.0 } else { 0.0 };
let top_v = chroma_420_top_sited_v(&chroma_location);
let need_output = luma.is_some()
|| rgb.is_some()
|| rgba.is_some()
|| hsv.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some();
let want_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_filter() {
plan.ensure_single_kernel_filter()?;
if top_v && want_color {
match yuv420p_top_reconstruct_filter::<BITS, BE>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p14_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p14_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p14_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p14_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != *native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
let insertion = select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: YUV420P_HIGH_BIT_NATIVE_ELIGIBLE,
with_native: *native,
area_plan: true,
},
);
match insertion {
InsertionPoint::NativeCodes => {
let stale_native = idx == 0
&& native_420_u16.as_ref().is_some_and(|join| {
(join.chroma_phase_centered() == Some(!center_sited)
|| join.chroma_bottom() == Some(!bottom_v)
|| join.chroma_top() == Some(!top_v))
&& join.next_y() == 0
});
let prev_native = if stale_native {
native_420_u16.take()
} else {
None
};
let native_result = yuv420p16_process_native::<BITS, BE>(
plan,
native_420_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
y,
u_half,
v_half,
matrix,
full_range,
idx,
w,
h,
|| {
ResamplePlan::area_chroma_420(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
chroma_v_phase,
top_v,
)
},
use_simd,
);
if stale_native && native_result.is_err() {
*native_420_u16 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::EncodedOutput => {
if top_v && want_color {
match yuv420p_top_reconstruct_area::<BITS, BE>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p14_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p14_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
if bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
} else {
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p14_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p14_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::LinearLight => {
unreachable!("encoded domain never selects the linear-light splice")
}
}
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
let want_color = want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16;
let need_output = want_color || luma.is_some();
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let top_v = chroma_420_top_sited_v(&chroma_location);
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
let need_centered_chroma = (center_sited || bottom_v) && want_color && !top_v;
if need_centered_chroma || (top_v && want_color) {
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
}
if top_v && want_color {
reserve_420_chroma_top_y_u16(chroma_top_y_u16, w, h)?;
}
if bottom_v || top_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if (bottom_v || top_v) && !want_color {
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
}
let centered = if need_centered_chroma {
Some(upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
true,
w,
BE,
use_simd,
))
} else {
None
};
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
let matrix = row.matrix();
let full_range = row.full_range();
if let Some(luma) = luma.as_deref_mut() {
let dst = &mut luma[one_plane_start..one_plane_end];
for (d, &s) in dst.iter_mut().zip(row.y().iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = (logical >> (BITS - 8)) as u8;
}
}
if top_v && want_color {
let is_last = idx + 1 == h;
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
true,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
&chroma_top_y_u16[..w],
u_full,
v_full,
p_idx * w,
p_idx * w + w,
w,
h,
p_matrix,
p_full_range,
use_simd,
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
} else if !is_last {
chroma_top_y_u16[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
}
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
return Ok(());
}
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p14_to_rgba_u16_row_endian(
row.y(),
u_full,
v_full,
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p14_to_rgba_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end =
one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
if let Some((u_full, v_full)) = centered {
yuv444p14_to_rgb_u16_row_endian(
row.y(),
u_full,
v_full,
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p14_to_rgb_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if let Some((u_full, v_full)) = centered {
yuv444p14_to_hsv_row_endian(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p14_to_hsv_row_endian(
row.y(),
row.u_half(),
row.v_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p14_to_rgba_row_endian(
row.y(),
u_full,
v_full,
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p14_to_rgba_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if let Some((u_full, v_full)) = centered {
yuv444p14_to_rgb_row_endian(
row.y(),
u_full,
v_full,
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p14_to_rgb_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R, const BE: bool> MixedSinker<'a, Yuv420p16<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected_elements = self.frame_elems(3)?;
if buf.len() < expected_elements {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected_elements, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> Yuv420p16Sink<BE> for MixedSinker<'_, Yuv420p16<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, Yuv420p16<BE>, R> {
type Input<'r> = Yuv420p16Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
reset_high_bit_yuv_streams(self);
Ok(())
}
#[allow(clippy::too_many_lines)]
fn process(&mut self, row: Yuv420p16Row<'_>) -> Result<(), Self::Error> {
const BITS: u32 = 16;
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y16,
idx,
w,
row.y().len(),
)));
}
if row.u_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UHalf16,
idx,
w / 2,
row.u_half().len(),
)));
}
if row.v_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VHalf16,
idx,
w / 2,
row.v_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let chroma_location = self.chroma_location.clone();
let Self {
rgb,
rgb_u16,
rgba,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
luma_scratch_u16,
rgb_stream,
rgb_stream_u16,
luma_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
luma_filter_stream_u16,
resample_outputs,
plan,
native,
native_420_u16,
frozen_native_route,
frozen_chroma_centered,
frozen_chroma_bottom_v,
frozen_chroma_top_v,
chroma_top_pending,
chroma_top_y_u16,
chroma_prev_u16,
chroma_prev_row,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let (y, u_half, v_half) = (row.y(), row.u_half(), row.v_half());
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
let chroma_v_phase = if bottom_v { 1.0 } else { 0.0 };
let top_v = chroma_420_top_sited_v(&chroma_location);
let need_output = luma.is_some()
|| rgb.is_some()
|| rgba.is_some()
|| hsv.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some();
let want_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_filter() {
plan.ensure_single_kernel_filter()?;
if top_v && want_color {
match yuv420p_top_reconstruct_filter::<BITS, BE>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p16_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p16_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
let r = packed_yuv422_triple_filter_resample::<BITS>(
luma_filter_stream_u16,
rgb_filter_stream,
rgb_filter_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p16_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p16_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != *native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
let insertion = select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: YUV420P_HIGH_BIT_NATIVE_ELIGIBLE,
with_native: *native,
area_plan: true,
},
);
match insertion {
InsertionPoint::NativeCodes => {
let stale_native = idx == 0
&& native_420_u16.as_ref().is_some_and(|join| {
(join.chroma_phase_centered() == Some(!center_sited)
|| join.chroma_bottom() == Some(!bottom_v)
|| join.chroma_top() == Some(!top_v))
&& join.next_y() == 0
});
let prev_native = if stale_native {
native_420_u16.take()
} else {
None
};
let native_result = yuv420p16_process_native::<BITS, BE>(
plan,
native_420_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
rgb_scratch,
rgb_scratch_u16,
y,
u_half,
v_half,
matrix,
full_range,
idx,
w,
h,
|| {
ResamplePlan::area_chroma_420(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
chroma_v_phase,
top_v,
)
},
use_simd,
);
if stale_native && native_result.is_err() {
*native_420_u16 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::EncodedOutput => {
if top_v && want_color {
match yuv420p_top_reconstruct_area::<BITS, BE>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
chroma_top_pending,
chroma_top_y_u16,
y,
u_half,
v_half,
w,
h,
plan,
idx,
use_simd,
matrix,
full_range,
center_sited,
)? {
core::ops::ControlFlow::Break(()) => return Ok(()),
core::ops::ControlFlow::Continue(()) => {}
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let expected = if luma.is_some() {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if rgb.is_some() || rgba.is_some() || hsv.is_some() {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
&None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
if bottom_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
u_half,
v_half,
idx,
bottom_v,
center_sited,
false,
w,
BE,
use_simd,
);
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv444p16_to_rgb_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv444p16_to_rgb_u16_row_endian(
y, u_full, v_full, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
if bottom_v {
stage_420_chroma_prev_u16(chroma_prev_u16, chroma_prev_row, u_half, v_half, idx, w);
}
} else {
packed_yuv422_triple_resample::<BITS>(
luma_stream_u16,
rgb_stream,
rgb_stream_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
&mut None,
hsv,
luma_scratch_u16,
rgb_scratch,
rgb_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
|scratch| deinterleave_y_high_bit_masked::<BITS, BE>(y, scratch, w),
|scratch| {
yuv420p16_to_rgb_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
|scratch| {
yuv420p16_to_rgb_u16_row_endian(
y, u_half, v_half, scratch, w, matrix, full_range, use_simd, BE,
)
},
)?;
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::LinearLight => {
unreachable!("encoded domain never selects the linear-light splice")
}
}
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_rgb_u16 = rgb_u16.is_some();
let want_rgba_u16 = rgba_u16.is_some();
let want_color = want_rgb || want_rgba || want_hsv || want_rgb_u16 || want_rgba_u16;
let need_output = want_color || luma.is_some();
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let top_v = chroma_420_top_sited_v(&chroma_location);
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
let need_centered_chroma = (center_sited || bottom_v) && want_color && !top_v;
if need_centered_chroma || (top_v && want_color) {
reserve_420_chroma_full_u16(chroma_full_u16, w, h)?;
}
if top_v && want_color {
reserve_420_chroma_top_y_u16(chroma_top_y_u16, w, h)?;
}
if bottom_v || top_v {
reserve_420_chroma_prev_u16(chroma_prev_u16, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if (bottom_v || top_v) && !want_color {
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
}
let centered = if need_centered_chroma {
Some(upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
true,
w,
BE,
use_simd,
))
} else {
None
};
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
let matrix = row.matrix();
let full_range = row.full_range();
if let Some(luma) = luma.as_deref_mut() {
let dst = &mut luma[one_plane_start..one_plane_end];
for (d, &s) in dst.iter_mut().zip(row.y().iter()) {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
*d = (logical >> (BITS - 8)) as u8;
}
}
if top_v && want_color {
let is_last = idx + 1 == h;
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
true,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
&chroma_top_y_u16[..w],
u_full,
v_full,
p_idx * w,
p_idx * w + w,
w,
h,
p_matrix,
p_full_range,
use_simd,
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
} else if !is_last {
chroma_top_y_u16[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited_u16::<BITS>(
chroma_full_u16,
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
BE,
use_simd,
);
yuv444p_top_identity_color_row::<BITS, BE>(
rgb,
rgba,
rgb_u16,
rgba_u16,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
matrix,
full_range,
use_simd,
)?;
}
stage_420_chroma_prev_u16(
chroma_prev_u16,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
return Ok(());
}
if want_rgba_u16 && !want_rgb_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p16_to_rgba_u16_row_endian(
row.y(),
u_full,
v_full,
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p16_to_rgba_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
} else if want_rgb_u16 {
let rgb_u16_buf = rgb_u16.as_deref_mut().unwrap();
let rgb_plane_end =
one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 3,
)))?;
let rgb_plane_start = one_plane_start * 3;
let rgb_u16_row = &mut rgb_u16_buf[rgb_plane_start..rgb_plane_end];
if let Some((u_full, v_full)) = centered {
yuv444p16_to_rgb_u16_row_endian(
row.y(),
u_full,
v_full,
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p16_to_rgb_u16_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_u16_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if want_rgba_u16 {
let rgba_u16_buf = rgba_u16.as_deref_mut().unwrap();
let rgba_u16_row =
rgba_u16_plane_row_slice(rgba_u16_buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_u16_to_rgba_u16_row::<BITS>(rgb_u16_row, rgba_u16_row, w);
}
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if let Some((u_full, v_full)) = centered {
yuv444p16_to_hsv_row_endian(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p16_to_hsv_row_endian(
row.y(),
row.u_half(),
row.v_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if let Some((u_full, v_full)) = centered {
yuv444p16_to_rgba_row_endian(
row.y(),
u_full,
v_full,
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p16_to_rgba_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgba_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if let Some((u_full, v_full)) = centered {
yuv444p16_to_rgb_row_endian(
row.y(),
u_full,
v_full,
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
} else {
yuv420p16_to_rgb_row_endian(
row.y(),
row.u_half(),
row.v_half(),
rgb_row,
w,
matrix,
full_range,
use_simd,
BE,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}