use super::{
AveragingDomainChanged, Chroma411U8, ChromaSitingChanged, ChromaU8, GeometryOverflow,
HsvFrameMut, InsufficientBuffer, MixedSinker, MixedSinkerError, NativeRouteChanged,
RowIndexOutOfRange, RowShapeMismatch, RowSlice, WidthAlignment, check_dimensions_match,
chroma_410_center_sited_h, chroma_411_center_sited_h, chroma_420_bottom_sited_v,
chroma_420_center_sited_h, chroma_420_top_sited_v, chroma_422_center_sited_h,
chroma_440_bottom_sited_v, chroma_440_top_sited_v, frozen_outputs_check,
planar_resample::{planar_dual_filter_resample, planar_dual_resample},
reconstruct_chroma, rgb_row_buf_or_scratch, rgba_plane_row_slice, st428_chroma_derived_guard,
};
use crate::{
KernelMatrix, PixelSink,
resample::{
AreaStream, AveragingDomain, FilterStream, InsertionContext, InsertionPoint, OutOfSequenceRow,
PlanGeometry, ResampleError, ResamplePlan, select_insertion_point, try_zeroed,
},
row::*,
source::*,
};
#[cfg(feature = "rgb")]
use super::linear_light;
#[cfg(feature = "rgb")]
use crate::resample::TransferFunction;
const YUV420P_NATIVE_ELIGIBLE: bool = true;
const YUV_PLANAR_8BIT_NATIVE_ELIGIBLE: bool = true;
pub(super) const YUV422P_CENTERED_H_PHASE: f64 = 0.25;
pub(super) const YUV41X_CENTERED_H_PHASE: f64 = 0.375;
pub(super) fn reserve_420_chroma_full(
chroma_full: &mut std::vec::Vec<u8>,
width: usize,
height: usize,
) -> Result<(), MixedSinkerError> {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if super::FORCE_CHROMA_FULL_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(width, height, width, height),
)));
}
let needed = width
.checked_mul(2)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width, height, 2,
)))?;
if chroma_full.len() < needed {
chroma_full
.try_reserve_exact(needed - chroma_full.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
width, height, width, height,
)))
})?;
chroma_full.resize(needed, 0);
}
Ok(())
}
#[cfg(feature = "yuv-planar")]
pub(super) fn reserve_420_chroma_prev(
chroma_prev: &mut std::vec::Vec<u8>,
width: usize,
height: usize,
) -> Result<(), MixedSinkerError> {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if super::FORCE_CHROMA_PREV_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(width, height, width, height),
)));
}
if chroma_prev.len() < width {
chroma_prev
.try_reserve_exact(width - chroma_prev.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
width, height, width, height,
)))
})?;
chroma_prev.resize(width, 0);
}
Ok(())
}
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) fn stage_420_chroma_prev(
chroma_prev: &mut [u8],
chroma_prev_row: &mut Option<usize>,
u_half: &[u8],
v_half: &[u8],
idx: usize,
width: usize,
) {
debug_assert!(
chroma_prev.len() >= width,
"chroma_prev must be reserved via reserve_420_chroma_prev first"
);
let half = width / 2;
chroma_prev[..half].copy_from_slice(&u_half[..half]);
chroma_prev[half..width].copy_from_slice(&v_half[..half]);
*chroma_prev_row = Some(idx / 2);
}
pub(super) fn upsample_420_chroma_center_h<'s>(
chroma_full: &'s mut [u8],
u_half: &[u8],
v_half: &[u8],
width: usize,
use_simd: bool,
) -> (&'s [u8], &'s [u8]) {
debug_assert!(
chroma_full.len() >= 2 * width,
"chroma_full must be reserved via reserve_420_chroma_full first"
);
let (u_full, v_full) = chroma_full[..2 * width].split_at_mut(width);
crate::row::chroma_upsample_2to1_center_h_row(u_half, u_full, width, use_simd);
crate::row::chroma_upsample_2to1_center_h_row(v_half, v_full, width, use_simd);
(u_full, v_full)
}
#[cfg(feature = "yuv-planar")]
pub(super) fn upsample_420_chroma_cosited_h<'s>(
chroma_full: &'s mut [u8],
u_half: &[u8],
v_half: &[u8],
width: usize,
) -> (&'s [u8], &'s [u8]) {
debug_assert!(
chroma_full.len() >= 2 * width,
"chroma_full must be reserved via reserve_420_chroma_full first"
);
let (u_full, v_full) = chroma_full[..2 * width].split_at_mut(width);
crate::row::scalar::chroma_upsample_2to1_cosited_h(u_half, u_full, width);
crate::row::scalar::chroma_upsample_2to1_cosited_h(v_half, v_full, width);
(u_full, v_full)
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
pub(super) fn upsample_420_chroma_sited<'s>(
chroma_full: &'s mut [u8],
chroma_prev: &mut [u8],
chroma_prev_row: &mut Option<usize>,
u_half: &[u8],
v_half: &[u8],
idx: usize,
bottom_v: bool,
center_h: bool,
stage: bool,
width: usize,
use_simd: bool,
) -> (&'s [u8], &'s [u8]) {
debug_assert!(
chroma_full.len() >= 2 * width,
"chroma_full must be reserved via reserve_420_chroma_full first"
);
let half = width / 2;
let chroma_row = idx / 2;
let do_vblend =
bottom_v && idx & 1 == 0 && chroma_row > 0 && *chroma_prev_row == Some(chroma_row - 1);
let result = if do_vblend {
debug_assert!(
chroma_prev.len() >= width,
"chroma_prev must be reserved via reserve_420_chroma_prev first"
);
let (u_full, v_full) = chroma_full[..2 * width].split_at_mut(width);
let (u_prev, v_prev) = chroma_prev[..width].split_at(half);
if center_h {
crate::row::chroma_upsample_420_bottom_even_h_row(u_prev, u_half, u_full, width, use_simd);
crate::row::chroma_upsample_420_bottom_even_h_row(v_prev, v_half, v_full, width, use_simd);
} else {
crate::row::chroma_upsample_420_bottomleft_even_h_row(
u_prev, u_half, u_full, width, use_simd,
);
crate::row::chroma_upsample_420_bottomleft_even_h_row(
v_prev, v_half, v_full, width, use_simd,
);
}
(&*u_full, &*v_full)
} else if center_h {
upsample_420_chroma_center_h(chroma_full, u_half, v_half, width, use_simd)
} else {
upsample_420_chroma_cosited_h(chroma_full, u_half, v_half, width)
};
if bottom_v && stage {
stage_420_chroma_prev(chroma_prev, chroma_prev_row, u_half, v_half, idx, width);
}
result
}
#[cfg(feature = "yuv-planar")]
pub(super) fn reserve_440_chroma_prev(
chroma_prev: &mut std::vec::Vec<u8>,
width: usize,
height: usize,
) -> Result<(), MixedSinkerError> {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if super::FORCE_CHROMA_PREV_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(width, height, width, height),
)));
}
let needed = width
.checked_mul(2)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width, height, 2,
)))?;
if chroma_prev.len() < needed {
chroma_prev
.try_reserve_exact(needed - chroma_prev.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
width, height, width, height,
)))
})?;
chroma_prev.resize(needed, 0);
}
Ok(())
}
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) fn stage_440_chroma_prev(
chroma_prev: &mut [u8],
chroma_prev_row: &mut Option<usize>,
u: &[u8],
v: &[u8],
idx: usize,
width: usize,
) {
debug_assert!(
chroma_prev.len() >= 2 * width,
"chroma_prev must be reserved via reserve_440_chroma_prev first"
);
chroma_prev[..width].copy_from_slice(&u[..width]);
chroma_prev[width..2 * width].copy_from_slice(&v[..width]);
*chroma_prev_row = Some(idx / 2);
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
pub(super) fn upsample_440_chroma_sited<'s>(
chroma_full: &'s mut [u8],
chroma_prev: &mut [u8],
chroma_prev_row: &mut Option<usize>,
u: &[u8],
v: &[u8],
idx: usize,
bottom_v: bool,
stage: bool,
width: usize,
use_simd: bool,
) -> (&'s [u8], &'s [u8]) {
debug_assert!(
chroma_full.len() >= 2 * width,
"chroma_full must be reserved via reserve_420_chroma_full first"
);
let chroma_row = idx / 2;
let do_vblend =
bottom_v && idx & 1 == 0 && chroma_row > 0 && *chroma_prev_row == Some(chroma_row - 1);
let (u_full, v_full) = chroma_full[..2 * width].split_at_mut(width);
if do_vblend {
debug_assert!(
chroma_prev.len() >= 2 * width,
"chroma_prev must be reserved via reserve_440_chroma_prev first"
);
let (u_prev, v_prev) = chroma_prev[..2 * width].split_at(width);
crate::row::chroma_upsample_440_bottom_v_row(u_prev, u, u_full, width, use_simd);
crate::row::chroma_upsample_440_bottom_v_row(v_prev, v, v_full, width, use_simd);
} else {
u_full.copy_from_slice(&u[..width]);
v_full.copy_from_slice(&v[..width]);
}
if bottom_v && stage {
stage_440_chroma_prev(chroma_prev, chroma_prev_row, u, v, idx, width);
}
(&*u_full, &*v_full)
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
fn yuv444_identity_color_row(
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y: &[u8],
u_full: &[u8],
v_full: &[u8],
one_plane_start: usize,
one_plane_end: usize,
w: usize,
h: usize,
matrix: crate::KernelMatrix,
primaries: &crate::Primaries,
full_range: bool,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (hh, ss, vv) = hsv.hsv();
yuv_444_to_hsv_row(
y,
u_full,
v_full,
&mut hh[one_plane_start..one_plane_end],
&mut ss[one_plane_start..one_plane_end],
&mut vv[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
);
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
yuv_444_to_rgba_row_primaries(
y, u_full, v_full, rgba_row, w, matrix, primaries, full_range, use_simd,
);
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
yuv_444_to_rgb_row_primaries(
y, u_full, v_full, rgb_row, w, matrix, primaries, full_range, use_simd,
);
if let Some(hsv) = hsv.as_mut() {
let (hh, ss, vv) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut hh[one_plane_start..one_plane_end],
&mut ss[one_plane_start..one_plane_end],
&mut vv[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
fn yuv440_identity_color_row(
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y: &[u8],
u_full: &[u8],
v_full: &[u8],
one_plane_start: usize,
one_plane_end: usize,
w: usize,
h: usize,
matrix: crate::KernelMatrix,
full_range: bool,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (hh, ss, vv) = hsv.hsv();
yuv_444_to_hsv_row(
y,
u_full,
v_full,
&mut hh[one_plane_start..one_plane_end],
&mut ss[one_plane_start..one_plane_end],
&mut vv[one_plane_start..one_plane_end],
w,
matrix,
full_range,
use_simd,
);
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
yuv_444_to_rgba_row(y, u_full, v_full, rgba_row, w, matrix, full_range, use_simd);
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
yuv_444_to_rgb_row(y, u_full, v_full, rgb_row, w, matrix, full_range, use_simd);
if let Some(hsv) = hsv.as_mut() {
let (hh, ss, vv) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut hh[one_plane_start..one_plane_end],
&mut ss[one_plane_start..one_plane_end],
&mut vv[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
impl<'a, R> MixedSinker<'a, Yuv420p, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_luma_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_luma_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_luma_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_pixels()?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientLumaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.luma_u16 = Some(buf);
Ok(self)
}
}
impl<R> PixelSink for MixedSinker<'_, Yuv420p, R> {
type Input<'r> = Yuv420pRow<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
if let Some(stream) = self.rgb_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.rgb_filter_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_filter_stream.as_mut() {
stream.reset();
}
if let Some(native) = self.native_420.as_mut() {
native.reset();
}
if let Some(bicublin) = self.bicublin_420.as_mut() {
bicublin.reset();
}
if let Some(hsv) = self.hsv_planar.as_mut() {
hsv.reset();
}
self.frozen_native_route = None;
self.frozen_domain = None;
self.resample_outputs = None;
self.frozen_chroma_centered = None;
self.frozen_chroma_bottom_v = None;
self.frozen_chroma_top_v = None;
self.chroma_prev_row = None;
self.chroma_top_pending = None;
self.chroma_top_st428 = None;
#[cfg(feature = "rgb")]
{
self.linear_light_frame = None;
}
Ok(())
}
fn process(&mut self, row: Yuv420pRow<'_>) -> Result<(), Self::Error> {
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
let chroma_location = self.chroma_location.clone();
let primaries = self.primaries.clone();
let st428_interpretation = self.st428_interpretation;
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y,
idx,
w,
row.y().len(),
)));
}
if row.u_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UHalf,
idx,
w / 2,
row.u_half().len(),
)));
}
if row.v_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VHalf,
idx,
w / 2,
row.v_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
st428_chroma_derived_guard(row.matrix(), &primaries, self.st428_interpretation)
.map_err(MixedSinkerError::St428CieXyzUnsupported)?;
let Self {
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
chroma_full,
chroma_prev,
chroma_prev_row,
plan,
rgb_stream,
luma_stream,
rgb_filter_stream,
luma_filter_stream,
resample_outputs,
native,
native_420,
hsv_planar,
bicublin_420,
frozen_native_route,
frozen_domain,
averaging_domain,
#[cfg(feature = "rgb")]
linear_mode,
#[cfg(feature = "rgb")]
linear_light_frame,
#[cfg(feature = "rgb")]
linear_scene_scratch,
#[cfg(feature = "rgb")]
transfer_function,
frozen_chroma_centered,
frozen_chroma_bottom_v,
frozen_chroma_top_v,
chroma_top_pending,
chroma_top_st428,
chroma_top_y,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let top_v = chroma_420_top_sited_v(&chroma_location);
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
let chroma_v_phase = if bottom_v { 1.0 } else { 0.0 };
let want_color = rgb.is_some() || rgba.is_some() || hsv.is_some();
let need_output =
luma.is_some() || luma_u16.is_some() || rgb.is_some() || rgba.is_some() || hsv.is_some();
if need_output
&& let Some(frozen) = *frozen_domain
&& frozen != *averaging_domain
{
return Err(MixedSinkerError::AveragingDomainChanged(
AveragingDomainChanged::new(idx),
));
}
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
match *averaging_domain {
AveragingDomain::Encoded => {}
AveragingDomain::Linear => {
#[cfg(feature = "rgb")]
{
let tf = transfer_function
.unwrap_or_else(|| TransferFunction::for_matrix(&crate::ColorMatrix::from(matrix)));
if top_v && want_color {
let is_last = idx + 1 == h;
let want_luma_tail = luma.is_some() || luma_u16.is_some();
let cursor = linear_light_frame.as_ref().map_or(0, |b| b.next_y());
let expected_source = cursor + usize::from(chroma_top_pending.is_some());
if idx != expected_source {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected_source, idx),
)));
}
linear_light::linear_light_preflight(
linear_light_frame,
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
hsv,
tf,
*linear_mode,
plan,
cursor,
)?;
reserve_420_chroma_full(chroma_full, w, h)?;
reserve_420_chroma_prev(chroma_prev, w, h)?;
if chroma_top_y.len() < w {
chroma_top_y
.try_reserve_exact(w - chroma_top_y.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w, h, w, h,
)))
})?;
chroma_top_y.resize(w, 0);
}
if idx & 1 == 0 && is_last && chroma_top_pending.is_some() {
linear_light::linear_light_reserve_final_tail(
linear_light_frame,
plan,
w,
h,
want_luma_tail,
)?;
}
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
true,
center_sited,
false,
w,
use_simd,
);
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
&chroma_top_y[..w],
p_idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_444_to_rgb_row(
&chroma_top_y[..w],
u_full,
v_full,
dst,
w,
p_matrix,
p_full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
&chroma_top_y[..w],
u_full,
v_full,
dst,
w,
p_matrix,
p_full_range,
);
},
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
use_simd,
);
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
);
},
)?;
} else if !is_last {
chroma_top_y[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
use_simd,
);
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
);
},
)?;
}
stage_420_chroma_prev(
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
if need_output && frozen_domain.is_none() {
*frozen_domain = Some(AveragingDomain::Linear);
}
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
let r = if (center_sited || bottom_v) && want_color {
linear_light::linear_light_preflight(
linear_light_frame,
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
hsv,
tf,
*linear_mode,
plan,
idx,
)?;
reserve_420_chroma_full(chroma_full, w, h)?;
if bottom_v {
reserve_420_chroma_prev(chroma_prev, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
false,
w,
use_simd,
);
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
);
},
)
} else {
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_420_to_rgb_row(
row.y(),
row.u_half(),
row.v_half(),
dst,
w,
matrix,
full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_420_to_rgb_f32_unclamped_row(
row.y(),
row.u_half(),
row.v_half(),
dst,
w,
matrix,
full_range,
);
},
)
};
if r.is_ok() && bottom_v && want_color {
stage_420_chroma_prev(
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
}
if r.is_ok() && need_output && frozen_domain.is_none() {
*frozen_domain = Some(AveragingDomain::Linear);
}
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
#[cfg(not(feature = "rgb"))]
{
return Err(plan.linear_domain_unsupported().into());
}
}
AveragingDomain::Premultiplied => {
return Err(plan.premultiplied_domain_unsupported().into());
}
}
if plan.kind().is_filter() {
if plan.is_bicublin() {
return bicublin_yuv420p_resample(
plan,
bicublin_420,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
row.u_half(),
row.v_half(),
matrix,
full_range,
idx,
w,
h,
use_simd,
);
}
if top_v && want_color {
let is_last = idx + 1 == h;
let need_luma = luma.is_some() || luma_u16.is_some();
let cursor = if need_luma {
luma_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
};
let expected_source = cursor + usize::from(chroma_top_pending.is_some());
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected_source),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
reserve_420_chroma_prev(chroma_prev, w, h)?;
if chroma_top_y.len() < w {
chroma_top_y
.try_reserve_exact(w - chroma_top_y.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w, h, w, h,
)))
})?;
chroma_top_y.resize(w, 0);
}
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
true,
center_sited,
false,
w,
use_simd,
);
planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
&chroma_top_y[..w],
w,
plan,
p_idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
&chroma_top_y[..w],
u_full,
v_full,
scratch,
w,
p_matrix,
p_full_range,
use_simd,
);
},
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
use_simd,
);
planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
} else if !is_last {
chroma_top_y[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
use_simd,
);
planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
}
stage_420_chroma_prev(
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
if (center_sited || bottom_v) && want_color {
let need_luma = luma.is_some() || luma_u16.is_some();
let expected = if need_luma {
luma_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
if bottom_v {
reserve_420_chroma_prev(chroma_prev, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
false,
w,
use_simd,
);
let r = planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
);
if r.is_ok() && bottom_v {
stage_420_chroma_prev(
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
}
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
let r = planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_420_to_rgb_row(
row.y(),
row.u_half(),
row.v_half(),
scratch,
w,
matrix,
full_range,
use_simd,
);
},
);
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != *native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
let insertion = select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: YUV420P_NATIVE_ELIGIBLE,
with_native: *native,
area_plan: true,
},
);
match insertion {
InsertionPoint::NativeCodes => {
let stale_native = idx == 0
&& native_420.as_ref().is_some_and(|join| {
join.chroma.is_some()
&& (join.chroma_centered != center_sited
|| join.chroma_bottom != bottom_v
|| join.chroma_top != top_v)
&& join.y.next_y() == 0
});
let prev_native = if stale_native {
native_420.take()
} else {
None
};
let native_result = yuv420p_process_native(
plan,
native_420,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
row.u_half(),
row.v_half(),
matrix,
full_range,
idx,
w,
h,
|| {
ResamplePlan::area_chroma_420(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
chroma_v_phase,
top_v,
)
},
use_simd,
);
if stale_native && native_result.is_err() {
*native_420 = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_domain.is_none() && need_output {
*frozen_domain = Some(AveragingDomain::Encoded);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::EncodedOutput => {
if hsv.is_some() && rgb.is_none() && rgba.is_none() {
let stale_hsv = idx == 0
&& hsv_planar.as_ref().is_some_and(|join| {
(join.chroma_centered() != center_sited
|| join.chroma_bottom() != bottom_v
|| join.chroma_top() != top_v)
&& join.next_y() == 0
});
let prev_hsv = if stale_hsv { hsv_planar.take() } else { None };
let hsv_result = super::planar_resample::hsv_direct_resample(
hsv_planar,
resample_outputs,
luma,
luma_u16,
hsv,
row.y(),
row.u_half(),
row.v_half(),
matrix,
full_range,
2,
|| {
ResamplePlan::area_chroma_420(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
chroma_v_phase,
top_v,
)
},
w,
plan,
idx,
use_simd,
);
if stale_hsv && hsv_result.is_err() {
*hsv_planar = prev_hsv;
}
hsv_result?;
} else if top_v && want_color {
let is_last = idx + 1 == h;
let need_luma = luma.is_some() || luma_u16.is_some();
let cursor = if need_luma {
luma_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
};
let expected_source = cursor + usize::from(chroma_top_pending.is_some());
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected_source),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
reserve_420_chroma_prev(chroma_prev, w, h)?;
if chroma_top_y.len() < w {
chroma_top_y
.try_reserve_exact(w - chroma_top_y.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w, h, w, h,
)))
})?;
chroma_top_y.resize(w, 0);
}
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
true,
center_sited,
false,
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
&chroma_top_y[..w],
w,
plan,
p_idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
&chroma_top_y[..w],
u_full,
v_full,
scratch,
w,
p_matrix,
p_full_range,
use_simd,
);
},
)?;
}
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
} else if !is_last {
chroma_top_y[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
}
stage_420_chroma_prev(
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
} else if (center_sited || bottom_v) && want_color {
let need_luma = luma.is_some() || luma_u16.is_some();
let expected = if need_luma {
luma_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
if bottom_v {
reserve_420_chroma_prev(chroma_prev, w, h)?;
}
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
false,
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
if bottom_v {
stage_420_chroma_prev(
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
}
} else {
yuv420p_process_resampled(
plan,
rgb_stream,
luma_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
row.u_half(),
row.v_half(),
matrix,
full_range,
idx,
w,
use_simd,
)?;
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_domain.is_none() && need_output {
*frozen_domain = Some(AveragingDomain::Encoded);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::LinearLight => {
unreachable!("encoded domain never selects the linear-light splice")
}
}
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_color = want_rgb || want_rgba || want_hsv;
let need_output = want_color || luma.is_some() || luma_u16.is_some();
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let center_sited = chroma_420_center_sited_h(&chroma_location);
let bottom_v = chroma_420_bottom_sited_v(&chroma_location);
let top_v = chroma_420_top_sited_v(&chroma_location);
if let Some(frozen) = *frozen_chroma_centered
&& (frozen != center_sited
|| *frozen_chroma_bottom_v != Some(bottom_v)
|| *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if (center_sited || bottom_v || top_v) && want_color {
reserve_420_chroma_full(chroma_full, w, h)?;
}
if bottom_v || top_v {
reserve_420_chroma_prev(chroma_prev, w, h)?;
}
if top_v && want_color && chroma_top_y.len() < w {
chroma_top_y
.try_reserve_exact(w - chroma_top_y.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w, h, w, h,
)))
})?;
chroma_top_y.resize(w, 0);
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if (bottom_v || top_v) && !want_color {
stage_420_chroma_prev(
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
}
if frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
if let Some(luma) = luma.as_deref_mut() {
luma[one_plane_start..one_plane_end].copy_from_slice(&row.y()[..w]);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::y_plane_to_luma_u16_row(
row.y(),
&mut buf[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if top_v && want_color {
let is_last = idx + 1 == h;
if idx & 1 == 0
&& let Some((p_idx, p_matrix, p_full_range)) = *chroma_top_pending
{
let (frozen_primaries, frozen_interp) = chroma_top_st428
.clone()
.expect("Top forward-delay freezes ST 428 colorimetry with chroma_top_pending");
st428_chroma_derived_guard(p_matrix, &frozen_primaries, frozen_interp)
.map_err(MixedSinkerError::St428CieXyzUnsupported)?;
*chroma_top_pending = None;
*chroma_top_st428 = None;
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
true,
center_sited,
false,
w,
use_simd,
);
yuv444_identity_color_row(
rgb,
rgba,
hsv,
rgb_scratch,
&chroma_top_y[..w],
u_full,
v_full,
p_idx * w,
p_idx * w + w,
w,
h,
p_matrix,
&frozen_primaries,
p_full_range,
use_simd,
)?;
}
if idx & 1 == 1 && !is_last {
chroma_top_y[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, row.matrix(), row.full_range()));
*chroma_top_st428 = Some((primaries, st428_interpretation));
} else {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
false,
center_sited,
false,
w,
use_simd,
);
yuv444_identity_color_row(
rgb,
rgba,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
row.matrix(),
&primaries,
row.full_range(),
use_simd,
)?;
}
stage_420_chroma_prev(
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
w,
);
return Ok(());
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if center_sited || bottom_v {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
true,
w,
use_simd,
);
yuv_444_to_hsv_row(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_420_to_hsv_row(
row.y(),
row.u_half(),
row.v_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if center_sited || bottom_v {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
true,
w,
use_simd,
);
yuv_444_to_rgba_row_primaries(
row.y(),
u_full,
v_full,
rgba_row,
w,
row.matrix(),
&primaries,
row.full_range(),
use_simd,
);
} else {
yuv_420_to_rgba_row_primaries(
row.y(),
row.u_half(),
row.v_half(),
rgba_row,
w,
row.matrix(),
&primaries,
row.full_range(),
use_simd,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if center_sited || bottom_v {
let (u_full, v_full) = upsample_420_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u_half(),
row.v_half(),
idx,
bottom_v,
center_sited,
true,
w,
use_simd,
);
yuv_444_to_rgb_row_primaries(
row.y(),
u_full,
v_full,
rgb_row,
w,
row.matrix(),
&primaries,
row.full_range(),
use_simd,
);
} else {
yuv_420_to_rgb_row_primaries(
row.y(),
row.u_half(),
row.v_half(),
rgb_row,
w,
row.matrix(),
&primaries,
row.full_range(),
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<R> Yuv420pSink for MixedSinker<'_, Yuv420p, R> {}
pub(super) struct NativeYuv420 {
y: AreaStream<u8>,
y_stage: std::vec::Vec<u8>,
chroma: Option<NativeChroma>,
staged: [[bool; 2]; 3],
next_emit: usize,
chroma_centered: bool,
chroma_bottom: bool,
chroma_top: bool,
}
pub(super) struct NativeChroma {
u: AreaStream<u8>,
v: AreaStream<u8>,
u_stage: std::vec::Vec<u8>,
v_stage: std::vec::Vec<u8>,
}
impl NativeYuv420 {
fn new(
plan: &ResamplePlan,
build_chroma_plan: impl FnOnce() -> Result<ResamplePlan, ResampleError>,
w: usize,
h: usize,
need_color: bool,
) -> Result<Self, ResampleError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter());
}
let y = AreaStream::new(plan.h(), plan.v(), w, h, 1)?;
let alloc =
|_| ResampleError::AllocationFailed(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()));
let stage_len = plan.out_w().checked_mul(2).ok_or_else(|| {
ResampleError::Overflow(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()))
})?;
let mut chroma_centered = false;
let mut chroma_bottom = false;
let mut chroma_top = false;
let chroma = if need_color {
let cplan = build_chroma_plan()?;
chroma_centered = cplan.has_chroma_h_phase();
chroma_bottom = cplan.has_chroma_v_phase();
chroma_top = cplan.has_chroma_v_top();
Some(NativeChroma {
u: AreaStream::new(cplan.h(), cplan.v(), cplan.src_w(), cplan.src_h(), 1)?,
v: AreaStream::new(cplan.h(), cplan.v(), cplan.src_w(), cplan.src_h(), 1)?,
u_stage: try_zeroed(stage_len).map_err(alloc)?,
v_stage: try_zeroed(stage_len).map_err(alloc)?,
})
} else {
None
};
Ok(Self {
y,
y_stage: try_zeroed(stage_len).map_err(alloc)?,
chroma,
staged: [[false; 2]; 3],
next_emit: 0,
chroma_centered,
chroma_bottom,
chroma_top,
})
}
pub(super) fn reset(&mut self) {
self.y.reset();
if let Some(chroma) = self.chroma.as_mut() {
chroma.u.reset();
chroma.v.reset();
}
self.staged = [[false; 2]; 3];
self.next_emit = 0;
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn has_chroma(&self) -> bool {
self.chroma.is_some()
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_centered(&self) -> bool {
self.chroma_centered
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_bottom(&self) -> bool {
self.chroma_bottom
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_top(&self) -> bool {
self.chroma_top
}
#[cfg(feature = "yuv-semi-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn next_y(&self) -> usize {
self.y.next_y()
}
fn check_sequence(&self, idx: usize) -> Result<(), MixedSinkerError> {
if self.y.next_y() != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(self.y.next_y(), idx),
)));
}
if let Some(chroma) = self.chroma.as_ref() {
let chroma_expected = idx.div_ceil(2);
for stream in [&chroma.u, &chroma.v] {
if stream.next_y() != chroma_expected {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(stream.next_y().saturating_mul(2), idx),
)));
}
}
}
Ok(())
}
}
#[allow(clippy::too_many_arguments)]
pub(super) fn yuv420p_native_preflight_check_only(
native_420: &Option<std::boxed::Box<NativeYuv420>>,
resample_outputs: &Option<super::FrozenOutputs>,
rgb: &Option<&mut [u8]>,
rgba: &Option<&mut [u8]>,
luma: &Option<&mut [u8]>,
luma_u16: &Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
idx: usize,
need_luma: bool,
need_color: bool,
) -> Result<bool, MixedSinkerError> {
native_preflight_core_check_only(
native_420.as_ref().map_or(0, |join| join.y.next_y()),
resample_outputs,
rgb,
rgba,
&None,
&None,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)
}
#[allow(clippy::too_many_arguments)]
pub(super) fn native_preflight_core_check_only(
expected: usize,
resample_outputs: &Option<super::FrozenOutputs>,
rgb: &Option<&mut [u8]>,
rgba: &Option<&mut [u8]>,
rgb_u16: &Option<&mut [u16]>,
rgba_u16: &Option<&mut [u16]>,
luma: &Option<&mut [u8]>,
luma_u16: &Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
idx: usize,
need_luma: bool,
need_color: bool,
) -> Result<bool, MixedSinkerError> {
if !need_luma && !need_color {
return Ok(false);
}
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(expected, idx),
)));
}
if let Some(frozen) = resample_outputs {
let snapshot = super::FrozenOutputs::snapshot(
luma.as_deref(),
luma_u16.as_deref(),
rgb.as_deref(),
rgba.as_deref(),
rgb_u16.as_deref(),
rgba_u16.as_deref(),
None,
None,
None,
None,
None,
hsv.as_mut().map(|f| {
let (h, s, v) = f.hsv();
(&h[..], &s[..], &v[..])
}),
None,
);
if *frozen != snapshot {
return Err(MixedSinkerError::ResampleOutputsChanged(
super::ResampleOutputsChanged::new(idx),
));
}
}
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(expected, idx),
)));
}
Ok(true)
}
#[allow(clippy::too_many_arguments)]
pub(super) fn yuv420p_process_native(
plan: &ResamplePlan,
native_420: &mut Option<std::boxed::Box<NativeYuv420>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y_row: &[u8],
u_half: &[u8],
v_half: &[u8],
matrix: KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
build_chroma_plan: impl FnOnce() -> Result<ResamplePlan, ResampleError>,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || hsv.is_some() || rgba.is_some();
let want_hsv_direct = hsv.is_some() && rgb.is_none() && rgba.is_none();
if !yuv420p_native_preflight_check_only(
native_420,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
let rebuild = native_420.is_none()
|| native_420
.as_ref()
.is_some_and(|join| join.chroma.is_some() != need_color);
let new_join = if rebuild {
let join = NativeYuv420::new(plan, build_chroma_plan, w, h, need_color)?;
Some(crate::resample::try_box(join).map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?)
} else {
None
};
if need_color && !want_hsv_direct {
let row_bytes =
ow.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
ow,
plan.out_h(),
3,
)))?;
if rgb_scratch.len() < row_bytes {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if FORCE_NATIVE_RGB_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
)));
}
rgb_scratch
.try_reserve_exact(row_bytes - rgb_scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?;
rgb_scratch.resize(row_bytes, 0);
}
}
if let Some(boxed) = new_join {
*native_420 = Some(boxed);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let join = native_420
.as_mut()
.expect("built or reused in the preflight");
join.check_sequence(idx)?;
let NativeYuv420 {
y,
y_stage,
chroma,
staged,
next_emit,
chroma_centered: _,
chroma_bottom: _,
chroma_top: _,
} = &mut **join;
y.feed_row(idx, y_row, use_simd, |oy, out_row| {
let slot = oy & 1;
y_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[0][slot] = true;
})?;
if let Some(c) = chroma.as_mut()
&& idx.is_multiple_of(2)
{
let cidx = idx / 2;
let NativeChroma {
u,
v,
u_stage,
v_stage,
} = c;
u.feed_row(cidx, u_half, use_simd, |oy, out_row| {
let slot = oy & 1;
u_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[1][slot] = true;
})?;
v.feed_row(cidx, v_half, use_simd, |oy, out_row| {
let slot = oy & 1;
v_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[2][slot] = true;
})?;
}
while *next_emit < plan.out_h() {
let slot = *next_emit & 1;
let chroma_ready = match chroma.as_ref() {
Some(_) => staged[1][slot] && staged[2][slot],
None => true,
};
if !(staged[0][slot] && chroma_ready) {
break;
}
let oy = *next_emit;
let y_out = &y_stage[slot * ow..slot * ow + ow];
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(y_out);
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(y_out) {
*dst = src as u16;
}
}
if let Some(c) = chroma.as_ref() {
let u_row = &c.u_stage[slot * ow..slot * ow + ow];
let v_row = &c.v_stage[slot * ow..slot * ow + ow];
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (hp, sp, vp) = hsv.hsv();
yuv_444_to_hsv_row(
y_out,
u_row,
v_row,
&mut hp[oy * ow..(oy + 1) * ow],
&mut sp[oy * ow..(oy + 1) * ow],
&mut vp[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
} else {
let out_rgb = &mut rgb_scratch[..ow * 3];
yuv_444_to_rgb_row(
y_out, u_row, v_row, out_rgb, ow, matrix, full_range, use_simd,
);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_rgb);
}
if let Some(hsv) = hsv.as_mut() {
let (hp, sp, vp) = hsv.hsv();
rgb_to_hsv_row(
out_rgb,
&mut hp[oy * ow..(oy + 1) * ow],
&mut sp[oy * ow..(oy + 1) * ow],
&mut vp[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
expand_rgb_to_rgba_row(out_rgb, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
}
}
staged[0][slot] = false;
staged[1][slot] = false;
staged[2][slot] = false;
*next_emit += 1;
}
Ok(())
}
pub(super) struct BicublinYuv420 {
y: FilterStream<u8>,
y_plane: std::vec::Vec<u8>,
y_emitted: usize,
chroma: Option<BicublinChroma>,
next_emit: usize,
}
struct BicublinChroma {
u: FilterStream<u8>,
v: FilterStream<u8>,
u_plane: std::vec::Vec<u8>,
v_plane: std::vec::Vec<u8>,
u_emitted: usize,
v_emitted: usize,
}
impl BicublinYuv420 {
fn new(plan: &ResamplePlan, w: usize, h: usize, need_color: bool) -> Result<Self, ResampleError> {
let (fh, fv) = (
plan
.filter_h()
.expect("bicublin plan carries luma horizontal windows"),
plan
.filter_v()
.expect("bicublin plan carries luma vertical windows"),
);
let y = FilterStream::<u8>::new(fh, fv, w, h, 1)?;
let alloc =
|_| ResampleError::AllocationFailed(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()));
let plane_len = plan.out_w().checked_mul(plan.out_h()).ok_or_else(|| {
ResampleError::Overflow(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()))
})?;
let chroma = if need_color {
let (fhc, fvc) = (
plan
.filter_h_chroma()
.expect("bicublin plan carries chroma horizontal windows"),
plan
.filter_v_chroma()
.expect("bicublin plan carries chroma vertical windows"),
);
let (cw, ch) = (w / 2, h.div_ceil(2));
Some(BicublinChroma {
u: FilterStream::<u8>::new(fhc, fvc, cw, ch, 1)?,
v: FilterStream::<u8>::new(fhc, fvc, cw, ch, 1)?,
u_plane: try_zeroed(plane_len).map_err(alloc)?,
v_plane: try_zeroed(plane_len).map_err(alloc)?,
u_emitted: 0,
v_emitted: 0,
})
} else {
None
};
Ok(Self {
y,
y_plane: try_zeroed(plane_len).map_err(alloc)?,
y_emitted: 0,
chroma,
next_emit: 0,
})
}
pub(super) fn reset(&mut self) {
self.y.reset();
self.y_emitted = 0;
if let Some(chroma) = self.chroma.as_mut() {
chroma.u.reset();
chroma.v.reset();
chroma.u_emitted = 0;
chroma.v_emitted = 0;
}
self.next_emit = 0;
}
fn check_sequence(&self, idx: usize) -> Result<(), MixedSinkerError> {
if self.y.next_y() != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(self.y.next_y(), idx),
)));
}
if let Some(chroma) = self.chroma.as_ref() {
let chroma_expected = idx.div_ceil(2);
for stream in [&chroma.u, &chroma.v] {
if stream.next_y() != chroma_expected {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(stream.next_y().saturating_mul(2), idx),
)));
}
}
}
Ok(())
}
}
#[allow(clippy::too_many_arguments)]
pub(super) fn bicublin_yuv420p_resample(
plan: &ResamplePlan,
bicublin_420: &mut Option<std::boxed::Box<BicublinYuv420>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y_row: &[u8],
u_half: &[u8],
v_half: &[u8],
matrix: KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || hsv.is_some() || rgba.is_some();
let want_hsv_direct = hsv.is_some() && rgb.is_none() && rgba.is_none();
if !yuv420p_native_preflight_bicublin_check_only(
bicublin_420,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
let rebuild = bicublin_420.is_none()
|| bicublin_420
.as_ref()
.is_some_and(|join| join.chroma.is_some() != need_color);
let new_join = if rebuild {
let join = BicublinYuv420::new(plan, w, h, need_color)?;
Some(crate::resample::try_box(join).map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?)
} else {
None
};
if need_color && !want_hsv_direct {
let row_bytes =
ow.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
ow,
plan.out_h(),
3,
)))?;
if rgb_scratch.len() < row_bytes {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if FORCE_NATIVE_RGB_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
)));
}
rgb_scratch
.try_reserve_exact(row_bytes - rgb_scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?;
rgb_scratch.resize(row_bytes, 0);
}
}
if let Some(boxed) = new_join {
*bicublin_420 = Some(boxed);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let join = bicublin_420
.as_mut()
.expect("built or reused in the preflight");
join.check_sequence(idx)?;
let BicublinYuv420 {
y,
y_plane,
y_emitted,
chroma,
next_emit,
} = &mut **join;
y.feed_row(idx, y_row, use_simd, |oy, out_row| {
y_plane[oy * ow..oy * ow + ow].copy_from_slice(out_row);
*y_emitted = oy + 1;
})?;
if let Some(c) = chroma.as_mut()
&& idx.is_multiple_of(2)
{
let cidx = idx / 2;
let BicublinChroma {
u,
v,
u_plane,
v_plane,
u_emitted,
v_emitted,
} = c;
u.feed_row(cidx, u_half, use_simd, |oy, out_row| {
u_plane[oy * ow..oy * ow + ow].copy_from_slice(out_row);
*u_emitted = oy + 1;
})?;
v.feed_row(cidx, v_half, use_simd, |oy, out_row| {
v_plane[oy * ow..oy * ow + ow].copy_from_slice(out_row);
*v_emitted = oy + 1;
})?;
}
let ready = match chroma.as_ref() {
Some(c) => (*y_emitted).min(c.u_emitted).min(c.v_emitted),
None => *y_emitted,
};
while *next_emit < ready {
let oy = *next_emit;
let y_out = &y_plane[oy * ow..oy * ow + ow];
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(y_out);
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(y_out) {
*dst = src as u16;
}
}
if let Some(c) = chroma.as_ref() {
let u_out = &c.u_plane[oy * ow..oy * ow + ow];
let v_out = &c.v_plane[oy * ow..oy * ow + ow];
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (hp, sp, vp) = hsv.hsv();
yuv_444_to_hsv_row(
y_out,
u_out,
v_out,
&mut hp[oy * ow..(oy + 1) * ow],
&mut sp[oy * ow..(oy + 1) * ow],
&mut vp[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
} else {
let out_rgb = &mut rgb_scratch[..ow * 3];
yuv_444_to_rgb_row(
y_out, u_out, v_out, out_rgb, ow, matrix, full_range, use_simd,
);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_rgb);
}
if let Some(hsv) = hsv.as_mut() {
let (hp, sp, vp) = hsv.hsv();
rgb_to_hsv_row(
out_rgb,
&mut hp[oy * ow..(oy + 1) * ow],
&mut sp[oy * ow..(oy + 1) * ow],
&mut vp[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
expand_rgb_to_rgba_row(out_rgb, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
}
}
*next_emit += 1;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn yuv420p_native_preflight_bicublin_check_only(
bicublin_420: &Option<std::boxed::Box<BicublinYuv420>>,
resample_outputs: &Option<super::FrozenOutputs>,
rgb: &Option<&mut [u8]>,
rgba: &Option<&mut [u8]>,
luma: &Option<&mut [u8]>,
luma_u16: &Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
idx: usize,
need_luma: bool,
need_color: bool,
) -> Result<bool, MixedSinkerError> {
native_preflight_core_check_only(
bicublin_420.as_ref().map_or(0, |join| join.y.next_y()),
resample_outputs,
rgb,
rgba,
&None,
&None,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)
}
#[allow(clippy::too_many_arguments)]
pub(super) fn yuv420p_process_resampled(
plan: &ResamplePlan,
rgb_stream: &mut Option<std::boxed::Box<AreaStream<u8>>>,
luma_stream: &mut Option<std::boxed::Box<AreaStream<u8>>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y_row: &[u8],
u_half: &[u8],
v_half: &[u8],
matrix: KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let ow = plan.out_w();
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || hsv.is_some() || rgba.is_some();
let expected = if need_luma {
luma_stream.as_ref().map_or(0, |stream| stream.next_y())
} else if need_color {
rgb_stream.as_ref().map_or(0, |stream| stream.next_y())
} else {
return Ok(());
};
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(expected, idx),
)));
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(expected, idx),
)));
}
if need_luma && luma_stream.is_none() {
*luma_stream = Some({
let stream = AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 1)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_color && rgb_stream.is_none() {
*rgb_stream = Some({
let stream = AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
let color_row = if need_color {
let row_bytes =
w.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w,
plan.src_h(),
3,
)))?;
if rgb_scratch.len() < row_bytes {
rgb_scratch
.try_reserve_exact(row_bytes - rgb_scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
)))
})?;
rgb_scratch.resize(row_bytes, 0);
}
let scratch = &mut rgb_scratch[..row_bytes];
yuv_420_to_rgb_row(
y_row, u_half, v_half, scratch, w, matrix, full_range, use_simd,
);
Some(scratch)
} else {
None
};
if need_luma {
let stream = luma_stream.as_mut().expect("created in the preflight");
stream.feed_row(idx, y_row, use_simd, |oy, out_row| {
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(out_row);
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(out_row) {
*dst = src as u16;
}
}
})?;
}
if let Some(scratch) = color_row {
let stream = rgb_stream.as_mut().expect("created in the preflight");
stream.feed_row(idx, scratch, use_simd, |oy, out_row| {
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_row);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
out_row,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
expand_rgb_to_rgba_row(out_row, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
})?;
}
Ok(())
}
#[cfg(feature = "yuva")]
pub(super) struct NativeYuva420 {
inner: NativeYuv420,
alpha: Option<NativeAlpha>,
}
#[cfg(feature = "yuva")]
struct NativeAlpha {
a: AreaStream<u8>,
a_stage: std::vec::Vec<u8>,
staged: [bool; 2],
}
#[cfg(feature = "yuva")]
impl NativeYuva420 {
#[allow(clippy::too_many_arguments)]
fn new(
plan: &ResamplePlan,
w: usize,
h: usize,
need_color: bool,
need_alpha: bool,
chroma_h_phase: f64,
chroma_v_phase: f64,
top_v: bool,
) -> Result<Self, ResampleError> {
let inner = NativeYuv420::new(
plan,
|| {
ResamplePlan::area_chroma_420(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
chroma_v_phase,
top_v,
)
},
w,
h,
need_color,
)?;
let alpha = if need_alpha {
let stage_len = plan.out_w().checked_mul(2).ok_or_else(|| {
ResampleError::Overflow(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()))
})?;
let alloc =
|_| ResampleError::AllocationFailed(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()));
Some(NativeAlpha {
a: AreaStream::new(plan.h(), plan.v(), w, h, 1)?,
a_stage: try_zeroed(stage_len).map_err(alloc)?,
staged: [false; 2],
})
} else {
None
};
Ok(Self { inner, alpha })
}
pub(super) fn reset(&mut self) {
self.inner.reset();
if let Some(alpha) = self.alpha.as_mut() {
alpha.a.reset();
alpha.staged = [false; 2];
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn has_chroma(&self) -> bool {
self.inner.chroma.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_centered(&self) -> bool {
self.inner.chroma_centered
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_bottom(&self) -> bool {
self.inner.chroma_bottom
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_top(&self) -> bool {
self.inner.chroma_top
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn next_y(&self) -> usize {
self.inner.y.next_y()
}
}
#[cfg(feature = "yuva")]
#[allow(clippy::too_many_arguments)]
pub(super) fn yuva420p_process_native(
plan: &ResamplePlan,
native: &mut Option<std::boxed::Box<NativeYuva420>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y_row: &[u8],
u_half: &[u8],
v_half: &[u8],
a_row: &[u8],
matrix: KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
chroma_h_phase: f64,
chroma_v_phase: f64,
top_v: bool,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || hsv.is_some() || rgba.is_some();
let need_alpha = rgba.is_some();
if !native_preflight_core_check_only(
native.as_ref().map_or(0, |n| n.inner.y.next_y()),
resample_outputs,
rgb,
rgba,
&None,
&None,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
let rebuild = native.is_none()
|| native
.as_ref()
.is_some_and(|n| n.inner.chroma.is_some() != need_color || n.alpha.is_some() != need_alpha);
let new_join = if rebuild {
let join = NativeYuva420::new(
plan,
w,
h,
need_color,
need_alpha,
chroma_h_phase,
chroma_v_phase,
top_v,
)?;
Some(crate::resample::try_box(join).map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?)
} else {
None
};
if need_color {
let row_bytes =
ow.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
ow,
plan.out_h(),
3,
)))?;
if rgb_scratch.len() < row_bytes {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if FORCE_NATIVE_RGB_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
)));
}
rgb_scratch
.try_reserve_exact(row_bytes - rgb_scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?;
rgb_scratch.resize(row_bytes, 0);
}
}
if let Some(boxed) = new_join {
*native = Some(boxed);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let join = native.as_mut().expect("built or reused in the preflight");
join.inner.check_sequence(idx)?;
if let Some(alpha) = join.alpha.as_ref()
&& alpha.a.next_y() != idx
{
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(alpha.a.next_y(), idx),
)));
}
if let Some(alpha) = join.alpha.as_mut() {
let NativeAlpha { a, a_stage, staged } = alpha;
a.feed_row(idx, a_row, use_simd, |oy, out_row| {
let slot = oy & 1;
a_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[slot] = true;
})?;
}
let NativeYuva420 { inner, alpha } = &mut **join;
let NativeYuv420 {
y,
y_stage,
chroma,
staged,
next_emit,
chroma_centered: _,
chroma_bottom: _,
chroma_top: _,
} = inner;
y.feed_row(idx, y_row, use_simd, |oy, out_row| {
let slot = oy & 1;
y_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[0][slot] = true;
})?;
if let Some(c) = chroma.as_mut()
&& idx.is_multiple_of(2)
{
let cidx = idx / 2;
let NativeChroma {
u,
v,
u_stage,
v_stage,
} = c;
u.feed_row(cidx, u_half, use_simd, |oy, out_row| {
let slot = oy & 1;
u_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[1][slot] = true;
})?;
v.feed_row(cidx, v_half, use_simd, |oy, out_row| {
let slot = oy & 1;
v_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[2][slot] = true;
})?;
}
while *next_emit < plan.out_h() {
let slot = *next_emit & 1;
let chroma_ready = match chroma.as_ref() {
Some(_) => staged[1][slot] && staged[2][slot],
None => true,
};
if !(staged[0][slot] && chroma_ready) {
break;
}
let oy = *next_emit;
let y_out = &y_stage[slot * ow..slot * ow + ow];
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(y_out);
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(y_out) {
*dst = src as u16;
}
}
if let Some(c) = chroma.as_ref() {
let u_row = &c.u_stage[slot * ow..slot * ow + ow];
let v_row = &c.v_stage[slot * ow..slot * ow + ow];
let out_rgb = &mut rgb_scratch[..ow * 3];
yuv_444_to_rgb_row(
y_out, u_row, v_row, out_rgb, ow, matrix, full_range, use_simd,
);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_rgb);
}
if let Some(hsv) = hsv.as_mut() {
let (hp, sp, vp) = hsv.hsv();
rgb_to_hsv_row(
out_rgb,
&mut hp[oy * ow..(oy + 1) * ow],
&mut sp[oy * ow..(oy + 1) * ow],
&mut vp[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
expand_rgb_to_rgba_row(out_rgb, rgba_row, ow);
if let Some(alpha) = alpha.as_ref() {
let a_out = &alpha.a_stage[slot * ow..slot * ow + ow];
crate::row::alpha_extract::copy_alpha_plane_u8(a_out, rgba_row, ow, use_simd);
}
}
}
staged[0][slot] = false;
staged[1][slot] = false;
staged[2][slot] = false;
if let Some(alpha) = alpha.as_mut() {
alpha.staged[slot] = false;
}
*next_emit += 1;
}
Ok(())
}
pub(super) struct NativePlanarYuv {
y: AreaStream<u8>,
y_stage: std::vec::Vec<u8>,
chroma: Option<NativePlanarChroma>,
chroma_vsub: usize,
staged: [[bool; 2]; 3],
next_emit: usize,
chroma_centered: bool,
#[cfg_attr(
not(any(feature = "yuv-semi-planar", feature = "yuv-packed")),
allow(dead_code)
)]
chroma_bottom: bool,
chroma_top: bool,
}
struct NativePlanarChroma {
u: AreaStream<u8>,
v: AreaStream<u8>,
u_stage: std::vec::Vec<u8>,
v_stage: std::vec::Vec<u8>,
}
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
std::thread_local! {
static FORCE_PLANAR_NATIVE_CHROMA_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
static FORCE_NATIVE_RGB_SCRATCH_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(
test,
feature = "std",
feature = "yuv-planar",
any(
feature = "yuv-packed",
feature = "yuv-444-packed",
all(feature = "yuv-semi-planar", feature = "rgb")
)
))]
pub(crate) fn arm_planar_native_chroma_failure() {
FORCE_PLANAR_NATIVE_CHROMA_FAILURE.with(|f| f.set(true));
}
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
pub(crate) fn arm_native_rgb_scratch_failure() {
FORCE_NATIVE_RGB_SCRATCH_FAILURE.with(|f| f.set(true));
}
impl NativePlanarYuv {
fn new(
plan: &ResamplePlan,
build_chroma_plan: impl FnOnce() -> Result<ResamplePlan, ResampleError>,
chroma_vsub: usize,
w: usize,
h: usize,
need_color: bool,
) -> Result<Self, ResampleError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter());
}
let y = AreaStream::new(plan.h(), plan.v(), w, h, 1)?;
let alloc =
|_| ResampleError::AllocationFailed(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()));
let stage_len = plan.out_w().checked_mul(2).ok_or_else(|| {
ResampleError::Overflow(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()))
})?;
let mut chroma_centered = false;
let mut chroma_bottom = false;
let mut chroma_top = false;
let chroma = if need_color {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if FORCE_PLANAR_NATIVE_CHROMA_FAILURE.with(|f| f.take()) {
return Err(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)));
}
let chroma_plan = build_chroma_plan()?;
chroma_centered = chroma_plan.has_chroma_phase();
chroma_bottom = chroma_plan.has_chroma_v_phase();
chroma_top = chroma_plan.has_chroma_v_top();
Some(NativePlanarChroma {
u: AreaStream::new(
chroma_plan.h(),
chroma_plan.v(),
chroma_plan.src_w(),
chroma_plan.src_h(),
1,
)?,
v: AreaStream::new(
chroma_plan.h(),
chroma_plan.v(),
chroma_plan.src_w(),
chroma_plan.src_h(),
1,
)?,
u_stage: try_zeroed(stage_len).map_err(alloc)?,
v_stage: try_zeroed(stage_len).map_err(alloc)?,
})
} else {
None
};
Ok(Self {
y,
y_stage: try_zeroed(stage_len).map_err(alloc)?,
chroma,
chroma_vsub,
staged: [[false; 2]; 3],
next_emit: 0,
chroma_centered,
chroma_bottom,
chroma_top,
})
}
pub(super) fn reset(&mut self) {
self.y.reset();
if let Some(chroma) = self.chroma.as_mut() {
chroma.u.reset();
chroma.v.reset();
}
self.staged = [[false; 2]; 3];
self.next_emit = 0;
}
#[cfg(any(feature = "yuv-semi-planar", feature = "yuv-packed"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn has_chroma(&self) -> bool {
self.chroma.is_some()
}
#[cfg(any(feature = "yuv-semi-planar", feature = "yuv-packed"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_centered(&self) -> bool {
self.chroma_centered
}
#[cfg(any(feature = "yuv-semi-planar", feature = "yuv-packed"))]
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(dead_code)]
pub(super) const fn chroma_bottom(&self) -> bool {
self.chroma_bottom
}
#[cfg(any(feature = "yuv-semi-planar", feature = "yuv-packed"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn next_y(&self) -> usize {
self.y.next_y()
}
fn check_sequence(&self, idx: usize) -> Result<(), MixedSinkerError> {
if self.y.next_y() != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(self.y.next_y(), idx),
)));
}
if let Some(chroma) = self.chroma.as_ref() {
let chroma_expected = idx.div_ceil(self.chroma_vsub);
for stream in [&chroma.u, &chroma.v] {
if stream.next_y() != chroma_expected {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(stream.next_y().saturating_mul(self.chroma_vsub), idx),
)));
}
}
}
Ok(())
}
}
#[allow(clippy::too_many_arguments)]
pub(super) fn native_planar_preflight_check_only(
join: &Option<std::boxed::Box<NativePlanarYuv>>,
resample_outputs: &Option<super::FrozenOutputs>,
rgb: &Option<&mut [u8]>,
rgba: &Option<&mut [u8]>,
luma: &Option<&mut [u8]>,
luma_u16: &Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
idx: usize,
need_luma: bool,
need_color: bool,
) -> Result<bool, MixedSinkerError> {
native_preflight_core_check_only(
join.as_ref().map_or(0, |join| join.y.next_y()),
resample_outputs,
rgb,
rgba,
&None,
&None,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)
}
#[allow(clippy::too_many_arguments)]
pub(super) fn yuv_planar_process_native(
plan: &ResamplePlan,
native: &mut Option<std::boxed::Box<NativePlanarYuv>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y_row: &[u8],
u_row: &[u8],
v_row: &[u8],
matrix: KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
chroma_vsub: usize,
build_chroma_plan: impl FnOnce() -> Result<ResamplePlan, ResampleError>,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || hsv.is_some() || rgba.is_some();
let want_hsv_direct = hsv.is_some() && rgb.is_none() && rgba.is_none();
if !native_planar_preflight_check_only(
native,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
let rebuild = native.is_none()
|| native
.as_ref()
.is_some_and(|join| join.chroma.is_some() != need_color);
let new_join = if rebuild {
let join = NativePlanarYuv::new(plan, build_chroma_plan, chroma_vsub, w, h, need_color)?;
Some(crate::resample::try_box(join).map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?)
} else {
None
};
if need_color && !want_hsv_direct {
let row_bytes =
ow.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
ow,
plan.out_h(),
3,
)))?;
if rgb_scratch.len() < row_bytes {
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
if FORCE_NATIVE_RGB_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
)));
}
rgb_scratch
.try_reserve_exact(row_bytes - rgb_scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)))
})?;
rgb_scratch.resize(row_bytes, 0);
}
}
if let Some(boxed) = new_join {
*native = Some(boxed);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let join = native.as_mut().expect("built or reused in the preflight");
join.check_sequence(idx)?;
let NativePlanarYuv {
y,
y_stage,
chroma,
chroma_vsub,
staged,
next_emit,
chroma_centered: _,
chroma_bottom: _,
chroma_top: _,
} = &mut **join;
y.feed_row(idx, y_row, use_simd, |oy, out_row| {
let slot = oy & 1;
y_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[0][slot] = true;
})?;
if let Some(c) = chroma.as_mut()
&& idx.is_multiple_of(*chroma_vsub)
{
let cidx = idx / *chroma_vsub;
let NativePlanarChroma {
u,
v,
u_stage,
v_stage,
} = c;
u.feed_row(cidx, u_row, use_simd, |oy, out_row| {
let slot = oy & 1;
u_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[1][slot] = true;
})?;
v.feed_row(cidx, v_row, use_simd, |oy, out_row| {
let slot = oy & 1;
v_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[2][slot] = true;
})?;
}
while *next_emit < plan.out_h() {
let slot = *next_emit & 1;
let chroma_ready = match chroma.as_ref() {
Some(_) => staged[1][slot] && staged[2][slot],
None => true,
};
if !(staged[0][slot] && chroma_ready) {
break;
}
let oy = *next_emit;
let y_out = &y_stage[slot * ow..slot * ow + ow];
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(y_out);
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(y_out) {
*dst = src as u16;
}
}
if let Some(c) = chroma.as_ref() {
let u_out = &c.u_stage[slot * ow..slot * ow + ow];
let v_out = &c.v_stage[slot * ow..slot * ow + ow];
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (hp, sp, vp) = hsv.hsv();
yuv_444_to_hsv_row(
y_out,
u_out,
v_out,
&mut hp[oy * ow..(oy + 1) * ow],
&mut sp[oy * ow..(oy + 1) * ow],
&mut vp[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
} else {
let out_rgb = &mut rgb_scratch[..ow * 3];
yuv_444_to_rgb_row(
y_out, u_out, v_out, out_rgb, ow, matrix, full_range, use_simd,
);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_rgb);
}
if let Some(hsv) = hsv.as_mut() {
let (hp, sp, vp) = hsv.hsv();
rgb_to_hsv_row(
out_rgb,
&mut hp[oy * ow..(oy + 1) * ow],
&mut sp[oy * ow..(oy + 1) * ow],
&mut vp[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
expand_rgb_to_rgba_row(out_rgb, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
}
}
staged[0][slot] = false;
staged[1][slot] = false;
staged[2][slot] = false;
*next_emit += 1;
}
Ok(())
}
impl<'a, R> MixedSinker<'a, Yuv410p, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_luma_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_luma_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_luma_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_pixels()?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientLumaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.luma_u16 = Some(buf);
Ok(self)
}
}
impl<R> Yuv410pSink for MixedSinker<'_, Yuv410p, R> {}
impl<R> PixelSink for MixedSinker<'_, Yuv410p, R> {
type Input<'r> = Yuv410pRow<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 3 != 0 {
return Err(MixedSinkerError::WidthAlignment(
WidthAlignment::multiple_of_four(self.width),
));
}
check_dimensions_match(self.width, self.height, width, height)?;
if let Some(stream) = self.rgb_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.rgb_filter_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_filter_stream.as_mut() {
stream.reset();
}
if let Some(hsv) = self.hsv_planar.as_mut() {
hsv.reset();
}
self.frozen_chroma_centered = None;
self.resample_outputs = None;
Ok(())
}
fn process(&mut self, row: Yuv410pRow<'_>) -> Result<(), Self::Error> {
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
let chroma_location = self.chroma_location.clone();
if w & 3 != 0 {
return Err(MixedSinkerError::WidthAlignment(
WidthAlignment::multiple_of_four(w),
));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y,
idx,
w,
row.y().len(),
)));
}
if row.u_quarter().len() != w / 4 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UQuarter,
idx,
w / 4,
row.u_quarter().len(),
)));
}
if row.v_quarter().len() != w / 4 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VQuarter,
idx,
w / 4,
row.v_quarter().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let Self {
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
plan,
rgb_stream,
luma_stream,
rgb_filter_stream,
luma_filter_stream,
hsv_planar,
resample_outputs,
chroma_full,
frozen_chroma_centered,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let center_sited = chroma_410_center_sited_h(&chroma_location);
let chroma_h_phase = if center_sited {
YUV41X_CENTERED_H_PHASE
} else {
0.0
};
let want_color = rgb.is_some() || rgba.is_some() || hsv.is_some();
let need_output =
luma.is_some() || luma_u16.is_some() || rgb.is_some() || rgba.is_some() || hsv.is_some();
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
let convert_rgb = |scratch: &mut [u8]| {
yuv_410_to_rgb_row(
row.y(),
row.u_quarter(),
row.v_quarter(),
scratch,
w,
matrix,
full_range,
use_simd,
);
};
if plan.kind().is_area() && hsv.is_some() && rgb.is_none() && rgba.is_none() {
let stale_hsv = idx == 0
&& hsv_planar
.as_ref()
.is_some_and(|join| join.chroma_centered() != center_sited && join.next_y() == 0);
let prev_hsv = if stale_hsv { hsv_planar.take() } else { None };
let hsv_result = super::planar_resample::hsv_direct_resample(
hsv_planar,
resample_outputs,
luma,
luma_u16,
hsv,
row.y(),
row.u_quarter(),
row.v_quarter(),
matrix,
full_range,
4,
|| ResamplePlan::area_chroma_410(w, h, plan.out_w(), plan.out_h(), chroma_h_phase, 0.0),
w,
plan,
idx,
use_simd,
);
if stale_hsv && hsv_result.is_err() {
*hsv_planar = prev_hsv;
}
hsv_result?;
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
let r = if center_sited && want_color {
if plan.kind().is_filter() {
plan.ensure_single_kernel_filter()?;
}
let need_luma = luma.is_some() || luma_u16.is_some();
let expected = if plan.kind().is_filter() {
if need_luma {
luma_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
}
} else if need_luma {
luma_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
let (u_full, v_full) = reconstruct_chroma(
Chroma411U8,
chroma_full,
row.u_quarter(),
row.v_quarter(),
w,
use_simd,
);
let convert_444 = |scratch: &mut [u8]| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
};
match plan.kind() {
crate::resample::SpanKind::Area => planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_444,
),
crate::resample::SpanKind::Filter => planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_444,
),
}
} else {
match plan.kind() {
crate::resample::SpanKind::Area => planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_rgb,
),
crate::resample::SpanKind::Filter => planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_rgb,
),
}
};
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_color = want_rgb || want_rgba || want_hsv;
let center_sited = chroma_410_center_sited_h(&chroma_location);
let need_output =
luma.is_some() || luma_u16.is_some() || rgb.is_some() || rgba.is_some() || hsv.is_some();
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if center_sited && want_color {
reserve_420_chroma_full(chroma_full, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
if let Some(luma) = luma.as_deref_mut() {
luma[one_plane_start..one_plane_end].copy_from_slice(&row.y()[..w]);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::y_plane_to_luma_u16_row(
row.y(),
&mut buf[one_plane_start..one_plane_end],
w,
use_simd,
);
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if center_sited {
let (u_full, v_full) = reconstruct_chroma(
Chroma411U8,
chroma_full,
row.u_quarter(),
row.v_quarter(),
w,
use_simd,
);
yuv_444_to_hsv_row(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_410_to_hsv_row(
row.y(),
row.u_quarter(),
row.v_quarter(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if center_sited {
let (u_full, v_full) = reconstruct_chroma(
Chroma411U8,
chroma_full,
row.u_quarter(),
row.v_quarter(),
w,
use_simd,
);
yuv_444_to_rgba_row(
row.y(),
u_full,
v_full,
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_410_to_rgba_row(
row.y(),
row.u_quarter(),
row.v_quarter(),
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if center_sited {
let (u_full, v_full) = reconstruct_chroma(
Chroma411U8,
chroma_full,
row.u_quarter(),
row.v_quarter(),
w,
use_simd,
);
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_410_to_rgb_row(
row.y(),
row.u_quarter(),
row.v_quarter(),
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R> MixedSinker<'a, Yuv422p, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_luma_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_luma_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_luma_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_pixels()?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientLumaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.luma_u16 = Some(buf);
Ok(self)
}
}
impl<R> Yuv422pSink for MixedSinker<'_, Yuv422p, R> {}
impl<R> PixelSink for MixedSinker<'_, Yuv422p, R> {
type Input<'r> = Yuv422pRow<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
if self.width & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(
self.width,
)));
}
check_dimensions_match(self.width, self.height, width, height)?;
if let Some(stream) = self.rgb_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.rgb_filter_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_filter_stream.as_mut() {
stream.reset();
}
if let Some(native) = self.native_planar.as_mut() {
native.reset();
}
if let Some(hsv) = self.hsv_planar.as_mut() {
hsv.reset();
}
self.frozen_native_route = None;
self.frozen_domain = None;
self.resample_outputs = None;
self.frozen_chroma_centered = None;
#[cfg(feature = "rgb")]
{
self.linear_light_frame = None;
}
Ok(())
}
fn process(&mut self, row: Yuv422pRow<'_>) -> Result<(), Self::Error> {
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
let chroma_location = self.chroma_location.clone();
if w & 1 != 0 {
return Err(MixedSinkerError::WidthAlignment(WidthAlignment::odd(w)));
}
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y,
idx,
w,
row.y().len(),
)));
}
if row.u_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UHalf,
idx,
w / 2,
row.u_half().len(),
)));
}
if row.v_half().len() != w / 2 {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VHalf,
idx,
w / 2,
row.v_half().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let Self {
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
plan,
rgb_stream,
luma_stream,
rgb_filter_stream,
luma_filter_stream,
resample_outputs,
native,
native_planar,
hsv_planar,
frozen_native_route,
frozen_domain,
averaging_domain,
#[cfg(feature = "rgb")]
linear_mode,
#[cfg(feature = "rgb")]
linear_light_frame,
#[cfg(feature = "rgb")]
linear_scene_scratch,
#[cfg(feature = "rgb")]
transfer_function,
chroma_full,
frozen_chroma_centered,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let center_sited = chroma_422_center_sited_h(&chroma_location);
let chroma_h_phase = if center_sited {
YUV422P_CENTERED_H_PHASE
} else {
0.0
};
let want_color = rgb.is_some() || rgba.is_some() || hsv.is_some();
let convert_rgb = |scratch: &mut [u8]| {
yuv_420_to_rgb_row(
row.y(),
row.u_half(),
row.v_half(),
scratch,
w,
matrix,
full_range,
use_simd,
);
};
#[cfg(feature = "rgb")]
let convert_rgb_unclamped = |dst: &mut [f32]| {
crate::row::scalar::yuv_420_to_rgb_f32_unclamped_row(
row.y(),
row.u_half(),
row.v_half(),
dst,
w,
matrix,
full_range,
);
};
let need_output =
luma.is_some() || luma_u16.is_some() || rgb.is_some() || rgba.is_some() || hsv.is_some();
if need_output
&& let Some(frozen) = *frozen_domain
&& frozen != *averaging_domain
{
return Err(MixedSinkerError::AveragingDomainChanged(
AveragingDomainChanged::new(idx),
));
}
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
match *averaging_domain {
AveragingDomain::Encoded => {}
AveragingDomain::Linear => {
#[cfg(feature = "rgb")]
{
let tf = transfer_function
.unwrap_or_else(|| TransferFunction::for_matrix(&crate::ColorMatrix::from(matrix)));
let r = if center_sited && want_color {
linear_light::linear_light_preflight(
linear_light_frame,
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
hsv,
tf,
*linear_mode,
plan,
idx,
)?;
reserve_420_chroma_full(chroma_full, w, h)?;
let (u_full, v_full) = reconstruct_chroma(
ChromaU8,
chroma_full,
row.u_half(),
row.v_half(),
w,
use_simd,
);
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
);
},
)
} else {
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| convert_rgb(dst),
|_idx, dst| convert_rgb_unclamped(dst),
)
};
if r.is_ok() && need_output && frozen_domain.is_none() {
*frozen_domain = Some(AveragingDomain::Linear);
}
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
#[cfg(not(feature = "rgb"))]
{
return Err(plan.linear_domain_unsupported().into());
}
}
AveragingDomain::Premultiplied => {
return Err(plan.premultiplied_domain_unsupported().into());
}
}
if plan.kind().is_filter() {
plan.ensure_single_kernel_filter()?;
if center_sited && want_color {
let need_luma = luma.is_some() || luma_u16.is_some();
let expected = if need_luma {
luma_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
let (u_full, v_full) = reconstruct_chroma(
ChromaU8,
chroma_full,
row.u_half(),
row.v_half(),
w,
use_simd,
);
let r = planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
);
if r.is_ok() && need_output {
if frozen_domain.is_none() {
*frozen_domain = Some(AveragingDomain::Encoded);
}
if frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
}
return r;
}
let r = planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_rgb,
);
if r.is_ok() && need_output {
if frozen_domain.is_none() {
*frozen_domain = Some(AveragingDomain::Encoded);
}
if frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
}
return r;
}
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != *native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
let insertion = select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: YUV_PLANAR_8BIT_NATIVE_ELIGIBLE,
with_native: *native,
area_plan: true,
},
);
match insertion {
InsertionPoint::NativeCodes => {
let stale_native = idx == 0
&& native_planar.as_ref().is_some_and(|join| {
join.chroma.is_some() && join.chroma_centered != center_sited && join.y.next_y() == 0
});
let prev_native = if stale_native {
native_planar.take()
} else {
None
};
let native_result = yuv_planar_process_native(
plan,
native_planar,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
row.u_half(),
row.v_half(),
matrix,
full_range,
idx,
w,
h,
1,
|| {
ResamplePlan::area_chroma_422(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
0.0,
)
},
use_simd,
);
if stale_native && native_result.is_err() {
*native_planar = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_domain.is_none() && need_output {
*frozen_domain = Some(AveragingDomain::Encoded);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
InsertionPoint::EncodedOutput => {
if hsv.is_some() && rgb.is_none() && rgba.is_none() {
let stale_hsv = idx == 0
&& hsv_planar
.as_ref()
.is_some_and(|join| join.chroma_centered() != center_sited && join.next_y() == 0);
let prev_hsv = if stale_hsv { hsv_planar.take() } else { None };
let hsv_result = super::planar_resample::hsv_direct_resample(
hsv_planar,
resample_outputs,
luma,
luma_u16,
hsv,
row.y(),
row.u_half(),
row.v_half(),
matrix,
full_range,
1,
|| {
ResamplePlan::area_chroma_422(
w / 2,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
0.0,
)
},
w,
plan,
idx,
use_simd,
);
if stale_hsv && hsv_result.is_err() {
*hsv_planar = prev_hsv;
}
hsv_result?;
} else if center_sited && want_color {
let need_luma = luma.is_some() || luma_u16.is_some();
let expected = if need_luma {
luma_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
let (u_full, v_full) = reconstruct_chroma(
ChromaU8,
chroma_full,
row.u_half(),
row.v_half(),
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
} else {
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_rgb,
)?;
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_domain.is_none() && need_output {
*frozen_domain = Some(AveragingDomain::Encoded);
}
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
InsertionPoint::LinearLight => {
unreachable!("encoded domain never selects the linear-light splice")
}
}
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_color = want_rgb || want_rgba || want_hsv;
let need_output = want_color || luma.is_some() || luma_u16.is_some();
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let center_sited = chroma_422_center_sited_h(&chroma_location);
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if center_sited && want_color {
reserve_420_chroma_full(chroma_full, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
if let Some(luma) = luma.as_deref_mut() {
luma[one_plane_start..one_plane_end].copy_from_slice(&row.y()[..w]);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::y_plane_to_luma_u16_row(
row.y(),
&mut buf[one_plane_start..one_plane_end],
w,
use_simd,
);
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if center_sited {
let (u_full, v_full) = reconstruct_chroma(
ChromaU8,
chroma_full,
row.u_half(),
row.v_half(),
w,
use_simd,
);
yuv_444_to_hsv_row(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_420_to_hsv_row(
row.y(),
row.u_half(),
row.v_half(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if center_sited {
let (u_full, v_full) = reconstruct_chroma(
ChromaU8,
chroma_full,
row.u_half(),
row.v_half(),
w,
use_simd,
);
yuv_444_to_rgba_row(
row.y(),
u_full,
v_full,
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_420_to_rgba_row(
row.y(),
row.u_half(),
row.v_half(),
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if center_sited {
let (u_full, v_full) = reconstruct_chroma(
ChromaU8,
chroma_full,
row.u_half(),
row.v_half(),
w,
use_simd,
);
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_420_to_rgb_row(
row.y(),
row.u_half(),
row.v_half(),
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R> MixedSinker<'a, Yuv444p, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_luma_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_luma_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_luma_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_pixels()?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientLumaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.luma_u16 = Some(buf);
Ok(self)
}
}
impl<R> Yuv444pSink for MixedSinker<'_, Yuv444p, R> {}
impl<R> PixelSink for MixedSinker<'_, Yuv444p, R> {
type Input<'r> = Yuv444pRow<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
check_dimensions_match(self.width, self.height, width, height)?;
if let Some(stream) = self.rgb_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.rgb_filter_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_filter_stream.as_mut() {
stream.reset();
}
if let Some(native) = self.native_planar.as_mut() {
native.reset();
}
if let Some(hsv) = self.hsv_planar.as_mut() {
hsv.reset();
}
self.frozen_native_route = None;
self.frozen_domain = None;
self.resample_outputs = None;
#[cfg(feature = "rgb")]
{
self.linear_light_frame = None;
}
Ok(())
}
fn process(&mut self, row: Yuv444pRow<'_>) -> Result<(), Self::Error> {
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y,
idx,
w,
row.y().len(),
)));
}
if row.u().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UFull,
idx,
w,
row.u().len(),
)));
}
if row.v().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VFull,
idx,
w,
row.v().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let Self {
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
plan,
rgb_stream,
luma_stream,
rgb_filter_stream,
luma_filter_stream,
resample_outputs,
native,
native_planar,
hsv_planar,
frozen_native_route,
frozen_domain,
averaging_domain,
#[cfg(feature = "rgb")]
linear_mode,
#[cfg(feature = "rgb")]
linear_light_frame,
#[cfg(feature = "rgb")]
linear_scene_scratch,
#[cfg(feature = "rgb")]
transfer_function,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let convert_rgb = |scratch: &mut [u8]| {
yuv_444_to_rgb_row(
row.y(),
row.u(),
row.v(),
scratch,
w,
matrix,
full_range,
use_simd,
);
};
#[cfg(feature = "rgb")]
let convert_rgb_unclamped = |dst: &mut [f32]| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
row.y(),
row.u(),
row.v(),
dst,
w,
matrix,
full_range,
);
};
let need_output =
luma.is_some() || luma_u16.is_some() || rgb.is_some() || rgba.is_some() || hsv.is_some();
if need_output
&& let Some(frozen) = *frozen_domain
&& frozen != *averaging_domain
{
return Err(MixedSinkerError::AveragingDomainChanged(
AveragingDomainChanged::new(idx),
));
}
match *averaging_domain {
AveragingDomain::Encoded => {}
AveragingDomain::Linear => {
#[cfg(feature = "rgb")]
{
let tf = transfer_function
.unwrap_or_else(|| TransferFunction::for_matrix(&crate::ColorMatrix::from(matrix)));
let r = linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| convert_rgb(dst),
|_idx, dst| convert_rgb_unclamped(dst),
);
if r.is_ok() && need_output && frozen_domain.is_none() {
*frozen_domain = Some(AveragingDomain::Linear);
}
return r;
}
#[cfg(not(feature = "rgb"))]
{
return Err(plan.linear_domain_unsupported().into());
}
}
AveragingDomain::Premultiplied => {
return Err(plan.premultiplied_domain_unsupported().into());
}
}
if plan.kind().is_filter() {
return planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_rgb,
);
}
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != *native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
let insertion = select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: YUV_PLANAR_8BIT_NATIVE_ELIGIBLE,
with_native: *native,
area_plan: true,
},
);
match insertion {
InsertionPoint::NativeCodes => {
yuv_planar_process_native(
plan,
native_planar,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
row.u(),
row.v(),
matrix,
full_range,
idx,
w,
h,
1,
|| ResamplePlan::area(w, h, plan.out_w(), plan.out_h()),
use_simd,
)?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_domain.is_none() && need_output {
*frozen_domain = Some(AveragingDomain::Encoded);
}
return Ok(());
}
InsertionPoint::EncodedOutput => {
if hsv.is_some() && rgb.is_none() && rgba.is_none() {
super::planar_resample::hsv_direct_resample(
hsv_planar,
resample_outputs,
luma,
luma_u16,
hsv,
row.y(),
row.u(),
row.v(),
matrix,
full_range,
1,
|| ResamplePlan::area(w, h, plan.out_w(), plan.out_h()),
w,
plan,
idx,
use_simd,
)?;
} else {
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_rgb,
)?;
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_domain.is_none() && need_output {
*frozen_domain = Some(AveragingDomain::Encoded);
}
return Ok(());
}
InsertionPoint::LinearLight => {
unreachable!("encoded domain never selects the linear-light splice")
}
}
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if let Some(luma) = luma.as_deref_mut() {
luma[one_plane_start..one_plane_end].copy_from_slice(&row.y()[..w]);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::y_plane_to_luma_u16_row(
row.y(),
&mut buf[one_plane_start..one_plane_end],
w,
use_simd,
);
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
yuv_444_to_hsv_row(
row.y(),
row.u(),
row.v(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
yuv_444_to_rgba_row(
row.y(),
row.u(),
row.v(),
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
yuv_444_to_rgb_row(
row.y(),
row.u(),
row.v(),
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R> MixedSinker<'a, Yuv440p, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_luma_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_luma_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_luma_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_pixels()?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientLumaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.luma_u16 = Some(buf);
Ok(self)
}
}
impl<R> Yuv440pSink for MixedSinker<'_, Yuv440p, R> {}
impl<R> PixelSink for MixedSinker<'_, Yuv440p, R> {
type Input<'r> = Yuv440pRow<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
check_dimensions_match(self.width, self.height, width, height)?;
if let Some(stream) = self.rgb_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.rgb_filter_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_filter_stream.as_mut() {
stream.reset();
}
if let Some(native) = self.native_planar.as_mut() {
native.reset();
}
if let Some(hsv) = self.hsv_planar.as_mut() {
hsv.reset();
}
self.frozen_native_route = None;
self.frozen_domain = None;
self.resample_outputs = None;
self.frozen_chroma_centered = None;
self.frozen_chroma_bottom_v = None;
self.frozen_chroma_top_v = None;
self.chroma_prev_row = None;
self.chroma_top_pending = None;
#[cfg(feature = "rgb")]
{
self.linear_light_frame = None;
}
Ok(())
}
fn process(&mut self, row: Yuv440pRow<'_>) -> Result<(), Self::Error> {
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
let chroma_location = self.chroma_location.clone();
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y,
idx,
w,
row.y().len(),
)));
}
if row.u().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UFull,
idx,
w,
row.u().len(),
)));
}
if row.v().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VFull,
idx,
w,
row.v().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let Self {
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
chroma_full,
chroma_prev,
chroma_prev_row,
plan,
rgb_stream,
luma_stream,
rgb_filter_stream,
luma_filter_stream,
resample_outputs,
native,
native_planar,
hsv_planar,
frozen_native_route,
frozen_domain,
averaging_domain,
#[cfg(feature = "rgb")]
linear_mode,
#[cfg(feature = "rgb")]
linear_light_frame,
#[cfg(feature = "rgb")]
linear_scene_scratch,
#[cfg(feature = "rgb")]
transfer_function,
frozen_chroma_bottom_v,
frozen_chroma_top_v,
chroma_top_pending,
chroma_top_y,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let bottom_v = chroma_440_bottom_sited_v(&chroma_location);
let chroma_v_phase = if bottom_v { 1.0 } else { 0.0 };
let top_v = chroma_440_top_sited_v(&chroma_location);
let want_color = rgb.is_some() || rgba.is_some() || hsv.is_some();
let convert_rgb = |scratch: &mut [u8]| {
yuv_444_to_rgb_row(
row.y(),
row.u(),
row.v(),
scratch,
w,
matrix,
full_range,
use_simd,
);
};
#[cfg(feature = "rgb")]
let convert_rgb_unclamped = |dst: &mut [f32]| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
row.y(),
row.u(),
row.v(),
dst,
w,
matrix,
full_range,
);
};
let need_output =
luma.is_some() || luma_u16.is_some() || rgb.is_some() || rgba.is_some() || hsv.is_some();
if need_output
&& let Some(frozen) = *frozen_domain
&& frozen != *averaging_domain
{
return Err(MixedSinkerError::AveragingDomainChanged(
AveragingDomainChanged::new(idx),
));
}
if need_output
&& let Some(frozen) = *frozen_chroma_bottom_v
&& (frozen != bottom_v || *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
match *averaging_domain {
AveragingDomain::Encoded => {}
AveragingDomain::Linear => {
#[cfg(feature = "rgb")]
{
let tf = transfer_function
.unwrap_or_else(|| TransferFunction::for_matrix(&crate::ColorMatrix::from(matrix)));
if top_v && want_color {
let is_last = idx + 1 == h;
let want_luma_tail = luma.is_some() || luma_u16.is_some();
let cursor = linear_light_frame.as_ref().map_or(0, |b| b.next_y());
let expected_source = cursor + usize::from(chroma_top_pending.is_some());
if idx != expected_source {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected_source, idx),
)));
}
linear_light::linear_light_preflight(
linear_light_frame,
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
hsv,
tf,
*linear_mode,
plan,
cursor,
)?;
reserve_420_chroma_full(chroma_full, w, h)?;
reserve_440_chroma_prev(chroma_prev, w, h)?;
if chroma_top_y.len() < w {
chroma_top_y
.try_reserve_exact(w - chroma_top_y.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w, h, w, h,
)))
})?;
chroma_top_y.resize(w, 0);
}
if idx & 1 == 0 && is_last && chroma_top_pending.is_some() {
linear_light::linear_light_reserve_final_tail(
linear_light_frame,
plan,
w,
h,
want_luma_tail,
)?;
}
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
true,
false,
w,
use_simd,
);
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
&chroma_top_y[..w],
p_idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_444_to_rgb_row(
&chroma_top_y[..w],
u_full,
v_full,
dst,
w,
p_matrix,
p_full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
&chroma_top_y[..w],
u_full,
v_full,
dst,
w,
p_matrix,
p_full_range,
);
},
)?;
}
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
false,
false,
w,
use_simd,
);
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
);
},
)?;
} else if !is_last {
chroma_top_y[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
false,
false,
w,
use_simd,
);
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
);
},
)?;
}
stage_440_chroma_prev(chroma_prev, chroma_prev_row, row.u(), row.v(), idx, w);
if need_output && frozen_domain.is_none() {
*frozen_domain = Some(AveragingDomain::Linear);
}
if need_output && frozen_chroma_bottom_v.is_none() {
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
let r = if bottom_v && want_color {
linear_light::linear_light_preflight(
linear_light_frame,
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
hsv,
tf,
*linear_mode,
plan,
idx,
)?;
reserve_420_chroma_full(chroma_full, w, h)?;
reserve_440_chroma_prev(chroma_prev, w, h)?;
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
bottom_v,
false,
w,
use_simd,
);
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
use_simd,
);
},
|_idx, dst| {
crate::row::scalar::yuv_444_to_rgb_f32_unclamped_row(
row.y(),
u_full,
v_full,
dst,
w,
matrix,
full_range,
);
},
)
} else {
linear_light::linear_light_resample(
linear_light_frame,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
linear_scene_scratch,
tf,
*linear_mode,
plan,
row.y(),
idx,
w,
h,
use_simd,
|_idx, dst| convert_rgb(dst),
|_idx, dst| convert_rgb_unclamped(dst),
)
};
if r.is_ok() && bottom_v && want_color {
stage_440_chroma_prev(chroma_prev, chroma_prev_row, row.u(), row.v(), idx, w);
}
if r.is_ok() && need_output && frozen_domain.is_none() {
*frozen_domain = Some(AveragingDomain::Linear);
}
if r.is_ok() && need_output && frozen_chroma_bottom_v.is_none() {
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
#[cfg(not(feature = "rgb"))]
{
return Err(plan.linear_domain_unsupported().into());
}
}
AveragingDomain::Premultiplied => {
return Err(plan.premultiplied_domain_unsupported().into());
}
}
if plan.kind().is_filter() {
if bottom_v && want_color {
let need_luma = luma.is_some() || luma_u16.is_some();
let expected = if need_luma {
luma_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
reserve_440_chroma_prev(chroma_prev, w, h)?;
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
bottom_v,
false,
w,
use_simd,
);
let r = planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
);
if r.is_ok() && bottom_v {
stage_440_chroma_prev(chroma_prev, chroma_prev_row, row.u(), row.v(), idx, w);
}
if r.is_ok() && need_output && frozen_chroma_bottom_v.is_none() {
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
if top_v && want_color {
let is_last = idx + 1 == h;
let need_luma = luma.is_some() || luma_u16.is_some();
let cursor = if need_luma {
luma_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
};
let expected_source = cursor + usize::from(chroma_top_pending.is_some());
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected_source),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
reserve_440_chroma_prev(chroma_prev, w, h)?;
if chroma_top_y.len() < w {
chroma_top_y
.try_reserve_exact(w - chroma_top_y.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w, h, w, h,
)))
})?;
chroma_top_y.resize(w, 0);
}
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
true,
false,
w,
use_simd,
);
planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
&chroma_top_y[..w],
w,
plan,
p_idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
&chroma_top_y[..w],
u_full,
v_full,
scratch,
w,
p_matrix,
p_full_range,
use_simd,
);
},
)?;
}
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
false,
false,
w,
use_simd,
);
planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
} else if !is_last {
chroma_top_y[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
false,
false,
w,
use_simd,
);
planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
}
stage_440_chroma_prev(chroma_prev, chroma_prev_row, row.u(), row.v(), idx, w);
if need_output && frozen_chroma_bottom_v.is_none() {
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
let r = planar_dual_filter_resample(
luma_filter_stream,
rgb_filter_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_rgb,
);
if r.is_ok() && need_output && frozen_chroma_bottom_v.is_none() {
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return r;
}
if need_output
&& let Some(frozen) = *frozen_native_route
&& frozen != *native
{
return Err(MixedSinkerError::NativeRouteChanged(
NativeRouteChanged::new(idx),
));
}
let insertion = select_insertion_point(
AveragingDomain::Encoded,
InsertionContext {
native_eligible: YUV_PLANAR_8BIT_NATIVE_ELIGIBLE,
with_native: *native,
area_plan: true,
},
);
match insertion {
InsertionPoint::NativeCodes => {
let stale_native = idx == 0
&& native_planar.as_ref().is_some_and(|join| {
join.chroma.is_some()
&& (join.chroma_bottom != bottom_v || join.chroma_top != top_v)
&& join.y.next_y() == 0
});
let prev_native = if stale_native {
native_planar.take()
} else {
None
};
let native_result = yuv_planar_process_native(
plan,
native_planar,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
row.u(),
row.v(),
matrix,
full_range,
idx,
w,
h,
2,
|| {
ResamplePlan::area_chroma_440(
w,
h,
plan.out_w(),
plan.out_h(),
0.0,
chroma_v_phase,
top_v,
)
},
use_simd,
);
if stale_native && native_result.is_err() {
*native_planar = prev_native;
}
native_result?;
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(true);
}
if frozen_domain.is_none() && need_output {
*frozen_domain = Some(AveragingDomain::Encoded);
}
if frozen_chroma_bottom_v.is_none() && need_output {
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::EncodedOutput => {
if hsv.is_some() && rgb.is_none() && rgba.is_none() {
let stale_hsv = idx == 0
&& hsv_planar.as_ref().is_some_and(|join| {
(join.chroma_bottom() != bottom_v || join.chroma_top() != top_v)
&& join.next_y() == 0
});
let prev_hsv = if stale_hsv { hsv_planar.take() } else { None };
let hsv_result = super::planar_resample::hsv_direct_resample(
hsv_planar,
resample_outputs,
luma,
luma_u16,
hsv,
row.y(),
row.u(),
row.v(),
matrix,
full_range,
2,
|| {
ResamplePlan::area_chroma_440(
w,
h,
plan.out_w(),
plan.out_h(),
0.0,
chroma_v_phase,
top_v,
)
},
w,
plan,
idx,
use_simd,
);
if stale_hsv && hsv_result.is_err() {
*hsv_planar = prev_hsv;
}
hsv_result?;
} else if bottom_v && want_color {
let need_luma = luma.is_some() || luma_u16.is_some();
let expected = if need_luma {
luma_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
reserve_440_chroma_prev(chroma_prev, w, h)?;
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
bottom_v,
false,
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
stage_440_chroma_prev(chroma_prev, chroma_prev_row, row.u(), row.v(), idx, w);
} else if top_v && want_color {
let is_last = idx + 1 == h;
let need_luma = luma.is_some() || luma_u16.is_some();
let cursor = if need_luma {
luma_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
};
let expected_source = cursor + usize::from(chroma_top_pending.is_some());
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected_source),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
reserve_440_chroma_prev(chroma_prev, w, h)?;
if chroma_top_y.len() < w {
chroma_top_y
.try_reserve_exact(w - chroma_top_y.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w, h, w, h,
)))
})?;
chroma_top_y.resize(w, 0);
}
if idx & 1 == 0 {
if let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take() {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
true,
false,
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
&chroma_top_y[..w],
w,
plan,
p_idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
&chroma_top_y[..w],
u_full,
v_full,
scratch,
w,
p_matrix,
p_full_range,
use_simd,
);
},
)?;
}
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
false,
false,
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
} else if !is_last {
chroma_top_y[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, matrix, full_range));
} else {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
false,
false,
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)?;
}
stage_440_chroma_prev(chroma_prev, chroma_prev_row, row.u(), row.v(), idx, w);
} else {
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
convert_rgb,
)?;
}
if frozen_native_route.is_none() && need_output {
*frozen_native_route = Some(false);
}
if frozen_domain.is_none() && need_output {
*frozen_domain = Some(AveragingDomain::Encoded);
}
if frozen_chroma_bottom_v.is_none() && need_output {
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
return Ok(());
}
InsertionPoint::LinearLight => {
unreachable!("encoded domain never selects the linear-light splice")
}
}
}
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_color = want_rgb || want_rgba || want_hsv;
let need_output = want_color || luma.is_some() || luma_u16.is_some();
if !need_output {
return Ok(());
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let bottom_v = chroma_440_bottom_sited_v(&chroma_location);
let top_v = chroma_440_top_sited_v(&chroma_location);
if let Some(frozen) = *frozen_chroma_bottom_v
&& (frozen != bottom_v || *frozen_chroma_top_v != Some(top_v))
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if (bottom_v || top_v) && want_color {
reserve_420_chroma_full(chroma_full, w, h)?;
}
if bottom_v || top_v {
reserve_440_chroma_prev(chroma_prev, w, h)?;
}
if top_v && want_color && chroma_top_y.len() < w {
chroma_top_y
.try_reserve_exact(w - chroma_top_y.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
w, h, w, h,
)))
})?;
chroma_top_y.resize(w, 0);
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if (bottom_v || top_v) && !want_color {
stage_440_chroma_prev(chroma_prev, chroma_prev_row, row.u(), row.v(), idx, w);
}
if frozen_chroma_bottom_v.is_none() {
*frozen_chroma_bottom_v = Some(bottom_v);
*frozen_chroma_top_v = Some(top_v);
}
if let Some(luma) = luma.as_deref_mut() {
luma[one_plane_start..one_plane_end].copy_from_slice(&row.y()[..w]);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::y_plane_to_luma_u16_row(
row.y(),
&mut buf[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if top_v && want_color {
let is_last = idx + 1 == h;
if idx & 1 == 0
&& let Some((p_idx, p_matrix, p_full_range)) = chroma_top_pending.take()
{
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
true,
false,
w,
use_simd,
);
yuv440_identity_color_row(
rgb,
rgba,
hsv,
rgb_scratch,
&chroma_top_y[..w],
u_full,
v_full,
p_idx * w,
p_idx * w + w,
w,
h,
p_matrix,
p_full_range,
use_simd,
)?;
}
if idx & 1 == 1 && !is_last {
chroma_top_y[..w].copy_from_slice(&row.y()[..w]);
*chroma_top_pending = Some((idx, row.matrix(), row.full_range()));
} else {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
false,
false,
w,
use_simd,
);
yuv440_identity_color_row(
rgb,
rgba,
hsv,
rgb_scratch,
row.y(),
u_full,
v_full,
one_plane_start,
one_plane_end,
w,
h,
row.matrix(),
row.full_range(),
use_simd,
)?;
}
stage_440_chroma_prev(chroma_prev, chroma_prev_row, row.u(), row.v(), idx, w);
return Ok(());
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if bottom_v {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
bottom_v,
true,
w,
use_simd,
);
yuv_444_to_hsv_row(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_444_to_hsv_row(
row.y(),
row.u(),
row.v(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if bottom_v {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
bottom_v,
true,
w,
use_simd,
);
yuv_444_to_rgba_row(
row.y(),
u_full,
v_full,
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_444_to_rgba_row(
row.y(),
row.u(),
row.v(),
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if bottom_v {
let (u_full, v_full) = upsample_440_chroma_sited(
chroma_full,
chroma_prev,
chroma_prev_row,
row.u(),
row.v(),
idx,
bottom_v,
true,
w,
use_simd,
);
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_444_to_rgb_row(
row.y(),
row.u(),
row.v(),
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
impl<'a, R> MixedSinker<'a, Yuv411p, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_luma_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_luma_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_luma_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_pixels()?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientLumaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.luma_u16 = Some(buf);
Ok(self)
}
}
impl<R> Yuv411pSink for MixedSinker<'_, Yuv411p, R> {}
impl<R> PixelSink for MixedSinker<'_, Yuv411p, R> {
type Input<'r> = Yuv411pRow<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
check_dimensions_match(self.width, self.height, width, height)?;
if let Some(stream) = self.rgb_stream.as_mut() {
stream.reset();
}
if let Some(stream) = self.luma_stream.as_mut() {
stream.reset();
}
if let Some(hsv) = self.hsv_planar.as_mut() {
hsv.reset();
}
self.frozen_chroma_centered = None;
self.resample_outputs = None;
Ok(())
}
fn process(&mut self, row: Yuv411pRow<'_>) -> Result<(), Self::Error> {
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
let chroma_location = self.chroma_location.clone();
let chroma_w = w.div_ceil(4);
if row.y().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::Y,
idx,
w,
row.y().len(),
)));
}
if row.u_quarter().len() != chroma_w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::UQuarter,
idx,
chroma_w,
row.u_quarter().len(),
)));
}
if row.v_quarter().len() != chroma_w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::VQuarter,
idx,
chroma_w,
row.v_quarter().len(),
)));
}
if idx >= self.height {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, self.height),
));
}
let Self {
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
plan,
rgb_stream,
luma_stream,
hsv_planar,
resample_outputs,
chroma_full,
frozen_chroma_centered,
..
} = self;
if let Some(plan) = plan.as_ref() {
let matrix = row.matrix();
let full_range = row.full_range();
let center_sited = chroma_411_center_sited_h(&chroma_location);
let chroma_h_phase = if center_sited {
YUV41X_CENTERED_H_PHASE
} else {
0.0
};
let want_color = rgb.is_some() || rgba.is_some() || hsv.is_some();
let need_output =
luma.is_some() || luma_u16.is_some() || rgb.is_some() || rgba.is_some() || hsv.is_some();
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if plan.kind().is_area() && hsv.is_some() && rgb.is_none() && rgba.is_none() {
let stale_hsv = idx == 0
&& hsv_planar
.as_ref()
.is_some_and(|join| join.chroma_centered() != center_sited && join.next_y() == 0);
let prev_hsv = if stale_hsv { hsv_planar.take() } else { None };
let hsv_result = super::planar_resample::hsv_direct_resample(
hsv_planar,
resample_outputs,
luma,
luma_u16,
hsv,
row.y(),
row.u_quarter(),
row.v_quarter(),
matrix,
full_range,
1,
|| ResamplePlan::area_chroma_411(w, h, plan.out_w(), plan.out_h(), chroma_h_phase, 0.0),
w,
plan,
idx,
use_simd,
);
if stale_hsv && hsv_result.is_err() {
*hsv_planar = prev_hsv;
}
hsv_result?;
if frozen_chroma_centered.is_none() && need_output {
*frozen_chroma_centered = Some(center_sited);
}
return Ok(());
}
let r = if center_sited && want_color {
let need_luma = luma.is_some() || luma_u16.is_some();
let expected = if need_luma {
luma_stream.as_ref().map_or(0, |s| s.next_y())
} else {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
};
if let core::ops::ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
reserve_420_chroma_full(chroma_full, w, h)?;
let (u_full, v_full) = reconstruct_chroma(
Chroma411U8,
chroma_full,
row.u_quarter(),
row.v_quarter(),
w,
use_simd,
);
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)
} else {
planar_dual_resample(
luma_stream,
rgb_stream,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
row.y(),
w,
plan,
idx,
use_simd,
|scratch| {
yuv_411_to_rgb_row(
row.y(),
row.u_quarter(),
row.v_quarter(),
scratch,
w,
matrix,
full_range,
use_simd,
);
},
)
};
if r.is_ok() && need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
return r;
}
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
let want_rgb = rgb.is_some();
let want_rgba = rgba.is_some();
let want_hsv = hsv.is_some();
let want_color = want_rgb || want_rgba || want_hsv;
let center_sited = chroma_411_center_sited_h(&chroma_location);
let need_output =
luma.is_some() || luma_u16.is_some() || rgb.is_some() || rgba.is_some() || hsv.is_some();
if need_output
&& let Some(frozen) = *frozen_chroma_centered
&& frozen != center_sited
{
return Err(MixedSinkerError::ChromaSitingChanged(
ChromaSitingChanged::new(idx),
));
}
if center_sited && want_color {
reserve_420_chroma_full(chroma_full, w, h)?;
}
if want_hsv && want_rgba && !want_rgb {
rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
}
if need_output && frozen_chroma_centered.is_none() {
*frozen_chroma_centered = Some(center_sited);
}
if let Some(luma) = luma.as_deref_mut() {
luma[one_plane_start..one_plane_end].copy_from_slice(&row.y()[..w]);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::y_plane_to_luma_u16_row(
row.y(),
&mut buf[one_plane_start..one_plane_end],
w,
use_simd,
);
}
let want_hsv_direct = want_hsv && !want_rgb && !want_rgba;
let need_rgb_kernel = want_rgb || (want_hsv && want_rgba);
if want_hsv_direct {
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
let (h, s, v) = hsv.hsv();
if center_sited {
let (u_full, v_full) = reconstruct_chroma(
Chroma411U8,
chroma_full,
row.u_quarter(),
row.v_quarter(),
w,
use_simd,
);
yuv_444_to_hsv_row(
row.y(),
u_full,
v_full,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_411_to_hsv_row(
row.y(),
row.u_quarter(),
row.v_quarter(),
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
return Ok(());
}
if want_rgba && !need_rgb_kernel {
let rgba_buf = rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
if center_sited {
let (u_full, v_full) = reconstruct_chroma(
Chroma411U8,
chroma_full,
row.u_quarter(),
row.v_quarter(),
w,
use_simd,
);
yuv_444_to_rgba_row(
row.y(),
u_full,
v_full,
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_411_to_rgba_row(
row.y(),
row.u_quarter(),
row.v_quarter(),
rgba_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
return Ok(());
}
if !need_rgb_kernel {
return Ok(());
}
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
if center_sited {
let (u_full, v_full) = reconstruct_chroma(
Chroma411U8,
chroma_full,
row.u_quarter(),
row.v_quarter(),
w,
use_simd,
);
yuv_444_to_rgb_row(
row.y(),
u_full,
v_full,
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
} else {
yuv_411_to_rgb_row(
row.y(),
row.u_quarter(),
row.v_quarter(),
rgb_row,
w,
row.matrix(),
row.full_range(),
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
rgb_row,
&mut h[one_plane_start..one_plane_end],
&mut s[one_plane_start..one_plane_end],
&mut v[one_plane_start..one_plane_end],
w,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}