use core::ops::ControlFlow;
use super::{HsvFrameMut, MixedSinkerError, frozen_outputs_check};
#[cfg(feature = "yuv-planar")]
use crate::{
KernelMatrix,
resample::{PlanGeometry, try_zeroed},
};
use crate::{
resample::{AreaStream, ResamplePlan, RowResampler},
row::*,
};
#[cfg(any(feature = "yuv-planar", feature = "yuv-packed"))]
use crate::resample::{OutOfSequenceRow, ResampleError};
#[cfg(feature = "yuv-planar")]
pub(super) struct HsvDirectPlanarYuv {
y: AreaStream<u8>,
y_stage: std::vec::Vec<u8>,
u: AreaStream<u8>,
v: AreaStream<u8>,
u_stage: std::vec::Vec<u8>,
v_stage: std::vec::Vec<u8>,
chroma_vsub: usize,
staged: [[bool; 4]; 3],
next_emit: usize,
chroma_centered: bool,
chroma_bottom: bool,
chroma_top: bool,
}
#[cfg(feature = "yuv-planar")]
impl HsvDirectPlanarYuv {
fn new(
plan: &ResamplePlan,
build_chroma_plan: impl FnOnce() -> Result<ResamplePlan, ResampleError>,
chroma_vsub: usize,
w: usize,
h: usize,
) -> Result<Self, ResampleError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter());
}
let alloc =
|_| ResampleError::AllocationFailed(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()));
let stage_len = plan.out_w().checked_mul(chroma_vsub).ok_or_else(|| {
ResampleError::Overflow(PlanGeometry::new(w, h, plan.out_w(), plan.out_h()))
})?;
let chroma_plan = build_chroma_plan()?;
let chroma_centered = chroma_plan.has_chroma_h_phase();
let chroma_bottom = chroma_plan.has_chroma_v_phase();
let chroma_top = chroma_plan.has_chroma_v_top();
Ok(Self {
y: AreaStream::new(plan.h(), plan.v(), w, h, 1)?,
y_stage: try_zeroed(stage_len).map_err(alloc)?,
u: AreaStream::new(
chroma_plan.h(),
chroma_plan.v(),
chroma_plan.src_w(),
chroma_plan.src_h(),
1,
)?,
v: AreaStream::new(
chroma_plan.h(),
chroma_plan.v(),
chroma_plan.src_w(),
chroma_plan.src_h(),
1,
)?,
u_stage: try_zeroed(stage_len).map_err(alloc)?,
v_stage: try_zeroed(stage_len).map_err(alloc)?,
chroma_vsub,
staged: [[false; 4]; 3],
next_emit: 0,
chroma_centered,
chroma_bottom,
chroma_top,
})
}
pub(super) fn reset(&mut self) {
self.y.reset();
self.u.reset();
self.v.reset();
self.staged = [[false; 4]; 3];
self.next_emit = 0;
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_centered(&self) -> bool {
self.chroma_centered
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_bottom(&self) -> bool {
self.chroma_bottom
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn chroma_top(&self) -> bool {
self.chroma_top
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn next_y(&self) -> usize {
self.y.next_y()
}
fn check_sequence(&self, idx: usize) -> Result<(), MixedSinkerError> {
if self.y.next_y() != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(self.y.next_y(), idx),
)));
}
let chroma_expected = idx.div_ceil(self.chroma_vsub);
for stream in [&self.u, &self.v] {
if stream.next_y() != chroma_expected {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(stream.next_y().saturating_mul(self.chroma_vsub), idx),
)));
}
}
Ok(())
}
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
fn hsv_direct_feed_emit(
join: &mut HsvDirectPlanarYuv,
hsv: &mut HsvFrameMut<'_>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
y_row: &[u8],
u_row: &[u8],
v_row: &[u8],
matrix: KernelMatrix,
full_range: bool,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
join.check_sequence(idx)?;
let HsvDirectPlanarYuv {
y,
y_stage,
u,
v,
u_stage,
v_stage,
chroma_vsub,
staged,
next_emit,
chroma_centered: _,
chroma_bottom: _,
chroma_top: _,
} = join;
let cv = *chroma_vsub;
y.feed_row(idx, y_row, use_simd, |oy, out_row| {
let slot = oy % cv;
y_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[0][slot] = true;
})?;
if idx.is_multiple_of(cv) {
let cidx = idx / cv;
u.feed_row(cidx, u_row, use_simd, |oy, out_row| {
let slot = oy % cv;
u_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[1][slot] = true;
})?;
v.feed_row(cidx, v_row, use_simd, |oy, out_row| {
let slot = oy % cv;
v_stage[slot * ow..slot * ow + ow].copy_from_slice(out_row);
staged[2][slot] = true;
})?;
}
let (hp, sp, vp) = hsv.hsv();
while *next_emit < plan.out_h() {
let slot = *next_emit % cv;
if !(staged[0][slot] && staged[1][slot] && staged[2][slot]) {
break;
}
let oy = *next_emit;
let y_out = &y_stage[slot * ow..slot * ow + ow];
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(y_out);
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(y_out) {
*dst = src as u16;
}
}
yuv_444_to_hsv_row(
y_out,
&u_stage[slot * ow..slot * ow + ow],
&v_stage[slot * ow..slot * ow + ow],
&mut hp[oy * ow..(oy + 1) * ow],
&mut sp[oy * ow..(oy + 1) * ow],
&mut vp[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
staged[0][slot] = false;
staged[1][slot] = false;
staged[2][slot] = false;
*next_emit += 1;
}
Ok(())
}
#[cfg(feature = "yuv-planar")]
#[allow(clippy::too_many_arguments)]
pub(super) fn hsv_direct_resample(
hsv_planar: &mut Option<std::boxed::Box<HsvDirectPlanarYuv>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
y_row: &[u8],
u_row: &[u8],
v_row: &[u8],
matrix: KernelMatrix,
full_range: bool,
chroma_vsub: usize,
build_chroma_plan: impl FnOnce() -> Result<ResamplePlan, ResampleError>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let expected = hsv_planar.as_ref().map_or(0, |join| join.y.next_y());
if let ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
Some(expected),
idx,
)? {
return Ok(());
}
if hsv_planar.is_none() {
let join = HsvDirectPlanarYuv::new(plan, build_chroma_plan, chroma_vsub, w, plan.src_h())?;
let boxed = crate::resample::try_box(join).map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
)))
})?;
*hsv_planar = Some(boxed);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let join = hsv_planar.as_mut().expect("created in the preflight");
let hsv = hsv.as_mut().expect("want_hsv_direct implies hsv attached");
hsv_direct_feed_emit(
join, hsv, luma, luma_u16, y_row, u_row, v_row, matrix, full_range, plan, idx, use_simd,
)
}
#[cfg_attr(
not(any(feature = "yuv-planar", feature = "yuv-semi-planar")),
allow(dead_code)
)]
#[allow(clippy::too_many_arguments)]
pub(super) fn planar_dual_resample(
luma_stream: &mut Option<std::boxed::Box<AreaStream<u8>>>,
rgb_stream: &mut Option<std::boxed::Box<AreaStream<u8>>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y_row: &[u8],
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
convert_rgb: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || hsv.is_some() || rgba.is_some();
let expected = if need_luma {
Some(luma_stream.as_ref().map_or(0, |stream| stream.next_y()))
} else if need_color {
Some(rgb_stream.as_ref().map_or(0, |stream| stream.next_y()))
} else {
None
};
if let ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let new_luma = if need_luma && luma_stream.is_none() {
let stream = AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 1)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
let new_rgb = if need_color && rgb_stream.is_none() {
let stream = AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if need_color {
super::source_rgb_scratch(rgb_scratch, w, plan)?;
}
if let Some(l) = new_luma {
*luma_stream = Some(l);
}
if let Some(r) = new_rgb {
*rgb_stream = Some(r);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
planar_dual_feed_emit(
luma_stream.as_mut(),
rgb_stream.as_mut(),
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
y_row,
w,
plan,
idx,
use_simd,
convert_rgb,
)
}
#[cfg_attr(
not(any(feature = "yuv-planar", feature = "yuv-semi-planar")),
allow(dead_code)
)]
#[allow(clippy::too_many_arguments)]
fn planar_dual_feed_emit<LS, CS>(
luma_stream: Option<&mut LS>,
rgb_stream: Option<&mut CS>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y_row: &[u8],
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
convert_rgb: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError>
where
LS: RowResampler<u8>,
CS: RowResampler<u8>,
{
let ow = plan.out_w();
let color_row = if rgb_stream.is_some() {
let scratch = super::source_rgb_scratch(rgb_scratch, w, plan)?;
convert_rgb(scratch);
Some(scratch)
} else {
None
};
if let Some(stream) = luma_stream {
stream.feed_row(idx, y_row, use_simd, |oy, out_row| {
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(out_row);
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(out_row) {
*dst = src as u16;
}
}
})?;
}
if let Some(scratch) = color_row {
let stream = rgb_stream.expect("staged only when present");
stream.feed_row(idx, scratch, use_simd, |oy, out_row| {
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_row);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
out_row,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
expand_rgb_to_rgba_row(out_row, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
})?;
}
Ok(())
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn planar_dual_filter_resample(
luma_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
rgb_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
y_row: &[u8],
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
convert_rgb: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
plan.ensure_single_kernel_filter()?;
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || hsv.is_some() || rgba.is_some();
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let expected = if need_luma {
Some(
luma_filter_stream
.as_ref()
.map_or(0, |stream| stream.next_y()),
)
} else if need_color {
Some(
rgb_filter_stream
.as_ref()
.map_or(0, |stream| stream.next_y()),
)
} else {
None
};
if let ControlFlow::Break(()) = super::resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let new_luma = if need_luma && luma_filter_stream.is_none() {
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 1)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
let new_rgb = if need_color && rgb_filter_stream.is_none() {
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if need_color {
super::source_rgb_scratch(rgb_scratch, w, plan)?;
}
if let Some(l) = new_luma {
*luma_filter_stream = Some(l);
}
if let Some(r) = new_rgb {
*rgb_filter_stream = Some(r);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
planar_dual_feed_emit(
luma_filter_stream.as_mut(),
rgb_filter_stream.as_mut(),
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
y_row,
w,
plan,
idx,
use_simd,
convert_rgb,
)
}
#[cfg(feature = "yuv-packed")]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_yuv_dual_resample(
luma_stream: &mut Option<std::boxed::Box<AreaStream<u8>>>,
rgb_stream: &mut Option<std::boxed::Box<AreaStream<u8>>>,
resample_outputs: &mut Option<super::FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
luma_scratch: &mut std::vec::Vec<u8>,
rgb_scratch: &mut std::vec::Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
convert_luma: impl FnOnce(&mut [u8]),
convert_rgb: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let ow = plan.out_w();
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || hsv.is_some() || rgba.is_some();
let expected = if need_luma {
luma_stream.as_ref().map_or(0, |stream| stream.next_y())
} else if need_color {
rgb_stream.as_ref().map_or(0, |stream| stream.next_y())
} else {
return Ok(());
};
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(expected, idx),
)));
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
OutOfSequenceRow::new(expected, idx),
)));
}
if need_luma && luma_stream.is_none() {
*luma_stream = Some({
let stream = AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 1)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_color && rgb_stream.is_none() {
*rgb_stream = Some({
let stream = AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
let luma_row = if need_luma {
let scratch = super::source_luma_scratch(luma_scratch, w, plan)?;
convert_luma(scratch);
Some(scratch)
} else {
None
};
let color_row = if need_color {
let scratch = super::source_rgb_scratch(rgb_scratch, w, plan)?;
convert_rgb(scratch);
Some(scratch)
} else {
None
};
if let Some(y_row) = luma_row {
let stream = luma_stream.as_mut().expect("created in the preflight");
stream.feed_row(idx, y_row, use_simd, |oy, out_row| {
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(out_row);
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(out_row) {
*dst = src as u16;
}
}
})?;
}
if let Some(scratch) = color_row {
let stream = rgb_stream.as_mut().expect("created in the preflight");
stream.feed_row(idx, scratch, use_simd, |oy, out_row| {
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_row);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
rgb_to_hsv_row(
out_row,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
expand_rgb_to_rgba_row(out_row, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
})?;
}
Ok(())
}