use super::{
FrozenOutputs, GeometryOverflow, HsvFrameMut, LinearModeChanged, MixedSinkerError,
ResampleOutputsChanged, TransferFunctionChanged,
};
use crate::{
resample::{AreaStream, LinearMode, ResamplePlan, TransferFunction, try_zeroed},
row::{expand_rgb_to_rgba_row, rgb_row_bytes, rgb_to_hsv_row, y_plane_to_luma_u16_row},
};
#[cfg(all(test, feature = "std"))]
mod tests;
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
std::thread_local! {
static FORCE_LINEAR_TAIL_ALLOC_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
static FORCE_LINEAR_SCRATCH_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
pub(crate) fn arm_linear_tail_alloc_failure() {
FORCE_LINEAR_TAIL_ALLOC_FAILURE.with(|f| f.set(true));
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn take_linear_tail_alloc_failure() -> bool {
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
{
FORCE_LINEAR_TAIL_ALLOC_FAILURE.with(|f| f.take())
}
#[cfg(not(all(test, feature = "std", feature = "yuv-planar", feature = "rgb")))]
{
false
}
}
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
pub(crate) fn arm_linear_scratch_failure() {
FORCE_LINEAR_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn take_linear_scratch_failure() -> bool {
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
{
FORCE_LINEAR_SCRATCH_FAILURE.with(|f| f.take())
}
#[cfg(not(all(test, feature = "std", feature = "yuv-planar", feature = "rgb")))]
{
false
}
}
fn linear_tail_rgb_len(out_w: usize, out_h: usize) -> Result<usize, MixedSinkerError> {
out_w
.checked_mul(out_h)
.and_then(|n| n.checked_mul(3))
.ok_or_else(|| MixedSinkerError::GeometryOverflow(GeometryOverflow::new(out_w, out_h, 3)))
}
type LinearTail = (
AreaStream<f32>,
std::vec::Vec<f32>,
Option<AreaStream<u8>>,
std::vec::Vec<u8>,
);
#[derive(Debug)]
pub(super) struct LinearLightFrame {
linear_rgb: std::vec::Vec<f32>,
y_plane: std::vec::Vec<u8>,
next_y: usize,
src_w: usize,
src_h: usize,
frozen_transfer: TransferFunction,
frozen_linear_mode: LinearMode,
reserved_tail: Option<LinearTail>,
}
impl LinearLightFrame {
fn new(
src_w: usize,
src_h: usize,
out_w: usize,
out_h: usize,
want_luma: bool,
frozen_transfer: TransferFunction,
frozen_linear_mode: LinearMode,
) -> Result<Self, MixedSinkerError> {
let overflow = || MixedSinkerError::GeometryOverflow(GeometryOverflow::new(src_w, src_h, 3));
let alloc_failed = || {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(src_w, src_h, out_w, out_h),
))
};
let luma = src_w.checked_mul(src_h).ok_or_else(overflow)?;
let n = luma.checked_mul(3).ok_or_else(overflow)?;
let y_plane = if want_luma {
try_zeroed(luma).map_err(|_| alloc_failed())?
} else {
std::vec::Vec::new()
};
Ok(Self {
linear_rgb: try_zeroed(n).map_err(|_| alloc_failed())?,
y_plane,
next_y: 0,
src_w,
src_h,
frozen_transfer,
frozen_linear_mode,
reserved_tail: None,
})
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) const fn next_y(&self) -> usize {
self.next_y
}
}
pub(super) fn linear_light_reserve_final_tail(
frame: &mut Option<LinearLightFrame>,
plan: &ResamplePlan,
w: usize,
h: usize,
want_luma: bool,
) -> Result<(), MixedSinkerError> {
let Some(buf) = frame.as_mut() else {
return Ok(());
};
if buf.reserved_tail.is_some() {
return Ok(());
}
let ow = plan.out_w();
let oh = plan.out_h();
if take_linear_tail_alloc_failure() {
return Err(MixedSinkerError::Resample(
crate::resample::ResampleError::AllocationFailed(crate::resample::PlanGeometry::new(
w, h, ow, oh,
)),
));
}
let rgb_len = linear_tail_rgb_len(ow, oh)?;
let alloc_failed = || {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(w, h, ow, oh),
))
};
let stream = AreaStream::<f32>::new(plan.h(), plan.v(), w, h, 3)?;
let binned = try_zeroed::<f32>(rgb_len).map_err(|_| alloc_failed())?;
let y_stream = if want_luma {
Some(AreaStream::<u8>::new(plan.h(), plan.v(), w, h, 1)?)
} else {
None
};
let enc_out = try_zeroed::<u8>(rgb_row_bytes(ow)).map_err(|_| alloc_failed())?;
buf.reserved_tail = Some((stream, binned, y_stream, enc_out));
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(super) fn linear_light_preflight(
frame: &Option<LinearLightFrame>,
resample_outputs: &Option<FrozenOutputs>,
luma: &Option<&mut [u8]>,
luma_u16: &Option<&mut [u16]>,
rgb: &Option<&mut [u8]>,
rgba: &Option<&mut [u8]>,
hsv: &mut Option<HsvFrameMut<'_>>,
tf: TransferFunction,
mode: LinearMode,
plan: &ResamplePlan,
idx: usize,
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let snapshot = FrozenOutputs::snapshot(
luma.as_deref(),
luma_u16.as_deref(),
rgb.as_deref(),
rgba.as_deref(),
None,
None,
None,
None,
None,
None,
None,
hsv.as_mut().map(|f| {
let (h, s, v) = f.hsv();
(&h[..], &s[..], &v[..])
}),
None,
);
if let Some(frozen) = resample_outputs
&& *frozen != snapshot
{
return Err(MixedSinkerError::ResampleOutputsChanged(
ResampleOutputsChanged::new(idx),
));
}
let expected = frame.as_ref().map_or(0, |b| b.next_y);
if idx != expected {
return Err(MixedSinkerError::Resample(
crate::resample::ResampleError::OutOfSequenceRow(crate::resample::OutOfSequenceRow::new(
expected, idx,
)),
));
}
if let Some(b) = frame.as_ref()
&& b.frozen_transfer != tf
{
return Err(MixedSinkerError::TransferFunctionChanged(
TransferFunctionChanged::new(idx),
));
}
if let Some(b) = frame.as_ref()
&& b.frozen_linear_mode != mode
{
return Err(MixedSinkerError::LinearModeChanged(LinearModeChanged::new(
idx,
)));
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(super) fn linear_light_resample(
frame: &mut Option<LinearLightFrame>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut std::vec::Vec<u8>,
scene_scratch: &mut std::vec::Vec<f32>,
tf: TransferFunction,
mode: LinearMode,
plan: &ResamplePlan,
y_row: &[u8],
idx: usize,
w: usize,
h: usize,
use_simd: bool,
mut decode_rgb_row: impl FnMut(usize, &mut [u8]),
mut decode_unclamped_f32: impl FnMut(usize, &mut [f32]),
) -> Result<(), MixedSinkerError> {
let need_output =
luma.is_some() || luma_u16.is_some() || rgb.is_some() || rgba.is_some() || hsv.is_some();
if !need_output {
return Ok(());
}
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let snapshot = FrozenOutputs::snapshot(
luma.as_deref(),
luma_u16.as_deref(),
rgb.as_deref(),
rgba.as_deref(),
None, None, None, None, None, None, None, hsv.as_mut().map(|f| {
let (h, s, v) = f.hsv();
(&h[..], &s[..], &v[..])
}),
None, );
let pending_outputs = match resample_outputs {
Some(frozen) if *frozen != snapshot => {
return Err(MixedSinkerError::ResampleOutputsChanged(
ResampleOutputsChanged::new(idx),
));
}
Some(_) => None,
None => Some(snapshot),
};
let expected = frame.as_ref().map_or(0, |b| b.next_y);
if idx != expected {
return Err(MixedSinkerError::Resample(
crate::resample::ResampleError::OutOfSequenceRow(crate::resample::OutOfSequenceRow::new(
expected, idx,
)),
));
}
if let Some(b) = frame.as_ref()
&& b.frozen_transfer != tf
{
return Err(MixedSinkerError::TransferFunctionChanged(
TransferFunctionChanged::new(idx),
));
}
if let Some(b) = frame.as_ref()
&& b.frozen_linear_mode != mode
{
return Err(MixedSinkerError::LinearModeChanged(LinearModeChanged::new(
idx,
)));
}
let want_luma = luma.is_some() || luma_u16.is_some();
let mut new_frame = if frame.is_none() {
Some(LinearLightFrame::new(
w,
h,
plan.out_w(),
plan.out_h(),
want_luma,
tf,
mode,
)?)
} else {
None
};
if take_linear_scratch_failure() {
return Err(MixedSinkerError::Resample(
crate::resample::ResampleError::AllocationFailed(crate::resample::PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)),
));
}
let scratch_alloc_failed = || {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
))
};
match mode {
LinearMode::DisplayReferred => {
let rgb_bytes = rgb_row_bytes(w);
if rgb_scratch.len() < rgb_bytes {
rgb_scratch
.try_reserve(rgb_bytes - rgb_scratch.len())
.map_err(|_| scratch_alloc_failed())?;
rgb_scratch.resize(rgb_bytes, 0);
}
decode_rgb_row(idx, &mut rgb_scratch[..rgb_bytes]);
}
LinearMode::SceneReferred => {
let n = w * 3;
if scene_scratch.len() < n {
scene_scratch
.try_reserve(n - scene_scratch.len())
.map_err(|_| scratch_alloc_failed())?;
scene_scratch.resize(n, 0.0);
}
decode_unclamped_f32(idx, &mut scene_scratch[..n]);
}
}
let is_final = idx + 1 == h;
let tail = if is_final && let Some(pre) = frame.as_mut().and_then(|b| b.reserved_tail.take()) {
let (stream, binned, y_stream, enc_out) = pre;
Some((
plan.out_w(),
plan.out_h(),
stream,
binned,
y_stream,
enc_out,
))
} else if is_final {
let ow = plan.out_w();
let oh = plan.out_h();
if take_linear_tail_alloc_failure() {
return Err(MixedSinkerError::Resample(
crate::resample::ResampleError::AllocationFailed(crate::resample::PlanGeometry::new(
w, h, ow, oh,
)),
));
}
let rgb_len = linear_tail_rgb_len(ow, oh)?;
let alloc_failed = || {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(w, h, ow, oh),
))
};
let stream = AreaStream::<f32>::new(plan.h(), plan.v(), w, h, 3)?;
let binned = try_zeroed::<f32>(rgb_len).map_err(|_| alloc_failed())?;
let y_stream = if want_luma {
Some(AreaStream::<u8>::new(plan.h(), plan.v(), w, h, 1)?)
} else {
None
};
let enc_out = try_zeroed::<u8>(rgb_row_bytes(ow)).map_err(|_| alloc_failed())?;
Some((ow, oh, stream, binned, y_stream, enc_out))
} else {
None
};
if let Some(new_frame) = new_frame.take() {
*frame = Some(new_frame);
}
if let Some(pending_outputs) = pending_outputs {
*resample_outputs = Some(pending_outputs);
}
let buf = frame
.as_mut()
.expect("frame committed above (or pre-existing)");
let lin = &mut buf.linear_rgb[idx * w * 3..(idx + 1) * w * 3];
match mode {
LinearMode::DisplayReferred => {
let enc_row = &rgb_scratch[..w * 3];
for (l, &e) in lin.iter_mut().zip(enc_row.iter()) {
*l = tf.eotf(e as f32 / 255.0);
}
}
LinearMode::SceneReferred => {
let enc_row = &scene_scratch[..w * 3];
for (l, &e) in lin.iter_mut().zip(enc_row.iter()) {
*l = tf.eotf(e);
}
}
}
if want_luma {
buf.y_plane[idx * w..(idx + 1) * w].copy_from_slice(&y_row[..w]);
}
buf.next_y += 1;
let Some((ow, oh, mut stream, mut binned, mut y_stream, mut enc_out)) = tail else {
return Ok(());
};
let stride = buf.src_w * 3;
for sy in 0..buf.src_h {
stream.feed_row(
sy,
&buf.linear_rgb[sy * stride..(sy + 1) * stride],
use_simd,
|oy, finalized| {
binned[oy * ow * 3..(oy + 1) * ow * 3].copy_from_slice(finalized);
},
)?;
}
if let Some(y_stream) = y_stream.as_mut() {
for sy in 0..buf.src_h {
y_stream.feed_row(
sy,
&buf.y_plane[sy * buf.src_w..(sy + 1) * buf.src_w],
use_simd,
|oy, finalized| {
if let Some(luma) = luma.as_deref_mut() {
luma[oy * ow..(oy + 1) * ow].copy_from_slice(finalized);
}
if let Some(luma_u16) = luma_u16.as_deref_mut() {
y_plane_to_luma_u16_row(
finalized,
&mut luma_u16[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
},
)?;
}
}
for oy in 0..oh {
let lin_row = &binned[oy * ow * 3..(oy + 1) * ow * 3];
for (o, &l) in enc_out[..ow * 3].iter_mut().zip(lin_row.iter()) {
*o = (tf.oetf(l) * 255.0 + 0.5).clamp(0.0, 255.0) as u8;
}
if let Some(rgb) = rgb.as_deref_mut() {
rgb[oy * ow * 3..(oy + 1) * ow * 3].copy_from_slice(&enc_out[..ow * 3]);
}
if let Some(rgba) = rgba.as_deref_mut() {
expand_rgb_to_rgba_row(
&enc_out[..ow * 3],
&mut rgba[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
);
}
if let Some(hsv) = hsv.as_mut() {
let (hh, hs, hv) = hsv.hsv();
rgb_to_hsv_row(
&enc_out[..ow * 3],
&mut hh[oy * ow..(oy + 1) * ow],
&mut hs[oy * ow..(oy + 1) * ow],
&mut hv[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
Ok(())
}