use core::marker::PhantomData;
#[allow(unused_imports)]
use std::vec::Vec;
use derive_more::{Display, IsVariant, TryUnwrap, Unwrap};
use thiserror::Error;
use crate::{
SourceFormat,
resample::{NoopResampler, ResampleError, ResamplePlan, Resampler},
};
#[cfg(feature = "yuv-planar")]
use crate::resample::{AveragingDomain, LinearMode, TransferFunction};
#[allow(unused_imports)]
use crate::PixelSink;
pub use mediaframe::{
frame::{WidthAlignment, WidthAlignmentRequirement},
source::{HsvFrame, HsvFrameMut, HsvPlane},
};
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) const fn chroma_420_center_sited_h(loc: &crate::ChromaLocation) -> bool {
matches!(
loc,
crate::ChromaLocation::Center | crate::ChromaLocation::Top | crate::ChromaLocation::Bottom
)
}
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) const fn chroma_420_bottom_sited_v(loc: &crate::ChromaLocation) -> bool {
matches!(
loc,
crate::ChromaLocation::Bottom | crate::ChromaLocation::BottomLeft
)
}
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) const fn chroma_420_top_sited_v(loc: &crate::ChromaLocation) -> bool {
matches!(
loc,
crate::ChromaLocation::Top | crate::ChromaLocation::TopLeft
)
}
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) const fn chroma_440_bottom_sited_v(loc: &crate::ChromaLocation) -> bool {
matches!(
loc,
crate::ChromaLocation::Bottom | crate::ChromaLocation::BottomLeft
)
}
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) const fn chroma_440_top_sited_v(loc: &crate::ChromaLocation) -> bool {
matches!(
loc,
crate::ChromaLocation::Top | crate::ChromaLocation::TopLeft
)
}
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) const fn chroma_422_center_sited_h(loc: &crate::ChromaLocation) -> bool {
matches!(
loc,
crate::ChromaLocation::Center | crate::ChromaLocation::Top | crate::ChromaLocation::Bottom
)
}
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) const fn chroma_411_center_sited_h(loc: &crate::ChromaLocation) -> bool {
matches!(
loc,
crate::ChromaLocation::Center | crate::ChromaLocation::Top | crate::ChromaLocation::Bottom
)
}
#[cfg(feature = "yuv-planar")]
#[inline]
pub(super) const fn chroma_410_center_sited_h(loc: &crate::ChromaLocation) -> bool {
matches!(
loc,
crate::ChromaLocation::Center | crate::ChromaLocation::Top | crate::ChromaLocation::Bottom
)
}
#[cfg(feature = "yuv-planar")]
pub(super) use chroma_reconstruct::{Chroma411U8, ChromaU8, ChromaU16, reconstruct_chroma};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DimensionMismatch {
configured_w: usize,
configured_h: usize,
frame_w: u32,
frame_h: u32,
}
impl DimensionMismatch {
#[inline]
pub const fn new(configured_w: usize, configured_h: usize, frame_w: u32, frame_h: u32) -> Self {
Self {
configured_w,
configured_h,
frame_w,
frame_h,
}
}
#[inline]
pub const fn configured_w(&self) -> usize {
self.configured_w
}
#[inline]
pub const fn configured_h(&self) -> usize {
self.configured_h
}
#[inline]
pub const fn frame_w(&self) -> u32 {
self.frame_w
}
#[inline]
pub const fn frame_h(&self) -> u32 {
self.frame_h
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InsufficientBuffer {
expected: usize,
actual: usize,
}
impl InsufficientBuffer {
#[inline]
pub const fn new(expected: usize, actual: usize) -> Self {
Self { expected, actual }
}
#[inline]
pub const fn expected(&self) -> usize {
self.expected
}
#[inline]
pub const fn actual(&self) -> usize {
self.actual
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InsufficientHsvPlane {
which: HsvPlane,
expected: usize,
actual: usize,
}
impl InsufficientHsvPlane {
#[inline]
pub const fn new(which: HsvPlane, expected: usize, actual: usize) -> Self {
Self {
which,
expected,
actual,
}
}
#[inline]
pub const fn which(&self) -> HsvPlane {
self.which
}
#[inline]
pub const fn expected(&self) -> usize {
self.expected
}
#[inline]
pub const fn actual(&self) -> usize {
self.actual
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GeometryOverflow {
width: usize,
height: usize,
channels: usize,
}
impl GeometryOverflow {
#[inline]
pub const fn new(width: usize, height: usize, channels: usize) -> Self {
Self {
width,
height,
channels,
}
}
#[inline]
pub const fn width(&self) -> usize {
self.width
}
#[inline]
pub const fn height(&self) -> usize {
self.height
}
#[inline]
pub const fn channels(&self) -> usize {
self.channels
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RowShapeMismatch {
which: RowSlice,
row: usize,
expected: usize,
actual: usize,
}
impl RowShapeMismatch {
#[inline]
pub const fn new(which: RowSlice, row: usize, expected: usize, actual: usize) -> Self {
Self {
which,
row,
expected,
actual,
}
}
#[inline]
pub const fn which(&self) -> RowSlice {
self.which
}
#[inline]
pub const fn row(&self) -> usize {
self.row
}
#[inline]
pub const fn expected(&self) -> usize {
self.expected
}
#[inline]
pub const fn actual(&self) -> usize {
self.actual
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RowIndexOutOfRange {
row: usize,
configured_height: usize,
}
impl RowIndexOutOfRange {
#[inline]
pub const fn new(row: usize, configured_height: usize) -> Self {
Self {
row,
configured_height,
}
}
#[inline]
pub const fn row(&self) -> usize {
self.row
}
#[inline]
pub const fn configured_height(&self) -> usize {
self.configured_height
}
}
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
#[cfg_attr(not(any(feature = "yuv-planar", feature = "rgb")), allow(dead_code))]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct FrozenOutputs {
idents: [(usize, usize); 15],
}
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
#[cfg_attr(not(any(feature = "yuv-planar", feature = "rgb")), allow(dead_code))]
impl FrozenOutputs {
fn ident<T>(buf: Option<&[T]>) -> (usize, usize) {
buf.map_or((0, 0), |b| (b.as_ptr() as usize, b.len()))
}
#[allow(clippy::too_many_arguments)]
pub(super) fn snapshot(
luma: Option<&[u8]>,
luma_u16: Option<&[u16]>,
rgb: Option<&[u8]>,
rgba: Option<&[u8]>,
rgb_u16: Option<&[u16]>,
rgba_u16: Option<&[u16]>,
rgb_f32: Option<&[f32]>,
rgba_f32: Option<&[f32]>,
xyz_f32: Option<&[f32]>,
rgb_f16: Option<&[half::f16]>,
rgba_f16: Option<&[half::f16]>,
hsv: Option<(&[u8], &[u8], &[u8])>,
luma_f32: Option<&[f32]>,
) -> Self {
let (h, s, v) = match hsv {
Some((h, s, v)) => (
Self::ident(Some(h)),
Self::ident(Some(s)),
Self::ident(Some(v)),
),
None => ((0, 0), (0, 0), (0, 0)),
};
Self {
idents: [
Self::ident(luma),
Self::ident(luma_u16),
Self::ident(rgb),
Self::ident(rgba),
Self::ident(rgb_u16),
Self::ident(rgba_u16),
Self::ident(rgb_f32),
Self::ident(rgba_f32),
Self::ident(xyz_f32),
Self::ident(rgb_f16),
Self::ident(rgba_f16),
h,
s,
v,
Self::ident(luma_f32),
],
}
}
}
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
#[cfg_attr(not(any(feature = "yuv-planar", feature = "rgb")), allow(dead_code))]
#[allow(clippy::too_many_arguments)]
pub(super) fn resample_preflight_check_only(
resample_outputs: &Option<FrozenOutputs>,
luma: &Option<&mut [u8]>,
luma_u16: &Option<&mut [u16]>,
rgb: &Option<&mut [u8]>,
rgba: &Option<&mut [u8]>,
rgb_u16: &Option<&mut [u16]>,
rgba_u16: &Option<&mut [u16]>,
rgb_f32: &Option<&mut [f32]>,
rgba_f32: &Option<&mut [f32]>,
xyz_f32: &Option<&mut [f32]>,
rgb_f16: &Option<&mut [half::f16]>,
rgba_f16: &Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
luma_f32: &Option<&mut [f32]>,
expected: Option<usize>,
idx: usize,
) -> Result<core::ops::ControlFlow<()>, MixedSinkerError> {
let Some(expected) = expected else {
return Ok(core::ops::ControlFlow::Break(()));
};
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
if let Some(frozen) = resample_outputs {
let snapshot = FrozenOutputs::snapshot(
luma.as_deref(),
luma_u16.as_deref(),
rgb.as_deref(),
rgba.as_deref(),
rgb_u16.as_deref(),
rgba_u16.as_deref(),
rgb_f32.as_deref(),
rgba_f32.as_deref(),
xyz_f32.as_deref(),
rgb_f16.as_deref(),
rgba_f16.as_deref(),
hsv.as_mut().map(|f| {
let (h, s, v) = f.hsv();
(&h[..], &s[..], &v[..])
}),
luma_f32.as_deref(),
);
if *frozen != snapshot {
return Err(MixedSinkerError::ResampleOutputsChanged(
ResampleOutputsChanged::new(idx),
));
}
}
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
Ok(core::ops::ControlFlow::Continue(()))
}
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
#[cfg_attr(not(any(feature = "yuv-planar", feature = "rgb")), allow(dead_code))]
#[allow(clippy::too_many_arguments)]
pub(super) fn frozen_outputs_check(
resample_outputs: &mut Option<FrozenOutputs>,
luma: &Option<&mut [u8]>,
luma_u16: &Option<&mut [u16]>,
rgb: &Option<&mut [u8]>,
rgba: &Option<&mut [u8]>,
rgb_u16: &Option<&mut [u16]>,
rgba_u16: &Option<&mut [u16]>,
rgb_f32: &Option<&mut [f32]>,
rgba_f32: &Option<&mut [f32]>,
xyz_f32: &Option<&mut [f32]>,
rgb_f16: &Option<&mut [half::f16]>,
rgba_f16: &Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
luma_f32: &Option<&mut [f32]>,
idx: usize,
) -> Result<(), MixedSinkerError> {
let snapshot = FrozenOutputs::snapshot(
luma.as_deref(),
luma_u16.as_deref(),
rgb.as_deref(),
rgba.as_deref(),
rgb_u16.as_deref(),
rgba_u16.as_deref(),
rgb_f32.as_deref(),
rgba_f32.as_deref(),
xyz_f32.as_deref(),
rgb_f16.as_deref(),
rgba_f16.as_deref(),
hsv.as_mut().map(|f| {
let (h, s, v) = f.hsv();
(&h[..], &s[..], &v[..])
}),
luma_f32.as_deref(),
);
match resample_outputs {
None => *resample_outputs = Some(snapshot),
Some(frozen) if *frozen != snapshot => {
return Err(MixedSinkerError::ResampleOutputsChanged(
ResampleOutputsChanged::new(idx),
));
}
Some(_) => {}
}
Ok(())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ResampleOutputsChanged {
row: usize,
}
impl ResampleOutputsChanged {
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn new(row: usize) -> Self {
Self { row }
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn row(&self) -> usize {
self.row
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct NativeRouteChanged {
row: usize,
}
impl NativeRouteChanged {
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn new(row: usize) -> Self {
Self { row }
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn row(&self) -> usize {
self.row
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct UnsupportedMatrixResample {
row: usize,
matrix: &'static str,
}
impl UnsupportedMatrixResample {
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn new(row: usize, matrix: &'static str) -> Self {
Self { row, matrix }
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn row(&self) -> usize {
self.row
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn matrix(&self) -> &'static str {
self.matrix
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TransferFunctionChanged {
row: usize,
}
impl TransferFunctionChanged {
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn new(row: usize) -> Self {
Self { row }
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn row(&self) -> usize {
self.row
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LinearModeChanged {
row: usize,
}
impl LinearModeChanged {
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn new(row: usize) -> Self {
Self { row }
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn row(&self) -> usize {
self.row
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AveragingDomainChanged {
row: usize,
}
impl AveragingDomainChanged {
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn new(row: usize) -> Self {
Self { row }
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn row(&self) -> usize {
self.row
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ChromaSitingChanged {
row: usize,
}
impl ChromaSitingChanged {
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn new(row: usize) -> Self {
Self { row }
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn row(&self) -> usize {
self.row
}
}
#[derive(Debug, Clone, Copy, PartialEq, IsVariant, TryUnwrap, Unwrap, Error)]
#[non_exhaustive]
pub enum MixedSinkerError {
#[error(
"MixedSinker frame dimensions mismatch: configured {}x{} but got {}x{}",
.0.configured_w(), .0.configured_h(), .0.frame_w(), .0.frame_h()
)]
DimensionMismatch(DimensionMismatch),
#[error("MixedSinker insufficient rgb buffer: expected >= {} bytes, got {}", .0.expected(), .0.actual())]
InsufficientRgbBuffer(InsufficientBuffer),
#[error("MixedSinker insufficient rgb_u16 buffer: expected >= {} elements, got {}", .0.expected(), .0.actual())]
InsufficientRgbU16Buffer(InsufficientBuffer),
#[error("MixedSinker insufficient luma_u16 buffer: expected >= {} elements, got {}", .0.expected(), .0.actual())]
InsufficientLumaU16Buffer(InsufficientBuffer),
#[error("MixedSinker insufficient rgba buffer: expected >= {} bytes, got {}", .0.expected(), .0.actual())]
InsufficientRgbaBuffer(InsufficientBuffer),
#[error("MixedSinker insufficient rgba_u16 buffer: expected >= {} elements, got {}", .0.expected(), .0.actual())]
InsufficientRgbaU16Buffer(InsufficientBuffer),
#[error("MixedSinker insufficient rgb_f32 buffer: expected >= {} elements, got {}", .0.expected(), .0.actual())]
InsufficientRgbF32Buffer(InsufficientBuffer),
#[error("MixedSinker insufficient rgb_f16 buffer: expected >= {} elements, got {}", .0.expected(), .0.actual())]
InsufficientRgbF16Buffer(InsufficientBuffer),
#[error("MixedSinker insufficient rgba_f32 buffer: expected >= {} elements, got {}", .0.expected(), .0.actual())]
InsufficientRgbaF32Buffer(InsufficientBuffer),
#[error("MixedSinker insufficient rgba_f16 buffer: expected >= {} elements, got {}", .0.expected(), .0.actual())]
InsufficientRgbaF16Buffer(InsufficientBuffer),
#[error("MixedSinker insufficient xyz_f32 buffer: expected >= {} elements, got {}", .0.expected(), .0.actual())]
InsufficientXyzF32Buffer(InsufficientBuffer),
#[error("MixedSinker insufficient luma_f32 buffer: expected >= {} elements, got {}", .0.expected(), .0.actual())]
InsufficientLumaF32Buffer(InsufficientBuffer),
#[error("MixedSinker insufficient luma buffer: expected >= {} bytes, got {}", .0.expected(), .0.actual())]
InsufficientLumaBuffer(InsufficientBuffer),
#[error("MixedSinker insufficient hsv {:?} plane: expected >= {} bytes, got {}", .0.which(), .0.expected(), .0.actual())]
InsufficientHsvPlane(InsufficientHsvPlane),
#[error("MixedSinker frame size overflows usize: {} x {} x channels={}", .0.width(), .0.height(), .0.channels())]
GeometryOverflow(GeometryOverflow),
#[error(
"MixedSinker row shape mismatch at row {}: {} slice has {} elements, expected {}",
.0.row(), .0.which(), .0.actual(), .0.expected()
)]
RowShapeMismatch(RowShapeMismatch),
#[error(
"MixedSinker row index {} is out of range for configured height {}",
.0.row(), .0.configured_height()
)]
RowIndexOutOfRange(RowIndexOutOfRange),
#[error("MixedSinker configured width {} {}", .0.width(), .0.required())]
WidthAlignment(WidthAlignment),
#[error(transparent)]
Resample(#[from] ResampleError),
#[error(
"MixedSinker resampled output set changed mid-frame at source row {}; \
restart the frame via begin_frame",
.0.row()
)]
ResampleOutputsChanged(ResampleOutputsChanged),
#[error(
"MixedSinker resample route (native vs row-stage) changed mid-frame at \
source row {}; restart the frame via begin_frame",
.0.row()
)]
NativeRouteChanged(NativeRouteChanged),
#[error(
"MixedSinker Linear transfer function changed mid-frame at source row \
{}; restart the frame via begin_frame",
.0.row()
)]
TransferFunctionChanged(TransferFunctionChanged),
#[error(
"MixedSinker Linear mode changed mid-frame at source row \
{}; restart the frame via begin_frame",
.0.row()
)]
LinearModeChanged(LinearModeChanged),
#[error(
"MixedSinker averaging domain changed mid-frame at source row \
{}; restart the frame via begin_frame",
.0.row()
)]
AveragingDomainChanged(AveragingDomainChanged),
#[error(
"MixedSinker chroma siting changed mid-frame at source row \
{}; restart the frame via begin_frame",
.0.row()
)]
ChromaSitingChanged(ChromaSitingChanged),
#[error(
"MixedSinker cannot resample the non-affine {} matrix at source row {}; \
decode on an identity sink or downscale after the colour convert",
.0.matrix(),
.0.row()
)]
UnsupportedMatrixResample(UnsupportedMatrixResample),
#[cfg(feature = "yuv-planar")]
#[error(transparent)]
St428CieXyzUnsupported(St428CieXyzUnsupported),
#[error(transparent)]
UnsupportedColorMatrix(#[from] crate::UnsupportedKernelMatrixError),
#[cfg(feature = "xyz")]
#[error(transparent)]
UnsupportedTargetGamut(#[from] crate::UnsupportedKernelGamutError),
}
#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash, IsVariant)]
#[non_exhaustive]
pub enum RowSlice {
#[display("Y")]
Y,
#[display("U Half")]
UHalf,
#[display("V Half")]
VHalf,
#[display("U Quarter")]
UQuarter,
#[display("V Quarter")]
VQuarter,
#[display("UV Half")]
UvHalf,
#[display("VU Half")]
VuHalf,
#[display("U Full")]
UFull,
#[display("V Full")]
VFull,
#[display("A Full")]
AFull,
#[display("U Full 10")]
UFull10,
#[display("V Full 10")]
VFull10,
#[display("A Full 10")]
AFull10,
#[display("U Full 12")]
UFull12,
#[display("V Full 12")]
VFull12,
#[display("A Full 12")]
AFull12,
#[display("U Full 14")]
UFull14,
#[display("V Full 14")]
VFull14,
#[display("A Full 14")]
AFull14,
#[display("UV Full")]
UvFull,
#[display("VU Full")]
VuFull,
#[display("Y9")]
Y9,
#[display("U Half 9")]
UHalf9,
#[display("V Half 9")]
VHalf9,
#[display("U Full 9")]
UFull9,
#[display("V Full 9")]
VFull9,
#[display("A Full 9")]
AFull9,
#[display("Y10")]
Y10,
#[display("U Half 10")]
UHalf10,
#[display("V Half 10")]
VHalf10,
#[display("UV Half 10")]
UvHalf10,
#[display("Y12")]
Y12,
#[display("U Half 12")]
UHalf12,
#[display("V Half 12")]
VHalf12,
#[display("UV Half 12")]
UvHalf12,
#[display("Y14")]
Y14,
#[display("U Half 14")]
UHalf14,
#[display("V Half 14")]
VHalf14,
#[display("Y16")]
Y16,
#[display("U Half 16")]
UHalf16,
#[display("V Half 16")]
VHalf16,
#[display("UV Half 16")]
UvHalf16,
#[display("A Full 16")]
AFull16,
#[display("U Full 16")]
UFull16,
#[display("V Full 16")]
VFull16,
#[display("UV Full 10")]
UvFull10,
#[display("UV Full 12")]
UvFull12,
#[display("UV Full 16")]
UvFull16,
#[display("Bayer Above")]
BayerAbove,
#[display("Bayer Mid")]
BayerMid,
#[display("Bayer Below")]
BayerBelow,
#[display("Bayer16 Above")]
Bayer16Above,
#[display("Bayer16 Mid")]
Bayer16Mid,
#[display("Bayer16 Below")]
Bayer16Below,
#[display("Pal8 index row")]
Pal8IndexRow,
#[display("RGB565 packed")]
Rgb565Packed,
#[display("BGR565 packed")]
Bgr565Packed,
#[display("RGB555 packed")]
Rgb555Packed,
#[display("BGR555 packed")]
Bgr555Packed,
#[display("RGB444 packed")]
Rgb444Packed,
#[display("BGR444 packed")]
Bgr444Packed,
#[display("RGB8 packed")]
Rgb8Packed,
#[display("BGR8 packed")]
Bgr8Packed,
#[display("RGB4_BYTE packed")]
Rgb4BytePacked,
#[display("BGR4_BYTE packed")]
Bgr4BytePacked,
#[display("RGB4 packed")]
Rgb4Packed,
#[display("BGR4 packed")]
Bgr4Packed,
#[display("RGB packed")]
RgbPacked,
#[display("BGR packed")]
BgrPacked,
#[display("RGBA packed")]
RgbaPacked,
#[display("BGRA packed")]
BgraPacked,
#[display("ARGB packed")]
ArgbPacked,
#[display("ABGR packed")]
AbgrPacked,
#[display("XRGB packed")]
XrgbPacked,
#[display("RGBX packed")]
RgbxPacked,
#[display("XBGR packed")]
XbgrPacked,
#[display("BGRX packed")]
BgrxPacked,
#[display("X2RGB10 packed")]
X2Rgb10Packed,
#[display("X2BGR10 packed")]
X2Bgr10Packed,
#[display("YUYV422 packed")]
Yuyv422Packed,
#[display("UYVY422 packed")]
Uyvy422Packed,
#[display("YVYU422 packed")]
Yvyu422Packed,
#[display("UYYVYY411 packed")]
Uyyvyy411Packed,
#[display("V210 packed")]
V210Packed,
#[display("Y210 packed")]
Y210Packed,
#[display("Y212 packed")]
Y212Packed,
#[display("Y216 packed")]
Y216Packed,
#[display("V410 packed")]
V410Packed,
#[display("V30X packed")]
V30XPacked,
#[display("XV36 packed")]
Xv36Packed,
#[display("XV48 packed")]
Xv48Packed,
#[display("VUYA packed")]
VuyaPacked,
#[display("VUYX packed")]
VuyxPacked,
#[display("AYUV64 packed")]
Ayuv64Packed,
#[display("AYUV packed")]
AyuvPacked,
#[display("UYVA packed")]
UyvaPacked,
#[display("VYU444 packed")]
Vyu444Packed,
#[display("RGBF32 packed")]
RgbF32Packed,
#[display("RGBF16 packed")]
RgbF16Packed,
#[display("RGBAF32 packed")]
RgbaF32Packed,
#[display("RGBAF16 packed")]
RgbaF16Packed,
#[display("XYZ12 packed")]
Xyz12Packed,
#[display("G plane")]
GPlane,
#[display("B plane")]
BPlane,
#[display("R plane")]
RPlane,
#[display("GBR f32 plane")]
GbrF32Plane,
#[display("GBR f16 plane")]
GbrF16Plane,
#[display("RGB48 packed")]
Rgb48Packed,
#[display("BGR48 packed")]
Bgr48Packed,
#[display("RGBA64 packed")]
Rgba64Packed,
#[display("BGRA64 packed")]
Bgra64Packed,
#[display("RGB96 packed")]
Rgb96Packed,
#[display("RGBA128 packed")]
Rgba128Packed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, IsVariant)]
pub enum AlphaMode {
#[default]
Straight,
Premultiplied,
}
impl AlphaMode {
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn as_str(self) -> &'static str {
match self {
Self::Straight => "straight",
Self::Premultiplied => "premultiplied",
}
}
}
pub trait DefaultAlphaMode: SourceFormat {
const DEFAULT_ALPHA_MODE: AlphaMode = AlphaMode::Straight;
}
impl<F: SourceFormat> DefaultAlphaMode for F {}
#[cfg(feature = "yuv-planar")]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, IsVariant)]
pub enum St428Interpretation {
#[default]
FfmpegTabulated,
CieXyz,
}
#[cfg(feature = "yuv-planar")]
impl St428Interpretation {
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn as_str(self) -> &'static str {
match self {
Self::FfmpegTabulated => "ffmpeg-tabulated",
Self::CieXyz => "cie-xyz",
}
}
}
#[cfg(feature = "yuv-planar")]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
#[error(
"ChromaDerivedNcl over SmpteSt428 primaries is unsupported under the CIE-XYZ \
interpretation: ST 428-1 is CIE XYZ, not RGB — decode via the xyz12 path, or \
select St428Interpretation::FfmpegTabulated"
)]
pub struct St428CieXyzUnsupported(());
#[cfg(feature = "yuv-planar")]
impl St428CieXyzUnsupported {
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) const fn new() -> Self {
Self(())
}
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
const fn st428_chroma_derived_guard(
matrix: crate::KernelMatrix,
primaries: &crate::Primaries,
interp: St428Interpretation,
) -> Result<(), St428CieXyzUnsupported> {
if matches!(interp, St428Interpretation::CieXyz)
&& matches!(matrix, crate::KernelMatrix::ChromaDerivedNcl)
&& primaries.is_cie_xyz()
{
return Err(St428CieXyzUnsupported::new());
}
Ok(())
}
pub struct MixedSinker<'a, F: SourceFormat, R = NoopResampler> {
rgb: Option<&'a mut [u8]>,
rgb_u16: Option<&'a mut [u16]>,
rgb_f32: Option<&'a mut [f32]>,
rgb_f16: Option<&'a mut [half::f16]>,
rgba: Option<&'a mut [u8]>,
rgba_u16: Option<&'a mut [u16]>,
rgba_f32: Option<&'a mut [f32]>,
rgba_f16: Option<&'a mut [half::f16]>,
luma: Option<&'a mut [u8]>,
luma_u16: Option<&'a mut [u16]>,
luma_f32: Option<&'a mut [f32]>,
#[cfg(any(
feature = "bayer",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
hsv: Option<HsvFrameMut<'a>>,
#[cfg(feature = "xyz")]
xyz_f32: Option<&'a mut [f32]>,
width: usize,
height: usize,
out_width: usize,
out_height: usize,
plan: Option<ResamplePlan>,
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
#[cfg_attr(not(any(feature = "yuv-planar", feature = "rgb")), allow(dead_code))]
rgb_stream: Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
rgb_stream_u16: Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_stream: Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_stream_u16: Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgb_stream_u32: Option<std::boxed::Box<crate::resample::AreaStream<u32>>>,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgb_filter_stream_u32: Option<std::boxed::Box<crate::resample::FilterStream<u32>>>,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgba_stream_u32: Option<std::boxed::Box<crate::resample::AreaStream<u32>>>,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgba_filter_stream_u32: Option<std::boxed::Box<crate::resample::FilterStream<u32>>>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
frozen_alpha_mode: Option<AlphaMode>,
#[cfg(any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
))]
rgb_stream_f32: Option<std::boxed::Box<crate::resample::AreaStream<f32>>>,
#[cfg(any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
))]
rgb_filter_stream_f32: Option<std::boxed::Box<crate::resample::FilterStream<f32>>>,
#[cfg(any(
feature = "gbr",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
))]
rgba_stream_f32: Option<std::boxed::Box<crate::resample::AreaStream<f32>>>,
#[cfg(any(
feature = "gbr",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
))]
rgba_filter_stream_f32: Option<std::boxed::Box<crate::resample::FilterStream<f32>>>,
#[cfg(feature = "xyz")]
xyz_stream_f32: Option<std::boxed::Box<crate::resample::AreaStream<f32>>>,
#[cfg(feature = "xyz")]
xyz_filter_stream_f32: Option<std::boxed::Box<crate::resample::FilterStream<f32>>>,
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
#[cfg_attr(
not(any(feature = "yuv-planar", feature = "gray", feature = "mono")),
allow(dead_code)
)]
luma_stream: Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
#[cfg(any(
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
luma_stream_u16: Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
#[cfg(feature = "gray")]
luma_stream_u32: Option<std::boxed::Box<crate::resample::AreaStream<u32>>>,
#[cfg(feature = "gray")]
luma_stream_f32: Option<std::boxed::Box<crate::resample::AreaStream<f32>>>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuv-packed"
))]
rgb_filter_stream: Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "mono",
feature = "yuva",
feature = "gray"
))]
rgba_filter_stream: Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
rgb_filter_stream_u16: Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "yuva",
feature = "gray"
))]
rgba_filter_stream_u16: Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
#[cfg(any(
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "mono",
feature = "gray"
))]
luma_filter_stream: Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
#[cfg(feature = "gray")]
luma_filter_stream_f32: Option<std::boxed::Box<crate::resample::FilterStream<f32>>>,
#[cfg(any(
feature = "yuv-444-packed",
feature = "y2xx",
feature = "yuv-planar",
feature = "gray",
feature = "yuva",
feature = "yuv-semi-planar"
))]
luma_filter_stream_u16: Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
#[cfg(feature = "gray")]
luma_filter_stream_u32: Option<std::boxed::Box<crate::resample::FilterStream<u32>>>,
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
#[cfg_attr(not(any(feature = "yuv-planar", feature = "rgb")), allow(dead_code))]
resample_outputs: Option<FrozenOutputs>,
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(feature = "yuv-planar"), allow(dead_code))]
native: bool,
#[cfg(feature = "yuv-planar")]
native_420: Option<std::boxed::Box<planar_8bit::NativeYuv420>>,
#[cfg(feature = "yuva")]
native_yuva_420: Option<std::boxed::Box<planar_8bit::NativeYuva420>>,
#[cfg(feature = "yuv-planar")]
bicublin_420: Option<std::boxed::Box<planar_8bit::BicublinYuv420>>,
#[cfg(feature = "yuv-planar")]
native_planar: Option<std::boxed::Box<planar_8bit::NativePlanarYuv>>,
#[cfg(feature = "yuv-planar")]
hsv_planar: Option<std::boxed::Box<planar_resample::HsvDirectPlanarYuv>>,
#[cfg(feature = "yuv-planar")]
native_420_u16: Option<std::boxed::Box<subsampled_4_2_0_high_bit::NativeYuv420U16>>,
#[cfg(feature = "yuv-planar")]
native_planar_u16: Option<std::boxed::Box<planar_high_bit_native::NativePlanarYuvU16>>,
#[cfg(feature = "yuv-planar")]
yuv444_msb_y_depack: Vec<u16>,
#[cfg(feature = "yuv-planar")]
yuv444_msb_u_depack: Vec<u16>,
#[cfg(feature = "yuv-planar")]
yuv444_msb_v_depack: Vec<u16>,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
semi_planar_u_half: Vec<u8>,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
semi_planar_v_half: Vec<u8>,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_y_half: Vec<u16>,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_u_half: Vec<u16>,
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_v_half: Vec<u16>,
#[cfg(all(feature = "yuv-packed", feature = "yuv-planar"))]
packed_yuv_y_full: Vec<u8>,
#[cfg(all(feature = "yuv-packed", feature = "yuv-planar"))]
packed_yuv_u_half: Vec<u8>,
#[cfg(all(feature = "yuv-packed", feature = "yuv-planar"))]
packed_yuv_v_half: Vec<u8>,
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
y2xx_y_full: Vec<u16>,
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
y2xx_u_half: Vec<u16>,
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
y2xx_v_half: Vec<u16>,
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
v210_y_full: Vec<u16>,
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
v210_u_half: Vec<u16>,
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
v210_v_half: Vec<u16>,
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_y_full: Vec<u8>,
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_u_full: Vec<u8>,
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_v_full: Vec<u8>,
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_y_full_u16: Vec<u16>,
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_u_full_u16: Vec<u16>,
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_v_full_u16: Vec<u16>,
#[cfg(feature = "yuv-planar")]
frozen_native_route: Option<bool>,
#[cfg(feature = "yuv-planar")]
frozen_domain: Option<crate::resample::AveragingDomain>,
#[cfg(feature = "yuv-planar")]
frozen_chroma_centered: Option<bool>,
#[cfg(feature = "yuv-planar")]
frozen_chroma_bottom_v: Option<bool>,
#[cfg(feature = "yuv-planar")]
frozen_chroma_top_v: Option<bool>,
#[cfg(feature = "yuv-planar")]
chroma_top_pending: Option<(usize, crate::KernelMatrix, bool)>,
#[cfg(feature = "yuv-planar")]
chroma_top_st428: Option<(crate::Primaries, St428Interpretation)>,
#[cfg(feature = "yuv-planar")]
chroma_top_y: Vec<u8>,
#[cfg(feature = "yuva")]
chroma_top_a: Vec<u8>,
#[cfg(any(
feature = "bayer",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
rgb_scratch: Vec<u8>,
#[cfg(feature = "yuv-planar")]
chroma_full: Vec<u8>,
#[cfg(feature = "yuv-planar")]
chroma_prev: Vec<u8>,
#[cfg(feature = "yuv-planar")]
chroma_prev_row: Option<usize>,
#[cfg(feature = "yuv-planar")]
chroma_full_u16: Vec<u16>,
#[cfg(feature = "yuv-planar")]
chroma_prev_u16: Vec<u16>,
#[cfg(feature = "yuv-planar")]
chroma_top_y_u16: Vec<u16>,
#[cfg(feature = "yuva")]
chroma_top_a_u16: Vec<u16>,
#[cfg(any(feature = "yuv-packed", feature = "gray"))]
luma_scratch: Vec<u8>,
#[cfg(feature = "rgb-legacy")]
legacy_rgb_native_scratch: Vec<u8>,
#[cfg(feature = "rgb-legacy")]
legacy_rgb_packed_scratch: Vec<u8>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
rgb_scratch_u16: Vec<u16>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_scratch: Vec<u8>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_scratch_u16: Vec<u16>,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_color_scratch_u16: Vec<u16>,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgb_scratch_u32: Vec<u32>,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgba_scratch_u32: Vec<u32>,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgba_color_scratch_u32: Vec<u32>,
#[cfg(any(
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
luma_scratch_u16: Vec<u16>,
#[cfg(feature = "gray")]
luma_scratch_u32: Vec<u32>,
#[cfg(feature = "gray")]
luma_scratch_f32: Vec<f32>,
#[cfg(feature = "gray")]
yaf_stream_f32: Option<std::boxed::Box<crate::resample::AreaStream<f32>>>,
#[cfg(feature = "gray")]
yaf_filter_stream_f32: Option<std::boxed::Box<crate::resample::FilterStream<f32>>>,
#[cfg(feature = "gray")]
yaf_stage_scratch_f32: Vec<f32>,
#[cfg(feature = "gray")]
yaf_plane_scratch_f32: Vec<f32>,
#[cfg(any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
))]
rgb_scratch_f32: Vec<f32>,
#[cfg(feature = "gbr")]
rgb_plane_scratch_f32: Vec<f32>,
#[cfg(feature = "gbr")]
rgb_plane_scratch_f16: Vec<half::f16>,
#[cfg(any(
feature = "gbr",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
))]
rgba_scratch_f32: Vec<f32>,
#[cfg(feature = "gbr")]
rgba_color_scratch_f32: Vec<f32>,
#[cfg(feature = "gbr")]
rgba_plane_scratch_f32: Vec<f32>,
#[cfg(feature = "gbr")]
rgba_plane_scratch_f16: Vec<half::f16>,
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
rgb_packed_scratch_f16: Vec<half::f16>,
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
rgba_packed_scratch_f16: Vec<half::f16>,
#[cfg(feature = "xyz")]
xyz_scratch_f32: Vec<f32>,
simd: bool,
alpha_mode: AlphaMode,
#[cfg(feature = "yuv-planar")]
averaging_domain: AveragingDomain,
#[cfg(feature = "yuv-planar")]
transfer_function: Option<TransferFunction>,
#[cfg(feature = "yuv-planar")]
chroma_location: crate::ChromaLocation,
#[cfg(feature = "yuv-planar")]
primaries: crate::Primaries,
#[cfg(feature = "yuv-planar")]
st428_interpretation: St428Interpretation,
#[cfg(feature = "yuv-planar")]
transfer: crate::Transfer,
#[cfg(feature = "yuv-planar")]
matrix: crate::ColorMatrix,
#[cfg(all(feature = "yuv-planar", feature = "rgb"))]
linear_light_frame: Option<linear_light::LinearLightFrame>,
#[cfg(feature = "yuv-planar")]
linear_mode: LinearMode,
#[cfg(all(feature = "yuv-planar", feature = "rgb"))]
linear_scene_scratch: Vec<f32>,
#[cfg(any(feature = "bayer", feature = "mono"))]
luma_coefficients_q8: (u32, u32, u32),
kernel_matrix: crate::KernelMatrix,
#[cfg(feature = "xyz")]
target_gamut: crate::KernelGamut,
_fmt: PhantomData<F>,
_resampler: PhantomData<R>,
}
#[derive(Debug, Clone, Copy, PartialEq, IsVariant)]
#[non_exhaustive]
pub enum LumaCoefficients {
Bt709,
Bt2020,
Bt601,
DciP3,
AcesAp1,
Custom(CustomLumaCoefficients),
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct CustomLumaCoefficients {
r: f32,
g: f32,
b: f32,
}
impl CustomLumaCoefficients {
pub const MAX_COEFFICIENT: f32 = 10.0;
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn try_new(r: f32, g: f32, b: f32) -> Result<Self, LumaCoefficientsError> {
if !r.is_finite() {
return Err(LumaCoefficientsError::NonFinite {
channel: LumaChannel::R,
value: r,
});
}
if !g.is_finite() {
return Err(LumaCoefficientsError::NonFinite {
channel: LumaChannel::G,
value: g,
});
}
if !b.is_finite() {
return Err(LumaCoefficientsError::NonFinite {
channel: LumaChannel::B,
value: b,
});
}
if r < 0.0 {
return Err(LumaCoefficientsError::Negative {
channel: LumaChannel::R,
value: r,
});
}
if g < 0.0 {
return Err(LumaCoefficientsError::Negative {
channel: LumaChannel::G,
value: g,
});
}
if b < 0.0 {
return Err(LumaCoefficientsError::Negative {
channel: LumaChannel::B,
value: b,
});
}
if r > Self::MAX_COEFFICIENT {
return Err(LumaCoefficientsError::OutOfBounds {
channel: LumaChannel::R,
value: r,
max: Self::MAX_COEFFICIENT,
});
}
if g > Self::MAX_COEFFICIENT {
return Err(LumaCoefficientsError::OutOfBounds {
channel: LumaChannel::G,
value: g,
max: Self::MAX_COEFFICIENT,
});
}
if b > Self::MAX_COEFFICIENT {
return Err(LumaCoefficientsError::OutOfBounds {
channel: LumaChannel::B,
value: b,
max: Self::MAX_COEFFICIENT,
});
}
Ok(Self { r, g, b })
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn new(r: f32, g: f32, b: f32) -> Self {
match Self::try_new(r, g, b) {
Ok(c) => c,
Err(_) => panic!("invalid CustomLumaCoefficients (non-finite, negative, or out of range)"),
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn r(&self) -> f32 {
self.r
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn g(&self) -> f32 {
self.g
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn b(&self) -> f32 {
self.b
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, IsVariant)]
#[non_exhaustive]
pub enum LumaChannel {
R,
G,
B,
}
#[derive(Debug, Clone, Copy, PartialEq, IsVariant, Error)]
#[non_exhaustive]
pub enum LumaCoefficientsError {
#[error("CustomLumaCoefficients.{channel:?} is non-finite (got {value})")]
NonFinite {
channel: LumaChannel,
value: f32,
},
#[error("CustomLumaCoefficients.{channel:?} is negative (got {value})")]
Negative {
channel: LumaChannel,
value: f32,
},
#[error("CustomLumaCoefficients.{channel:?} = {value} exceeds the magnitude bound ({max})")]
OutOfBounds {
channel: LumaChannel,
value: f32,
max: f32,
},
}
impl LumaCoefficients {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn to_q8(self) -> (u32, u32, u32) {
match self {
Self::Bt709 => (54, 183, 19),
Self::Bt2020 => (67, 174, 15),
Self::Bt601 => (77, 150, 29),
Self::DciP3 => (59, 177, 20),
Self::AcesAp1 => (70, 172, 14),
Self::Custom(c) => (
(c.r * 256.0 + 0.5) as u32,
(c.g * 256.0 + 0.5) as u32,
(c.b * 256.0 + 0.5) as u32,
),
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn try_custom(r: f32, g: f32, b: f32) -> Result<Self, LumaCoefficientsError> {
match CustomLumaCoefficients::try_new(r, g, b) {
Ok(c) => Ok(Self::Custom(c)),
Err(e) => Err(e),
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn custom(r: f32, g: f32, b: f32) -> Self {
Self::Custom(CustomLumaCoefficients::new(r, g, b))
}
}
impl Default for LumaCoefficients {
fn default() -> Self {
Self::Bt709
}
}
impl<F: SourceFormat> MixedSinker<'_, F, NoopResampler> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn new(width: usize, height: usize) -> Self {
Self::with_geometry(width, height, width, height)
}
}
impl<F: SourceFormat, R> MixedSinker<'_, F, R> {
fn with_geometry(width: usize, height: usize, out_width: usize, out_height: usize) -> Self {
Self {
rgb: None,
rgb_u16: None,
rgb_f32: None,
rgb_f16: None,
rgba: None,
rgba_u16: None,
rgba_f32: None,
rgba_f16: None,
luma: None,
luma_u16: None,
luma_f32: None,
#[cfg(any(
feature = "bayer",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
hsv: None,
#[cfg(feature = "xyz")]
xyz_f32: None,
width,
height,
out_width,
out_height,
plan: None,
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
rgb_stream: None,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
rgb_stream_u16: None,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_stream: None,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_stream_u16: None,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgb_stream_u32: None,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgb_filter_stream_u32: None,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgba_stream_u32: None,
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgba_filter_stream_u32: None,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
frozen_alpha_mode: None,
#[cfg(any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
))]
rgb_stream_f32: None,
#[cfg(any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
))]
rgb_filter_stream_f32: None,
#[cfg(any(
feature = "gbr",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
))]
rgba_stream_f32: None,
#[cfg(any(
feature = "gbr",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
))]
rgba_filter_stream_f32: None,
#[cfg(feature = "xyz")]
xyz_stream_f32: None,
#[cfg(feature = "xyz")]
xyz_filter_stream_f32: None,
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
luma_stream: None,
#[cfg(any(
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
luma_stream_u16: None,
#[cfg(feature = "gray")]
luma_stream_u32: None,
#[cfg(feature = "gray")]
luma_stream_f32: None,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuv-packed"
))]
rgb_filter_stream: None,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "mono",
feature = "yuva",
feature = "gray"
))]
rgba_filter_stream: None,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
rgb_filter_stream_u16: None,
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "yuva",
feature = "gray"
))]
rgba_filter_stream_u16: None,
#[cfg(any(
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "mono",
feature = "gray"
))]
luma_filter_stream: None,
#[cfg(feature = "gray")]
luma_filter_stream_f32: None,
#[cfg(any(
feature = "yuv-444-packed",
feature = "y2xx",
feature = "yuv-planar",
feature = "gray",
feature = "yuva",
feature = "yuv-semi-planar"
))]
luma_filter_stream_u16: None,
#[cfg(feature = "gray")]
luma_filter_stream_u32: None,
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
resample_outputs: None,
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
native: true,
#[cfg(feature = "yuv-planar")]
native_420: None,
#[cfg(feature = "yuva")]
native_yuva_420: None,
#[cfg(feature = "yuv-planar")]
bicublin_420: None,
#[cfg(feature = "yuv-planar")]
native_planar: None,
#[cfg(feature = "yuv-planar")]
hsv_planar: None,
#[cfg(feature = "yuv-planar")]
native_420_u16: None,
#[cfg(feature = "yuv-planar")]
native_planar_u16: None,
#[cfg(feature = "yuv-planar")]
yuv444_msb_y_depack: Vec::new(),
#[cfg(feature = "yuv-planar")]
yuv444_msb_u_depack: Vec::new(),
#[cfg(feature = "yuv-planar")]
yuv444_msb_v_depack: Vec::new(),
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
semi_planar_u_half: Vec::new(),
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
semi_planar_v_half: Vec::new(),
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_y_half: Vec::new(),
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_u_half: Vec::new(),
#[cfg(all(feature = "yuv-semi-planar", feature = "yuv-planar"))]
p0xx_v_half: Vec::new(),
#[cfg(all(feature = "yuv-packed", feature = "yuv-planar"))]
packed_yuv_y_full: Vec::new(),
#[cfg(all(feature = "yuv-packed", feature = "yuv-planar"))]
packed_yuv_u_half: Vec::new(),
#[cfg(all(feature = "yuv-packed", feature = "yuv-planar"))]
packed_yuv_v_half: Vec::new(),
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
y2xx_y_full: Vec::new(),
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
y2xx_u_half: Vec::new(),
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
y2xx_v_half: Vec::new(),
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
v210_y_full: Vec::new(),
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
v210_u_half: Vec::new(),
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
v210_v_half: Vec::new(),
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_y_full: Vec::new(),
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_u_full: Vec::new(),
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_v_full: Vec::new(),
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_y_full_u16: Vec::new(),
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_u_full_u16: Vec::new(),
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
packed_444_v_full_u16: Vec::new(),
#[cfg(feature = "yuv-planar")]
frozen_native_route: None,
#[cfg(feature = "yuv-planar")]
frozen_domain: None,
#[cfg(feature = "yuv-planar")]
frozen_chroma_centered: None,
#[cfg(feature = "yuv-planar")]
frozen_chroma_bottom_v: None,
#[cfg(feature = "yuv-planar")]
frozen_chroma_top_v: None,
#[cfg(feature = "yuv-planar")]
chroma_top_pending: None,
#[cfg(feature = "yuv-planar")]
chroma_top_st428: None,
#[cfg(feature = "yuv-planar")]
chroma_top_y: Vec::new(),
#[cfg(feature = "yuva")]
chroma_top_a: Vec::new(),
#[cfg(any(
feature = "bayer",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
rgb_scratch: Vec::new(),
#[cfg(feature = "yuv-planar")]
chroma_full: Vec::new(),
#[cfg(feature = "yuv-planar")]
chroma_prev: Vec::new(),
#[cfg(feature = "yuv-planar")]
chroma_prev_row: None,
#[cfg(feature = "yuv-planar")]
chroma_full_u16: Vec::new(),
#[cfg(feature = "yuv-planar")]
chroma_prev_u16: Vec::new(),
#[cfg(feature = "yuv-planar")]
chroma_top_y_u16: Vec::new(),
#[cfg(feature = "yuva")]
chroma_top_a_u16: Vec::new(),
#[cfg(any(feature = "yuv-packed", feature = "gray"))]
luma_scratch: Vec::new(),
#[cfg(feature = "rgb-legacy")]
legacy_rgb_native_scratch: Vec::new(),
#[cfg(feature = "rgb-legacy")]
legacy_rgb_packed_scratch: Vec::new(),
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
rgb_scratch_u16: Vec::new(),
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_scratch: Vec::new(),
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_scratch_u16: Vec::new(),
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed"
))]
rgba_color_scratch_u16: Vec::new(),
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgb_scratch_u32: Vec::new(),
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgba_scratch_u32: Vec::new(),
#[cfg(any(feature = "rgb", feature = "gbr"))]
rgba_color_scratch_u32: Vec::new(),
#[cfg(any(
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
luma_scratch_u16: Vec::new(),
#[cfg(feature = "gray")]
luma_scratch_u32: Vec::new(),
#[cfg(feature = "gray")]
luma_scratch_f32: Vec::new(),
#[cfg(feature = "gray")]
yaf_stream_f32: None,
#[cfg(feature = "gray")]
yaf_filter_stream_f32: None,
#[cfg(feature = "gray")]
yaf_stage_scratch_f32: Vec::new(),
#[cfg(feature = "gray")]
yaf_plane_scratch_f32: Vec::new(),
#[cfg(any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
))]
rgb_scratch_f32: Vec::new(),
#[cfg(feature = "gbr")]
rgb_plane_scratch_f32: Vec::new(),
#[cfg(feature = "gbr")]
rgb_plane_scratch_f16: Vec::new(),
#[cfg(any(
feature = "gbr",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
))]
rgba_scratch_f32: Vec::new(),
#[cfg(feature = "gbr")]
rgba_color_scratch_f32: Vec::new(),
#[cfg(feature = "gbr")]
rgba_plane_scratch_f32: Vec::new(),
#[cfg(feature = "gbr")]
rgba_plane_scratch_f16: Vec::new(),
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
rgb_packed_scratch_f16: Vec::new(),
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
rgba_packed_scratch_f16: Vec::new(),
#[cfg(feature = "xyz")]
xyz_scratch_f32: Vec::new(),
simd: true,
alpha_mode: F::DEFAULT_ALPHA_MODE,
#[cfg(feature = "yuv-planar")]
averaging_domain: AveragingDomain::Encoded,
#[cfg(feature = "yuv-planar")]
transfer_function: None,
#[cfg(feature = "yuv-planar")]
chroma_location: crate::ChromaLocation::Unspecified,
#[cfg(feature = "yuv-planar")]
primaries: crate::Primaries::Unspecified,
#[cfg(feature = "yuv-planar")]
st428_interpretation: St428Interpretation::FfmpegTabulated,
#[cfg(feature = "yuv-planar")]
transfer: crate::Transfer::Unspecified,
#[cfg(feature = "yuv-planar")]
matrix: crate::ColorMatrix::Unspecified,
#[cfg(feature = "yuv-planar")]
linear_mode: LinearMode::DisplayReferred,
#[cfg(all(feature = "yuv-planar", feature = "rgb"))]
linear_light_frame: None,
#[cfg(all(feature = "yuv-planar", feature = "rgb"))]
linear_scene_scratch: Vec::new(),
#[cfg(any(feature = "bayer", feature = "mono"))]
luma_coefficients_q8: (54, 183, 19),
kernel_matrix: crate::KernelMatrix::Bt709,
#[cfg(feature = "xyz")]
target_gamut: crate::KernelGamut::DciP3,
_fmt: PhantomData,
_resampler: PhantomData,
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_rgb(&self) -> bool {
self.rgb.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_rgb_u16(&self) -> bool {
self.rgb_u16.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_rgb_f32(&self) -> bool {
self.rgb_f32.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_rgb_f16(&self) -> bool {
self.rgb_f16.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_rgba_f32(&self) -> bool {
self.rgba_f32.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_rgba_f16(&self) -> bool {
self.rgba_f16.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_rgba(&self) -> bool {
self.rgba.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_rgba_u16(&self) -> bool {
self.rgba_u16.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_luma(&self) -> bool {
self.luma.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_luma_u16(&self) -> bool {
self.luma_u16.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_luma_f32(&self) -> bool {
self.luma_f32.is_some()
}
#[cfg(any(
feature = "bayer",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn produces_hsv(&self) -> bool {
self.hsv.is_some()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn width(&self) -> usize {
self.width
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn height(&self) -> usize {
self.height
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn out_width(&self) -> usize {
self.out_width
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn out_height(&self) -> usize {
self.out_height
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn resample_plan(&self) -> Option<&ResamplePlan> {
self.plan.as_ref()
}
#[cfg(all(
test,
feature = "std",
any(
feature = "rgb",
feature = "xyz",
feature = "bayer",
feature = "gray",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "yuva",
feature = "rgb-legacy",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
)
))]
pub(crate) fn rgb_scratch_capacity(&self) -> usize {
self.rgb_scratch.capacity()
}
#[cfg(all(test, feature = "rgb-legacy", feature = "std"))]
pub(crate) fn legacy_rgb_native_scratch_capacity(&self) -> usize {
self.legacy_rgb_native_scratch.capacity()
}
#[cfg(all(test, feature = "rgb-legacy", feature = "std"))]
pub(crate) fn legacy_rgb_packed_scratch_capacity(&self) -> usize {
self.legacy_rgb_packed_scratch.capacity()
}
#[cfg(all(
test,
feature = "std",
any(
feature = "rgb",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
)
))]
pub(crate) fn rgb_stream_u16_allocated(&self) -> bool {
self.rgb_stream_u16.is_some()
}
#[cfg(all(test, feature = "std", feature = "rgb"))]
pub(crate) fn rgba_stream_u16_allocated(&self) -> bool {
self.rgba_stream_u16.is_some()
}
#[cfg(all(test, feature = "std", feature = "rgb"))]
pub(crate) fn rgb_stream_u32_allocated(&self) -> bool {
self.rgb_stream_u32.is_some()
}
#[cfg(all(test, feature = "std", feature = "rgb"))]
pub(crate) fn rgba_stream_u32_allocated(&self) -> bool {
self.rgba_stream_u32.is_some()
}
#[cfg(all(
test,
feature = "std",
any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
)
))]
pub(crate) fn rgb_stream_f32_allocated(&self) -> bool {
self.rgb_stream_f32.is_some()
}
#[cfg(all(
test,
feature = "std",
any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
)
))]
pub(crate) fn rgba_stream_f32_allocated(&self) -> bool {
self.rgba_stream_f32.is_some()
}
#[cfg(all(test, feature = "gbr", feature = "std"))]
pub(crate) fn rgb_filter_stream_f32_allocated(&self) -> bool {
self.rgb_filter_stream_f32.is_some()
}
#[cfg(all(test, feature = "gbr", feature = "std"))]
pub(crate) fn rgba_filter_stream_f32_allocated(&self) -> bool {
self.rgba_filter_stream_f32.is_some()
}
#[cfg(all(test, feature = "gbr", feature = "std"))]
pub(crate) fn rgb_plane_scratch_capacity(&self) -> usize {
self.rgb_plane_scratch_f32.capacity()
}
#[cfg(all(test, feature = "gbr", feature = "std"))]
pub(crate) fn rgb_plane_scratch_f16_capacity(&self) -> usize {
self.rgb_plane_scratch_f16.capacity()
}
#[cfg(all(
test,
feature = "std",
feature = "rgb-float",
any(feature = "yuv-planar", feature = "rgb")
))]
pub(crate) fn rgb_packed_scratch_f16_capacity(&self) -> usize {
self.rgb_packed_scratch_f16.capacity()
}
#[cfg(all(test, feature = "xyz", feature = "std"))]
pub(crate) fn xyz_stream_f32_allocated(&self) -> bool {
self.xyz_stream_f32.is_some()
}
#[cfg(all(test, feature = "xyz", feature = "std"))]
pub(crate) fn xyz_scratch_f32_capacity(&self) -> usize {
self.xyz_scratch_f32.capacity()
}
#[cfg(all(
test,
feature = "std",
any(
feature = "xyz",
feature = "mono",
feature = "rgb-legacy",
feature = "yuv-planar"
)
))]
pub(crate) fn resample_outputs_frozen(&self) -> bool {
self.resample_outputs.is_some()
}
#[cfg(all(
test,
feature = "std",
any(feature = "gray", feature = "mono", feature = "yuv-packed")
))]
pub(crate) fn luma_stream_allocated(&self) -> bool {
self.luma_stream.is_some()
}
#[cfg(all(
test,
feature = "std",
any(
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "rgb",
feature = "bayer"
)
))]
pub(crate) fn rgb_stream_allocated(&self) -> bool {
self.rgb_stream.is_some()
}
#[cfg(all(test, feature = "std", any(feature = "rgb", feature = "mono")))]
pub(crate) fn rgba_stream_allocated(&self) -> bool {
self.rgba_stream.is_some()
}
#[cfg(all(test, feature = "std", feature = "yuv-packed"))]
pub(crate) fn rgb_filter_stream_allocated(&self) -> bool {
self.rgb_filter_stream.is_some()
}
#[cfg(all(test, feature = "std", feature = "mono"))]
pub(crate) fn rgba_filter_stream_allocated(&self) -> bool {
self.rgba_filter_stream.is_some()
}
#[cfg(all(test, feature = "std", any(feature = "mono", feature = "yuv-packed")))]
pub(crate) fn luma_filter_stream_allocated(&self) -> bool {
self.luma_filter_stream.is_some()
}
#[cfg(all(test, feature = "std", feature = "yuv-packed"))]
pub(crate) fn luma_scratch_capacity(&self) -> usize {
self.luma_scratch.capacity()
}
#[cfg(all(test, feature = "std", feature = "yuv-444-packed"))]
pub(crate) fn rgb_scratch_u16_capacity(&self) -> usize {
self.rgb_scratch_u16.capacity()
}
#[cfg(all(test, feature = "std", feature = "yuv-444-packed"))]
pub(crate) fn luma_scratch_u16_capacity(&self) -> usize {
self.luma_scratch_u16.capacity()
}
#[cfg(all(
test,
feature = "std",
any(
feature = "gray",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
)
))]
pub(crate) fn luma_stream_u16_allocated(&self) -> bool {
self.luma_stream_u16.is_some()
}
#[cfg(all(test, feature = "std", feature = "gray"))]
pub(crate) fn luma_stream_f32_allocated(&self) -> bool {
self.luma_stream_f32.is_some()
}
#[cfg(all(test, feature = "std", feature = "gray"))]
pub(crate) fn luma_scratch_f32_capacity(&self) -> usize {
self.luma_scratch_f32.capacity()
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn simd(&self) -> bool {
self.simd
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(feature = "yuv-planar"), allow(dead_code))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn native(&self) -> bool {
self.native
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn set_native(&mut self, native: bool) -> &mut Self {
self.native = native;
self
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn with_native(mut self, native: bool) -> Self {
self.set_native(native);
self
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn set_simd(&mut self, simd: bool) -> &mut Self {
self.simd = simd;
self
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn with_simd(mut self, simd: bool) -> Self {
self.set_simd(simd);
self
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn alpha_mode(&self) -> AlphaMode {
self.alpha_mode
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn set_alpha_mode(&mut self, mode: AlphaMode) -> &mut Self {
self.alpha_mode = mode;
self
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn with_alpha_mode(mut self, mode: AlphaMode) -> Self {
self.set_alpha_mode(mode);
self
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn averaging_domain(&self) -> AveragingDomain {
self.averaging_domain
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn set_averaging_domain(&mut self, domain: AveragingDomain) -> &mut Self {
self.averaging_domain = domain;
self
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn with_averaging_domain(mut self, domain: AveragingDomain) -> Self {
self.set_averaging_domain(domain);
self
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn transfer_function(&self) -> Option<TransferFunction> {
self.transfer_function
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn set_transfer_function(&mut self, tf: TransferFunction) -> &mut Self {
self.transfer_function = Some(tf);
self
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn with_transfer_function(mut self, tf: TransferFunction) -> Self {
self.set_transfer_function(tf);
self
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn chroma_location(&self) -> crate::ChromaLocation {
self.chroma_location.clone()
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn primaries(&self) -> crate::Primaries {
self.primaries.clone()
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn st428_interpretation(&self) -> St428Interpretation {
self.st428_interpretation
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn transfer(&self) -> crate::Transfer {
self.transfer.clone()
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_chroma_location(&mut self, loc: crate::ChromaLocation) -> &mut Self {
self.chroma_location = loc;
self
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_chroma_location(mut self, loc: crate::ChromaLocation) -> Self {
self.set_chroma_location(loc.clone());
self
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_color_spec(
&mut self,
spec: &crate::ColorSpec,
) -> Result<&mut Self, crate::UnsupportedKernelMatrixError> {
self.kernel_matrix = spec.kernel_matrix()?;
#[cfg(feature = "yuv-planar")]
{
self.chroma_location = spec.chroma_location();
self.primaries = spec.primaries();
self.transfer = spec.transfer();
self.matrix = spec.matrix();
}
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_color_spec(
mut self,
spec: &crate::ColorSpec,
) -> Result<Self, crate::UnsupportedKernelMatrixError> {
self.set_color_spec(spec)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
#[cfg(all(
test,
any(
feature = "yuv-planar",
all(
any(feature = "std", feature = "alloc"),
any(
feature = "xyz",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuva",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "gray",
feature = "gbr",
),
),
),
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) const fn set_kernel_matrix(&mut self, matrix: crate::KernelMatrix) -> &mut Self {
self.kernel_matrix = matrix;
self
}
#[cfg(all(test, feature = "xyz"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) const fn set_kernel_gamut(&mut self, gamut: crate::KernelGamut) -> &mut Self {
self.target_gamut = gamut;
self
}
#[cfg(feature = "xyz")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn target_gamut(&self) -> crate::KernelGamut {
self.target_gamut
}
#[cfg(feature = "xyz")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_target_gamut(
&mut self,
gamut: &crate::DcpTargetGamut,
) -> Result<&mut Self, crate::UnsupportedKernelGamutError> {
self.target_gamut = crate::KernelGamut::try_from(gamut)?;
Ok(self)
}
#[cfg(feature = "xyz")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_target_gamut(
mut self,
gamut: &crate::DcpTargetGamut,
) -> Result<Self, crate::UnsupportedKernelGamutError> {
self.set_target_gamut(gamut)?;
Ok(self)
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn linear_mode(&self) -> LinearMode {
self.linear_mode
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn set_linear_mode(&mut self, mode: LinearMode) -> &mut Self {
self.linear_mode = mode;
self
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn with_linear_mode(mut self, mode: LinearMode) -> Self {
self.set_linear_mode(mode);
self
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub const fn set_st428_interpretation(&mut self, interp: St428Interpretation) -> &mut Self {
self.st428_interpretation = interp;
self
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
#[must_use]
pub const fn with_st428_interpretation(mut self, interp: St428Interpretation) -> Self {
self.set_st428_interpretation(interp);
self
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn frame_elems(&self, channels: usize) -> Result<usize, MixedSinkerError> {
self
.out_width
.checked_mul(self.out_height)
.and_then(|n| n.checked_mul(channels))
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
self.out_width,
self.out_height,
channels,
)))
}
#[cfg(any(
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn frame_pixels(&self) -> Result<usize, MixedSinkerError> {
self
.out_width
.checked_mul(self.out_height)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
self.out_width,
self.out_height,
1,
)))
}
}
impl<'a, F: SourceFormat, R> MixedSinker<'a, F, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_resampler(width: usize, height: usize, resampler: R) -> Result<Self, MixedSinkerError>
where
R: Resampler,
{
let plan = resampler.plan(width, height)?;
let (out_width, out_height) = match plan.as_ref() {
Some(plan) => plan.out_dims(),
None => (width, height),
};
let mut sink = Self::with_geometry(width, height, out_width, out_height);
sink.plan = plan;
Ok(sink)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgb(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_luma(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_luma(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_luma(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(1)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientLumaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.luma = Some(buf);
Ok(self)
}
#[cfg(any(
feature = "bayer",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_hsv(
mut self,
h: &'a mut [u8],
s: &'a mut [u8],
v: &'a mut [u8],
) -> Result<Self, MixedSinkerError> {
self.set_hsv(h, s, v)?;
Ok(self)
}
#[cfg(any(
feature = "bayer",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_hsv(
&mut self,
h: &'a mut [u8],
s: &'a mut [u8],
v: &'a mut [u8],
) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(1)?;
if h.len() < expected {
return Err(MixedSinkerError::InsufficientHsvPlane(
InsufficientHsvPlane::new(HsvPlane::H, expected, h.len()),
));
}
if s.len() < expected {
return Err(MixedSinkerError::InsufficientHsvPlane(
InsufficientHsvPlane::new(HsvPlane::S, expected, s.len()),
));
}
if v.len() < expected {
return Err(MixedSinkerError::InsufficientHsvPlane(
InsufficientHsvPlane::new(HsvPlane::V, expected, v.len()),
));
}
self.hsv = Some(HsvFrameMut::new(h, s, v));
Ok(self)
}
}
#[cfg(any(
feature = "bayer",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn check_dimensions_match(
configured_w: usize,
configured_h: usize,
frame_w: u32,
frame_h: u32,
) -> Result<(), MixedSinkerError> {
let fw = frame_w as usize;
let fh = frame_h as usize;
if fw != configured_w || fh != configured_h {
return Err(MixedSinkerError::DimensionMismatch(DimensionMismatch::new(
configured_w,
configured_h,
frame_w,
frame_h,
)));
}
Ok(())
}
#[cfg(any(
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn rgba_plane_row_slice(
buf: &mut [u8],
one_plane_start: usize,
one_plane_end: usize,
width: usize,
height: usize,
) -> Result<&mut [u8], MixedSinkerError> {
let end = one_plane_end
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width, height, 4,
)))?;
let start = one_plane_start * 4; Ok(&mut buf[start..end])
}
#[cfg(any(
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn rgba_u16_plane_row_slice(
buf: &mut [u16],
one_plane_start: usize,
one_plane_end: usize,
width: usize,
height: usize,
) -> Result<&mut [u16], MixedSinkerError> {
let end = one_plane_end
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width, height, 4,
)))?;
let start = one_plane_start * 4; Ok(&mut buf[start..end])
}
#[cfg(all(
test,
feature = "std",
any(feature = "yuva", all(feature = "gray", feature = "rgb"))
))]
std::thread_local! {
static FORCE_RGB_SCRATCH_ALLOC_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(
test,
feature = "std",
any(feature = "yuva", all(feature = "gray", feature = "rgb"))
))]
pub(crate) fn arm_rgb_scratch_alloc_failure() {
FORCE_RGB_SCRATCH_ALLOC_FAILURE.with(|f| f.set(true));
}
#[cfg(all(test, feature = "std", feature = "yuva"))]
pub(crate) fn disarm_rgb_scratch_alloc_failure() {
FORCE_RGB_SCRATCH_ALLOC_FAILURE.with(|f| f.set(false));
}
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
std::thread_local! {
static FORCE_CHROMA_FULL_ALLOC_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
pub(crate) fn arm_chroma_full_alloc_failure() {
FORCE_CHROMA_FULL_ALLOC_FAILURE.with(|f| f.set(true));
}
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
std::thread_local! {
static FORCE_CHROMA_PREV_ALLOC_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
pub(crate) fn arm_chroma_prev_alloc_failure() {
FORCE_CHROMA_PREV_ALLOC_FAILURE.with(|f| f.set(true));
}
#[cfg(any(
feature = "bayer",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "rgb",
feature = "rgb-float",
feature = "rgb-legacy",
feature = "v210",
feature = "xyz",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuva",
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn rgb_row_buf_or_scratch<'a>(
rgb: Option<&'a mut [u8]>,
rgb_scratch: &'a mut Vec<u8>,
one_plane_start: usize,
one_plane_end: usize,
width: usize,
height: usize,
) -> Result<&'a mut [u8], MixedSinkerError> {
#[cfg(all(
test,
feature = "std",
any(feature = "yuva", all(feature = "gray", feature = "rgb"))
))]
if FORCE_RGB_SCRATCH_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(width, height, width, height),
)));
}
match rgb {
Some(buf) => {
let end = one_plane_end
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width, height, 3,
)))?;
let start = one_plane_start * 3;
Ok(&mut buf[start..end])
}
None => {
let row_bytes = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width, height, 3,
)))?;
if rgb_scratch.len() < row_bytes {
rgb_scratch
.try_reserve_exact(row_bytes - rgb_scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(width, height, width, height),
))
})?;
rgb_scratch.resize(row_bytes, 0);
}
Ok(&mut rgb_scratch[..row_bytes])
}
}
}
#[cfg(feature = "rgb")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn direct_rgb_u16_scratch(
scratch: &mut Vec<u16>,
width: usize,
height: usize,
) -> Result<&mut [u16], MixedSinkerError> {
let row = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width, height, 3,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(width, height, width, height),
))
})?;
scratch.resize(row, 0);
}
Ok(&mut scratch[..row])
}
#[cfg(any(
feature = "yuv-planar",
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "xyz",
feature = "bayer",
feature = "mono",
feature = "yuv-semi-planar",
feature = "yuv-packed",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb-legacy"
))]
#[cfg_attr(not(any(feature = "yuv-planar", feature = "rgb")), allow(dead_code))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn source_rgb_scratch<'s>(
rgb_scratch: &'s mut Vec<u8>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [u8], MixedSinkerError> {
#[cfg(all(
test,
feature = "std",
any(
feature = "rgb",
feature = "gbr",
feature = "bayer",
feature = "mono",
feature = "xyz",
feature = "rgb-legacy",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
)
))]
if FORCE_SOURCE_RGB_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
let row_bytes = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.src_h(),
3,
)))?;
if rgb_scratch.len() < row_bytes {
rgb_scratch
.try_reserve_exact(row_bytes - rgb_scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
rgb_scratch.resize(row_bytes, 0);
}
Ok(&mut rgb_scratch[..row_bytes])
}
#[cfg(all(
test,
feature = "std",
any(
feature = "rgb",
feature = "gbr",
feature = "bayer",
feature = "mono",
feature = "xyz",
feature = "rgb-legacy",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
)
))]
std::thread_local! {
static FORCE_SOURCE_RGB_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(
test,
feature = "std",
any(
feature = "rgb",
feature = "bayer",
feature = "mono",
feature = "xyz",
feature = "rgb-legacy",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
)
))]
pub(crate) fn arm_source_rgb_scratch_failure() {
FORCE_SOURCE_RGB_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(any(
feature = "mono",
feature = "yuv-packed",
feature = "gray",
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "yuva"
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn source_luma_scratch<'s>(
scratch: &'s mut Vec<u8>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [u8], MixedSinkerError> {
#[cfg(all(test, feature = "std", feature = "mono"))]
if FORCE_SOURCE_LUMA_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
if scratch.len() < width {
scratch
.try_reserve_exact(width - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(width, 0);
}
Ok(&mut scratch[..width])
}
#[cfg(all(test, feature = "std", feature = "mono"))]
std::thread_local! {
static FORCE_SOURCE_LUMA_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "mono"))]
pub(crate) fn arm_source_luma_scratch_failure() {
FORCE_SOURCE_LUMA_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(any(
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "rgb",
feature = "gbr",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn source_luma_u16_scratch<'s>(
scratch: &'s mut Vec<u16>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [u16], MixedSinkerError> {
#[cfg(all(test, feature = "std", feature = "gray"))]
if FORCE_SOURCE_LUMA_U16_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
if scratch.len() < width {
scratch
.try_reserve_exact(width - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(width, 0);
}
Ok(&mut scratch[..width])
}
#[cfg(all(test, feature = "std", feature = "gray"))]
std::thread_local! {
static FORCE_SOURCE_LUMA_U16_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "gray"))]
pub(crate) fn arm_source_luma_u16_scratch_failure() {
FORCE_SOURCE_LUMA_U16_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(feature = "gray")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn source_luma_u32_scratch<'s>(
scratch: &'s mut Vec<u32>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [u32], MixedSinkerError> {
if scratch.len() < width {
scratch
.try_reserve_exact(width - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(width, 0);
}
Ok(&mut scratch[..width])
}
#[cfg(feature = "gray")]
pub(super) enum GrayLumaStreamMut<
'a,
E: crate::resample::AreaSample + crate::resample::FilterSample,
> {
Area(&'a mut crate::resample::AreaStream<E>),
Filter(&'a mut crate::resample::FilterStream<E>),
}
#[cfg(feature = "gray")]
#[allow(clippy::too_many_arguments)]
pub(super) fn gray_luma_resample<E: crate::resample::AreaSample + crate::resample::FilterSample>(
luma_stream: &mut Option<std::boxed::Box<crate::resample::AreaStream<E>>>,
luma_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<E>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
luma_f32: &mut Option<&mut [f32]>,
hsv: &mut Option<HsvFrameMut<'_>>,
plan: &ResamplePlan,
idx: usize,
mut emit: impl FnMut(
Option<GrayLumaStreamMut<'_, E>>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [f32]>,
&mut Option<&mut [f32]>,
&mut Option<HsvFrameMut<'_>>,
) -> Result<(), MixedSinkerError>,
) -> Result<(), MixedSinkerError> {
plan.ensure_single_kernel_filter()?;
let need_any = rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| luma_f32.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(match plan.kind() {
crate::resample::SpanKind::Area => luma_stream.as_ref().map_or(0, |s| s.next_y()),
crate::resample::SpanKind::Filter => luma_filter_stream.as_ref().map_or(0, |s| s.next_y()),
})
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
&None,
&None,
&None,
&None,
hsv,
luma_f32,
expected,
idx,
)? {
return Ok(());
}
match plan.kind() {
crate::resample::SpanKind::Area => {
let new_stream = if luma_stream.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 1)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?)
} else {
None
};
emit(
None, rgb, rgba, rgb_u16, rgba_u16, luma, luma_u16, rgb_f32, luma_f32, hsv,
)?;
if let Some(s) = new_stream {
*luma_stream = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
&None,
&None,
&None,
&None,
hsv,
luma_f32,
idx,
)?;
let stream = luma_stream.as_deref_mut().expect("created above");
emit(
Some(GrayLumaStreamMut::Area(stream)),
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
rgb_f32,
luma_f32,
hsv,
)
}
crate::resample::SpanKind::Filter => {
let new_stream = if luma_filter_stream.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 1)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?)
} else {
None
};
emit(
None, rgb, rgba, rgb_u16, rgba_u16, luma, luma_u16, rgb_f32, luma_f32, hsv,
)?;
if let Some(s) = new_stream {
*luma_filter_stream = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
&None,
&None,
&None,
&None,
hsv,
luma_f32,
idx,
)?;
let stream = luma_filter_stream.as_deref_mut().expect("created above");
emit(
Some(GrayLumaStreamMut::Filter(stream)),
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
rgb_f32,
luma_f32,
hsv,
)
}
}
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
#[cfg_attr(
not(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
)),
allow(dead_code)
)]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_resample_emit(
stream: &mut impl crate::resample::RowResampler<u8>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_rgb: &[u8],
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
stream.feed_row(idx, src_rgb, use_simd, |oy, out_row| {
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(out_row);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
out_row,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
out_row,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
out_row,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::expand_rgb_to_rgba_row(out_row, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
})?;
Ok(())
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_resample(
rgb_stream: &mut Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
direct: Option<&[u8]>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
let need_any =
rgb.is_some() || rgba.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let expected = if need_any {
Some(rgb_stream.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let new_stream = if rgb_stream.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if direct.is_none() {
source_rgb_scratch(rgb_scratch, w, plan)?;
}
if let Some(s) = new_stream {
*rgb_stream = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let stream = rgb_stream.as_mut().expect("created above");
let src: &[u8] = match direct {
Some(row) => row,
None => {
let scratch = source_rgb_scratch(rgb_scratch, w, plan)?;
convert(scratch);
scratch
}
};
packed_rgb_resample_emit(
stream, plan, rgb, rgba, luma, luma_u16, hsv, src, matrix, full_range, idx, use_simd,
)
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_resample_filter(
rgb_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
direct: Option<&[u8]>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
let need_any =
rgb.is_some() || rgba.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let expected = if need_any {
Some(rgb_filter_stream.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
plan.ensure_single_kernel_filter()?;
let new_stream = if rgb_filter_stream.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if direct.is_none() {
source_rgb_scratch(rgb_scratch, w, plan)?;
}
if let Some(s) = new_stream {
*rgb_filter_stream = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let stream = rgb_filter_stream.as_mut().expect("created above");
let src: &[u8] = match direct {
Some(row) => row,
None => {
let scratch = source_rgb_scratch(rgb_scratch, w, plan)?;
convert(scratch);
scratch
}
};
packed_rgb_resample_emit(
stream, plan, rgb, rgba, luma, luma_u16, hsv, src, matrix, full_range, idx, use_simd,
)
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_filter_resample(
rgba_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<u8>,
rgb_drop_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
plan.ensure_single_kernel_filter()?;
let ow = plan.out_w();
let need_any =
rgb.is_some() || rgba.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let expected = if need_any {
Some(rgba_filter_stream.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let need_rgb_drop = rgb.is_some() || hsv.is_some() || luma.is_some() || luma_u16.is_some();
let new_stream = if rgba_filter_stream.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if need_rgb_drop {
source_rgb_scratch(rgb_drop_scratch, ow, plan)?;
}
source_rgba_scratch(rgba_scratch, w, plan)?;
if let Some(s) = new_stream {
*rgba_filter_stream = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
&None,
&None,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let rgb_drop: &mut [u8] = if need_rgb_drop {
source_rgb_scratch(rgb_drop_scratch, ow, plan)?
} else {
&mut []
};
let src_rgba = source_rgba_scratch(rgba_scratch, w, plan)?;
convert_rgba(src_rgba);
let stream = rgba_filter_stream.as_mut().expect("created above");
stream.feed_row(idx, src_rgba, use_simd, |oy, finalized| {
if let Some(buf) = rgba.as_deref_mut() {
buf[oy * 4 * ow..(oy + 1) * 4 * ow].copy_from_slice(finalized);
}
if need_rgb_drop {
let nrow = &mut rgb_drop[..3 * ow];
drop_alpha_rgba_to_rgb_row(finalized, nrow, ow);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_u16_filter_resample<const SRC_BITS: u32, const NATIVE_LUMA16: bool>(
rgba_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch_u16: &mut Vec<u16>,
color_scratch_u16: &mut Vec<u16>,
narrow_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba_u16: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError> {
const {
assert!(
SRC_BITS > 8 && SRC_BITS <= 16,
"SRC_BITS must be in (8, 16] for the high-bit packed RGBA filter tail"
)
};
plan.ensure_single_kernel_filter()?;
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_filter_stream_u16.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let narrowed_luma_u16 = !NATIVE_LUMA16 && luma_u16.is_some();
let need_narrow =
rgb.is_some() || rgba.is_some() || luma.is_some() || narrowed_luma_u16 || hsv.is_some();
let native_luma = NATIVE_LUMA16 && luma_u16.is_some();
let need_native_rgb = need_narrow || native_luma;
let new_stream = if rgba_filter_stream_u16.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if need_native_rgb {
source_rgb_u16_scratch(color_scratch_u16, ow, plan)?;
}
if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?;
}
source_rgba_u16_scratch(rgba_scratch_u16, w, plan)?;
if let Some(s) = new_stream {
*rgba_filter_stream_u16 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let color_rgb: &mut [u16] = if need_native_rgb {
source_rgb_u16_scratch(color_scratch_u16, ow, plan)?
} else {
&mut []
};
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
let src_rgba = source_rgba_u16_scratch(rgba_scratch_u16, w, plan)?;
convert_rgba_u16(src_rgba);
let native_max: u16 = ((1u32 << SRC_BITS) - 1) as u16;
let stream = rgba_filter_stream_u16.as_mut().expect("created above");
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
if let Some(buf) = rgba_u16.as_deref_mut() {
let out = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
if SRC_BITS < 16 {
for (d, &s) in out.iter_mut().zip(binned.iter()) {
*d = s.min(native_max);
}
} else {
out.copy_from_slice(binned);
}
}
if let Some(buf) = rgb_u16.as_deref_mut() {
let out = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
for (rgb_px, rgba_px) in out
.as_chunks_mut::<3>()
.0
.iter_mut()
.zip(binned.as_chunks::<4>().0.iter())
{
if SRC_BITS < 16 {
rgb_px[0] = rgba_px[0].min(native_max);
rgb_px[1] = rgba_px[1].min(native_max);
rgb_px[2] = rgba_px[2].min(native_max);
} else {
rgb_px.copy_from_slice(&rgba_px[..3]);
}
}
}
if need_native_rgb {
let rgb_row = &mut color_rgb[..3 * ow];
for (out_px, in_px) in rgb_row
.as_chunks_mut::<3>()
.0
.iter_mut()
.zip(binned.as_chunks::<4>().0.iter())
{
if SRC_BITS < 16 {
out_px[0] = in_px[0].min(native_max);
out_px[1] = in_px[1].min(native_max);
out_px[2] = in_px[2].min(native_max);
} else {
out_px.copy_from_slice(&in_px[..3]);
}
}
}
if native_luma && let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_native_row(
&color_rgb[..3 * ow],
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
SRC_BITS,
);
}
if need_narrow {
let rgb_row = &color_rgb[..3 * ow];
let nrow = &mut narrow[..3 * ow];
for (d, &s) in nrow.iter_mut().zip(rgb_row.iter()) {
*d = (s >> (SRC_BITS - 8)) as u8;
}
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = rgba.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
for (d, px) in dst
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(binned.as_chunks::<4>().0.iter())
{
if SRC_BITS < 16 {
d[0] = (px[0].min(native_max) >> (SRC_BITS - 8)) as u8;
d[1] = (px[1].min(native_max) >> (SRC_BITS - 8)) as u8;
d[2] = (px[2].min(native_max) >> (SRC_BITS - 8)) as u8;
d[3] = (px[3].min(native_max) >> (SRC_BITS - 8)) as u8;
} else {
d[0] = (px[0] >> (SRC_BITS - 8)) as u8;
d[1] = (px[1] >> (SRC_BITS - 8)) as u8;
d[2] = (px[2] >> (SRC_BITS - 8)) as u8;
d[3] = (px[3] >> (SRC_BITS - 8)) as u8;
}
}
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if narrowed_luma_u16 && let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
pub(super) fn source_rgb_u16_scratch<'s>(
scratch: &'s mut Vec<u16>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [u16], MixedSinkerError> {
#[cfg(all(test, feature = "std", any(feature = "rgb", feature = "gbr")))]
if FORCE_SOURCE_RGB_U16_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
let row = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.src_h(),
3,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0);
}
Ok(&mut scratch[..row])
}
#[cfg(all(test, feature = "std", any(feature = "rgb", feature = "gbr")))]
std::thread_local! {
static FORCE_SOURCE_RGB_U16_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "rgb"))]
pub(crate) fn arm_source_rgb_u16_scratch_failure() {
FORCE_SOURCE_RGB_U16_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "yuv-444-packed",
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_u16_resample_emit<const SRC_BITS: u32, const NATIVE_LUMA16: bool>(
stream: &mut impl crate::resample::RowResampler<u16>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_u16: &[u16],
narrow_scratch: &mut Vec<u8>,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
const {
assert!(
SRC_BITS >= 8 && SRC_BITS <= 16,
"SRC_BITS must be in [8, 16]"
)
};
let ow = plan.out_w();
let need_narrow = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| hsv.is_some()
|| (!NATIVE_LUMA16 && luma_u16.is_some());
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
let native_max: u16 = ((1u32 << SRC_BITS) - 1) as u16;
stream.feed_row(idx, src_u16, use_simd, |oy, binned| {
if let Some(buf) = rgb_u16.as_deref_mut() {
let out = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
if SRC_BITS < 16 {
for (d, &s) in out.iter_mut().zip(binned.iter()) {
*d = s.min(native_max);
}
} else {
out.copy_from_slice(binned);
}
}
if let Some(buf) = rgba_u16.as_deref_mut() {
let out = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
if SRC_BITS < 16 {
for (rgba_px, rgb_px) in out
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(binned.as_chunks::<3>().0.iter())
{
rgba_px[0] = rgb_px[0].min(native_max);
rgba_px[1] = rgb_px[1].min(native_max);
rgba_px[2] = rgb_px[2].min(native_max);
rgba_px[3] = native_max;
}
} else {
crate::row::expand_rgb_u16_to_rgba_u16_row::<SRC_BITS>(binned, out, ow);
}
}
if NATIVE_LUMA16 && let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_native_row(
binned,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
SRC_BITS,
);
}
if need_narrow {
let nrow = &mut narrow[..3 * ow];
if SRC_BITS < 16 {
for (d, &s) in nrow.iter_mut().zip(binned.iter()) {
*d = (s.min(native_max) >> (SRC_BITS - 8)) as u8;
}
} else {
for (d, &s) in nrow.iter_mut().zip(binned.iter()) {
*d = (s >> (SRC_BITS - 8)) as u8;
}
}
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::expand_rgb_to_rgba_row(nrow, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if !NATIVE_LUMA16 && let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_u16_resample<const SRC_BITS: u32, const NATIVE_LUMA16: bool>(
rgb_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch_u16: &mut Vec<u16>,
narrow_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError> {
const {
assert!(
SRC_BITS >= 8 && SRC_BITS <= 16,
"SRC_BITS must be in [8, 16]"
)
};
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgb_stream_u16.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let need_narrow = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| hsv.is_some()
|| (!NATIVE_LUMA16 && luma_u16.is_some());
let new_stream = if rgb_stream_u16.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgb_u16_scratch(rgb_scratch_u16, w, plan)?;
if need_narrow {
source_rgb_scratch(narrow_scratch, plan.out_w(), plan)?;
}
if let Some(s) = new_stream {
*rgb_stream_u16 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let stream = rgb_stream_u16.as_mut().expect("created above");
let src_u16 = source_rgb_u16_scratch(rgb_scratch_u16, w, plan)?;
convert(src_u16);
packed_rgb_u16_resample_emit::<SRC_BITS, NATIVE_LUMA16>(
stream,
plan,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
hsv,
src_u16,
narrow_scratch,
matrix,
full_range,
idx,
use_simd,
)
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_u16_resample_filter<const SRC_BITS: u32, const NATIVE_LUMA16: bool>(
rgb_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch_u16: &mut Vec<u16>,
narrow_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError> {
const {
assert!(
SRC_BITS >= 8 && SRC_BITS <= 16,
"SRC_BITS must be in [8, 16]"
)
};
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
plan.ensure_single_kernel_filter()?;
let need_narrow = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| hsv.is_some()
|| (!NATIVE_LUMA16 && luma_u16.is_some());
let new_stream = if rgb_filter_stream_u16.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgb_u16_scratch(rgb_scratch_u16, w, plan)?;
if need_narrow {
source_rgb_scratch(narrow_scratch, plan.out_w(), plan)?;
}
if let Some(s) = new_stream {
*rgb_filter_stream_u16 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let stream = rgb_filter_stream_u16.as_mut().expect("created above");
let src_u16 = source_rgb_u16_scratch(rgb_scratch_u16, w, plan)?;
convert(src_u16);
packed_rgb_u16_resample_emit::<SRC_BITS, NATIVE_LUMA16>(
stream,
plan,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
hsv,
src_u16,
narrow_scratch,
matrix,
full_range,
idx,
use_simd,
)
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
pub(super) fn source_rgb_u32_scratch<'s>(
scratch: &'s mut Vec<u32>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [u32], MixedSinkerError> {
#[cfg(all(test, feature = "std", any(feature = "rgb", feature = "gbr")))]
if FORCE_SOURCE_RGB_U32_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
let row = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.src_h(),
3,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0);
}
Ok(&mut scratch[..row])
}
#[cfg(all(test, feature = "std", any(feature = "rgb", feature = "gbr")))]
std::thread_local! {
static FORCE_SOURCE_RGB_U32_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "rgb"))]
pub(crate) fn arm_source_rgb_u32_scratch_failure() {
FORCE_SOURCE_RGB_U32_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
pub(super) fn source_rgba_u32_scratch<'s>(
scratch: &'s mut Vec<u32>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [u32], MixedSinkerError> {
#[cfg(all(test, feature = "std", any(feature = "rgb", feature = "gbr")))]
if FORCE_SOURCE_RGBA_U32_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
let row = width
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.src_h(),
4,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0);
}
Ok(&mut scratch[..row])
}
#[cfg(all(test, feature = "std", any(feature = "rgb", feature = "gbr")))]
std::thread_local! {
static FORCE_SOURCE_RGBA_U32_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "rgb"))]
pub(crate) fn arm_source_rgba_u32_scratch_failure() {
FORCE_SOURCE_RGBA_U32_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_u32_resample<const NATIVE_LUMA16: bool>(
rgb_stream_u32: &mut Option<std::boxed::Box<crate::resample::AreaStream<u32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch_u32: &mut Vec<u32>,
binned_scratch_u16: &mut Vec<u16>,
narrow_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert: impl FnOnce(&mut [u32]),
) -> Result<(), MixedSinkerError> {
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgb_stream_u32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let need_narrow = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| hsv.is_some()
|| (!NATIVE_LUMA16 && luma_u16.is_some());
let new_stream = if rgb_stream_u32.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgb_u32_scratch(rgb_scratch_u32, w, plan)?;
source_rgb_u16_scratch(binned_scratch_u16, plan.out_w(), plan)?;
if need_narrow {
source_rgb_scratch(narrow_scratch, plan.out_w(), plan)?;
}
if let Some(s) = new_stream {
*rgb_stream_u32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let stream = rgb_stream_u32.as_mut().expect("created above");
let src_u32 = source_rgb_u32_scratch(rgb_scratch_u32, w, plan)?;
convert(src_u32);
packed_rgb_u32_resample_emit::<NATIVE_LUMA16>(
stream,
plan,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
hsv,
src_u32,
binned_scratch_u16,
narrow_scratch,
matrix,
full_range,
idx,
use_simd,
)
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_u32_resample_filter<const NATIVE_LUMA16: bool>(
rgb_filter_stream_u32: &mut Option<std::boxed::Box<crate::resample::FilterStream<u32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch_u32: &mut Vec<u32>,
binned_scratch_u16: &mut Vec<u16>,
narrow_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert: impl FnOnce(&mut [u32]),
) -> Result<(), MixedSinkerError> {
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgb_filter_stream_u32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
plan.ensure_single_kernel_filter()?;
let need_narrow = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| hsv.is_some()
|| (!NATIVE_LUMA16 && luma_u16.is_some());
let new_stream = if rgb_filter_stream_u32.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgb_u32_scratch(rgb_scratch_u32, w, plan)?;
source_rgb_u16_scratch(binned_scratch_u16, plan.out_w(), plan)?;
if need_narrow {
source_rgb_scratch(narrow_scratch, plan.out_w(), plan)?;
}
if let Some(s) = new_stream {
*rgb_filter_stream_u32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let stream = rgb_filter_stream_u32.as_mut().expect("created above");
let src_u32 = source_rgb_u32_scratch(rgb_scratch_u32, w, plan)?;
convert(src_u32);
packed_rgb_u32_resample_emit::<NATIVE_LUMA16>(
stream,
plan,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
hsv,
src_u32,
binned_scratch_u16,
narrow_scratch,
matrix,
full_range,
idx,
use_simd,
)
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_u32_resample_emit<const NATIVE_LUMA16: bool>(
stream: &mut impl crate::resample::RowResampler<u32>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_u32: &[u32],
binned_scratch_u16: &mut Vec<u16>,
narrow_scratch: &mut Vec<u8>,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
let narrowed_luma_u16 = !NATIVE_LUMA16 && luma_u16.is_some();
let need_narrow =
rgb.is_some() || rgba.is_some() || luma.is_some() || narrowed_luma_u16 || hsv.is_some();
let binned16 = source_rgb_u16_scratch(binned_scratch_u16, ow, plan)?;
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
stream.feed_row(idx, src_u32, use_simd, |oy, binned| {
let b16 = &mut binned16[..3 * ow];
for (d, &s) in b16.iter_mut().zip(binned.iter()) {
*d = (s >> 16) as u16;
}
let b16: &[u16] = b16;
if let Some(buf) = rgb_u16.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(b16);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
crate::row::expand_rgb_u16_to_rgba_u16_row::<16>(
b16,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
);
}
if NATIVE_LUMA16 && let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_native_row(
b16,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
16,
);
}
if need_narrow {
let nrow = &mut narrow[..3 * ow];
for (d, &s) in nrow.iter_mut().zip(b16.iter()) {
*d = (s >> 8) as u8;
}
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::expand_rgb_to_rgba_row(nrow, &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow], ow);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if narrowed_luma_u16 && let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn premultiply_rgba_u32_row_in_place(row: &mut [u32], width: usize) {
const MAX: u64 = u32::MAX as u64;
const HALF: u64 = MAX / 2;
for px in row[..width * 4].as_chunks_mut::<4>().0 {
let a = u64::from(px[3]);
for c in &mut px[..3] {
let v = u64::from(*c);
*c = ((v * a + HALF) / MAX) as u32;
}
}
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn unpremultiply_binned_rgba_u32_into(binned: &[u32], dst: &mut [u32], width: usize) {
const MAX: u64 = u32::MAX as u64;
for (out_px, in_px) in dst[..width * 4]
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(binned[..width * 4].as_chunks::<4>().0.iter())
{
let a = u64::from(in_px[3]);
for c in 0..3 {
let pm = u64::from(in_px[c]);
out_px[c] = (pm * MAX + a / 2).checked_div(a).map_or(0, |q| q.min(MAX)) as u32;
}
out_px[3] = in_px[3];
}
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
fn derive_packed_rgba_u32_outputs<const NATIVE_LUMA16: bool>(
oy: usize,
ow: usize,
straight: &[u32],
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
color16: &mut [u16],
rgb16: &mut [u16],
narrow: &mut [u8],
need_narrow: bool,
native_luma: bool,
narrowed_luma_u16: bool,
matrix: crate::KernelMatrix,
full_range: bool,
use_simd: bool,
) {
let c16 = &mut color16[..4 * ow];
for (d, &s) in c16.iter_mut().zip(straight.iter()) {
*d = (s >> 16) as u16;
}
let c16: &[u16] = c16;
if let Some(buf) = rgba_u16.as_deref_mut() {
buf[oy * 4 * ow..(oy + 1) * 4 * ow].copy_from_slice(c16);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
drop_alpha_rgba_to_rgb_row(c16, &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow], ow);
}
if native_luma && let Some(buf) = luma_u16.as_deref_mut() {
let rgb_row = &mut rgb16[..3 * ow];
drop_alpha_rgba_to_rgb_row(c16, rgb_row, ow);
crate::row::rgb_to_luma_u16_native_row(
rgb_row,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
16,
);
}
if need_narrow {
let nrow = &mut narrow[..3 * ow];
for (d, px) in nrow
.as_chunks_mut::<3>()
.0
.iter_mut()
.zip(c16.as_chunks::<4>().0.iter())
{
d[0] = (px[0] >> 8) as u8;
d[1] = (px[1] >> 8) as u8;
d[2] = (px[2] >> 8) as u8;
}
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = rgba.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
for (d, px) in dst
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(c16.as_chunks::<4>().0.iter())
{
d[0] = (px[0] >> 8) as u8;
d[1] = (px[1] >> 8) as u8;
d[2] = (px[2] >> 8) as u8;
d[3] = (px[3] >> 8) as u8;
}
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if narrowed_luma_u16 && let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_u32_resample<const NATIVE_LUMA16: bool>(
rgba_stream_u32: &mut Option<std::boxed::Box<crate::resample::AreaStream<u32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch_u32: &mut Vec<u32>,
color_scratch_u32: &mut Vec<u32>,
color_scratch_u16: &mut Vec<u16>,
rgb_scratch_u16: &mut Vec<u16>,
narrow_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
alpha_mode: AlphaMode,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba_u32: impl FnOnce(&mut [u32]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_stream_u32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let narrowed_luma_u16 = !NATIVE_LUMA16 && luma_u16.is_some();
let need_narrow =
rgb.is_some() || rgba.is_some() || luma.is_some() || narrowed_luma_u16 || hsv.is_some();
let native_luma = NATIVE_LUMA16 && luma_u16.is_some();
let premult = alpha_mode.is_premultiplied();
let new_stream = if rgba_stream_u32.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if premult {
source_rgba_u32_scratch(color_scratch_u32, ow, plan)?;
}
source_rgba_u16_scratch(color_scratch_u16, ow, plan)?;
if native_luma {
source_rgb_u16_scratch(rgb_scratch_u16, ow, plan)?;
}
if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?;
}
source_rgba_u32_scratch(rgba_scratch_u32, w, plan)?;
if let Some(s) = new_stream {
*rgba_stream_u32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let color_u32: &mut [u32] = if premult {
source_rgba_u32_scratch(color_scratch_u32, ow, plan)?
} else {
&mut []
};
let color16 = source_rgba_u16_scratch(color_scratch_u16, ow, plan)?;
let rgb16: &mut [u16] = if native_luma {
source_rgb_u16_scratch(rgb_scratch_u16, ow, plan)?
} else {
&mut []
};
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
let src_rgba = source_rgba_u32_scratch(rgba_scratch_u32, w, plan)?;
convert_rgba_u32(src_rgba);
if premult {
premultiply_rgba_u32_row_in_place(src_rgba, w);
}
let stream = rgba_stream_u32.as_mut().expect("created above");
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
let straight: &[u32] = if premult {
let c = &mut color_u32[..4 * ow];
unpremultiply_binned_rgba_u32_into(binned, c, ow);
c
} else {
binned
};
derive_packed_rgba_u32_outputs::<NATIVE_LUMA16>(
oy,
ow,
straight,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
color16,
rgb16,
narrow,
need_narrow,
native_luma,
narrowed_luma_u16,
matrix,
full_range,
use_simd,
);
})?;
Ok(())
}
#[cfg(any(feature = "rgb", feature = "gbr"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_u32_filter_resample<const NATIVE_LUMA16: bool>(
rgba_filter_stream_u32: &mut Option<std::boxed::Box<crate::resample::FilterStream<u32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch_u32: &mut Vec<u32>,
color_scratch_u16: &mut Vec<u16>,
rgb_scratch_u16: &mut Vec<u16>,
narrow_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba_u32: impl FnOnce(&mut [u32]),
) -> Result<(), MixedSinkerError> {
plan.ensure_single_kernel_filter()?;
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_filter_stream_u32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let narrowed_luma_u16 = !NATIVE_LUMA16 && luma_u16.is_some();
let need_narrow =
rgb.is_some() || rgba.is_some() || luma.is_some() || narrowed_luma_u16 || hsv.is_some();
let native_luma = NATIVE_LUMA16 && luma_u16.is_some();
let new_stream = if rgba_filter_stream_u32.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgba_u16_scratch(color_scratch_u16, ow, plan)?;
if native_luma {
source_rgb_u16_scratch(rgb_scratch_u16, ow, plan)?;
}
if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?;
}
source_rgba_u32_scratch(rgba_scratch_u32, w, plan)?;
if let Some(s) = new_stream {
*rgba_filter_stream_u32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let color16 = source_rgba_u16_scratch(color_scratch_u16, ow, plan)?;
let rgb16: &mut [u16] = if native_luma {
source_rgb_u16_scratch(rgb_scratch_u16, ow, plan)?
} else {
&mut []
};
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
let src_rgba = source_rgba_u32_scratch(rgba_scratch_u32, w, plan)?;
convert_rgba_u32(src_rgba);
let stream = rgba_filter_stream_u32.as_mut().expect("created above");
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
derive_packed_rgba_u32_outputs::<NATIVE_LUMA16>(
oy,
ow,
binned,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
color16,
rgb16,
narrow,
need_narrow,
native_luma,
narrowed_luma_u16,
matrix,
full_range,
use_simd,
);
})?;
Ok(())
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
pub(super) fn source_rgba_scratch<'s>(
scratch: &'s mut Vec<u8>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [u8], MixedSinkerError> {
let row = width
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.src_h(),
4,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0);
}
Ok(&mut scratch[..row])
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "yuva",
feature = "yuv-444-packed"
))]
pub(super) fn source_rgba_u16_scratch<'s>(
scratch: &'s mut Vec<u16>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [u16], MixedSinkerError> {
#[cfg(all(
test,
feature = "std",
any(feature = "rgb", feature = "gbr", feature = "gray")
))]
if FORCE_SOURCE_RGBA_U16_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
let row = width
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.src_h(),
4,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0);
}
Ok(&mut scratch[..row])
}
#[cfg(all(
test,
feature = "std",
any(feature = "rgb", feature = "gbr", feature = "gray")
))]
std::thread_local! {
static FORCE_SOURCE_RGBA_U16_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "rgb"))]
pub(crate) fn arm_source_rgba_u16_scratch_failure() {
FORCE_SOURCE_RGBA_U16_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn premultiply_rgba_row_in_place<T>(row: &mut [T], width: usize, max: u32)
where
T: Copy + TryFrom<u32> + Into<u32>,
{
let half = max / 2;
for px in row[..width * 4].as_chunks_mut::<4>().0 {
let a: u32 = px[3].into();
for c in &mut px[..3] {
let v: u32 = (*c).into();
let pm = (v * a + half) / max;
*c = T::try_from(pm).unwrap_or_else(|_| unreachable!("premultiplied value <= max"));
}
}
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn unpremultiply_channel(pm: u32, a: u32, max: u32) -> u32 {
(pm * max + a / 2).checked_div(a).map_or(0, |q| q.min(max))
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn unpremultiply_binned_rgba_into<T>(binned: &[T], dst: &mut [T], width: usize, max: u32)
where
T: Copy + TryFrom<u32> + Into<u32>,
{
for (out_px, in_px) in dst[..width * 4]
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(binned[..width * 4].as_chunks::<4>().0.iter())
{
let a: u32 = in_px[3].into();
for c in 0..3 {
let straight = unpremultiply_channel(in_px[c].into(), a, max);
out_px[c] =
T::try_from(straight).unwrap_or_else(|_| unreachable!("un-premultiplied value <= max"));
}
out_px[3] = in_px[3];
}
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn unpremultiply_binned_rgb_into<T>(binned: &[T], dst: &mut [T], width: usize, max: u32)
where
T: Copy + TryFrom<u32> + Into<u32>,
{
for (out_px, in_px) in dst[..width * 3]
.as_chunks_mut::<3>()
.0
.iter_mut()
.zip(binned[..width * 4].as_chunks::<4>().0.iter())
{
let a: u32 = in_px[3].into();
for c in 0..3 {
let straight = unpremultiply_channel(in_px[c].into(), a, max);
out_px[c] =
T::try_from(straight).unwrap_or_else(|_| unreachable!("un-premultiplied value <= max"));
}
}
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn drop_alpha_rgba_to_rgb_row<T: Copy>(rgba: &[T], dst: &mut [T], width: usize) {
for (out_px, in_px) in dst[..width * 3]
.as_chunks_mut::<3>()
.0
.iter_mut()
.zip(rgba[..width * 4].as_chunks::<4>().0.iter())
{
out_px.copy_from_slice(&in_px[..3]);
}
}
#[cfg(any(
feature = "rgb",
feature = "gbr",
feature = "gray",
feature = "mono",
feature = "yuva",
feature = "yuv-444-packed"
))]
pub(super) fn check_frozen_alpha_mode(
frozen: Option<AlphaMode>,
current: AlphaMode,
idx: usize,
) -> Result<(), MixedSinkerError> {
if frozen != Some(current) {
return Err(MixedSinkerError::ResampleOutputsChanged(
ResampleOutputsChanged::new(idx),
));
}
Ok(())
}
#[cfg(any(feature = "rgb", feature = "gbr", feature = "gray"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_resample<const NATIVE_Y_LUMA: bool>(
rgba_stream: &mut Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
y_luma_stream: &mut Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<u8>,
rgb_drop_scratch: &mut Vec<u8>,
y_luma_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
alpha_mode: AlphaMode,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba: impl FnOnce(&mut [u8]),
deinterleave_y: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_stream.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let premult = alpha_mode.is_premultiplied();
let need_y_luma = NATIVE_Y_LUMA && (luma.is_some() || luma_u16.is_some());
let need_rgb_drop = rgb.is_some()
|| rgb_u16.is_some()
|| hsv.is_some()
|| (!NATIVE_Y_LUMA && (luma.is_some() || luma_u16.is_some()));
let new_rgba = if rgba_stream.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
let new_y_luma = if need_y_luma && y_luma_stream.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 1)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if need_rgb_drop {
source_rgb_scratch(rgb_drop_scratch, ow, plan)?;
}
if need_y_luma {
source_luma_scratch(y_luma_scratch, w, plan)?;
}
source_rgba_scratch(rgba_scratch, w, plan)?;
if let Some(s) = new_rgba {
*rgba_stream = Some(s);
}
if let Some(s) = new_y_luma {
*y_luma_stream = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let rgb_drop: &mut [u8] = if need_rgb_drop {
source_rgb_scratch(rgb_drop_scratch, ow, plan)?
} else {
&mut []
};
let y_src: &mut [u8] = if need_y_luma {
let scratch = source_luma_scratch(y_luma_scratch, w, plan)?;
deinterleave_y(scratch);
scratch
} else {
&mut []
};
let src_rgba = source_rgba_scratch(rgba_scratch, w, plan)?;
convert_rgba(src_rgba);
if premult {
premultiply_rgba_row_in_place::<u8>(src_rgba, w, 255);
}
let stream = rgba_stream.as_mut().expect("created above");
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
if let Some(buf) = rgba.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
if premult {
unpremultiply_binned_rgba_into::<u8>(binned, dst, ow, 255);
} else {
dst.copy_from_slice(binned);
}
}
if let Some(buf) = rgba_u16.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
if premult {
for (out_px, in_px) in dst
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(binned.as_chunks::<4>().0.iter())
{
let a = in_px[3] as u32;
for c in 0..3 {
out_px[c] = unpremultiply_channel(in_px[c] as u32, a, 255) as u16;
}
out_px[3] = in_px[3] as u16;
}
} else {
for (d, &s) in dst.iter_mut().zip(binned.iter()) {
*d = s as u16;
}
}
}
if need_rgb_drop {
let nrow = &mut rgb_drop[..3 * ow];
if premult {
unpremultiply_binned_rgb_into::<u8>(binned, nrow, ow, 255);
} else {
drop_alpha_rgba_to_rgb_row(binned, nrow, ow);
}
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
for (d, &s) in dst.iter_mut().zip(nrow.iter()) {
*d = s as u16;
}
}
if !NATIVE_Y_LUMA {
if let Some(buf) = luma.as_deref_mut() {
let dst = &mut buf[oy * ow..(oy + 1) * ow];
crate::row::rgb_to_luma_row(nrow, dst, ow, matrix, full_range, use_simd);
}
if let Some(buf) = luma_u16.as_deref_mut() {
let dst = &mut buf[oy * ow..(oy + 1) * ow];
crate::row::rgb_to_luma_u16_row(nrow, dst, ow, matrix, full_range, use_simd);
}
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
if need_y_luma {
let stream = y_luma_stream.as_mut().expect("created in the preflight");
stream.feed_row(idx, y_src, use_simd, |oy, binned_y| {
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(binned_y);
}
if let Some(buf) = luma_u16.as_deref_mut() {
let dst = &mut buf[oy * ow..(oy + 1) * ow];
for (d, &y) in dst.iter_mut().zip(binned_y.iter()) {
*d = y as u16;
}
}
})?;
}
Ok(())
}
#[cfg(feature = "mono")]
#[allow(clippy::too_many_arguments)]
pub(super) fn pal8_rgba_resample(
rgba_stream: &mut Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<u8>,
rgb_drop_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
alpha_mode: AlphaMode,
luma_coeffs_q8: (u32, u32, u32),
convert_rgba: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_stream.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let premult = alpha_mode.is_premultiplied();
let need_rgb_drop =
rgb.is_some() || rgb_u16.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let new_stream = if rgba_stream.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if need_rgb_drop {
source_rgb_scratch(rgb_drop_scratch, ow, plan)?;
}
source_rgba_scratch(rgba_scratch, w, plan)?;
if let Some(s) = new_stream {
*rgba_stream = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let rgb_drop: &mut [u8] = if need_rgb_drop {
source_rgb_scratch(rgb_drop_scratch, ow, plan)?
} else {
&mut []
};
let src_rgba = source_rgba_scratch(rgba_scratch, w, plan)?;
convert_rgba(src_rgba);
if premult {
premultiply_rgba_row_in_place::<u8>(src_rgba, w, 255);
}
let stream = rgba_stream.as_mut().expect("created above");
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
if let Some(buf) = rgba.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
if premult {
unpremultiply_binned_rgba_into::<u8>(binned, dst, ow, 255);
} else {
dst.copy_from_slice(binned);
}
}
if let Some(buf) = rgba_u16.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
if premult {
for (out_px, in_px) in dst
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(binned.as_chunks::<4>().0.iter())
{
let a = in_px[3] as u32;
for c in 0..3 {
let s = unpremultiply_channel(in_px[c] as u32, a, 255) as u16;
out_px[c] = (s << 8) | s;
}
let a16 = in_px[3] as u16;
out_px[3] = (a16 << 8) | a16;
}
} else {
for (d, &s) in dst.iter_mut().zip(binned.iter()) {
let s = s as u16;
*d = (s << 8) | s;
}
}
}
if need_rgb_drop {
let nrow = &mut rgb_drop[..3 * ow];
if premult {
unpremultiply_binned_rgb_into::<u8>(binned, nrow, ow, 255);
} else {
drop_alpha_rgba_to_rgb_row(binned, nrow, ow);
}
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
for (d, &s) in dst.iter_mut().zip(nrow.iter()) {
let s = s as u16;
*d = (s << 8) | s;
}
}
if let Some(buf) = luma.as_deref_mut() {
rgb_row_to_luma_row(nrow, &mut buf[oy * ow..(oy + 1) * ow], luma_coeffs_q8);
}
if let Some(buf) = luma_u16.as_deref_mut() {
rgb_row_to_luma_u16_row(nrow, &mut buf[oy * ow..(oy + 1) * ow], luma_coeffs_q8);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(feature = "mono")]
#[allow(clippy::too_many_arguments)]
pub(super) fn pal8_rgba_filter_resample(
rgba_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<u8>,
rgb_drop_scratch: &mut Vec<u8>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
luma_coeffs_q8: (u32, u32, u32),
convert_rgba: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
plan.ensure_single_kernel_filter()?;
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_filter_stream.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let need_rgb_drop =
rgb.is_some() || rgb_u16.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let new_stream = if rgba_filter_stream.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
if need_rgb_drop {
source_rgb_scratch(rgb_drop_scratch, ow, plan)?;
}
source_rgba_scratch(rgba_scratch, w, plan)?;
if let Some(s) = new_stream {
*rgba_filter_stream = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let rgb_drop: &mut [u8] = if need_rgb_drop {
source_rgb_scratch(rgb_drop_scratch, ow, plan)?
} else {
&mut []
};
let src_rgba = source_rgba_scratch(rgba_scratch, w, plan)?;
convert_rgba(src_rgba);
let stream = rgba_filter_stream.as_mut().expect("created above");
stream.feed_row(idx, src_rgba, use_simd, |oy, finalized| {
if let Some(buf) = rgba.as_deref_mut() {
buf[oy * 4 * ow..(oy + 1) * 4 * ow].copy_from_slice(finalized);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
for (d, &s) in dst.iter_mut().zip(finalized.iter()) {
let s = s as u16;
*d = (s << 8) | s;
}
}
if need_rgb_drop {
let nrow = &mut rgb_drop[..3 * ow];
drop_alpha_rgba_to_rgb_row(finalized, nrow, ow);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
for (d, &s) in dst.iter_mut().zip(nrow.iter()) {
let s = s as u16;
*d = (s << 8) | s;
}
}
if let Some(buf) = luma.as_deref_mut() {
rgb_row_to_luma_row(nrow, &mut buf[oy * ow..(oy + 1) * ow], luma_coeffs_q8);
}
if let Some(buf) = luma_u16.as_deref_mut() {
rgb_row_to_luma_u16_row(nrow, &mut buf[oy * ow..(oy + 1) * ow], luma_coeffs_q8);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(any(feature = "rgb", feature = "gbr", feature = "gray"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_u16_resample<
const SRC_BITS: u32,
const NATIVE_LUMA16: bool,
const NATIVE_Y_LUMA: bool,
>(
rgba_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
y_luma_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch_u16: &mut Vec<u16>,
color_scratch_u16: &mut Vec<u16>,
narrow_scratch: &mut Vec<u8>,
luma_rgb_scratch_u16: &mut Vec<u16>,
y_luma_scratch_u16: &mut Vec<u16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
alpha_mode: AlphaMode,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba_u16: impl FnOnce(&mut [u16]),
deinterleave_y: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
const {
assert!(
SRC_BITS > 8 && SRC_BITS <= 16,
"SRC_BITS must be in (8, 16] for the high-bit packed RGBA tail"
);
assert!(
!(NATIVE_LUMA16 && NATIVE_Y_LUMA),
"luma_u16 has a single source-of-truth: NATIVE_LUMA16 and NATIVE_Y_LUMA are mutually exclusive"
);
};
let max: u32 = (1u32 << SRC_BITS) - 1;
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_stream_u16.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let need_y_luma = NATIVE_Y_LUMA && (luma.is_some() || luma_u16.is_some());
let narrowed_luma_u16 = !NATIVE_LUMA16 && !NATIVE_Y_LUMA && luma_u16.is_some();
let narrowed_luma = !NATIVE_Y_LUMA && luma.is_some();
let need_narrow =
rgb.is_some() || rgba.is_some() || narrowed_luma || narrowed_luma_u16 || hsv.is_some();
let native_luma = NATIVE_LUMA16 && luma_u16.is_some();
let new_rgba = if rgba_stream_u16.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
let new_y_luma = if need_y_luma && y_luma_stream_u16.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 1)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgba_u16_scratch(color_scratch_u16, ow, plan)?;
if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?;
}
if native_luma {
source_rgb_u16_scratch(luma_rgb_scratch_u16, ow, plan)?;
}
if need_y_luma {
source_luma_u16_scratch(y_luma_scratch_u16, w, plan)?;
}
source_rgba_u16_scratch(rgba_scratch_u16, w, plan)?;
if let Some(s) = new_rgba {
*rgba_stream_u16 = Some(s);
}
if let Some(s) = new_y_luma {
*y_luma_stream_u16 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
let color = source_rgba_u16_scratch(color_scratch_u16, ow, plan)?;
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
let luma_rgb: &mut [u16] = if native_luma {
source_rgb_u16_scratch(luma_rgb_scratch_u16, ow, plan)?
} else {
&mut []
};
let y_src: &mut [u16] = if need_y_luma {
let scratch = source_luma_u16_scratch(y_luma_scratch_u16, w, plan)?;
deinterleave_y(scratch);
scratch
} else {
&mut []
};
let premult = alpha_mode.is_premultiplied();
let src_rgba = source_rgba_u16_scratch(rgba_scratch_u16, w, plan)?;
convert_rgba_u16(src_rgba);
if premult {
premultiply_rgba_row_in_place::<u16>(src_rgba, w, max);
}
let stream = rgba_stream_u16.as_mut().expect("created above");
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
let color = &mut color[..4 * ow];
if premult {
unpremultiply_binned_rgba_into::<u16>(binned, color, ow, max);
} else {
color.copy_from_slice(binned);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
buf[oy * 4 * ow..(oy + 1) * 4 * ow].copy_from_slice(color);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
drop_alpha_rgba_to_rgb_row(color, dst, ow);
}
if native_luma && let Some(buf) = luma_u16.as_deref_mut() {
let rgb_row = &mut luma_rgb[..3 * ow];
drop_alpha_rgba_to_rgb_row(color, rgb_row, ow);
crate::row::rgb_to_luma_u16_native_row(
rgb_row,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
SRC_BITS,
);
}
if need_narrow {
let nrow = &mut narrow[..3 * ow];
for (d, px) in nrow
.as_chunks_mut::<3>()
.0
.iter_mut()
.zip(color.as_chunks::<4>().0.iter())
{
d[0] = (px[0] >> (SRC_BITS - 8)) as u8;
d[1] = (px[1] >> (SRC_BITS - 8)) as u8;
d[2] = (px[2] >> (SRC_BITS - 8)) as u8;
}
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = rgba.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
for (d, px) in dst
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(color.as_chunks::<4>().0.iter())
{
d[0] = (px[0] >> (SRC_BITS - 8)) as u8;
d[1] = (px[1] >> (SRC_BITS - 8)) as u8;
d[2] = (px[2] >> (SRC_BITS - 8)) as u8;
d[3] = (px[3] >> (SRC_BITS - 8)) as u8;
}
}
if narrowed_luma && let Some(buf) = luma.as_deref_mut() {
let dst = &mut buf[oy * ow..(oy + 1) * ow];
crate::row::rgb_to_luma_row(nrow, dst, ow, matrix, full_range, use_simd);
}
if narrowed_luma_u16 && let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
if need_y_luma {
let stream = y_luma_stream_u16
.as_mut()
.expect("created in the preflight");
stream.feed_row(idx, y_src, use_simd, |oy, binned_y| {
if let Some(buf) = luma_u16.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(binned_y);
}
if let Some(buf) = luma.as_deref_mut() {
let dst = &mut buf[oy * ow..(oy + 1) * ow];
for (d, &y) in dst.iter_mut().zip(binned_y.iter()) {
*d = (y >> 8) as u8;
}
}
})?;
}
Ok(())
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
pub(super) fn reset_high_bit_yuv_streams<F: SourceFormat, R>(sink: &mut MixedSinker<'_, F, R>) {
if let Some(stream) = sink.rgb_stream.as_mut() {
stream.reset();
}
if let Some(stream) = sink.rgb_stream_u16.as_mut() {
stream.reset();
}
if let Some(stream) = sink.luma_stream_u16.as_mut() {
stream.reset();
}
if let Some(stream) = sink.rgb_filter_stream.as_mut() {
stream.reset();
}
if let Some(stream) = sink.rgb_filter_stream_u16.as_mut() {
stream.reset();
}
if let Some(stream) = sink.luma_filter_stream_u16.as_mut() {
stream.reset();
}
#[cfg(feature = "yuv-planar")]
if let Some(join) = sink.native_420_u16.as_mut() {
join.reset();
}
#[cfg(feature = "yuv-planar")]
if let Some(join) = sink.native_planar_u16.as_mut() {
join.reset();
}
#[cfg(feature = "yuv-planar")]
{
sink.frozen_native_route = None;
sink.frozen_domain = None;
sink.frozen_chroma_centered = None;
sink.frozen_chroma_bottom_v = None;
sink.chroma_prev_row = None;
sink.frozen_chroma_top_v = None;
sink.chroma_top_pending = None;
}
sink.resample_outputs = None;
}
#[cfg(feature = "yuv-planar")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn deinterleave_y_high_bit_masked<const BITS: u32, const BE: bool>(
y: &[u16],
scratch: &mut [u16],
width: usize,
) {
let mask = ((1u32 << BITS) - 1) as u16;
for (dst, &s) in scratch[..width].iter_mut().zip(y.iter()) {
*dst = (if BE { u16::from_be(s) } else { u16::from_le(s) }) & mask;
}
}
#[cfg(any(
feature = "yuv-444-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_yuv444_triple_resample<const SRC_BITS: u32>(
rgb_stream: &mut Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
rgb_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
luma_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
rgb_scratch_u16: &mut Vec<u16>,
luma_scratch_u16: &mut Vec<u16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgb_u8: impl FnOnce(&mut [u8]),
convert_rgb_u16: impl FnOnce(&mut [u16]),
deinterleave_y: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
const {
assert!(
SRC_BITS >= 8 && SRC_BITS <= 16,
"SRC_BITS must be in [8, 16]"
)
};
let need_luma = luma.is_some() || luma_u16.is_some();
let need_u8_color = rgb.is_some() || rgba.is_some() || hsv.is_some();
let need_u16_color = rgb_u16.is_some() || rgba_u16.is_some();
let expected = if need_luma {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if need_u8_color {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else if need_u16_color {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else {
return Ok(());
};
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
if need_u8_color && rgb_stream.is_none() {
*rgb_stream = Some({
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_u16_color && rgb_stream_u16.is_none() {
*rgb_stream_u16 = Some({
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_luma && luma_stream_u16.is_none() {
*luma_stream_u16 = Some({
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 1)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
packed_yuv444_triple_feed_emit::<_, _, _, SRC_BITS>(
rgb_stream.as_mut(),
rgb_stream_u16.as_mut(),
luma_stream_u16.as_mut(),
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
rgb_scratch,
rgb_scratch_u16,
luma_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
convert_rgb_u8,
convert_rgb_u16,
deinterleave_y,
)
}
#[cfg(any(
feature = "yuv-444-packed",
feature = "yuv-planar",
feature = "y2xx",
feature = "yuv-semi-planar"
))]
#[allow(clippy::too_many_arguments)]
fn packed_yuv444_triple_feed_emit<U8S, U16S, Y16S, const SRC_BITS: u32>(
rgb_stream: Option<&mut U8S>,
rgb_stream_u16: Option<&mut U16S>,
luma_stream_u16: Option<&mut Y16S>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
rgb_scratch_u16: &mut Vec<u16>,
luma_scratch_u16: &mut Vec<u16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgb_u8: impl FnOnce(&mut [u8]),
convert_rgb_u16: impl FnOnce(&mut [u16]),
deinterleave_y: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError>
where
U8S: crate::resample::RowResampler<u8>,
U16S: crate::resample::RowResampler<u16>,
Y16S: crate::resample::RowResampler<u16>,
{
let ow = plan.out_w();
let u8_color_row = if rgb_stream.is_some() {
let scratch = source_rgb_scratch(rgb_scratch, w, plan)?;
convert_rgb_u8(scratch);
Some(scratch)
} else {
None
};
let u16_color_row = if rgb_stream_u16.is_some() {
let scratch = source_rgb_u16_scratch(rgb_scratch_u16, w, plan)?;
convert_rgb_u16(scratch);
Some(scratch)
} else {
None
};
let y_row = if luma_stream_u16.is_some() {
let scratch = source_luma_u16_scratch(luma_scratch_u16, w, plan)?;
deinterleave_y(scratch);
Some(scratch)
} else {
None
};
if let Some(scratch) = u8_color_row {
let stream = rgb_stream.expect("staged only when present");
packed_rgb_resample_emit(
stream, plan, rgb, rgba, &mut None, &mut None, hsv, scratch, matrix, full_range, idx,
use_simd,
)?;
}
if let Some(scratch) = u16_color_row {
let stream = rgb_stream_u16.expect("staged only when present");
packed_rgb_u16_resample_emit::<SRC_BITS, false>(
stream,
plan,
&mut None,
&mut None,
&mut None,
rgb_u16,
rgba_u16,
&mut None,
&mut None,
scratch,
rgb_scratch,
matrix,
full_range,
idx,
use_simd,
)?;
}
if let Some(scratch) = y_row {
let stream = luma_stream_u16.expect("staged only when present");
let native_max: u16 = ((1u32 << SRC_BITS) - 1) as u16;
stream.feed_row(idx, scratch, use_simd, |oy, binned_y| {
if let Some(buf) = luma_u16.as_deref_mut() {
let out = &mut buf[oy * ow..(oy + 1) * ow];
if SRC_BITS < 16 {
for (dst, &src) in out.iter_mut().zip(binned_y) {
*dst = src.min(native_max);
}
} else {
out.copy_from_slice(binned_y);
}
}
if let Some(buf) = luma.as_deref_mut() {
if SRC_BITS < 16 {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(binned_y) {
*dst = (src.min(native_max) >> (SRC_BITS - 8)) as u8;
}
} else {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(binned_y) {
*dst = (src >> (SRC_BITS - 8)) as u8;
}
}
}
})?;
}
Ok(())
}
#[cfg(any(
feature = "yuv-444-packed",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_yuv444_triple_filter_resample<const SRC_BITS: u32>(
rgb_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
rgb_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
luma_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
rgb_scratch_u16: &mut Vec<u16>,
luma_scratch_u16: &mut Vec<u16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgb_u8: impl FnOnce(&mut [u8]),
convert_rgb_u16: impl FnOnce(&mut [u16]),
deinterleave_y: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError> {
const {
assert!(
SRC_BITS >= 8 && SRC_BITS <= 16,
"SRC_BITS must be in [8, 16]"
)
};
plan.ensure_single_kernel_filter()?;
let need_luma = luma.is_some() || luma_u16.is_some();
let need_u8_color = rgb.is_some() || rgba.is_some() || hsv.is_some();
let need_u16_color = rgb_u16.is_some() || rgba_u16.is_some();
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let expected = if need_luma {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if need_u8_color {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else if need_u16_color {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else {
return Ok(());
};
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
if need_u8_color && rgb_filter_stream.is_none() {
*rgb_filter_stream = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_u16_color && rgb_filter_stream_u16.is_none() {
*rgb_filter_stream_u16 = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_luma && luma_filter_stream_u16.is_none() {
*luma_filter_stream_u16 = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 1)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
packed_yuv444_triple_feed_emit::<_, _, _, SRC_BITS>(
rgb_filter_stream.as_mut(),
rgb_filter_stream_u16.as_mut(),
luma_filter_stream_u16.as_mut(),
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
rgb_scratch,
rgb_scratch_u16,
luma_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
convert_rgb_u8,
convert_rgb_u16,
deinterleave_y,
)
}
#[cfg(any(feature = "yuv-444-packed", feature = "yuva", feature = "gray"))]
#[allow(clippy::too_many_arguments)]
fn packed_yuva444_feed_emit<C8S, C16S, Y16S, const SRC_BITS: u32, const ZEXT_U16_COLOR: bool>(
rgba_stream: Option<&mut C8S>,
rgba_stream_u16: Option<&mut C16S>,
luma_stream_u16: Option<&mut Y16S>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<u8>,
rgb_drop_scratch: &mut Vec<u8>,
rgba_scratch_u16: &mut Vec<u16>,
color_scratch_u16: &mut Vec<u16>,
luma_scratch_u16: &mut Vec<u16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
premult: bool,
convert_rgba_u8: impl FnOnce(&mut [u8]),
convert_rgba_u16: impl FnOnce(&mut [u16]),
deinterleave_y: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError>
where
C8S: crate::resample::RowResampler<u8>,
C16S: crate::resample::RowResampler<u16>,
Y16S: crate::resample::RowResampler<u16>,
{
let max_u16: u32 = (1u32 << SRC_BITS) - 1;
let native_max: u16 = max_u16 as u16;
let ow = plan.out_w();
let need_colour_u8 = rgba_stream.is_some();
let need_colour_u16 = rgba_stream_u16.is_some();
let zext_u16 = ZEXT_U16_COLOR && (rgb_u16.is_some() || rgba_u16.is_some());
debug_assert!(
!ZEXT_U16_COLOR || !need_colour_u16,
"ZEXT_U16_COLOR derives the u16 colour from the u8 binning, so no u16 colour stream is fed"
);
let need_rgb_drop = rgb.is_some() || hsv.is_some() || (zext_u16 && rgb_u16.is_some());
let rgb_drop: &mut [u8] = if need_rgb_drop {
source_rgb_scratch(rgb_drop_scratch, ow, plan)?
} else {
&mut []
};
let colour_u16: &mut [u16] = if need_colour_u16 {
source_rgba_u16_scratch(color_scratch_u16, ow, plan)?
} else {
&mut []
};
let src_rgba_u8 = if need_colour_u8 {
let scratch = source_rgba_scratch(rgba_scratch, w, plan)?;
convert_rgba_u8(scratch);
if premult {
premultiply_rgba_row_in_place::<u8>(scratch, w, 255);
}
Some(scratch)
} else {
None
};
let src_rgba_u16 = if need_colour_u16 {
let scratch = source_rgba_u16_scratch(rgba_scratch_u16, w, plan)?;
convert_rgba_u16(scratch);
if premult {
premultiply_rgba_row_in_place::<u16>(scratch, w, max_u16);
}
Some(scratch)
} else {
None
};
let y_row = if luma_stream_u16.is_some() {
let scratch = source_luma_u16_scratch(luma_scratch_u16, w, plan)?;
deinterleave_y(scratch);
Some(scratch)
} else {
None
};
if let Some(scratch) = src_rgba_u8 {
let stream = rgba_stream.expect("staged only when present");
stream.feed_row(idx, scratch, use_simd, |oy, binned| {
if let Some(buf) = rgba.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
if premult {
unpremultiply_binned_rgba_into::<u8>(binned, dst, ow, 255);
} else {
dst.copy_from_slice(binned);
}
}
if zext_u16 && let Some(buf) = rgba_u16.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
for (d, &s) in dst.iter_mut().zip(binned.iter()) {
*d = s as u16;
}
}
if need_rgb_drop {
let nrow = &mut rgb_drop[..3 * ow];
if premult {
unpremultiply_binned_rgb_into::<u8>(binned, nrow, ow, 255);
} else {
drop_alpha_rgba_to_rgb_row(binned, nrow, ow);
}
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if zext_u16 && let Some(buf) = rgb_u16.as_deref_mut() {
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
for (d, &s) in dst.iter_mut().zip(nrow.iter()) {
*d = s as u16;
}
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
}
if let Some(scratch) = src_rgba_u16 {
let stream = rgba_stream_u16.expect("staged only when present");
stream.feed_row(idx, scratch, use_simd, |oy, binned| {
let colour = &mut colour_u16[..4 * ow];
if premult {
unpremultiply_binned_rgba_into::<u16>(binned, colour, ow, max_u16);
} else if SRC_BITS < 16 {
for (d, &s) in colour.iter_mut().zip(binned.iter()) {
*d = s.min(native_max);
}
} else {
colour.copy_from_slice(binned);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
buf[oy * 4 * ow..(oy + 1) * 4 * ow].copy_from_slice(colour);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
drop_alpha_rgba_to_rgb_row(colour, dst, ow);
}
})?;
}
if let Some(y_row) = y_row {
let stream = luma_stream_u16.expect("staged only when present");
stream.feed_row(idx, y_row, use_simd, |oy, binned_y| {
if let Some(buf) = luma_u16.as_deref_mut() {
let out = &mut buf[oy * ow..(oy + 1) * ow];
if SRC_BITS < 16 {
for (dst, &y) in out.iter_mut().zip(binned_y) {
*dst = y.min(native_max);
}
} else {
out.copy_from_slice(binned_y);
}
}
if let Some(buf) = luma.as_deref_mut() {
if SRC_BITS < 16 {
for (dst, &y) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(binned_y) {
*dst = (y.min(native_max) >> (SRC_BITS - 8)) as u8;
}
} else {
for (dst, &y) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(binned_y) {
*dst = (y >> (SRC_BITS - 8)) as u8;
}
}
}
})?;
}
Ok(())
}
#[cfg(any(feature = "yuv-444-packed", feature = "yuva"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_yuva444_resample<const SRC_BITS: u32>(
rgba_stream: &mut Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
rgba_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
luma_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<u8>,
rgb_drop_scratch: &mut Vec<u8>,
rgba_scratch_u16: &mut Vec<u16>,
color_scratch_u16: &mut Vec<u16>,
luma_scratch_u16: &mut Vec<u16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
alpha_mode: AlphaMode,
convert_rgba_u8: impl FnOnce(&mut [u8]),
convert_rgba_u16: impl FnOnce(&mut [u16]),
deinterleave_y: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
const {
assert!(
SRC_BITS >= 8 && SRC_BITS <= 16,
"SRC_BITS must be in [8, 16] for packed YUVA 4:4:4"
)
};
let premult = alpha_mode.is_premultiplied();
let need_colour_u8 = rgb.is_some() || rgba.is_some() || hsv.is_some();
let need_colour_u16 = rgb_u16.is_some() || rgba_u16.is_some();
let need_luma = luma.is_some() || luma_u16.is_some();
let expected = if need_colour_u8 {
rgba_stream.as_ref().map_or(0, |s| s.next_y())
} else if need_colour_u16 {
rgba_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if need_luma {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else {
return Ok(());
};
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
if need_colour_u8 && rgba_stream.is_none() {
*rgba_stream = Some({
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 4)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_colour_u16 && rgba_stream_u16.is_none() {
*rgba_stream_u16 = Some({
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 4)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_luma && luma_stream_u16.is_none() {
*luma_stream_u16 = Some({
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 1)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
packed_yuva444_feed_emit::<_, _, _, SRC_BITS, false>(
rgba_stream.as_mut(),
rgba_stream_u16.as_mut(),
luma_stream_u16.as_mut(),
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
rgba_scratch,
rgb_drop_scratch,
rgba_scratch_u16,
color_scratch_u16,
luma_scratch_u16,
w,
plan,
idx,
use_simd,
premult,
convert_rgba_u8,
convert_rgba_u16,
deinterleave_y,
)
}
#[cfg(any(feature = "yuv-444-packed", feature = "yuva", feature = "gray"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_yuva444_filter_resample<
const SRC_BITS: u32,
const NATIVE_LUMA_U8: bool,
const ZEXT_U16_COLOR: bool,
>(
rgba_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
rgba_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
luma_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
luma_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<u8>,
rgb_drop_scratch: &mut Vec<u8>,
rgba_scratch_u16: &mut Vec<u16>,
color_scratch_u16: &mut Vec<u16>,
luma_scratch_u16: &mut Vec<u16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
y_row_u8: &[u8],
y_u8_scratch: Option<&mut Vec<u8>>,
convert_rgba_u8: impl FnOnce(&mut [u8]),
convert_rgba_u16: impl FnOnce(&mut [u16]),
deinterleave_y: impl FnOnce(&mut [u16]),
deinterleave_y_u8: impl FnOnce(&mut [u8]),
) -> Result<(), MixedSinkerError> {
const {
assert!(
SRC_BITS >= 8 && SRC_BITS <= 16,
"SRC_BITS must be in [8, 16] for packed YUVA 4:4:4"
)
};
const {
assert!(
!NATIVE_LUMA_U8 || SRC_BITS == 8,
"NATIVE_LUMA_U8 (the u8 native-Y luma stream) is only valid for an 8-bit source"
)
};
const {
assert!(
!ZEXT_U16_COLOR || SRC_BITS == 8,
"ZEXT_U16_COLOR (u16 colour = zero-extended u8 colour) is only valid for an 8-bit source"
)
};
plan.ensure_single_kernel_filter()?;
let zext_u16_color = ZEXT_U16_COLOR && (rgb_u16.is_some() || rgba_u16.is_some());
let need_colour_u8 = rgb.is_some() || rgba.is_some() || hsv.is_some() || zext_u16_color;
let need_colour_u16 = (rgb_u16.is_some() || rgba_u16.is_some()) && !ZEXT_U16_COLOR;
let need_luma = luma.is_some() || luma_u16.is_some();
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let expected = if need_colour_u8 {
rgba_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else if need_colour_u16 {
rgba_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if need_luma {
if NATIVE_LUMA_U8 {
luma_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
}
} else {
return Ok(());
};
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
if need_colour_u8 && rgba_filter_stream.is_none() {
*rgba_filter_stream = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 4)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_colour_u16 && rgba_filter_stream_u16.is_none() {
*rgba_filter_stream_u16 = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 4)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_luma {
if NATIVE_LUMA_U8 {
if luma_filter_stream.is_none() {
*luma_filter_stream = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 1)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
} else if luma_filter_stream_u16.is_none() {
*luma_filter_stream_u16 = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 1)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
}
let native_y_u8: Option<&[u8]> = if NATIVE_LUMA_U8 && need_luma {
if y_row_u8.is_empty() {
let scratch = source_luma_scratch(
y_u8_scratch.expect("packed NATIVE_LUMA_U8 caller supplies a u8 Y scratch"),
w,
plan,
)?;
deinterleave_y_u8(scratch);
Some(&scratch[..w])
} else {
Some(&y_row_u8[..w])
}
} else {
None
};
if NATIVE_LUMA_U8 {
packed_yuva444_feed_emit::<_, _, crate::resample::FilterStream<u16>, SRC_BITS, ZEXT_U16_COLOR>(
rgba_filter_stream.as_mut(),
rgba_filter_stream_u16.as_mut(),
None,
rgb,
rgba,
rgb_u16,
rgba_u16,
&mut None,
&mut None,
hsv,
rgba_scratch,
rgb_drop_scratch,
rgba_scratch_u16,
color_scratch_u16,
luma_scratch_u16,
w,
plan,
idx,
use_simd,
false,
convert_rgba_u8,
convert_rgba_u16,
deinterleave_y,
)?;
if let Some(y_src) = native_y_u8 {
let stream = luma_filter_stream
.as_mut()
.expect("created in the preflight when need_luma");
let ow = plan.out_w();
stream.feed_row(idx, y_src, use_simd, |oy, out_row| {
if let Some(buf) = luma.as_deref_mut() {
buf[oy * ow..(oy + 1) * ow].copy_from_slice(out_row);
}
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &src) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(out_row) {
*dst = src as u16;
}
}
})?;
}
Ok(())
} else {
packed_yuva444_feed_emit::<_, _, _, SRC_BITS, ZEXT_U16_COLOR>(
rgba_filter_stream.as_mut(),
rgba_filter_stream_u16.as_mut(),
luma_filter_stream_u16.as_mut(),
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
rgba_scratch,
rgb_drop_scratch,
rgba_scratch_u16,
color_scratch_u16,
luma_scratch_u16,
w,
plan,
idx,
use_simd,
false,
convert_rgba_u8,
convert_rgba_u16,
deinterleave_y,
)
}
}
#[cfg(feature = "yuva")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn reset_high_bit_yuva_streams<F: SourceFormat, R>(sink: &mut MixedSinker<'_, F, R>) {
if let Some(stream) = sink.rgba_stream.as_mut() {
stream.reset();
}
if let Some(stream) = sink.rgba_stream_u16.as_mut() {
stream.reset();
}
if let Some(stream) = sink.luma_stream_u16.as_mut() {
stream.reset();
}
if let Some(stream) = sink.rgba_filter_stream.as_mut() {
stream.reset();
}
if let Some(stream) = sink.rgba_filter_stream_u16.as_mut() {
stream.reset();
}
if let Some(stream) = sink.luma_filter_stream_u16.as_mut() {
stream.reset();
}
sink.resample_outputs = None;
sink.frozen_chroma_centered = None;
sink.frozen_chroma_bottom_v = None;
sink.chroma_prev_row = None;
sink.frozen_chroma_top_v = None;
sink.chroma_top_pending = None;
sink.frozen_alpha_mode = Some(sink.alpha_mode);
}
#[cfg(all(test, feature = "std", feature = "y2xx", feature = "yuv-planar"))]
std::thread_local! {
static FORCE_Y2XX_ALLOC_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "y2xx", feature = "yuv-planar"))]
pub(crate) fn arm_y2xx_alloc_failure() {
FORCE_Y2XX_ALLOC_FAILURE.with(|f| f.set(true));
}
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn grow_y2xx_depack_scratch(
scratch: &mut Vec<u16>,
len: usize,
fail: bool,
w: usize,
h: usize,
plan: &crate::resample::ResamplePlan,
) -> Result<(), MixedSinkerError> {
let _ = fail;
if scratch.len() < len {
#[cfg(all(test, feature = "std", feature = "y2xx", feature = "yuv-planar"))]
if fail && FORCE_Y2XX_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(
crate::resample::ResampleError::AllocationFailed(crate::resample::PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)),
));
}
scratch
.try_reserve_exact(len - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
))
})?;
scratch.resize(len, 0);
}
Ok(())
}
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
#[allow(clippy::too_many_arguments)]
fn y2xx_process_native<const BITS: u32, const BE: bool>(
plan: &crate::resample::ResamplePlan,
native_planar_u16: &mut Option<std::boxed::Box<NativePlanarYuvU16>>,
y_scratch: &mut Vec<u16>,
u_scratch: &mut Vec<u16>,
v_scratch: &mut Vec<u16>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
rgb_scratch_u16: &mut Vec<u16>,
packed: &[u16],
chroma_h_phase: f64,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
const {
assert!(
BITS > 8 && BITS <= 16,
"BITS must be in (8, 16] for high-bit packed 4:2:2 Y2xx"
)
};
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
let cw = w / 2;
if !native_planar_hb_preflight_check_only(
native_planar_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
grow_y2xx_depack_scratch(y_scratch, w, true, w, h, plan)?;
if need_color {
grow_y2xx_depack_scratch(u_scratch, cw, false, w, h, plan)?;
grow_y2xx_depack_scratch(v_scratch, cw, false, w, h, plan)?;
}
let depack = |s: u16| -> u16 {
let logical = if BE { u16::from_be(s) } else { u16::from_le(s) };
logical >> (16 - BITS)
};
for (i, group) in packed.as_chunks::<4>().0.iter().enumerate() {
y_scratch[i * 2] = depack(group[0]);
y_scratch[i * 2 + 1] = depack(group[2]);
}
if need_color {
for (i, group) in packed.as_chunks::<4>().0.iter().enumerate() {
u_scratch[i] = depack(group[1]);
v_scratch[i] = depack(group[3]);
}
}
let (u_plane, v_plane): (&[u16], &[u16]) = if need_color {
(&u_scratch[..cw], &v_scratch[..cw])
} else {
(&[], &[])
};
planar_high_bit_native::yuv_planar16_process_native::<BITS, HIGH_BIT_HOST_NATIVE_BE>(
plan,
native_planar_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
rgb_scratch,
rgb_scratch_u16,
&y_scratch[..w],
u_plane,
v_plane,
matrix,
full_range,
idx,
w,
h,
1,
cw,
|| {
crate::resample::ResamplePlan::area_chroma_422(
cw,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
0.0,
)
},
use_simd,
)
}
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
fn reserve_y2xx_center_planes(
y_full: &mut Vec<u16>,
u_half: &mut Vec<u16>,
v_half: &mut Vec<u16>,
width: usize,
height: usize,
) -> Result<(), MixedSinkerError> {
let grow = |scratch: &mut Vec<u16>, len: usize| -> Result<(), MixedSinkerError> {
if scratch.len() < len {
scratch
.try_reserve_exact(len - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(width, height, width, height),
))
})?;
scratch.resize(len, 0);
}
Ok(())
};
grow(y_full, width)?;
let cw = width / 2;
grow(u_half, cw)?;
grow(v_half, cw)?;
Ok(())
}
#[cfg(all(feature = "y2xx", feature = "yuv-planar"))]
#[allow(clippy::too_many_arguments)]
fn y2xx_center_reconstruct<'c, const BITS: u32>(
packed: &[u16],
y_full: &mut Vec<u16>,
u_half: &mut Vec<u16>,
v_half: &mut Vec<u16>,
chroma_full: &'c mut Vec<u16>,
w: usize,
h: usize,
be: bool,
use_simd: bool,
) -> Result<(&'c [u16], &'c [u16]), MixedSinkerError> {
let cw = w / 2;
subsampled_4_2_0_high_bit::reserve_420_chroma_full_u16(chroma_full, w, h)?;
reserve_y2xx_center_planes(y_full, u_half, v_half, w, h)?;
let repack = |wire: u16| -> u16 {
let logical = (if be {
u16::from_be(wire)
} else {
u16::from_le(wire)
}) >> (16 - BITS);
if be { logical.to_be() } else { logical.to_le() }
};
for (i, group) in packed.as_chunks::<4>().0.iter().take(cw).enumerate() {
y_full[2 * i] = repack(group[0]);
y_full[2 * i + 1] = repack(group[2]);
u_half[i] = repack(group[1]);
v_half[i] = repack(group[3]);
}
Ok(reconstruct_chroma(
ChromaU16::<BITS> { big_endian: be },
chroma_full,
&u_half[..cw],
&v_half[..cw],
w,
use_simd,
))
}
#[cfg(all(test, feature = "std", feature = "v210", feature = "yuv-planar"))]
std::thread_local! {
static FORCE_V210_ALLOC_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "std", feature = "v210", feature = "yuv-planar"))]
pub(crate) fn arm_v210_alloc_failure() {
FORCE_V210_ALLOC_FAILURE.with(|f| f.set(true));
}
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn grow_v210_depack_scratch(
scratch: &mut Vec<u16>,
len: usize,
fail: bool,
w: usize,
h: usize,
plan: &crate::resample::ResamplePlan,
) -> Result<(), MixedSinkerError> {
let _ = fail;
if scratch.len() < len {
#[cfg(all(test, feature = "std", feature = "v210", feature = "yuv-planar"))]
if fail && FORCE_V210_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(
crate::resample::ResampleError::AllocationFailed(crate::resample::PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)),
));
}
scratch
.try_reserve_exact(len - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
))
})?;
scratch.resize(len, 0);
}
Ok(())
}
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
#[allow(clippy::too_many_arguments)]
fn v210_process_native<const BE: bool>(
plan: &crate::resample::ResamplePlan,
native_planar_u16: &mut Option<std::boxed::Box<NativePlanarYuvU16>>,
y_scratch: &mut Vec<u16>,
u_scratch: &mut Vec<u16>,
v_scratch: &mut Vec<u16>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
rgb_scratch_u16: &mut Vec<u16>,
packed: &[u8],
chroma_h_phase: f64,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
const BITS: u32 = 10;
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
let cw = w / 2;
if !native_planar_hb_preflight_check_only(
native_planar_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
grow_v210_depack_scratch(y_scratch, w, true, w, h, plan)?;
if need_color {
grow_v210_depack_scratch(u_scratch, cw, false, w, h, plan)?;
grow_v210_depack_scratch(v_scratch, cw, false, w, h, plan)?;
}
let load = |word: &[u8], lane: usize| -> u32 {
let bytes = [
word[lane * 4],
word[lane * 4 + 1],
word[lane * 4 + 2],
word[lane * 4 + 3],
];
if BE {
u32::from_be_bytes(bytes)
} else {
u32::from_le_bytes(bytes)
}
};
for (wi, word) in packed.as_chunks::<16>().0.iter().enumerate() {
let px = wi * 6;
let w0 = load(word, 0);
let w1 = load(word, 1);
let w2 = load(word, 2);
let w3 = load(word, 3);
let ys = [
((w0 >> 10) & 0x3FF) as u16, (w1 & 0x3FF) as u16, ((w1 >> 20) & 0x3FF) as u16, ((w2 >> 10) & 0x3FF) as u16, (w3 & 0x3FF) as u16, ((w3 >> 20) & 0x3FF) as u16, ];
for (k, &y) in ys.iter().enumerate() {
if px + k < w {
y_scratch[px + k] = y;
}
}
if need_color {
let cu = px / 2;
let us = [
(w0 & 0x3FF) as u16, ((w1 >> 10) & 0x3FF) as u16, ((w2 >> 20) & 0x3FF) as u16, ];
let vs = [
((w0 >> 20) & 0x3FF) as u16, (w2 & 0x3FF) as u16, ((w3 >> 10) & 0x3FF) as u16, ];
for k in 0..3 {
if cu + k < cw {
u_scratch[cu + k] = us[k];
v_scratch[cu + k] = vs[k];
}
}
}
}
let (u_plane, v_plane): (&[u16], &[u16]) = if need_color {
(&u_scratch[..cw], &v_scratch[..cw])
} else {
(&[], &[])
};
planar_high_bit_native::yuv_planar16_process_native::<BITS, HIGH_BIT_HOST_NATIVE_BE>(
plan,
native_planar_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
rgb_scratch,
rgb_scratch_u16,
&y_scratch[..w],
u_plane,
v_plane,
matrix,
full_range,
idx,
w,
h,
1,
cw,
|| {
crate::resample::ResamplePlan::area_chroma_422(
cw,
h,
plan.out_w(),
plan.out_h(),
chroma_h_phase,
0.0,
)
},
use_simd,
)
}
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
fn reserve_v210_center_planes(
y_full: &mut Vec<u16>,
u_half: &mut Vec<u16>,
v_half: &mut Vec<u16>,
width: usize,
height: usize,
) -> Result<(), MixedSinkerError> {
let grow = |scratch: &mut Vec<u16>, len: usize| -> Result<(), MixedSinkerError> {
if scratch.len() < len {
scratch
.try_reserve_exact(len - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(width, height, width, height),
))
})?;
scratch.resize(len, 0);
}
Ok(())
};
grow(y_full, width)?;
let cw = width / 2;
grow(u_half, cw)?;
grow(v_half, cw)?;
Ok(())
}
#[cfg(all(feature = "v210", feature = "yuv-planar"))]
#[allow(clippy::too_many_arguments)]
fn v210_center_reconstruct<'c>(
packed: &[u8],
y_full: &mut Vec<u16>,
u_half: &mut Vec<u16>,
v_half: &mut Vec<u16>,
chroma_full: &'c mut Vec<u16>,
w: usize,
h: usize,
be: bool,
use_simd: bool,
) -> Result<(&'c [u16], &'c [u16]), MixedSinkerError> {
const BITS: u32 = 10;
let cw = w / 2;
subsampled_4_2_0_high_bit::reserve_420_chroma_full_u16(chroma_full, w, h)?;
reserve_v210_center_planes(y_full, u_half, v_half, w, h)?;
let encode = |logical: u16| -> u16 { if be { logical.to_be() } else { logical.to_le() } };
let load = |word: &[u8], lane: usize| -> u32 {
let bytes = [
word[lane * 4],
word[lane * 4 + 1],
word[lane * 4 + 2],
word[lane * 4 + 3],
];
if be {
u32::from_be_bytes(bytes)
} else {
u32::from_le_bytes(bytes)
}
};
for (wi, word) in packed.as_chunks::<16>().0.iter().enumerate() {
let px = wi * 6;
let w0 = load(word, 0);
let w1 = load(word, 1);
let w2 = load(word, 2);
let w3 = load(word, 3);
let ys = [
((w0 >> 10) & 0x3FF) as u16, (w1 & 0x3FF) as u16, ((w1 >> 20) & 0x3FF) as u16, ((w2 >> 10) & 0x3FF) as u16, (w3 & 0x3FF) as u16, ((w3 >> 20) & 0x3FF) as u16, ];
for (k, &y) in ys.iter().enumerate() {
if px + k < w {
y_full[px + k] = encode(y);
}
}
let cu = px / 2;
let us = [
(w0 & 0x3FF) as u16, ((w1 >> 10) & 0x3FF) as u16, ((w2 >> 20) & 0x3FF) as u16, ];
let vs = [
((w0 >> 20) & 0x3FF) as u16, (w2 & 0x3FF) as u16, ((w3 >> 10) & 0x3FF) as u16, ];
for k in 0..3 {
if cu + k < cw {
u_half[cu + k] = encode(us[k]);
v_half[cu + k] = encode(vs[k]);
}
}
}
Ok(reconstruct_chroma(
ChromaU16::<BITS> { big_endian: be },
chroma_full,
&u_half[..cw],
&v_half[..cw],
w,
use_simd,
))
}
#[cfg(all(
any(feature = "y2xx", feature = "yuv-444-packed", feature = "v210"),
feature = "yuv-planar"
))]
const HIGH_BIT_HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
#[cfg(all(
test,
feature = "std",
feature = "yuv-444-packed",
feature = "yuv-planar"
))]
std::thread_local! {
static FORCE_PACKED_444_ALLOC_FAILURE: core::cell::Cell<bool> =
const { core::cell::Cell::new(false) };
}
#[cfg(all(
test,
feature = "std",
feature = "yuv-444-packed",
feature = "yuv-planar"
))]
pub(crate) fn arm_packed_444_alloc_failure() {
FORCE_PACKED_444_ALLOC_FAILURE.with(|f| f.set(true));
}
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn grow_packed_444_u8_scratch(
scratch: &mut Vec<u8>,
len: usize,
fail: bool,
w: usize,
h: usize,
plan: &crate::resample::ResamplePlan,
) -> Result<(), MixedSinkerError> {
let _ = fail;
if scratch.len() < len {
#[cfg(all(
test,
feature = "std",
feature = "yuv-444-packed",
feature = "yuv-planar"
))]
if fail && FORCE_PACKED_444_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(
crate::resample::ResampleError::AllocationFailed(crate::resample::PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)),
));
}
scratch
.try_reserve_exact(len - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
))
})?;
scratch.resize(len, 0);
}
Ok(())
}
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn grow_packed_444_u16_scratch(
scratch: &mut Vec<u16>,
len: usize,
fail: bool,
w: usize,
h: usize,
plan: &crate::resample::ResamplePlan,
) -> Result<(), MixedSinkerError> {
let _ = fail;
if scratch.len() < len {
#[cfg(all(
test,
feature = "std",
feature = "yuv-444-packed",
feature = "yuv-planar"
))]
if fail && FORCE_PACKED_444_ALLOC_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(
crate::resample::ResampleError::AllocationFailed(crate::resample::PlanGeometry::new(
w,
h,
plan.out_w(),
plan.out_h(),
)),
));
}
scratch
.try_reserve_exact(len - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(w, h, plan.out_w(), plan.out_h()),
))
})?;
scratch.resize(len, 0);
}
Ok(())
}
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
#[allow(clippy::too_many_arguments)]
fn packed_vuyx_process_native(
plan: &crate::resample::ResamplePlan,
native_planar: &mut Option<std::boxed::Box<planar_8bit::NativePlanarYuv>>,
y_scratch: &mut Vec<u8>,
u_scratch: &mut Vec<u8>,
v_scratch: &mut Vec<u8>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
packed: &[u8],
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || rgba.is_some() || hsv.is_some();
if !planar_8bit::native_planar_preflight_check_only(
native_planar,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
grow_packed_444_u8_scratch(y_scratch, w, true, w, h, plan)?;
for (i, group) in packed.as_chunks::<4>().0.iter().enumerate() {
y_scratch[i] = group[2];
}
if need_color {
grow_packed_444_u8_scratch(u_scratch, w, false, w, h, plan)?;
grow_packed_444_u8_scratch(v_scratch, w, false, w, h, plan)?;
for (i, group) in packed.as_chunks::<4>().0.iter().enumerate() {
u_scratch[i] = group[1];
v_scratch[i] = group[0];
}
}
let (u_plane, v_plane): (&[u8], &[u8]) = if need_color {
(&u_scratch[..w], &v_scratch[..w])
} else {
(&[], &[])
};
planar_8bit::yuv_planar_process_native(
plan,
native_planar,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
&y_scratch[..w],
u_plane,
v_plane,
matrix,
full_range,
idx,
w,
h,
1,
|| crate::resample::ResamplePlan::area(w, h, plan.out_w(), plan.out_h()),
use_simd,
)
}
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
#[allow(clippy::too_many_arguments)]
fn packed_vyu444_process_native(
plan: &crate::resample::ResamplePlan,
native_planar: &mut Option<std::boxed::Box<planar_8bit::NativePlanarYuv>>,
y_scratch: &mut Vec<u8>,
u_scratch: &mut Vec<u8>,
v_scratch: &mut Vec<u8>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
packed: &[u8],
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color = rgb.is_some() || rgba.is_some() || hsv.is_some();
if !planar_8bit::native_planar_preflight_check_only(
native_planar,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
grow_packed_444_u8_scratch(y_scratch, w, true, w, h, plan)?;
for (i, group) in packed.as_chunks::<3>().0.iter().enumerate() {
y_scratch[i] = group[1]; }
if need_color {
grow_packed_444_u8_scratch(u_scratch, w, false, w, h, plan)?;
grow_packed_444_u8_scratch(v_scratch, w, false, w, h, plan)?;
for (i, group) in packed.as_chunks::<3>().0.iter().enumerate() {
u_scratch[i] = group[2]; v_scratch[i] = group[0]; }
}
let (u_plane, v_plane): (&[u8], &[u8]) = if need_color {
(&u_scratch[..w], &v_scratch[..w])
} else {
(&[], &[])
};
planar_8bit::yuv_planar_process_native(
plan,
native_planar,
resample_outputs,
rgb,
rgba,
luma,
luma_u16,
hsv,
rgb_scratch,
&y_scratch[..w],
u_plane,
v_plane,
matrix,
full_range,
idx,
w,
h,
1,
|| crate::resample::ResamplePlan::area(w, h, plan.out_w(), plan.out_h()),
use_simd,
)
}
#[cfg(all(feature = "yuv-444-packed", feature = "yuv-planar"))]
#[allow(clippy::too_many_arguments)]
fn packed_yuv444_hb_process_native<const BITS: u32>(
plan: &crate::resample::ResamplePlan,
native_planar_u16: &mut Option<std::boxed::Box<NativePlanarYuvU16>>,
y_scratch: &mut Vec<u16>,
u_scratch: &mut Vec<u16>,
v_scratch: &mut Vec<u16>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch: &mut Vec<u8>,
rgb_scratch_u16: &mut Vec<u16>,
fill_y: impl FnOnce(&mut [u16]),
fill_uv: impl FnOnce(&mut [u16], &mut [u16]),
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
w: usize,
h: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
const {
assert!(
BITS > 8 && BITS <= 16,
"BITS must be in (8, 16] for high-bit packed 4:4:4 YUV"
)
};
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color =
rgb.is_some() || rgba.is_some() || hsv.is_some() || rgb_u16.is_some() || rgba_u16.is_some();
if !native_planar_hb_preflight_check_only(
native_planar_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
idx,
need_luma,
need_color,
)? {
return Ok(());
}
grow_packed_444_u16_scratch(y_scratch, w, true, w, h, plan)?;
if need_color {
grow_packed_444_u16_scratch(u_scratch, w, false, w, h, plan)?;
grow_packed_444_u16_scratch(v_scratch, w, false, w, h, plan)?;
}
fill_y(&mut y_scratch[..w]);
if need_color {
fill_uv(&mut u_scratch[..w], &mut v_scratch[..w]);
}
let (u_plane, v_plane): (&[u16], &[u16]) = if need_color {
(&u_scratch[..w], &v_scratch[..w])
} else {
(&[], &[])
};
planar_high_bit_native::yuv_planar16_process_native::<BITS, HIGH_BIT_HOST_NATIVE_BE>(
plan,
native_planar_u16,
resample_outputs,
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
rgb_scratch,
rgb_scratch_u16,
&y_scratch[..w],
u_plane,
v_plane,
matrix,
full_range,
idx,
w,
h,
1,
w,
|| crate::resample::ResamplePlan::area(w, h, plan.out_w(), plan.out_h()),
use_simd,
)
}
#[cfg(any(
feature = "y2xx",
feature = "v210",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_yuv422_triple_resample<const SRC_BITS: u32>(
luma_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
rgb_stream: &mut Option<std::boxed::Box<crate::resample::AreaStream<u8>>>,
rgb_stream_u16: &mut Option<std::boxed::Box<crate::resample::AreaStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
luma_scratch_u16: &mut Vec<u16>,
rgb_scratch: &mut Vec<u8>,
rgb_scratch_u16: &mut Vec<u16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
deinterleave_y_u16: impl FnOnce(&mut [u16]),
convert_rgb_u8: impl FnOnce(&mut [u8]),
convert_rgb_u16: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
const {
assert!(
SRC_BITS > 8 && SRC_BITS <= 16,
"SRC_BITS must be in (8, 16] for high-bit packed 4:2:2 YUV"
)
};
let ow = plan.out_w();
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color_u8 = rgb.is_some() || rgba.is_some() || hsv.is_some();
let need_color_u16 = rgb_u16.is_some() || rgba_u16.is_some();
let expected = if need_luma {
luma_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if need_color_u8 {
rgb_stream.as_ref().map_or(0, |s| s.next_y())
} else if need_color_u16 {
rgb_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else {
return Ok(());
};
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
if need_luma && luma_stream_u16.is_none() {
*luma_stream_u16 = Some({
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 1)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_color_u8 && rgb_stream.is_none() {
*rgb_stream = Some({
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_color_u16 && rgb_stream_u16.is_none() {
*rgb_stream_u16 = Some({
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
let luma_row = if need_luma {
let scratch = source_luma_u16_scratch(luma_scratch_u16, w, plan)?;
deinterleave_y_u16(scratch);
Some(scratch)
} else {
None
};
let color_u8_row = if need_color_u8 {
let scratch = source_rgb_scratch(rgb_scratch, w, plan)?;
convert_rgb_u8(scratch);
Some(scratch)
} else {
None
};
let color_u16_row = if need_color_u16 {
let scratch = source_rgb_u16_scratch(rgb_scratch_u16, w, plan)?;
convert_rgb_u16(scratch);
Some(scratch)
} else {
None
};
if let Some(y_row) = luma_row {
let native_max: u16 = ((1u32 << SRC_BITS) - 1) as u16;
let stream = luma_stream_u16.as_mut().expect("created in the preflight");
stream.feed_row(idx, y_row, use_simd, |oy, binned_y| {
if let Some(buf) = luma_u16.as_deref_mut() {
for (dst, &y) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(binned_y) {
*dst = y.min(native_max);
}
}
if let Some(buf) = luma.as_deref_mut() {
for (dst, &y) in buf[oy * ow..(oy + 1) * ow].iter_mut().zip(binned_y) {
*dst = (y.min(native_max) >> (SRC_BITS - 8)) as u8;
}
}
})?;
}
if let Some(scratch) = color_u8_row {
let stream = rgb_stream.as_mut().expect("created in the preflight");
packed_rgb_resample_emit(
stream, plan, rgb, rgba, &mut None, &mut None, hsv, scratch, matrix, full_range, idx,
use_simd,
)?;
}
if let Some(scratch) = color_u16_row {
let stream = rgb_stream_u16.as_mut().expect("created in the preflight");
packed_rgb_u16_resample_emit::<SRC_BITS, false>(
stream,
plan,
&mut None,
&mut None,
&mut None,
rgb_u16,
rgba_u16,
&mut None,
&mut None,
scratch,
rgb_scratch,
matrix,
full_range,
idx,
use_simd,
)?;
}
Ok(())
}
#[cfg(any(feature = "y2xx", feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_yuv422_triple_filter_resample<const SRC_BITS: u32>(
luma_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
rgb_filter_stream: &mut Option<std::boxed::Box<crate::resample::FilterStream<u8>>>,
rgb_filter_stream_u16: &mut Option<std::boxed::Box<crate::resample::FilterStream<u16>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
luma_scratch_u16: &mut Vec<u16>,
rgb_scratch: &mut Vec<u8>,
rgb_scratch_u16: &mut Vec<u16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
deinterleave_y_u16: impl FnOnce(&mut [u16]),
convert_rgb_u8: impl FnOnce(&mut [u8]),
convert_rgb_u16: impl FnOnce(&mut [u16]),
) -> Result<(), MixedSinkerError> {
const {
assert!(
SRC_BITS > 8 && SRC_BITS <= 16,
"SRC_BITS must be in (8, 16] for high-bit packed 4:2:2 YUV"
)
};
plan.ensure_single_kernel_filter()?;
let need_luma = luma.is_some() || luma_u16.is_some();
let need_color_u8 = rgb.is_some() || rgba.is_some() || hsv.is_some();
let need_color_u16 = rgb_u16.is_some() || rgba_u16.is_some();
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let expected = if need_luma {
luma_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else if need_color_u8 {
rgb_filter_stream.as_ref().map_or(0, |s| s.next_y())
} else if need_color_u16 {
rgb_filter_stream_u16.as_ref().map_or(0, |s| s.next_y())
} else {
return Ok(());
};
if resample_outputs.is_none() && expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
&None,
&None,
&None,
&None,
&None,
hsv,
&None,
idx,
)?;
if expected != idx {
return Err(MixedSinkerError::Resample(ResampleError::OutOfSequenceRow(
crate::resample::OutOfSequenceRow::new(expected, idx),
)));
}
if need_luma && luma_filter_stream_u16.is_none() {
*luma_filter_stream_u16 = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 1)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_color_u8 && rgb_filter_stream.is_none() {
*rgb_filter_stream = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
if need_color_u16 && rgb_filter_stream_u16.is_none() {
*rgb_filter_stream_u16 = Some({
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 3)?;
crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?
});
}
packed_yuv444_triple_feed_emit::<_, _, _, SRC_BITS>(
rgb_filter_stream.as_mut(),
rgb_filter_stream_u16.as_mut(),
luma_filter_stream_u16.as_mut(),
rgb,
rgba,
rgb_u16,
rgba_u16,
luma,
luma_u16,
hsv,
rgb_scratch,
rgb_scratch_u16,
luma_scratch_u16,
w,
plan,
idx,
use_simd,
matrix,
full_range,
convert_rgb_u8,
convert_rgb_u16,
deinterleave_y_u16,
)
}
#[cfg(any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
))]
pub(super) fn source_rgb_f32_scratch<'s>(
scratch: &'s mut Vec<f32>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [f32], MixedSinkerError> {
#[cfg(all(
test,
feature = "std",
any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
)
))]
if FORCE_SOURCE_RGB_F32_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
let row = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.src_h(),
3,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0.0);
}
Ok(&mut scratch[..row])
}
#[cfg(all(
test,
feature = "std",
any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
)
))]
std::thread_local! {
static FORCE_SOURCE_RGB_F32_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(
test,
feature = "std",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
))]
pub(crate) fn arm_source_rgb_f32_scratch_failure() {
FORCE_SOURCE_RGB_F32_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(feature = "gbr")]
pub(super) fn rgb_plane_f32_scratch<'s>(
scratch: &'s mut Vec<f32>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [f32], MixedSinkerError> {
let row = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.out_h(),
3,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0.0);
}
Ok(&mut scratch[..row])
}
#[cfg(feature = "gbr")]
pub(super) fn rgb_plane_f16_scratch<'s>(
scratch: &'s mut Vec<half::f16>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [half::f16], MixedSinkerError> {
let row = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.out_h(),
3,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, half::f16::ZERO);
}
Ok(&mut scratch[..row])
}
#[cfg(any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_f32_resample(
rgb_stream_f32: &mut Option<std::boxed::Box<crate::resample::AreaStream<f32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch_f32: &mut Vec<f32>,
w: usize,
plan: &ResamplePlan,
idx: usize,
convert: impl FnOnce(&mut [f32]),
mut emit: impl FnMut(
Option<&mut crate::resample::AreaStream<f32>>,
&[f32],
&mut Option<&mut [u8]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [f32]>,
&mut Option<&mut [f32]>,
&mut Option<&mut [half::f16]>,
&mut Option<&mut [half::f16]>,
&mut Option<HsvFrameMut<'_>>,
) -> Result<(), MixedSinkerError>,
) -> Result<(), MixedSinkerError> {
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgb_stream_f32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let new_stream = if rgb_stream_f32.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgb_f32_scratch(rgb_scratch_f32, w, plan)?;
emit(
None,
&[],
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
)?;
if let Some(s) = new_stream {
*rgb_stream_f32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
idx,
)?;
let stream = rgb_stream_f32.as_mut().expect("created above");
let src_f32 = source_rgb_f32_scratch(rgb_scratch_f32, w, plan)?;
convert(src_f32);
emit(
Some(stream),
src_f32,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
)
}
#[cfg(any(
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")),
feature = "gbr"
))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_f32_resample_filter(
rgb_filter_stream_f32: &mut Option<std::boxed::Box<crate::resample::FilterStream<f32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgb_scratch_f32: &mut Vec<f32>,
w: usize,
plan: &ResamplePlan,
idx: usize,
convert: impl FnOnce(&mut [f32]),
mut emit: impl FnMut(
Option<&mut crate::resample::FilterStream<f32>>,
&[f32],
&mut Option<&mut [u8]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [f32]>,
&mut Option<&mut [f32]>,
&mut Option<&mut [half::f16]>,
&mut Option<&mut [half::f16]>,
&mut Option<HsvFrameMut<'_>>,
) -> Result<(), MixedSinkerError>,
) -> Result<(), MixedSinkerError> {
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgb_filter_stream_f32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
plan.ensure_single_kernel_filter()?;
let new_stream = if rgb_filter_stream_f32.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 3)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgb_f32_scratch(rgb_scratch_f32, w, plan)?;
emit(
None,
&[],
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
)?;
if let Some(s) = new_stream {
*rgb_filter_stream_f32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
idx,
)?;
let stream = rgb_filter_stream_f32.as_mut().expect("created above");
let src_f32 = source_rgb_f32_scratch(rgb_scratch_f32, w, plan)?;
convert(src_f32);
emit(
Some(stream),
src_f32,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
)
}
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_f32_resample_emit(
stream: Option<&mut impl crate::resample::RowResampler<f32>>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_f32: &[f32],
narrow_scratch: &mut Vec<u8>,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
let need_narrow = rgb.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
let Some(stream) = stream else {
return Ok(());
};
stream.feed_row(idx, src_f32, use_simd, |oy, binned| {
if let Some(buf) = rgb_f32.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(binned);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
crate::row::rgbf32_to_rgb_u16_row::<HOST_NATIVE_BE>(
binned,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
crate::row::rgbf32_to_rgba_u16_row::<HOST_NATIVE_BE>(
binned,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::rgbf32_to_rgba_row::<HOST_NATIVE_BE>(
binned,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if need_narrow {
let nrow = &mut narrow[..3 * ow];
crate::row::rgbf32_to_rgb_row::<HOST_NATIVE_BE>(binned, nrow, ow, use_simd);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
pub(super) fn rgb_packed_f16_scratch<'s>(
scratch: &'s mut Vec<half::f16>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [half::f16], MixedSinkerError> {
let row = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.out_h(),
3,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, half::f16::ZERO);
}
Ok(&mut scratch[..row])
}
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgb_f16_resample_emit(
stream: Option<&mut impl crate::resample::RowResampler<f32>>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_f32: &[f32],
packed_scratch_f16: &mut Vec<half::f16>,
narrow_scratch: &mut Vec<u8>,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
let ow = plan.out_w();
let need_round = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgb_f16.is_some()
|| hsv.is_some();
let need_narrow = rgb.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let packed_f16: &mut [half::f16] = if need_round {
rgb_packed_f16_scratch(packed_scratch_f16, ow, plan)?
} else {
&mut []
};
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
let Some(stream) = stream else {
return Ok(());
};
stream.feed_row(idx, src_f32, use_simd, |oy, binned| {
if need_round {
let prow = &mut packed_f16[..3 * ow];
for (dst, &src) in prow.iter_mut().zip(binned.iter()) {
*dst = half::f16::from_f32(src);
}
let prow = &prow[..3 * ow];
if let Some(buf) = rgb_f16.as_deref_mut() {
crate::row::rgbf16_to_rgb_f16_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_f32.as_deref_mut() {
crate::row::rgbf16_to_rgb_f32_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
crate::row::rgbf16_to_rgb_u16_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
crate::row::rgbf16_to_rgba_u16_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::rgbf16_to_rgba_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if need_narrow {
let nrow = &mut narrow[..3 * ow];
crate::row::rgbf16_to_rgb_row::<HOST_NATIVE_BE>(prow, nrow, ow, use_simd);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
}
})?;
Ok(())
}
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_f32_packed_resample(
rgba_stream_f32: &mut Option<std::boxed::Box<crate::resample::AreaStream<f32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch_f32: &mut Vec<f32>,
w: usize,
plan: &ResamplePlan,
idx: usize,
convert: impl FnOnce(&mut [f32]),
mut emit: impl FnMut(
Option<&mut crate::resample::AreaStream<f32>>,
&[f32],
&mut Option<&mut [u8]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [f32]>,
&mut Option<&mut [f32]>,
&mut Option<&mut [half::f16]>,
&mut Option<&mut [half::f16]>,
&mut Option<HsvFrameMut<'_>>,
) -> Result<(), MixedSinkerError>,
) -> Result<(), MixedSinkerError> {
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_stream_f32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let new_stream = if rgba_stream_f32.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgba_f32_scratch(rgba_scratch_f32, w, plan)?;
emit(
None,
&[],
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
)?;
if let Some(s) = new_stream {
*rgba_stream_f32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
idx,
)?;
let stream = rgba_stream_f32.as_mut().expect("created above");
let src_rgba = source_rgba_f32_scratch(rgba_scratch_f32, w, plan)?;
convert(src_rgba);
emit(
Some(stream),
src_rgba,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
)
}
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_f32_packed_filter(
rgba_filter_stream_f32: &mut Option<std::boxed::Box<crate::resample::FilterStream<f32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch_f32: &mut Vec<f32>,
w: usize,
plan: &ResamplePlan,
idx: usize,
convert: impl FnOnce(&mut [f32]),
mut emit: impl FnMut(
Option<&mut crate::resample::FilterStream<f32>>,
&[f32],
&mut Option<&mut [u8]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u8]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [u16]>,
&mut Option<&mut [f32]>,
&mut Option<&mut [f32]>,
&mut Option<&mut [half::f16]>,
&mut Option<&mut [half::f16]>,
&mut Option<HsvFrameMut<'_>>,
) -> Result<(), MixedSinkerError>,
) -> Result<(), MixedSinkerError> {
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_filter_stream_f32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
plan.ensure_single_kernel_filter()?;
let new_stream = if rgba_filter_stream_f32.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
source_rgba_f32_scratch(rgba_scratch_f32, w, plan)?;
emit(
None,
&[],
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
)?;
if let Some(s) = new_stream {
*rgba_filter_stream_f32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
idx,
)?;
let stream = rgba_filter_stream_f32.as_mut().expect("created above");
let src_rgba = source_rgba_f32_scratch(rgba_scratch_f32, w, plan)?;
convert(src_rgba);
emit(
Some(stream),
src_rgba,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
)
}
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_f32_resample_emit(
stream: Option<&mut impl crate::resample::RowResampler<f32>>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_rgba: &[f32],
narrow_scratch: &mut Vec<u8>,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
let need_narrow = rgb.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
let Some(stream) = stream else {
return Ok(());
};
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
if let Some(buf) = rgba_f32.as_deref_mut() {
buf[oy * 4 * ow..(oy + 1) * 4 * ow].copy_from_slice(binned);
}
if let Some(buf) = rgb_f32.as_deref_mut() {
crate::row::rgbaf32_to_rgb_f32_row::<HOST_NATIVE_BE>(
binned,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
crate::row::rgbaf32_to_rgb_u16_row::<HOST_NATIVE_BE>(
binned,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
crate::row::rgbaf32_to_rgba_u16_row::<HOST_NATIVE_BE>(
binned,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::rgbaf32_to_rgba_row::<HOST_NATIVE_BE>(
binned,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if need_narrow {
let nrow = &mut narrow[..3 * ow];
crate::row::rgbaf32_to_rgb_row::<HOST_NATIVE_BE>(binned, nrow, ow, use_simd);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
pub(super) fn rgba_packed_f16_scratch<'s>(
scratch: &'s mut Vec<half::f16>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [half::f16], MixedSinkerError> {
let row = width
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.out_h(),
4,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, half::f16::ZERO);
}
Ok(&mut scratch[..row])
}
#[cfg(all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb")))]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_f16_resample_emit(
stream: Option<&mut impl crate::resample::RowResampler<f32>>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_rgba: &[f32],
packed_scratch_f16: &mut Vec<half::f16>,
narrow_scratch: &mut Vec<u8>,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
let ow = plan.out_w();
let need_round = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let need_narrow = rgb.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let packed_f16: &mut [half::f16] = if need_round {
rgba_packed_f16_scratch(packed_scratch_f16, ow, plan)?
} else {
&mut []
};
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
let Some(stream) = stream else {
return Ok(());
};
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
if need_round {
let prow = &mut packed_f16[..4 * ow];
for (dst, &src) in prow.iter_mut().zip(binned.iter()) {
*dst = half::f16::from_f32(src);
}
let prow = &prow[..4 * ow];
if let Some(buf) = rgba_f16.as_deref_mut() {
crate::row::rgbaf16_to_rgba_f16_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_f16.as_deref_mut() {
crate::row::rgbaf16_to_rgb_f16_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_f32.as_deref_mut() {
crate::row::rgbaf16_to_rgba_f32_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_f32.as_deref_mut() {
crate::row::rgbaf16_to_rgb_f32_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
crate::row::rgbaf16_to_rgb_u16_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
crate::row::rgbaf16_to_rgba_u16_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::rgbaf16_to_rgba_row::<HOST_NATIVE_BE>(
prow,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if need_narrow {
let nrow = &mut narrow[..3 * ow];
crate::row::rgbaf16_to_rgb_row::<HOST_NATIVE_BE>(prow, nrow, ow, use_simd);
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
}
})?;
Ok(())
}
#[cfg(feature = "gbr")]
#[allow(clippy::too_many_arguments)]
pub(super) fn planar_gbr_f32_resample_emit(
stream: Option<&mut impl crate::resample::RowResampler<f32>>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_f32: &[f32],
plane_scratch: &mut Vec<f32>,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
let ow = plan.out_w();
let need_planes = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let planes: &mut [f32] = if need_planes {
rgb_plane_f32_scratch(plane_scratch, ow, plan)?
} else {
&mut []
};
let Some(stream) = stream else {
return Ok(());
};
stream.feed_row(idx, src_f32, use_simd, |oy, binned| {
if let Some(buf) = rgb_f32.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(binned);
}
if need_planes {
let (g, rest) = planes.split_at_mut(ow);
let (b, r) = rest.split_at_mut(ow);
for x in 0..ow {
r[x] = binned[x * 3];
g[x] = binned[x * 3 + 1];
b[x] = binned[x * 3 + 2];
}
let g = &g[..ow];
let b = &b[..ow];
let r = &r[..ow];
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
if let Some(buf) = rgb_u16.as_deref_mut() {
crate::row::gbrpf32_to_rgb_u16_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
crate::row::gbrpf32_to_rgba_u16_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_f32.as_deref_mut() {
crate::row::gbrpf32_to_rgba_f32_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_f16.as_deref_mut() {
crate::row::gbrpf32_to_rgb_f16_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_f16.as_deref_mut() {
crate::row::gbrpf32_to_rgba_f16_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb.as_deref_mut() {
crate::row::gbrpf32_to_rgb_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::gbrpf32_to_rgba_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::gbrpf32_to_luma_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::gbrpf32_to_luma_u16_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::gbrpf32_to_hsv_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(feature = "gbr")]
#[allow(clippy::too_many_arguments)]
pub(super) fn planar_gbr_f16_resample_emit(
stream: Option<&mut impl crate::resample::RowResampler<f32>>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_f32: &[f32],
plane_scratch_f16: &mut Vec<half::f16>,
matrix: crate::KernelMatrix,
full_range: bool,
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
use crate::row::scalar::planar_gbr_f16::widen_f16_be_to_host_f32;
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
const WIDEN_CHUNK: usize = 64;
let ow = plan.out_w();
let need_planes = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let planes_f16: &mut [half::f16] = if need_planes {
rgb_plane_f16_scratch(plane_scratch_f16, ow, plan)?
} else {
&mut []
};
let Some(stream) = stream else {
return Ok(());
};
stream.feed_row(idx, src_f32, use_simd, |oy, binned| {
if need_planes {
let (g16, rest_f16) = planes_f16.split_at_mut(ow);
let (b16, r16) = rest_f16.split_at_mut(ow);
for x in 0..ow {
r16[x] = half::f16::from_f32(binned[x * 3]);
g16[x] = half::f16::from_f32(binned[x * 3 + 1]);
b16[x] = half::f16::from_f32(binned[x * 3 + 2]);
}
let g16 = &g16[..ow];
let b16 = &b16[..ow];
let r16 = &r16[..ow];
if let Some(buf) = rgb_f16.as_deref_mut() {
crate::row::gbrpf16_to_rgb_f16_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_f16.as_deref_mut() {
crate::row::gbrpf16_to_rgba_f16_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
crate::row::gbrpf16_to_rgb_u16_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_u16.as_deref_mut() {
crate::row::gbrpf16_to_rgba_u16_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb.as_deref_mut() {
crate::row::gbrpf16_to_rgb_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::gbrpf16_to_rgba_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
let need_wide_back = rgb_f32.is_some()
|| rgba_f32.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
if need_wide_back {
let mut gw = [0.0f32; WIDEN_CHUNK];
let mut bw = [0.0f32; WIDEN_CHUNK];
let mut rw = [0.0f32; WIDEN_CHUNK];
let mut off = 0;
while off < ow {
let n = (ow - off).min(WIDEN_CHUNK);
widen_f16_be_to_host_f32::<HOST_NATIVE_BE>(g16, off, &mut gw, n);
widen_f16_be_to_host_f32::<HOST_NATIVE_BE>(b16, off, &mut bw, n);
widen_f16_be_to_host_f32::<HOST_NATIVE_BE>(r16, off, &mut rw, n);
let gwn = &gw[..n];
let bwn = &bw[..n];
let rwn = &rw[..n];
let cps = oy * ow + off;
let cpe = cps + n;
if let Some(buf) = rgb_f32.as_deref_mut() {
crate::row::gbrpf32_to_rgb_f32_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut buf[cps * 3..cpe * 3],
n,
use_simd,
);
}
if let Some(buf) = rgba_f32.as_deref_mut() {
crate::row::gbrpf32_to_rgba_f32_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut buf[cps * 4..cpe * 4],
n,
use_simd,
);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::gbrpf32_to_luma_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut buf[cps..cpe],
n,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::gbrpf32_to_luma_u16_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut buf[cps..cpe],
n,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::gbrpf32_to_hsv_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut h[cps..cpe],
&mut s[cps..cpe],
&mut v[cps..cpe],
n,
use_simd,
);
}
off += n;
}
}
}
})?;
Ok(())
}
#[cfg(any(
feature = "gbr",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
))]
pub(super) fn source_rgba_f32_scratch<'s>(
scratch: &'s mut Vec<f32>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [f32], MixedSinkerError> {
#[cfg(all(
test,
feature = "std",
any(
feature = "gbr",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
)
))]
if FORCE_SOURCE_RGBA_F32_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
let row = width
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.src_h(),
4,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0.0);
}
Ok(&mut scratch[..row])
}
#[cfg(all(
test,
feature = "std",
any(
feature = "gbr",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
)
))]
std::thread_local! {
static FORCE_SOURCE_RGBA_F32_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(
test,
feature = "std",
all(feature = "rgb-float", any(feature = "yuv-planar", feature = "rgb"))
))]
pub(crate) fn arm_source_rgba_f32_scratch_failure() {
FORCE_SOURCE_RGBA_F32_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(feature = "gbr")]
pub(super) fn out_rgba_f32_scratch<'s>(
scratch: &'s mut Vec<f32>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [f32], MixedSinkerError> {
let row = width
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.out_h(),
4,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0.0);
}
Ok(&mut scratch[..row])
}
#[cfg(feature = "gbr")]
pub(super) fn rgba_plane_f32_scratch<'s>(
scratch: &'s mut Vec<f32>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [f32], MixedSinkerError> {
let row = width
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.out_h(),
4,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0.0);
}
Ok(&mut scratch[..row])
}
#[cfg(feature = "gbr")]
pub(super) fn rgba_plane_f16_scratch<'s>(
scratch: &'s mut Vec<half::f16>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [half::f16], MixedSinkerError> {
let row = width
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.out_h(),
4,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, half::f16::ZERO);
}
Ok(&mut scratch[..row])
}
#[cfg(feature = "gbr")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn premultiply_rgba_f32_row_in_place(row: &mut [f32], width: usize) {
for px in row[..width * 4].as_chunks_mut::<4>().0 {
let a = px[3];
for c in &mut px[..3] {
*c *= a;
}
}
}
#[cfg(feature = "gbr")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn unpremultiply_channel_f32(pm: f32, a: f32) -> f32 {
if a == 0.0 { 0.0 } else { pm / a }
}
#[cfg(feature = "gbr")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn resolve_straight_rgba_f32_into(binned: &[f32], dst: &mut [f32], width: usize, premult: bool) {
if !premult {
dst[..width * 4].copy_from_slice(&binned[..width * 4]);
return;
}
for (out_px, in_px) in dst[..width * 4]
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(binned[..width * 4].as_chunks::<4>().0.iter())
{
let a = in_px[3];
for c in 0..3 {
out_px[c] = unpremultiply_channel_f32(in_px[c], a);
}
out_px[3] = a;
}
}
#[cfg(feature = "gbr")]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_f32_resample(
rgba_stream_f32: &mut Option<std::boxed::Box<crate::resample::AreaStream<f32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<f32>,
color_scratch: &mut Vec<f32>,
plane_scratch: &mut Vec<f32>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
alpha_mode: AlphaMode,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba: impl FnOnce(&mut [f32]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_stream_f32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let premult = alpha_mode.is_premultiplied();
let need_planes = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let new_stream = if rgba_stream_f32.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
out_rgba_f32_scratch(color_scratch, ow, plan)?;
if need_planes {
rgba_plane_f32_scratch(plane_scratch, ow, plan)?;
}
source_rgba_f32_scratch(rgba_scratch, w, plan)?;
if let Some(s) = new_stream {
*rgba_stream_f32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
idx,
)?;
let stream = rgba_stream_f32.as_mut().expect("created above");
packed_rgba_f32_emit(
stream,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
rgba_scratch,
color_scratch,
plane_scratch,
w,
plan,
idx,
use_simd,
premult,
need_planes,
matrix,
full_range,
convert_rgba,
)
}
#[cfg(feature = "gbr")]
#[allow(clippy::too_many_arguments)]
fn packed_rgba_f32_emit(
stream: &mut impl crate::resample::RowResampler<f32>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<f32>,
color_scratch: &mut Vec<f32>,
plane_scratch: &mut Vec<f32>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
premult: bool,
need_planes: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba: impl FnOnce(&mut [f32]),
) -> Result<(), MixedSinkerError> {
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
let ow = plan.out_w();
let color = out_rgba_f32_scratch(color_scratch, ow, plan)?;
let planes: &mut [f32] = if need_planes {
rgba_plane_f32_scratch(plane_scratch, ow, plan)?
} else {
&mut []
};
let src_rgba = source_rgba_f32_scratch(rgba_scratch, w, plan)?;
convert_rgba(src_rgba);
if premult {
premultiply_rgba_f32_row_in_place(src_rgba, w);
}
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
let color = &mut color[..4 * ow];
resolve_straight_rgba_f32_into(binned, color, ow, premult);
if let Some(buf) = rgba_f32.as_deref_mut() {
buf[oy * 4 * ow..(oy + 1) * 4 * ow].copy_from_slice(color);
}
if need_planes {
let (g, rest) = planes.split_at_mut(ow);
let (b, rest) = rest.split_at_mut(ow);
let (r, a) = rest.split_at_mut(ow);
for x in 0..ow {
r[x] = color[x * 4];
g[x] = color[x * 4 + 1];
b[x] = color[x * 4 + 2];
a[x] = color[x * 4 + 3];
}
let g = &g[..ow];
let b = &b[..ow];
let r = &r[..ow];
let a = &a[..ow];
if let Some(buf) = rgba_u16.as_deref_mut() {
crate::row::gbrapf32_to_rgba_u16_row::<HOST_NATIVE_BE>(
g,
b,
r,
a,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba_f16.as_deref_mut() {
crate::row::gbrapf32_to_rgba_f16_row::<HOST_NATIVE_BE>(
g,
b,
r,
a,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::gbrapf32_to_rgba_row::<HOST_NATIVE_BE>(
g,
b,
r,
a,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
crate::row::gbrpf32_to_rgb_u16_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_f16.as_deref_mut() {
crate::row::gbrpf32_to_rgb_f16_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_f32.as_deref_mut() {
crate::row::gbrpf32_to_rgb_f32_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb.as_deref_mut() {
crate::row::gbrpf32_to_rgb_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::gbrpf32_to_luma_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::gbrpf32_to_luma_u16_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::gbrpf32_to_hsv_row::<HOST_NATIVE_BE>(
g,
b,
r,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
} else if rgb_f32.as_deref().is_some() {
let buf = rgb_f32.as_deref_mut().unwrap();
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
for x in 0..ow {
dst[x * 3] = color[x * 4];
dst[x * 3 + 1] = color[x * 4 + 1];
dst[x * 3 + 2] = color[x * 4 + 2];
}
}
})?;
Ok(())
}
#[cfg(feature = "gbr")]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_f32_filter_resample(
rgba_filter_stream_f32: &mut Option<std::boxed::Box<crate::resample::FilterStream<f32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<f32>,
color_scratch: &mut Vec<f32>,
plane_scratch: &mut Vec<f32>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba: impl FnOnce(&mut [f32]),
) -> Result<(), MixedSinkerError> {
plan.ensure_single_kernel_filter()?;
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_filter_stream_f32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let need_planes = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let new_stream = if rgba_filter_stream_f32.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
out_rgba_f32_scratch(color_scratch, ow, plan)?;
if need_planes {
rgba_plane_f32_scratch(plane_scratch, ow, plan)?;
}
source_rgba_f32_scratch(rgba_scratch, w, plan)?;
if let Some(s) = new_stream {
*rgba_filter_stream_f32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
idx,
)?;
let stream = rgba_filter_stream_f32.as_mut().expect("created above");
packed_rgba_f32_emit(
stream,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
rgba_scratch,
color_scratch,
plane_scratch,
w,
plan,
idx,
use_simd,
false,
need_planes,
matrix,
full_range,
convert_rgba,
)
}
#[cfg(feature = "gbr")]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_f16_resample(
rgba_stream_f32: &mut Option<std::boxed::Box<crate::resample::AreaStream<f32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<f32>,
color_scratch: &mut Vec<f32>,
plane_scratch_f16: &mut Vec<half::f16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
alpha_mode: AlphaMode,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba: impl FnOnce(&mut [f32]),
) -> Result<(), MixedSinkerError> {
if plan.kind().is_filter() {
return Err(plan.unsupported_filter().into());
}
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_stream_f32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let premult = alpha_mode.is_premultiplied();
let need_planes = need_any;
let new_stream = if rgba_stream_f32.is_none() {
let stream =
crate::resample::AreaStream::new(plan.h(), plan.v(), plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
out_rgba_f32_scratch(color_scratch, ow, plan)?;
if need_planes {
rgba_plane_f16_scratch(plane_scratch_f16, ow, plan)?;
}
source_rgba_f32_scratch(rgba_scratch, w, plan)?;
if let Some(s) = new_stream {
*rgba_stream_f32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
idx,
)?;
let stream = rgba_stream_f32.as_mut().expect("created above");
packed_rgba_f16_emit(
stream,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
rgba_scratch,
color_scratch,
plane_scratch_f16,
w,
plan,
idx,
use_simd,
premult,
need_planes,
matrix,
full_range,
convert_rgba,
)
}
#[cfg(feature = "gbr")]
#[allow(clippy::too_many_arguments)]
fn packed_rgba_f16_emit(
stream: &mut impl crate::resample::RowResampler<f32>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<f32>,
color_scratch: &mut Vec<f32>,
plane_scratch_f16: &mut Vec<half::f16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
premult: bool,
need_planes: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba: impl FnOnce(&mut [f32]),
) -> Result<(), MixedSinkerError> {
use crate::row::scalar::planar_gbr_f16::widen_f16_be_to_host_f32;
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
const WIDEN_CHUNK: usize = 64;
let ow = plan.out_w();
let color = out_rgba_f32_scratch(color_scratch, ow, plan)?;
let planes_f16: &mut [half::f16] = if need_planes {
rgba_plane_f16_scratch(plane_scratch_f16, ow, plan)?
} else {
&mut []
};
let src_rgba = source_rgba_f32_scratch(rgba_scratch, w, plan)?;
convert_rgba(src_rgba);
if premult {
premultiply_rgba_f32_row_in_place(src_rgba, w);
}
stream.feed_row(idx, src_rgba, use_simd, |oy, binned| {
if need_planes {
let color = &mut color[..4 * ow];
resolve_straight_rgba_f32_into(binned, color, ow, premult);
let (g16, rest) = planes_f16.split_at_mut(ow);
let (b16, rest) = rest.split_at_mut(ow);
let (r16, a16) = rest.split_at_mut(ow);
for x in 0..ow {
r16[x] = half::f16::from_f32(color[x * 4]);
g16[x] = half::f16::from_f32(color[x * 4 + 1]);
b16[x] = half::f16::from_f32(color[x * 4 + 2]);
a16[x] = half::f16::from_f32(color[x * 4 + 3]);
}
let g16 = &g16[..ow];
let b16 = &b16[..ow];
let r16 = &r16[..ow];
let a16 = &a16[..ow];
if let Some(buf) = rgba_f16.as_deref_mut() {
crate::row::gbrapf16_to_rgba_f16_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
a16,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb_f16.as_deref_mut() {
crate::row::gbrpf16_to_rgb_f16_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::gbrpf16_to_rgba_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
&mut buf[oy * 4 * ow..(oy + 1) * 4 * ow],
ow,
use_simd,
);
}
if let Some(buf) = rgb.as_deref_mut() {
crate::row::gbrpf16_to_rgb_row::<HOST_NATIVE_BE>(
g16,
b16,
r16,
&mut buf[oy * 3 * ow..(oy + 1) * 3 * ow],
ow,
use_simd,
);
}
let need_wide_back = rgb_u16.is_some()
|| rgba_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgba.is_some()
|| luma.is_some()
|| luma_u16.is_some()
|| hsv.is_some();
if need_wide_back {
let mut gw = [0.0f32; WIDEN_CHUNK];
let mut bw = [0.0f32; WIDEN_CHUNK];
let mut rw = [0.0f32; WIDEN_CHUNK];
let mut aw = [0.0f32; WIDEN_CHUNK];
let mut off = 0;
while off < ow {
let n = (ow - off).min(WIDEN_CHUNK);
widen_f16_be_to_host_f32::<HOST_NATIVE_BE>(g16, off, &mut gw, n);
widen_f16_be_to_host_f32::<HOST_NATIVE_BE>(b16, off, &mut bw, n);
widen_f16_be_to_host_f32::<HOST_NATIVE_BE>(r16, off, &mut rw, n);
widen_f16_be_to_host_f32::<HOST_NATIVE_BE>(a16, off, &mut aw, n);
let gwn = &gw[..n];
let bwn = &bw[..n];
let rwn = &rw[..n];
let awn = &aw[..n];
let cps = oy * ow + off;
let cpe = cps + n;
if let Some(buf) = rgba_u16.as_deref_mut() {
crate::row::gbrapf32_to_rgba_u16_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
awn,
&mut buf[cps * 4..cpe * 4],
n,
use_simd,
);
}
if let Some(buf) = rgb_u16.as_deref_mut() {
crate::row::gbrpf32_to_rgb_u16_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut buf[cps * 3..cpe * 3],
n,
use_simd,
);
}
if let Some(buf) = rgba.as_deref_mut() {
crate::row::scalar::alpha_extract::copy_alpha_plane_f32_to_u8::<HOST_NATIVE_BE>(
awn,
&mut buf[cps * 4..cpe * 4],
n,
);
}
if let Some(buf) = rgba_f32.as_deref_mut() {
crate::row::gbrapf32_to_rgba_f32_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
awn,
&mut buf[cps * 4..cpe * 4],
n,
use_simd,
);
}
if let Some(buf) = rgb_f32.as_deref_mut() {
crate::row::gbrpf32_to_rgb_f32_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut buf[cps * 3..cpe * 3],
n,
use_simd,
);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::gbrpf32_to_luma_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut buf[cps..cpe],
n,
matrix,
full_range,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::gbrpf32_to_luma_u16_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut buf[cps..cpe],
n,
matrix,
full_range,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::gbrpf32_to_hsv_row::<HOST_NATIVE_BE>(
gwn,
bwn,
rwn,
&mut h[cps..cpe],
&mut s[cps..cpe],
&mut v[cps..cpe],
n,
use_simd,
);
}
off += n;
}
}
}
})?;
Ok(())
}
#[cfg(feature = "gbr")]
#[allow(clippy::too_many_arguments)]
pub(super) fn packed_rgba_f16_filter_resample(
rgba_filter_stream_f32: &mut Option<std::boxed::Box<crate::resample::FilterStream<f32>>>,
resample_outputs: &mut Option<FrozenOutputs>,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
rgba_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
rgba_scratch: &mut Vec<f32>,
color_scratch: &mut Vec<f32>,
plane_scratch_f16: &mut Vec<half::f16>,
w: usize,
plan: &ResamplePlan,
idx: usize,
use_simd: bool,
matrix: crate::KernelMatrix,
full_range: bool,
convert_rgba: impl FnOnce(&mut [f32]),
) -> Result<(), MixedSinkerError> {
plan.ensure_single_kernel_filter()?;
let ow = plan.out_w();
let need_any = rgb.is_some()
|| rgba.is_some()
|| luma.is_some()
|| rgb_u16.is_some()
|| rgba_u16.is_some()
|| luma_u16.is_some()
|| rgb_f32.is_some()
|| rgba_f32.is_some()
|| rgb_f16.is_some()
|| rgba_f16.is_some()
|| hsv.is_some();
let expected = if need_any {
Some(rgba_filter_stream_f32.as_ref().map_or(0, |s| s.next_y()))
} else {
None
};
if let core::ops::ControlFlow::Break(()) = resample_preflight_check_only(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
expected,
idx,
)? {
return Ok(());
}
let need_planes = need_any;
let new_stream = if rgba_filter_stream_f32.is_none() {
let (fh, fv) = (
plan
.filter_h()
.expect("filter plan carries horizontal windows"),
plan
.filter_v()
.expect("filter plan carries vertical windows"),
);
let stream = crate::resample::FilterStream::new(fh, fv, plan.src_w(), plan.src_h(), 4)?;
Some(crate::resample::try_box(stream).map_err(|_| {
MixedSinkerError::Resample(crate::resample::ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
))
})?)
} else {
None
};
out_rgba_f32_scratch(color_scratch, ow, plan)?;
if need_planes {
rgba_plane_f16_scratch(plane_scratch_f16, ow, plan)?;
}
source_rgba_f32_scratch(rgba_scratch, w, plan)?;
if let Some(s) = new_stream {
*rgba_filter_stream_f32 = Some(s);
}
frozen_outputs_check(
resample_outputs,
luma,
luma_u16,
rgb,
rgba,
rgb_u16,
rgba_u16,
rgb_f32,
rgba_f32,
&None,
rgb_f16,
rgba_f16,
hsv,
&None,
idx,
)?;
let stream = rgba_filter_stream_f32.as_mut().expect("created above");
packed_rgba_f16_emit(
stream,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
rgba_scratch,
color_scratch,
plane_scratch_f16,
w,
plan,
idx,
use_simd,
false,
need_planes,
matrix,
full_range,
convert_rgba,
)
}
#[cfg(feature = "xyz")]
pub(super) fn source_xyz_f32_scratch<'s>(
scratch: &'s mut Vec<f32>,
width: usize,
plan: &ResamplePlan,
) -> Result<&'s mut [f32], MixedSinkerError> {
#[cfg(all(test, feature = "xyz", feature = "std"))]
if FORCE_SOURCE_XYZ_F32_SCRATCH_FAILURE.with(|f| f.take()) {
return Err(MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(plan.src_w(), plan.src_h(), plan.out_w(), plan.out_h()),
)));
}
let row = width
.checked_mul(3)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
width,
plan.src_h(),
3,
)))?;
if scratch.len() < row {
scratch
.try_reserve_exact(row - scratch.len())
.map_err(|_| {
MixedSinkerError::Resample(ResampleError::AllocationFailed(
crate::resample::PlanGeometry::new(
plan.src_w(),
plan.src_h(),
plan.out_w(),
plan.out_h(),
),
))
})?;
scratch.resize(row, 0.0);
}
Ok(&mut scratch[..row])
}
#[cfg(all(test, feature = "xyz", feature = "std"))]
std::thread_local! {
static FORCE_SOURCE_XYZ_F32_SCRATCH_FAILURE: core::cell::Cell<bool> = const { core::cell::Cell::new(false) };
}
#[cfg(all(test, feature = "xyz", feature = "std"))]
pub(crate) fn arm_source_xyz_f32_scratch_failure() {
FORCE_SOURCE_XYZ_F32_SCRATCH_FAILURE.with(|f| f.set(true));
}
#[cfg(feature = "xyz")]
#[allow(clippy::too_many_arguments)]
pub(super) fn xyz12_resample_emit(
stream: &mut impl crate::resample::RowResampler<f32>,
plan: &ResamplePlan,
rgb: &mut Option<&mut [u8]>,
rgba: &mut Option<&mut [u8]>,
luma: &mut Option<&mut [u8]>,
luma_u16: &mut Option<&mut [u16]>,
rgb_u16: &mut Option<&mut [u16]>,
rgba_u16: &mut Option<&mut [u16]>,
rgb_f32: &mut Option<&mut [f32]>,
xyz_f32: &mut Option<&mut [f32]>,
rgb_f16: &mut Option<&mut [half::f16]>,
rgba_f16: &mut Option<&mut [half::f16]>,
hsv: &mut Option<HsvFrameMut<'_>>,
src_xyz: &[f32],
narrow_scratch: &mut Vec<u8>,
target_gamut: crate::KernelGamut,
luma_q15: (i32, i32, i32),
idx: usize,
use_simd: bool,
) -> Result<(), MixedSinkerError> {
use crate::row::scalar::{
xyz12::{matmul3_xyz_rgb, narrow_unit_to_u8, narrow_unit_to_u16, oetf_srgb},
xyz12_constants::xyz_to_rgb_matrix,
};
let ow = plan.out_w();
let m = xyz_to_rgb_matrix(target_gamut);
let one_f16 = half::f16::from_f32(1.0);
let need_narrow = rgb.is_some() || luma.is_some() || luma_u16.is_some() || hsv.is_some();
let narrow: &mut [u8] = if need_narrow {
source_rgb_scratch(narrow_scratch, ow, plan)?
} else {
&mut []
};
stream.feed_row(idx, src_xyz, use_simd, |oy, binned| {
if let Some(buf) = xyz_f32.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(binned);
}
if let Some(buf) = rgb_f32.as_deref_mut() {
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
for x in 0..ow {
let i = x * 3;
let rgb_lin = matmul3_xyz_rgb(&m, [binned[i], binned[i + 1], binned[i + 2]]);
dst[i] = rgb_lin[0];
dst[i + 1] = rgb_lin[1];
dst[i + 2] = rgb_lin[2];
}
}
if let Some(buf) = rgb_f16.as_deref_mut() {
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
for x in 0..ow {
let i = x * 3;
let rgb_lin = matmul3_xyz_rgb(&m, [binned[i], binned[i + 1], binned[i + 2]]);
dst[i] = half::f16::from_f32(oetf_srgb(rgb_lin[0]).clamp(0.0, 1.0));
dst[i + 1] = half::f16::from_f32(oetf_srgb(rgb_lin[1]).clamp(0.0, 1.0));
dst[i + 2] = half::f16::from_f32(oetf_srgb(rgb_lin[2]).clamp(0.0, 1.0));
}
}
if let Some(buf) = rgba_f16.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
for x in 0..ow {
let i = x * 3;
let o = x * 4;
let rgb_lin = matmul3_xyz_rgb(&m, [binned[i], binned[i + 1], binned[i + 2]]);
dst[o] = half::f16::from_f32(oetf_srgb(rgb_lin[0]).clamp(0.0, 1.0));
dst[o + 1] = half::f16::from_f32(oetf_srgb(rgb_lin[1]).clamp(0.0, 1.0));
dst[o + 2] = half::f16::from_f32(oetf_srgb(rgb_lin[2]).clamp(0.0, 1.0));
dst[o + 3] = one_f16;
}
}
if let Some(buf) = rgb_u16.as_deref_mut() {
let dst = &mut buf[oy * 3 * ow..(oy + 1) * 3 * ow];
for x in 0..ow {
let i = x * 3;
let rgb_lin = matmul3_xyz_rgb(&m, [binned[i], binned[i + 1], binned[i + 2]]);
dst[i] = narrow_unit_to_u16(oetf_srgb(rgb_lin[0]));
dst[i + 1] = narrow_unit_to_u16(oetf_srgb(rgb_lin[1]));
dst[i + 2] = narrow_unit_to_u16(oetf_srgb(rgb_lin[2]));
}
}
if let Some(buf) = rgba_u16.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
for x in 0..ow {
let i = x * 3;
let o = x * 4;
let rgb_lin = matmul3_xyz_rgb(&m, [binned[i], binned[i + 1], binned[i + 2]]);
dst[o] = narrow_unit_to_u16(oetf_srgb(rgb_lin[0]));
dst[o + 1] = narrow_unit_to_u16(oetf_srgb(rgb_lin[1]));
dst[o + 2] = narrow_unit_to_u16(oetf_srgb(rgb_lin[2]));
dst[o + 3] = 0xFFFF;
}
}
if let Some(buf) = rgba.as_deref_mut() {
let dst = &mut buf[oy * 4 * ow..(oy + 1) * 4 * ow];
for x in 0..ow {
let i = x * 3;
let o = x * 4;
let rgb_lin = matmul3_xyz_rgb(&m, [binned[i], binned[i + 1], binned[i + 2]]);
dst[o] = narrow_unit_to_u8(oetf_srgb(rgb_lin[0]));
dst[o + 1] = narrow_unit_to_u8(oetf_srgb(rgb_lin[1]));
dst[o + 2] = narrow_unit_to_u8(oetf_srgb(rgb_lin[2]));
dst[o + 3] = 0xFF;
}
}
if need_narrow {
let nrow = &mut narrow[..3 * ow];
for x in 0..ow {
let i = x * 3;
let rgb_lin = matmul3_xyz_rgb(&m, [binned[i], binned[i + 1], binned[i + 2]]);
nrow[i] = narrow_unit_to_u8(oetf_srgb(rgb_lin[0]));
nrow[i + 1] = narrow_unit_to_u8(oetf_srgb(rgb_lin[1]));
nrow[i + 2] = narrow_unit_to_u8(oetf_srgb(rgb_lin[2]));
}
if let Some(buf) = rgb.as_deref_mut() {
buf[oy * 3 * ow..(oy + 1) * 3 * ow].copy_from_slice(nrow);
}
if let Some(buf) = luma.as_deref_mut() {
crate::row::xyz12_rgb_to_luma_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
luma_q15,
use_simd,
);
}
if let Some(buf) = luma_u16.as_deref_mut() {
crate::row::xyz12_rgb_to_luma_u16_row(
nrow,
&mut buf[oy * ow..(oy + 1) * ow],
ow,
luma_q15,
use_simd,
);
}
if let Some(hsv) = hsv.as_mut() {
let (h, s, v) = hsv.hsv();
crate::row::rgb_to_hsv_row(
nrow,
&mut h[oy * ow..(oy + 1) * ow],
&mut s[oy * ow..(oy + 1) * ow],
&mut v[oy * ow..(oy + 1) * ow],
ow,
use_simd,
);
}
}
})?;
Ok(())
}
#[cfg(any(feature = "bayer", feature = "mono"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn rgb_row_to_luma_row(rgb: &[u8], luma: &mut [u8], coeffs_q8: (u32, u32, u32)) {
debug_assert!(
luma
.len()
.checked_mul(3)
.is_some_and(|need| rgb.len() >= need),
"rgb_row_to_luma_row: rgb.len()={} but need {} (= 3 x luma.len()={})",
rgb.len(),
luma.len().saturating_mul(3),
luma.len(),
);
let (cr, cg, cb) = coeffs_q8;
for (i, d) in luma.iter_mut().enumerate() {
let r = rgb[3 * i] as u32;
let g = rgb[3 * i + 1] as u32;
let b = rgb[3 * i + 2] as u32;
*d = ((cr * r + cg * g + cb * b + 128) >> 8).min(255) as u8;
}
}
#[cfg(feature = "mono")]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(super) fn rgb_row_to_luma_u16_row(
rgb: &[u8],
luma_u16: &mut [u16],
coeffs_q8: (u32, u32, u32),
) {
debug_assert!(
luma_u16
.len()
.checked_mul(3)
.is_some_and(|need| rgb.len() >= need),
"rgb_row_to_luma_u16_row: rgb.len()={} but need {} (= 3 x luma_u16.len()={})",
rgb.len(),
luma_u16.len().saturating_mul(3),
luma_u16.len(),
);
let (cr, cg, cb) = coeffs_q8;
for (i, dst) in luma_u16.iter_mut().enumerate() {
let r = rgb[3 * i] as u32;
let g = rgb[3 * i + 1] as u32;
let b = rgb[3 * i + 2] as u32;
let y = ((cr * r + cg * g + cb * b + 128) >> 8).min(255) as u16;
*dst = (y << 8) | y;
}
}
#[cfg(feature = "yuv-444-packed")]
mod ayuv;
#[cfg(feature = "yuv-444-packed")]
mod ayuv64;
#[cfg(feature = "bayer")]
mod bayer;
#[cfg(feature = "yuv-planar")]
mod chroma_reconstruct;
#[cfg(feature = "gray")]
mod gray;
#[cfg(feature = "rgb-legacy")]
mod legacy_rgb;
#[cfg(feature = "mono")]
mod mono1bit;
#[cfg(feature = "rgb")]
mod packed_rgb_10bit;
#[cfg(feature = "rgb")]
mod packed_rgb_16bit;
#[cfg(feature = "rgb")]
mod packed_rgb_32bit;
#[cfg(feature = "rgb")]
mod packed_rgb_8bit;
#[cfg(feature = "rgb-float")]
mod packed_rgb_f16;
#[cfg(feature = "rgb-float")]
mod packed_rgb_float;
#[cfg(feature = "yuv-packed")]
mod packed_yuv_4_1_1;
#[cfg(feature = "yuv-packed")]
mod packed_yuv_8bit;
#[cfg(feature = "mono")]
mod pal8;
#[cfg(feature = "yuv-planar")]
mod planar_8bit;
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
pub(crate) use planar_8bit::arm_native_rgb_scratch_failure;
#[cfg(all(
test,
feature = "std",
feature = "yuv-planar",
any(
feature = "yuv-packed",
feature = "yuv-444-packed",
all(feature = "yuv-semi-planar", feature = "rgb")
)
))]
pub(crate) use planar_8bit::arm_planar_native_chroma_failure;
#[cfg(all(feature = "yuv-planar", feature = "rgb"))]
mod linear_light;
#[cfg(feature = "yuv-planar")]
mod planar_high_bit_native;
#[cfg(feature = "yuv-planar")]
use planar_high_bit_native::yuv_planar16_process_native;
#[cfg(feature = "yuv-planar")]
use planar_high_bit_native::{NativePlanarYuvU16, native_planar_hb_preflight_check_only};
#[cfg(all(test, feature = "std", feature = "yuv-planar", feature = "rgb"))]
pub(crate) use planar_high_bit_native::{
arm_planar_hb_native_alloc_failure, arm_planar_hb_native_alloc_failure_after,
arm_planar_hb_native_rgb_u16_scratch_failure,
};
#[cfg(all(
test,
feature = "std",
feature = "yuv-planar",
any(
feature = "rgb",
feature = "v210",
feature = "y2xx",
feature = "yuv-444-packed",
feature = "yuv-semi-planar"
)
))]
pub(crate) use planar_high_bit_native::arm_planar_hb_native_chroma_failure;
#[cfg(feature = "gbr")]
mod planar_gbr_32bit;
#[cfg(feature = "gbr")]
mod planar_gbr_8bit;
#[cfg(feature = "gbr")]
mod planar_gbr_f16;
#[cfg(feature = "gbr")]
mod planar_gbr_float;
#[cfg(feature = "gbr")]
mod planar_gbr_high_bit;
#[cfg(feature = "gbr")]
mod planar_gbr_msb;
#[cfg(any(
feature = "yuv-planar",
feature = "yuv-semi-planar",
feature = "yuv-packed"
))]
mod planar_resample;
#[cfg(feature = "yuv-semi-planar")]
mod semi_planar_8bit;
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
mod subsampled_4_2_0_high_bit;
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
mod subsampled_4_2_2_high_bit;
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
mod subsampled_4_4_4_high_bit;
#[cfg(feature = "yuv-444-packed")]
mod uyva;
#[cfg(feature = "v210")]
mod v210;
#[cfg(feature = "yuv-444-packed")]
mod v30x;
#[cfg(feature = "yuv-444-packed")]
mod v410;
#[cfg(feature = "yuv-444-packed")]
mod vuya;
#[cfg(feature = "yuv-444-packed")]
mod vuyx;
#[cfg(feature = "yuv-444-packed")]
mod vyu444;
#[cfg(feature = "yuv-444-packed")]
mod xv36;
#[cfg(feature = "yuv-444-packed")]
mod xv48;
#[cfg(feature = "xyz")]
mod xyz12;
#[cfg(feature = "y2xx")]
mod y210;
#[cfg(feature = "y2xx")]
mod y212;
#[cfg(feature = "y2xx")]
mod y216;
#[cfg(feature = "yuva")]
mod yuva_4_2_0;
#[cfg(feature = "yuva")]
mod yuva_4_2_2;
#[cfg(feature = "yuva")]
mod yuva_4_4_4;
#[cfg(all(test, feature = "std"))]
mod tests;
#[cfg(all(test, feature = "std"))]
mod api_smoke_tests {
use super::*;
#[cfg(feature = "v210")]
#[test]
fn mixed_sinker_default_does_not_produce_luma_u16() {
let sink: MixedSinker<'_, crate::source::V210> = MixedSinker::new(6, 1);
assert!(!sink.produces_luma_u16());
}
#[test]
fn luma_u16_buffer_too_short_error_displays() {
let err = MixedSinkerError::InsufficientLumaU16Buffer(InsufficientBuffer::new(100, 50));
let msg = format!("{err}");
assert!(msg.contains("100"));
assert!(msg.contains("50"));
}
#[cfg(feature = "yuv-planar")]
#[test]
fn mixed_sinker_inline_size_stays_small() {
use crate::{resample::AreaResampler, source::Yuv420p};
let area = core::mem::size_of::<MixedSinker<'_, Yuv420p, AreaResampler>>();
let noop = core::mem::size_of::<MixedSinker<'_, Yuv420p>>();
assert!(
area < 16384,
"MixedSinker<Yuv420p, AreaResampler> inline size {area} exceeds the 16 KiB \
stack-footprint ceiling; box the largest new field(s) (mirror the existing \
Option<Box<…>> stream / native-join fields)",
);
assert!(
noop < 16384,
"MixedSinker<Yuv420p, NoopResampler> inline size {noop} exceeds the 16 KiB ceiling",
);
}
}