use super::{
GeometryOverflow, InsufficientBuffer, MixedSinker, MixedSinkerError, RowIndexOutOfRange,
RowShapeMismatch, RowSlice, check_dimensions_match, check_frozen_alpha_mode,
packed_rgb_f32_resample, packed_rgb_f32_resample_filter, packed_rgba_f16_filter_resample,
packed_rgba_f16_resample, planar_gbr_f16_resample_emit, rgb_row_buf_or_scratch,
rgba_plane_row_slice,
};
use crate::{
KernelMatrix, PixelSink,
row::{
expand_rgb_to_rgba_row, gbrapf16_to_rgba_f16_row, gbrapf32_to_rgba_f32_row,
gbrapf32_to_rgba_u16_row, gbrpf16_to_rgb_f16_row, gbrpf16_to_rgb_row, gbrpf16_to_rgba_f16_row,
gbrpf16_to_rgba_row, gbrpf32_to_hsv_row, gbrpf32_to_luma_row, gbrpf32_to_luma_u16_row,
gbrpf32_to_rgb_f32_row, gbrpf32_to_rgb_u16_row, gbrpf32_to_rgba_f32_row,
gbrpf32_to_rgba_u16_row,
scalar::{alpha_extract::copy_alpha_plane_f32_to_u8, planar_gbr_f16::widen_f16_be_to_host_f32},
},
source::{Gbrapf16, Gbrapf16Row, Gbrapf16Sink, Gbrpf16, Gbrpf16Row, Gbrpf16Sink},
};
const GBR_F16_LUMA_MATRIX: KernelMatrix = KernelMatrix::Bt709;
const GBR_F16_FULL_RANGE: bool = true;
const WIDEN_CHUNK: usize = 64;
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
impl<'a, R, const BE: bool> MixedSinker<'a, Gbrpf16<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_f32(mut self, buf: &'a mut [f32]) -> Result<Self, MixedSinkerError> {
self.set_rgb_f32(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_f32(&mut self, buf: &'a mut [f32]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbF32Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_f32 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_f32(mut self, buf: &'a mut [f32]) -> Result<Self, MixedSinkerError> {
self.set_rgba_f32(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_f32(&mut self, buf: &'a mut [f32]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaF32Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_f32 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_f16(mut self, buf: &'a mut [half::f16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_f16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_f16(&mut self, buf: &'a mut [half::f16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbF16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_f16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_f16(mut self, buf: &'a mut [half::f16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_f16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_f16(&mut self, buf: &'a mut [half::f16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaF16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_f16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_luma_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_luma_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_luma_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_pixels()?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientLumaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.luma_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> Gbrpf16Sink<BE> for MixedSinker<'_, Gbrpf16<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, Gbrpf16<BE>, R> {
type Input<'r> = Gbrpf16Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
check_dimensions_match(self.width, self.height, width, height)?;
if let Some(stream) = self.rgb_stream_f32.as_mut() {
stream.reset();
}
if let Some(stream) = self.rgb_filter_stream_f32.as_mut() {
stream.reset();
}
self.resample_outputs = None;
Ok(())
}
fn process(&mut self, row: Gbrpf16Row<'_>) -> Result<(), Self::Error> {
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if row.g().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::GbrF16Plane,
idx,
w,
row.g().len(),
)));
}
if row.b().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::GbrF16Plane,
idx,
w,
row.b().len(),
)));
}
if row.r().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::GbrF16Plane,
idx,
w,
row.r().len(),
)));
}
if idx >= h {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, h),
));
}
if let Some(plan) = self.plan.as_ref() {
let Self {
rgb,
rgb_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
rgba,
rgba_u16,
luma,
luma_u16,
hsv,
rgb_scratch_f32,
rgb_plane_scratch_f16,
rgb_stream_f32,
rgb_filter_stream_f32,
resample_outputs,
..
} = self;
return match plan.kind() {
crate::resample::SpanKind::Area => packed_rgb_f32_resample(
rgb_stream_f32,
resample_outputs,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
rgb_scratch_f32,
w,
plan,
idx,
|src_f32| {
scatter_gbrpf16_to_packed_f32::<BE>(row.g(), row.b(), row.r(), src_f32, w, use_simd)
},
|stream,
src,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv| {
planar_gbr_f16_resample_emit(
stream,
plan,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
src,
rgb_plane_scratch_f16,
GBR_F16_LUMA_MATRIX,
GBR_F16_FULL_RANGE,
idx,
use_simd,
)
},
),
crate::resample::SpanKind::Filter => packed_rgb_f32_resample_filter(
rgb_filter_stream_f32,
resample_outputs,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
rgb_scratch_f32,
w,
plan,
idx,
|src_f32| {
scatter_gbrpf16_to_packed_f32::<BE>(row.g(), row.b(), row.r(), src_f32, w, use_simd)
},
|stream,
src,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv| {
planar_gbr_f16_resample_emit(
stream,
plan,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
src,
rgb_plane_scratch_f16,
GBR_F16_LUMA_MATRIX,
GBR_F16_FULL_RANGE,
idx,
use_simd,
)
},
),
};
}
let g_in = row.g();
let b_in = row.b();
let r_in = row.r();
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
if let Some(buf) = self.rgb_f16.as_deref_mut() {
let start = one_plane_start * 3;
let end = one_plane_end * 3;
gbrpf16_to_rgb_f16_row::<BE>(g_in, b_in, r_in, &mut buf[start..end], w, use_simd);
}
if let Some(buf) = self.rgba_f16.as_deref_mut() {
let start = one_plane_start * 4;
let end = one_plane_end
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 4,
)))?;
gbrpf16_to_rgba_f16_row::<BE>(g_in, b_in, r_in, &mut buf[start..end], w, use_simd);
}
let need_wide = self.rgb_f32.is_some()
|| self.rgba_f32.is_some()
|| self.rgb_u16.is_some()
|| self.rgba_u16.is_some()
|| self.luma.is_some()
|| self.luma_u16.is_some()
|| self.hsv.is_some();
if need_wide {
let mut gf_chunk = [0.0f32; WIDEN_CHUNK];
let mut bf_chunk = [0.0f32; WIDEN_CHUNK];
let mut rf_chunk = [0.0f32; WIDEN_CHUNK];
let mut offset = 0;
while offset < w {
let n = (w - offset).min(WIDEN_CHUNK);
widen_f16_be_to_host_f32::<BE>(g_in, offset, &mut gf_chunk, n);
widen_f16_be_to_host_f32::<BE>(b_in, offset, &mut bf_chunk, n);
widen_f16_be_to_host_f32::<BE>(r_in, offset, &mut rf_chunk, n);
let gf = &gf_chunk[..n];
let bf = &bf_chunk[..n];
let rf = &rf_chunk[..n];
let chunk_plane_start = one_plane_start + offset;
let chunk_plane_end = chunk_plane_start + n;
if let Some(buf) = self.rgb_f32.as_deref_mut() {
let start = chunk_plane_start * 3;
let end = chunk_plane_end * 3;
gbrpf32_to_rgb_f32_row::<HOST_NATIVE_BE>(gf, bf, rf, &mut buf[start..end], n, use_simd);
}
if let Some(buf) = self.rgba_f32.as_deref_mut() {
let start = chunk_plane_start * 4;
let end = chunk_plane_end * 4;
gbrpf32_to_rgba_f32_row::<HOST_NATIVE_BE>(gf, bf, rf, &mut buf[start..end], n, use_simd);
}
if let Some(buf) = self.rgb_u16.as_deref_mut() {
let start = chunk_plane_start * 3;
let end = chunk_plane_end * 3;
gbrpf32_to_rgb_u16_row::<HOST_NATIVE_BE>(gf, bf, rf, &mut buf[start..end], n, use_simd);
}
if let Some(buf) = self.rgba_u16.as_deref_mut() {
let start = chunk_plane_start * 4;
let end = chunk_plane_end * 4;
gbrpf32_to_rgba_u16_row::<HOST_NATIVE_BE>(gf, bf, rf, &mut buf[start..end], n, use_simd);
}
if let Some(buf) = self.luma.as_deref_mut() {
gbrpf32_to_luma_row::<HOST_NATIVE_BE>(
gf,
bf,
rf,
&mut buf[chunk_plane_start..chunk_plane_end],
n,
GBR_F16_LUMA_MATRIX,
GBR_F16_FULL_RANGE,
use_simd,
);
}
if let Some(buf) = self.luma_u16.as_deref_mut() {
gbrpf32_to_luma_u16_row::<HOST_NATIVE_BE>(
gf,
bf,
rf,
&mut buf[chunk_plane_start..chunk_plane_end],
n,
GBR_F16_LUMA_MATRIX,
GBR_F16_FULL_RANGE,
use_simd,
);
}
if let Some(hsv) = self.hsv.as_mut() {
let (h, s, v) = hsv.hsv();
gbrpf32_to_hsv_row::<HOST_NATIVE_BE>(
gf,
bf,
rf,
&mut h[chunk_plane_start..chunk_plane_end],
&mut s[chunk_plane_start..chunk_plane_end],
&mut v[chunk_plane_start..chunk_plane_end],
n,
use_simd,
);
}
offset += n;
}
}
let want_rgba = self.rgba.is_some();
let want_rgb = self.rgb.is_some();
let need_u8_rgb = want_rgb;
if want_rgba && !need_u8_rgb {
let rgba_buf = self.rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
gbrpf16_to_rgba_row::<BE>(g_in, b_in, r_in, rgba_row, w, use_simd);
return Ok(());
}
if !need_u8_rgb && !want_rgba {
return Ok(());
}
let Self {
rgb,
rgba,
rgb_scratch,
..
} = self;
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
gbrpf16_to_rgb_row::<BE>(g_in, b_in, r_in, rgb_row, w, use_simd);
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
}
Ok(())
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn scatter_gbrpf16_to_packed_f32<const BE: bool>(
g: &[half::f16],
b: &[half::f16],
r: &[half::f16],
rgb_out: &mut [f32],
width: usize,
use_simd: bool,
) {
let mut gf = [0.0f32; WIDEN_CHUNK];
let mut bf = [0.0f32; WIDEN_CHUNK];
let mut rf = [0.0f32; WIDEN_CHUNK];
let mut offset = 0;
while offset < width {
let n = (width - offset).min(WIDEN_CHUNK);
widen_f16_be_to_host_f32::<BE>(g, offset, &mut gf, n);
widen_f16_be_to_host_f32::<BE>(b, offset, &mut bf, n);
widen_f16_be_to_host_f32::<BE>(r, offset, &mut rf, n);
gbrpf32_to_rgb_f32_row::<HOST_NATIVE_BE>(
&gf[..n],
&bf[..n],
&rf[..n],
&mut rgb_out[offset * 3..],
n,
use_simd,
);
offset += n;
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn scatter_gbrapf16_to_packed_rgba_f32<const BE: bool>(
g: &[half::f16],
b: &[half::f16],
r: &[half::f16],
a: &[half::f16],
rgba_out: &mut [f32],
width: usize,
use_simd: bool,
) {
let mut gf = [0.0f32; WIDEN_CHUNK];
let mut bf = [0.0f32; WIDEN_CHUNK];
let mut rf = [0.0f32; WIDEN_CHUNK];
let mut af = [0.0f32; WIDEN_CHUNK];
let mut offset = 0;
while offset < width {
let n = (width - offset).min(WIDEN_CHUNK);
widen_f16_be_to_host_f32::<BE>(g, offset, &mut gf, n);
widen_f16_be_to_host_f32::<BE>(b, offset, &mut bf, n);
widen_f16_be_to_host_f32::<BE>(r, offset, &mut rf, n);
widen_f16_be_to_host_f32::<BE>(a, offset, &mut af, n);
gbrapf32_to_rgba_f32_row::<HOST_NATIVE_BE>(
&gf[..n],
&bf[..n],
&rf[..n],
&af[..n],
&mut rgba_out[offset * 4..],
n,
use_simd,
);
offset += n;
}
}
impl<'a, R, const BE: bool> MixedSinker<'a, Gbrapf16<BE>, R> {
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba(mut self, buf: &'a mut [u8]) -> Result<Self, MixedSinkerError> {
self.set_rgba(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba(&mut self, buf: &'a mut [u8]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaBuffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_u16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_f32(mut self, buf: &'a mut [f32]) -> Result<Self, MixedSinkerError> {
self.set_rgb_f32(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_f32(&mut self, buf: &'a mut [f32]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbF32Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_f32 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_f32(mut self, buf: &'a mut [f32]) -> Result<Self, MixedSinkerError> {
self.set_rgba_f32(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_f32(&mut self, buf: &'a mut [f32]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaF32Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_f32 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgb_f16(mut self, buf: &'a mut [half::f16]) -> Result<Self, MixedSinkerError> {
self.set_rgb_f16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgb_f16(&mut self, buf: &'a mut [half::f16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(3)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbF16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgb_f16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_rgba_f16(mut self, buf: &'a mut [half::f16]) -> Result<Self, MixedSinkerError> {
self.set_rgba_f16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_rgba_f16(&mut self, buf: &'a mut [half::f16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_elems(4)?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientRgbaF16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.rgba_f16 = Some(buf);
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn with_luma_u16(mut self, buf: &'a mut [u16]) -> Result<Self, MixedSinkerError> {
self.set_luma_u16(buf)?;
Ok(self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn set_luma_u16(&mut self, buf: &'a mut [u16]) -> Result<&mut Self, MixedSinkerError> {
let expected = self.frame_pixels()?;
if buf.len() < expected {
return Err(MixedSinkerError::InsufficientLumaU16Buffer(
InsufficientBuffer::new(expected, buf.len()),
));
}
self.luma_u16 = Some(buf);
Ok(self)
}
}
impl<R, const BE: bool> Gbrapf16Sink<BE> for MixedSinker<'_, Gbrapf16<BE>, R> {}
impl<R, const BE: bool> PixelSink for MixedSinker<'_, Gbrapf16<BE>, R> {
type Input<'r> = Gbrapf16Row<'r>;
type Error = MixedSinkerError;
#[cfg_attr(not(tarpaulin), inline(always))]
fn kernel_matrix(&self) -> crate::KernelMatrix {
self.kernel_matrix
}
fn begin_frame(&mut self, width: u32, height: u32) -> Result<(), Self::Error> {
check_dimensions_match(self.width, self.height, width, height)?;
if let Some(stream) = self.rgba_stream_f32.as_mut() {
stream.reset();
}
if let Some(stream) = self.rgba_filter_stream_f32.as_mut() {
stream.reset();
}
self.resample_outputs = None;
self.frozen_alpha_mode = Some(self.alpha_mode);
Ok(())
}
fn process(&mut self, row: Gbrapf16Row<'_>) -> Result<(), Self::Error> {
let w = self.width;
let h = self.height;
let idx = row.row();
let use_simd = self.simd;
if row.g().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::GbrF16Plane,
idx,
w,
row.g().len(),
)));
}
if row.b().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::GbrF16Plane,
idx,
w,
row.b().len(),
)));
}
if row.r().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::GbrF16Plane,
idx,
w,
row.r().len(),
)));
}
if row.a().len() != w {
return Err(MixedSinkerError::RowShapeMismatch(RowShapeMismatch::new(
RowSlice::GbrF16Plane,
idx,
w,
row.a().len(),
)));
}
if idx >= h {
return Err(MixedSinkerError::RowIndexOutOfRange(
RowIndexOutOfRange::new(idx, h),
));
}
if let Some(plan) = self.plan.as_ref() {
let alpha_mode = self.alpha_mode;
let g_in = row.g();
let b_in = row.b();
let r_in = row.r();
let a_in = row.a();
let Self {
rgb,
rgb_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
rgba,
rgba_u16,
luma,
luma_u16,
hsv,
rgba_scratch_f32,
rgba_color_scratch_f32,
rgba_plane_scratch_f16,
rgba_stream_f32,
rgba_filter_stream_f32,
resample_outputs,
frozen_alpha_mode,
..
} = self;
check_frozen_alpha_mode(*frozen_alpha_mode, alpha_mode, idx)?;
match plan.kind() {
crate::resample::SpanKind::Area => {
return packed_rgba_f16_resample(
rgba_stream_f32,
resample_outputs,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
rgba_scratch_f32,
rgba_color_scratch_f32,
rgba_plane_scratch_f16,
w,
plan,
idx,
use_simd,
alpha_mode,
GBR_F16_LUMA_MATRIX,
GBR_F16_FULL_RANGE,
|dst| {
scatter_gbrapf16_to_packed_rgba_f32::<BE>(g_in, b_in, r_in, a_in, dst, w, use_simd)
},
);
}
crate::resample::SpanKind::Filter => {
if alpha_mode.is_premultiplied() {
return packed_rgba_f16_resample(
rgba_stream_f32,
resample_outputs,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
rgba_scratch_f32,
rgba_color_scratch_f32,
rgba_plane_scratch_f16,
w,
plan,
idx,
use_simd,
alpha_mode,
GBR_F16_LUMA_MATRIX,
GBR_F16_FULL_RANGE,
|dst| {
scatter_gbrapf16_to_packed_rgba_f32::<BE>(g_in, b_in, r_in, a_in, dst, w, use_simd)
},
);
}
return packed_rgba_f16_filter_resample(
rgba_filter_stream_f32,
resample_outputs,
rgb,
rgba,
luma,
rgb_u16,
rgba_u16,
luma_u16,
rgb_f32,
rgba_f32,
rgb_f16,
rgba_f16,
hsv,
rgba_scratch_f32,
rgba_color_scratch_f32,
rgba_plane_scratch_f16,
w,
plan,
idx,
use_simd,
GBR_F16_LUMA_MATRIX,
GBR_F16_FULL_RANGE,
|dst| {
scatter_gbrapf16_to_packed_rgba_f32::<BE>(g_in, b_in, r_in, a_in, dst, w, use_simd)
},
);
}
}
}
let g_in = row.g();
let b_in = row.b();
let r_in = row.r();
let a_in = row.a();
let one_plane_start = idx * w;
let one_plane_end = one_plane_start + w;
if let Some(buf) = self.rgb_f16.as_deref_mut() {
let start = one_plane_start * 3;
let end = one_plane_end * 3;
gbrpf16_to_rgb_f16_row::<BE>(g_in, b_in, r_in, &mut buf[start..end], w, use_simd);
}
if let Some(buf) = self.rgba_f16.as_deref_mut() {
let start = one_plane_start * 4;
let end = one_plane_end
.checked_mul(4)
.ok_or(MixedSinkerError::GeometryOverflow(GeometryOverflow::new(
w, h, 4,
)))?;
gbrapf16_to_rgba_f16_row::<BE>(g_in, b_in, r_in, a_in, &mut buf[start..end], w, use_simd);
}
let need_wide = self.rgb_f32.is_some()
|| self.rgba_f32.is_some()
|| self.rgb_u16.is_some()
|| self.rgba_u16.is_some()
|| self.luma.is_some()
|| self.luma_u16.is_some()
|| self.hsv.is_some();
if need_wide {
let mut gf_chunk = [0.0f32; WIDEN_CHUNK];
let mut bf_chunk = [0.0f32; WIDEN_CHUNK];
let mut rf_chunk = [0.0f32; WIDEN_CHUNK];
let mut af_chunk = [0.0f32; WIDEN_CHUNK];
let mut offset = 0;
while offset < w {
let n = (w - offset).min(WIDEN_CHUNK);
widen_f16_be_to_host_f32::<BE>(g_in, offset, &mut gf_chunk, n);
widen_f16_be_to_host_f32::<BE>(b_in, offset, &mut bf_chunk, n);
widen_f16_be_to_host_f32::<BE>(r_in, offset, &mut rf_chunk, n);
widen_f16_be_to_host_f32::<BE>(a_in, offset, &mut af_chunk, n);
let gf = &gf_chunk[..n];
let bf = &bf_chunk[..n];
let rf = &rf_chunk[..n];
let af = &af_chunk[..n];
let chunk_plane_start = one_plane_start + offset;
let chunk_plane_end = chunk_plane_start + n;
if let Some(buf) = self.rgb_f32.as_deref_mut() {
let start = chunk_plane_start * 3;
let end = chunk_plane_end * 3;
gbrpf32_to_rgb_f32_row::<HOST_NATIVE_BE>(gf, bf, rf, &mut buf[start..end], n, use_simd);
}
if let Some(buf) = self.rgba_f32.as_deref_mut() {
let start = chunk_plane_start * 4;
let end = chunk_plane_end * 4;
gbrapf32_to_rgba_f32_row::<HOST_NATIVE_BE>(
gf,
bf,
rf,
af,
&mut buf[start..end],
n,
use_simd,
);
}
if let Some(buf) = self.rgb_u16.as_deref_mut() {
let start = chunk_plane_start * 3;
let end = chunk_plane_end * 3;
gbrpf32_to_rgb_u16_row::<HOST_NATIVE_BE>(gf, bf, rf, &mut buf[start..end], n, use_simd);
}
if let Some(buf) = self.rgba_u16.as_deref_mut() {
let start = chunk_plane_start * 4;
let end = chunk_plane_end * 4;
gbrapf32_to_rgba_u16_row::<HOST_NATIVE_BE>(
gf,
bf,
rf,
af,
&mut buf[start..end],
n,
use_simd,
);
}
if let Some(buf) = self.luma.as_deref_mut() {
gbrpf32_to_luma_row::<HOST_NATIVE_BE>(
gf,
bf,
rf,
&mut buf[chunk_plane_start..chunk_plane_end],
n,
GBR_F16_LUMA_MATRIX,
GBR_F16_FULL_RANGE,
use_simd,
);
}
if let Some(buf) = self.luma_u16.as_deref_mut() {
gbrpf32_to_luma_u16_row::<HOST_NATIVE_BE>(
gf,
bf,
rf,
&mut buf[chunk_plane_start..chunk_plane_end],
n,
GBR_F16_LUMA_MATRIX,
GBR_F16_FULL_RANGE,
use_simd,
);
}
if let Some(hsv) = self.hsv.as_mut() {
let (h, s, v) = hsv.hsv();
gbrpf32_to_hsv_row::<HOST_NATIVE_BE>(
gf,
bf,
rf,
&mut h[chunk_plane_start..chunk_plane_end],
&mut s[chunk_plane_start..chunk_plane_end],
&mut v[chunk_plane_start..chunk_plane_end],
n,
use_simd,
);
}
offset += n;
}
}
let want_rgba = self.rgba.is_some();
let want_rgb = self.rgb.is_some();
let need_u8_rgb = want_rgb;
if want_rgba && !need_u8_rgb {
let rgba_buf = self.rgba.as_deref_mut().unwrap();
let rgba_row = rgba_plane_row_slice(rgba_buf, one_plane_start, one_plane_end, w, h)?;
gbrpf16_to_rgba_row::<BE>(g_in, b_in, r_in, rgba_row, w, use_simd);
widen_and_scatter_f16_alpha_to_u8::<BE>(a_in, rgba_row, w);
return Ok(());
}
if !need_u8_rgb && !want_rgba {
return Ok(());
}
let Self {
rgb,
rgba,
rgb_scratch,
..
} = self;
let rgb_row = rgb_row_buf_or_scratch(
rgb.as_deref_mut(),
rgb_scratch,
one_plane_start,
one_plane_end,
w,
h,
)?;
gbrpf16_to_rgb_row::<BE>(g_in, b_in, r_in, rgb_row, w, use_simd);
if let Some(buf) = rgba.as_deref_mut() {
let rgba_row = rgba_plane_row_slice(buf, one_plane_start, one_plane_end, w, h)?;
expand_rgb_to_rgba_row(rgb_row, rgba_row, w);
widen_and_scatter_f16_alpha_to_u8::<BE>(a_in, rgba_row, w);
}
Ok(())
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn widen_and_scatter_f16_alpha_to_u8<const BE: bool>(
alpha_f16: &[half::f16],
rgba_out: &mut [u8],
width: usize,
) {
let mut af_chunk = [0.0f32; WIDEN_CHUNK];
let mut offset = 0;
while offset < width {
let n = (width - offset).min(WIDEN_CHUNK);
widen_f16_be_to_host_f32::<BE>(alpha_f16, offset, &mut af_chunk, n);
copy_alpha_plane_f32_to_u8::<HOST_NATIVE_BE>(&af_chunk[..n], &mut rgba_out[offset * 4..], n);
offset += n;
}
}