use super::*;
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_420_to_rgb_row(
y: &[u8],
u_half: &[u8],
v_half: &[u8],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_420_to_rgb_or_rgba_row::<false, false>(
y, u_half, v_half, None, rgb_out, width, matrix, full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_420_to_rgba_row(
y: &[u8],
u_half: &[u8],
v_half: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_420_to_rgb_or_rgba_row::<true, false>(
y, u_half, v_half, None, rgba_out, width, matrix, full_range,
);
}
#[allow(dead_code)]
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_420_to_rgba_with_alpha_src_row(
y: &[u8],
u_half: &[u8],
v_half: &[u8],
a_src: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_420_to_rgb_or_rgba_row::<true, true>(
y,
u_half,
v_half,
Some(a_src),
rgba_out,
width,
matrix,
full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_420_to_rgb_or_rgba_row<const ALPHA: bool, const ALPHA_SRC: bool>(
y: &[u8],
u_half: &[u8],
v_half: &[u8],
a_src: Option<&[u8]>,
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_420_to_rgb_or_rgba_row_with_coeffs::<ALPHA, ALPHA_SRC>(
y,
u_half,
v_half,
a_src,
out,
width,
Coefficients::for_matrix(matrix),
full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_420_to_rgb_or_rgba_row_with_coeffs<const ALPHA: bool, const ALPHA_SRC: bool>(
y: &[u8],
u_half: &[u8],
v_half: &[u8],
a_src: Option<&[u8]>,
out: &mut [u8],
width: usize,
coeffs: Coefficients,
full_range: bool,
) {
const { assert!(!ALPHA_SRC || ALPHA) };
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u_half.len() >= width / 2, "u_half row too short");
debug_assert!(v_half.len() >= width / 2, "v_half row too short");
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(out.len() >= width * bpp, "out row too short for {bpp}bpp");
if ALPHA_SRC {
debug_assert!(
a_src.as_ref().is_some_and(|s| s.len() >= width),
"a_src row too short"
);
}
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
const RND: i32 = 1 << 14;
let mut x = 0;
while x < width {
let c_idx = x / 2;
let u_d = ((u_half[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v_half[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
let y0 = ((y[x] as i32 - y_off) * y_scale + RND) >> 15;
let r0 = clamp_u8(y0 + r_chroma);
let g0 = clamp_u8(y0 + g_chroma);
let b0 = clamp_u8(y0 + b_chroma);
out[x * bpp] = r0;
out[x * bpp + 1] = g0;
out[x * bpp + 2] = b0;
if ALPHA_SRC {
out[x * bpp + 3] = a_src.as_ref().unwrap()[x];
} else if ALPHA {
out[x * bpp + 3] = 0xFF;
}
let y1 = ((y[x + 1] as i32 - y_off) * y_scale + RND) >> 15;
let r1 = clamp_u8(y1 + r_chroma);
let g1 = clamp_u8(y1 + g_chroma);
let b1 = clamp_u8(y1 + b_chroma);
out[(x + 1) * bpp] = r1;
out[(x + 1) * bpp + 1] = g1;
out[(x + 1) * bpp + 2] = b1;
if ALPHA_SRC {
out[(x + 1) * bpp + 3] = a_src.as_ref().unwrap()[x + 1];
} else if ALPHA {
out[(x + 1) * bpp + 3] = 0xFF;
}
x += 2;
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_420_to_rgb_row_with_coeffs(
y: &[u8],
u_half: &[u8],
v_half: &[u8],
rgb_out: &mut [u8],
width: usize,
coeffs: Coefficients,
full_range: bool,
) {
yuv_420_to_rgb_or_rgba_row_with_coeffs::<false, false>(
y, u_half, v_half, None, rgb_out, width, coeffs, full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_420_to_rgba_row_with_coeffs(
y: &[u8],
u_half: &[u8],
v_half: &[u8],
rgba_out: &mut [u8],
width: usize,
coeffs: Coefficients,
full_range: bool,
) {
yuv_420_to_rgb_or_rgba_row_with_coeffs::<true, false>(
y, u_half, v_half, None, rgba_out, width, coeffs, full_range,
);
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_2to1_center_h(c_half: &[u8], c_full: &mut [u8], width: usize) {
debug_assert_eq!(
width & 1,
0,
"2:1 horizontal chroma subsampling requires even width"
);
debug_assert!(c_half.len() >= width / 2, "c_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
for j in 0..half {
let left = c_half[j.saturating_sub(1)] as u32;
let mid = c_half[j] as u32;
let right = c_half[if j + 1 < half { j + 1 } else { j }] as u32;
c_full[2 * j] = ((left + 3 * mid + 2) >> 2) as u8;
c_full[2 * j + 1] = ((3 * mid + right + 2) >> 2) as u8;
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(
not(all(
feature = "yuv-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_2to1_center_h_pair(c: &[u8], full: &mut [u8], j: usize, half: usize) {
let left = c[j.saturating_sub(1)] as u32;
let mid = c[j] as u32;
let right = c[if j + 1 < half { j + 1 } else { j }] as u32;
full[2 * j] = ((left + 3 * mid + 2) >> 2) as u8;
full[2 * j + 1] = ((3 * mid + right + 2) >> 2) as u8;
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_4to1_center_h(c_quarter: &[u8], c_full: &mut [u8], width: usize) {
let quarter = width.div_ceil(4);
debug_assert!(c_quarter.len() >= quarter, "c_quarter row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
for j in 0..quarter {
let left = c_quarter[j.saturating_sub(1)] as u32;
let mid = c_quarter[j] as u32;
let right = c_quarter[if j + 1 < quarter { j + 1 } else { j }] as u32;
let base = 4 * j;
if base < width {
c_full[base] = ((3 * left + 5 * mid + 4) >> 3) as u8;
}
if base + 1 < width {
c_full[base + 1] = ((left + 7 * mid + 4) >> 3) as u8;
}
if base + 2 < width {
c_full[base + 2] = ((7 * mid + right + 4) >> 3) as u8;
}
if base + 3 < width {
c_full[base + 3] = ((5 * mid + 3 * right + 4) >> 3) as u8;
}
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(
not(all(
feature = "yuv-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_4to1_center_h_group(
c_quarter: &[u8],
c_full: &mut [u8],
j: usize,
quarter: usize,
width: usize,
) {
let left = c_quarter[j.saturating_sub(1)] as u32;
let mid = c_quarter[j] as u32;
let right = c_quarter[if j + 1 < quarter { j + 1 } else { j }] as u32;
let base = 4 * j;
if base < width {
c_full[base] = ((3 * left + 5 * mid + 4) >> 3) as u8;
}
if base + 1 < width {
c_full[base + 1] = ((left + 7 * mid + 4) >> 3) as u8;
}
if base + 2 < width {
c_full[base + 2] = ((7 * mid + right + 4) >> 3) as u8;
}
if base + 3 < width {
c_full[base + 3] = ((5 * mid + 3 * right + 4) >> 3) as u8;
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottom_even_h(
prev_half: &[u8],
cur_half: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
let vblend = |j: usize| -> u32 { ((prev_half[j] as u32) + (cur_half[j] as u32) + 1) >> 1 };
for j in 0..half {
let left = vblend(j.saturating_sub(1));
let mid = vblend(j);
let right = vblend(if j + 1 < half { j + 1 } else { j });
c_full[2 * j] = ((left + 3 * mid + 2) >> 2) as u8;
c_full[2 * j + 1] = ((3 * mid + right + 2) >> 2) as u8;
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(
not(all(
feature = "yuv-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottom_even_h_pair(
prev_half: &[u8],
cur_half: &[u8],
full: &mut [u8],
j: usize,
half: usize,
) {
let vblend = |k: usize| -> u32 { ((prev_half[k] as u32) + (cur_half[k] as u32) + 1) >> 1 };
let left = vblend(j.saturating_sub(1));
let mid = vblend(j);
let right = vblend(if j + 1 < half { j + 1 } else { j });
full[2 * j] = ((left + 3 * mid + 2) >> 2) as u8;
full[2 * j + 1] = ((3 * mid + right + 2) >> 2) as u8;
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_2to1_cosited_h(c_half: &[u8], c_full: &mut [u8], width: usize) {
debug_assert_eq!(
width & 1,
0,
"2:1 horizontal chroma subsampling requires even width"
);
debug_assert!(c_half.len() >= width / 2, "c_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
for j in 0..half {
let c = c_half[j];
c_full[2 * j] = c;
c_full[2 * j + 1] = c;
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottomleft_even_h(
prev_half: &[u8],
cur_half: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
for j in 0..half {
let e = (((prev_half[j] as u32) + (cur_half[j] as u32) + 1) >> 1) as u8;
c_full[2 * j] = e;
c_full[2 * j + 1] = e;
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(
not(all(
feature = "yuv-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottomleft_even_h_pair(
prev_half: &[u8],
cur_half: &[u8],
full: &mut [u8],
j: usize,
) {
let e = (((prev_half[j] as u32) + (cur_half[j] as u32) + 1) >> 1) as u8;
full[2 * j] = e;
full[2 * j + 1] = e;
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_2to1_center_h_u16<const BITS: u32>(
c_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(
width & 1,
0,
"2:1 horizontal chroma subsampling requires even width"
);
debug_assert!(c_half.len() >= width / 2, "c_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u32 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
(logical & mask) as u32
};
let store = |logical: u32| -> u16 {
let v = logical as u16;
if big_endian { v.to_be() } else { v.to_le() }
};
let half = width / 2;
for j in 0..half {
let left = load(c_half[j.saturating_sub(1)]);
let mid = load(c_half[j]);
let right = load(c_half[if j + 1 < half { j + 1 } else { j }]);
c_full[2 * j] = store((left + 3 * mid + 2) >> 2);
c_full[2 * j + 1] = store((3 * mid + right + 2) >> 2);
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(
not(all(
feature = "yuv-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_2to1_center_h_u16_pair<const BITS: u32>(
c: &[u16],
full: &mut [u16],
j: usize,
half: usize,
big_endian: bool,
) {
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u32 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
(logical & mask) as u32
};
let store = |logical: u32| -> u16 {
let v = logical as u16;
if big_endian { v.to_be() } else { v.to_le() }
};
let left = load(c[j.saturating_sub(1)]);
let mid = load(c[j]);
let right = load(c[if j + 1 < half { j + 1 } else { j }]);
full[2 * j] = store((left + 3 * mid + 2) >> 2);
full[2 * j + 1] = store((3 * mid + right + 2) >> 2);
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottom_even_h_u16<const BITS: u32>(
prev_half: &[u16],
cur_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u32 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
(logical & mask) as u32
};
let store = |logical: u32| -> u16 {
let v = logical as u16;
if big_endian { v.to_be() } else { v.to_le() }
};
let half = width / 2;
let vblend = |j: usize| -> u32 { (load(prev_half[j]) + load(cur_half[j]) + 1) >> 1 };
for j in 0..half {
let left = vblend(j.saturating_sub(1));
let mid = vblend(j);
let right = vblend(if j + 1 < half { j + 1 } else { j });
c_full[2 * j] = store((left + 3 * mid + 2) >> 2);
c_full[2 * j + 1] = store((3 * mid + right + 2) >> 2);
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(
not(all(
feature = "yuv-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottom_even_h_u16_pair<const BITS: u32>(
prev_half: &[u16],
cur_half: &[u16],
full: &mut [u16],
j: usize,
half: usize,
big_endian: bool,
) {
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u32 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
(logical & mask) as u32
};
let store = |logical: u32| -> u16 {
let v = logical as u16;
if big_endian { v.to_be() } else { v.to_le() }
};
let vblend = |k: usize| -> u32 { (load(prev_half[k]) + load(cur_half[k]) + 1) >> 1 };
let left = vblend(j.saturating_sub(1));
let mid = vblend(j);
let right = vblend(if j + 1 < half { j + 1 } else { j });
full[2 * j] = store((left + 3 * mid + 2) >> 2);
full[2 * j + 1] = store((3 * mid + right + 2) >> 2);
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_2to1_cosited_h_u16<const BITS: u32>(
c_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(
width & 1,
0,
"2:1 horizontal chroma subsampling requires even width"
);
debug_assert!(c_half.len() >= width / 2, "c_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u16 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
logical & mask
};
let store = |logical: u16| -> u16 {
if big_endian {
logical.to_be()
} else {
logical.to_le()
}
};
let half = width / 2;
for j in 0..half {
let c = store(load(c_half[j]));
c_full[2 * j] = c;
c_full[2 * j + 1] = c;
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottomleft_even_h_u16<const BITS: u32>(
prev_half: &[u16],
cur_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u32 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
(logical & mask) as u32
};
let store = |logical: u32| -> u16 {
let v = logical as u16;
if big_endian { v.to_be() } else { v.to_le() }
};
let half = width / 2;
for j in 0..half {
let e = store((load(prev_half[j]) + load(cur_half[j]) + 1) >> 1);
c_full[2 * j] = e;
c_full[2 * j + 1] = e;
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(
not(all(
feature = "yuv-planar",
any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
)
)),
allow(dead_code)
)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_420_bottomleft_even_h_u16_pair<const BITS: u32>(
prev_half: &[u16],
cur_half: &[u16],
full: &mut [u16],
j: usize,
big_endian: bool,
) {
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u32 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
(logical & mask) as u32
};
let store = |logical: u32| -> u16 {
let v = logical as u16;
if big_endian { v.to_be() } else { v.to_le() }
};
let e = store((load(prev_half[j]) + load(cur_half[j]) + 1) >> 1);
full[2 * j] = e;
full[2 * j + 1] = e;
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn chroma_upsample_440_bottom_v(prev: &[u8], cur: &[u8], out: &mut [u8], width: usize) {
debug_assert!(prev.len() >= width, "prev row too short");
debug_assert!(cur.len() >= width, "cur row too short");
debug_assert!(out.len() >= width, "out row too short");
for j in 0..width {
out[j] = (((prev[j] as u16) + (cur[j] as u16) + 1) >> 1) as u8;
}
}
#[cfg(any(feature = "std", feature = "alloc"))]
#[cfg_attr(not(tarpaulin), inline(always))]
#[cfg_attr(not(all(test, feature = "std")), allow(dead_code))]
pub(crate) fn chroma_upsample_440_bottom_v_u16(
prev: &[u16],
cur: &[u16],
out: &mut [u16],
width: usize,
) {
debug_assert!(prev.len() >= width, "prev row too short");
debug_assert!(cur.len() >= width, "cur row too short");
debug_assert!(out.len() >= width, "out row too short");
for j in 0..width {
out[j] = (((prev[j] as u32) + (cur[j] as u32) + 1) >> 1) as u16;
}
}
#[cfg(all(any(feature = "std", feature = "alloc"), feature = "yuv-planar"))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn chroma_upsample_440_bottom_v_u16_wire<const BITS: u32>(
prev: &[u16],
cur: &[u16],
out: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert!(prev.len() >= width, "prev row too short");
debug_assert!(cur.len() >= width, "cur row too short");
debug_assert!(out.len() >= width, "out row too short");
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u32 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
u32::from(logical & mask)
};
for j in 0..width {
let blended = ((load(prev[j]) + load(cur[j]) + 1) >> 1) as u16;
out[j] = if big_endian {
blended.to_be()
} else {
blended.to_le()
};
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_410_to_rgb_row(
y: &[u8],
u_quarter: &[u8],
v_quarter: &[u8],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_410_to_rgb_or_rgba_row::<false>(y, u_quarter, v_quarter, rgb_out, width, matrix, full_range);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_410_to_rgba_row(
y: &[u8],
u_quarter: &[u8],
v_quarter: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_410_to_rgb_or_rgba_row::<true>(y, u_quarter, v_quarter, rgba_out, width, matrix, full_range);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_410_to_rgb_or_rgba_row<const ALPHA: bool>(
y: &[u8],
u_quarter: &[u8],
v_quarter: &[u8],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert_eq!(width & 3, 0, "YUV 4:1:0 requires width % 4 == 0");
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u_quarter.len() >= width / 4, "u_quarter row too short");
debug_assert!(v_quarter.len() >= width / 4, "v_quarter row too short");
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(out.len() >= width * bpp, "out row too short for {bpp}bpp");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
const RND: i32 = 1 << 14;
let mut x = 0;
while x < width {
let c_idx = x / 4;
let u_d = ((u_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
for k in 0..4 {
let yk = ((y[x + k] as i32 - y_off) * y_scale + RND) >> 15;
out[(x + k) * bpp] = clamp_u8(yk + r_chroma);
out[(x + k) * bpp + 1] = clamp_u8(yk + g_chroma);
out[(x + k) * bpp + 2] = clamp_u8(yk + b_chroma);
if ALPHA {
out[(x + k) * bpp + 3] = 0xFF;
}
}
x += 4;
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_444_to_rgb_row(
y: &[u8],
u: &[u8],
v: &[u8],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_444_to_rgb_or_rgba_row::<false, false>(y, u, v, None, rgb_out, width, matrix, full_range);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_444_to_rgba_row(
y: &[u8],
u: &[u8],
v: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_444_to_rgb_or_rgba_row::<true, false>(y, u, v, None, rgba_out, width, matrix, full_range);
}
#[allow(dead_code)]
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_444_to_rgba_with_alpha_src_row(
y: &[u8],
u: &[u8],
v: &[u8],
a_src: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_444_to_rgb_or_rgba_row::<true, true>(
y,
u,
v,
Some(a_src),
rgba_out,
width,
matrix,
full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_444_to_rgb_or_rgba_row<const ALPHA: bool, const ALPHA_SRC: bool>(
y: &[u8],
u: &[u8],
v: &[u8],
a_src: Option<&[u8]>,
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_444_to_rgb_or_rgba_row_with_coeffs::<ALPHA, ALPHA_SRC>(
y,
u,
v,
a_src,
out,
width,
Coefficients::for_matrix(matrix),
full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_444_to_rgb_or_rgba_row_with_coeffs<const ALPHA: bool, const ALPHA_SRC: bool>(
y: &[u8],
u: &[u8],
v: &[u8],
a_src: Option<&[u8]>,
out: &mut [u8],
width: usize,
coeffs: Coefficients,
full_range: bool,
) {
const { assert!(!ALPHA_SRC || ALPHA) };
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u.len() >= width, "u row too short");
debug_assert!(v.len() >= width, "v row too short");
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(out.len() >= width * bpp, "out row too short for {bpp}bpp");
if ALPHA_SRC {
debug_assert!(
a_src.as_ref().is_some_and(|s| s.len() >= width),
"a_src row too short"
);
}
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
const RND: i32 = 1 << 14;
for x in 0..width {
let u_d = ((u[x] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v[x] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
let y0 = ((y[x] as i32 - y_off) * y_scale + RND) >> 15;
out[x * bpp] = clamp_u8(y0 + r_chroma);
out[x * bpp + 1] = clamp_u8(y0 + g_chroma);
out[x * bpp + 2] = clamp_u8(y0 + b_chroma);
if ALPHA_SRC {
out[x * bpp + 3] = a_src.as_ref().unwrap()[x];
} else if ALPHA {
out[x * bpp + 3] = 0xFF;
}
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_444_to_rgb_row_with_coeffs(
y: &[u8],
u: &[u8],
v: &[u8],
rgb_out: &mut [u8],
width: usize,
coeffs: Coefficients,
full_range: bool,
) {
yuv_444_to_rgb_or_rgba_row_with_coeffs::<false, false>(
y, u, v, None, rgb_out, width, coeffs, full_range,
);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_444_to_rgba_row_with_coeffs(
y: &[u8],
u: &[u8],
v: &[u8],
rgba_out: &mut [u8],
width: usize,
coeffs: Coefficients,
full_range: bool,
) {
yuv_444_to_rgb_or_rgba_row_with_coeffs::<true, false>(
y, u, v, None, rgba_out, width, coeffs, full_range,
);
}
#[cfg(all(feature = "rgb", any(feature = "std", feature = "alloc")))]
const Q15_TO_REAL: f32 = 1.0 / 32768.0;
#[cfg(all(feature = "rgb", any(feature = "std", feature = "alloc")))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_420_to_rgb_f32_unclamped_row(
y: &[u8],
u_half: &[u8],
v_half: &[u8],
out: &mut [f32],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u_half.len() >= width / 2, "u_half row too short");
debug_assert!(v_half.len() >= width / 2, "v_half row too short");
debug_assert!(out.len() >= width * 3, "out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
let y_off_f = y_off as f32;
let y_scale_f = y_scale as f32 * Q15_TO_REAL;
let c_scale_f = c_scale as f32 * Q15_TO_REAL;
let r_u = coeffs.r_u() as f32 * Q15_TO_REAL;
let r_v = coeffs.r_v() as f32 * Q15_TO_REAL;
let g_u = coeffs.g_u() as f32 * Q15_TO_REAL;
let g_v = coeffs.g_v() as f32 * Q15_TO_REAL;
let b_u = coeffs.b_u() as f32 * Q15_TO_REAL;
let b_v = coeffs.b_v() as f32 * Q15_TO_REAL;
let mut x = 0;
while x < width {
let c_idx = x / 2;
let u_d = (u_half[c_idx] as f32 - 128.0) * c_scale_f;
let v_d = (v_half[c_idx] as f32 - 128.0) * c_scale_f;
let r_chroma = r_u * u_d + r_v * v_d;
let g_chroma = g_u * u_d + g_v * v_d;
let b_chroma = b_u * u_d + b_v * v_d;
let y0 = (y[x] as f32 - y_off_f) * y_scale_f;
out[x * 3] = (y0 + r_chroma) / 255.0;
out[x * 3 + 1] = (y0 + g_chroma) / 255.0;
out[x * 3 + 2] = (y0 + b_chroma) / 255.0;
let y1 = (y[x + 1] as f32 - y_off_f) * y_scale_f;
out[(x + 1) * 3] = (y1 + r_chroma) / 255.0;
out[(x + 1) * 3 + 1] = (y1 + g_chroma) / 255.0;
out[(x + 1) * 3 + 2] = (y1 + b_chroma) / 255.0;
x += 2;
}
}
#[cfg(all(feature = "rgb", any(feature = "std", feature = "alloc")))]
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_444_to_rgb_f32_unclamped_row(
y: &[u8],
u: &[u8],
v: &[u8],
out: &mut [f32],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u.len() >= width, "u row too short");
debug_assert!(v.len() >= width, "v row too short");
debug_assert!(out.len() >= width * 3, "out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
let y_off_f = y_off as f32;
let y_scale_f = y_scale as f32 * Q15_TO_REAL;
let c_scale_f = c_scale as f32 * Q15_TO_REAL;
let r_u = coeffs.r_u() as f32 * Q15_TO_REAL;
let r_v = coeffs.r_v() as f32 * Q15_TO_REAL;
let g_u = coeffs.g_u() as f32 * Q15_TO_REAL;
let g_v = coeffs.g_v() as f32 * Q15_TO_REAL;
let b_u = coeffs.b_u() as f32 * Q15_TO_REAL;
let b_v = coeffs.b_v() as f32 * Q15_TO_REAL;
for x in 0..width {
let u_d = (u[x] as f32 - 128.0) * c_scale_f;
let v_d = (v[x] as f32 - 128.0) * c_scale_f;
let r_chroma = r_u * u_d + r_v * v_d;
let g_chroma = g_u * u_d + g_v * v_d;
let b_chroma = b_u * u_d + b_v * v_d;
let y0 = (y[x] as f32 - y_off_f) * y_scale_f;
out[x * 3] = (y0 + r_chroma) / 255.0;
out[x * 3 + 1] = (y0 + g_chroma) / 255.0;
out[x * 3 + 2] = (y0 + b_chroma) / 255.0;
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_411_to_rgb_row(
y: &[u8],
u_quarter: &[u8],
v_quarter: &[u8],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_411_to_rgb_or_rgba_row::<false>(y, u_quarter, v_quarter, rgb_out, width, matrix, full_range);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_411_to_rgba_row(
y: &[u8],
u_quarter: &[u8],
v_quarter: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
yuv_411_to_rgb_or_rgba_row::<true>(y, u_quarter, v_quarter, rgba_out, width, matrix, full_range);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn yuv_411_to_rgb_or_rgba_row<const ALPHA: bool>(
y: &[u8],
u_quarter: &[u8],
v_quarter: &[u8],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(
u_quarter.len() >= width.div_ceil(4),
"u_quarter row too short"
);
debug_assert!(
v_quarter.len() >= width.div_ceil(4),
"v_quarter row too short"
);
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(out.len() >= width * bpp, "out row too short for {bpp}bpp");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
const RND: i32 = 1 << 14;
let body_end = width & !3;
let mut x = 0;
while x < body_end {
let c_idx = x / 4;
let u_d = ((u_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
let mut k = 0;
while k < 4 {
let y_k = ((y[x + k] as i32 - y_off) * y_scale + RND) >> 15;
let pos = (x + k) * bpp;
out[pos] = clamp_u8(y_k + r_chroma);
out[pos + 1] = clamp_u8(y_k + g_chroma);
out[pos + 2] = clamp_u8(y_k + b_chroma);
if ALPHA {
out[pos + 3] = 0xFF;
}
k += 1;
}
x += 4;
}
if x < width {
let c_idx = x / 4; let u_d = ((u_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
while x < width {
let y_k = ((y[x] as i32 - y_off) * y_scale + RND) >> 15;
let pos = x * bpp;
out[pos] = clamp_u8(y_k + r_chroma);
out[pos + 1] = clamp_u8(y_k + g_chroma);
out[pos + 2] = clamp_u8(y_k + b_chroma);
if ALPHA {
out[pos + 3] = 0xFF;
}
x += 1;
}
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_420_to_hsv_row(
y: &[u8],
u_half: &[u8],
v_half: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u_half.len() >= width / 2, "u_half row too short");
debug_assert!(v_half.len() >= width / 2, "v_half row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
const RND: i32 = 1 << 14;
let mut x = 0;
while x < width {
let c_idx = x / 2;
let u_d = ((u_half[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v_half[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
let y0 = ((y[x] as i32 - y_off) * y_scale + RND) >> 15;
let (h0, s0, v0) = rgb_to_hsv_pixel(
clamp_u8(y0 + r_chroma) as i32,
clamp_u8(y0 + g_chroma) as i32,
clamp_u8(y0 + b_chroma) as i32,
);
h_out[x] = h0;
s_out[x] = s0;
v_out[x] = v0;
let y1 = ((y[x + 1] as i32 - y_off) * y_scale + RND) >> 15;
let (h1, s1, v1) = rgb_to_hsv_pixel(
clamp_u8(y1 + r_chroma) as i32,
clamp_u8(y1 + g_chroma) as i32,
clamp_u8(y1 + b_chroma) as i32,
);
h_out[x + 1] = h1;
s_out[x + 1] = s1;
v_out[x + 1] = v1;
x += 2;
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_444_to_hsv_row(
y: &[u8],
u: &[u8],
v: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u.len() >= width, "u row too short");
debug_assert!(v.len() >= width, "v row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
const RND: i32 = 1 << 14;
for x in 0..width {
let u_d = ((u[x] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v[x] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
let y0 = ((y[x] as i32 - y_off) * y_scale + RND) >> 15;
let (h, s, vv) = rgb_to_hsv_pixel(
clamp_u8(y0 + r_chroma) as i32,
clamp_u8(y0 + g_chroma) as i32,
clamp_u8(y0 + b_chroma) as i32,
);
h_out[x] = h;
s_out[x] = s;
v_out[x] = vv;
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_410_to_hsv_row(
y: &[u8],
u_quarter: &[u8],
v_quarter: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert_eq!(width & 3, 0, "YUV 4:1:0 requires width % 4 == 0");
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u_quarter.len() >= width / 4, "u_quarter row too short");
debug_assert!(v_quarter.len() >= width / 4, "v_quarter row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
const RND: i32 = 1 << 14;
let mut x = 0;
while x < width {
let c_idx = x / 4;
let u_d = ((u_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
for k in 0..4 {
let yk = ((y[x + k] as i32 - y_off) * y_scale + RND) >> 15;
let (h, s, vv) = rgb_to_hsv_pixel(
clamp_u8(yk + r_chroma) as i32,
clamp_u8(yk + g_chroma) as i32,
clamp_u8(yk + b_chroma) as i32,
);
h_out[x + k] = h;
s_out[x + k] = s;
v_out[x + k] = vv;
}
x += 4;
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn yuv_411_to_hsv_row(
y: &[u8],
u_quarter: &[u8],
v_quarter: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(
u_quarter.len() >= width.div_ceil(4),
"u_quarter row too short"
);
debug_assert!(
v_quarter.len() >= width.div_ceil(4),
"v_quarter row too short"
);
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
let coeffs = Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = range_params_n::<8, 8>(full_range);
const RND: i32 = 1 << 14;
let mut emit = |x: usize, r_chroma: i32, g_chroma: i32, b_chroma: i32| {
let yk = ((y[x] as i32 - y_off) * y_scale + RND) >> 15;
let (h, s, vv) = rgb_to_hsv_pixel(
clamp_u8(yk + r_chroma) as i32,
clamp_u8(yk + g_chroma) as i32,
clamp_u8(yk + b_chroma) as i32,
);
h_out[x] = h;
s_out[x] = s;
v_out[x] = vv;
};
let body_end = width & !3;
let mut x = 0;
while x < body_end {
let c_idx = x / 4;
let u_d = ((u_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
let mut k = 0;
while k < 4 {
emit(x + k, r_chroma, g_chroma, b_chroma);
k += 1;
}
x += 4;
}
if x < width {
let c_idx = x / 4;
let u_d = ((u_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let v_d = ((v_quarter[c_idx] as i32 - 128) * c_scale + RND) >> 15;
let r_chroma = (coeffs.r_u() * u_d + coeffs.r_v() * v_d + RND) >> 15;
let g_chroma = (coeffs.g_u() * u_d + coeffs.g_v() * v_d + RND) >> 15;
let b_chroma = (coeffs.b_u() * u_d + coeffs.b_v() * v_d + RND) >> 15;
while x < width {
emit(x, r_chroma, g_chroma, b_chroma);
x += 1;
}
}
}