fast_image_resize/convolution/f32x4/
sse4.rs

1use std::arch::x86_64::*;
2
3use crate::convolution::{Coefficients, CoefficientsChunk};
4use crate::pixels::F32x4;
5use crate::{simd_utils, ImageView, ImageViewMut};
6
7#[inline]
8pub(crate) fn horiz_convolution(
9    src_view: &impl ImageView<Pixel = F32x4>,
10    dst_view: &mut impl ImageViewMut<Pixel = F32x4>,
11    offset: u32,
12    coeffs: &Coefficients,
13) {
14    let coefficients_chunks = coeffs.get_chunks();
15    let dst_height = dst_view.height();
16
17    let src_iter = src_view.iter_4_rows(offset, dst_height + offset);
18    let dst_iter = dst_view.iter_4_rows_mut();
19    for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
20        unsafe {
21            horiz_convolution_rows(src_rows, dst_rows, &coefficients_chunks);
22        }
23    }
24
25    let yy = dst_height - dst_height % 4;
26    let src_rows = src_view.iter_rows(yy + offset);
27    let dst_rows = dst_view.iter_rows_mut(yy);
28    for (src_row, dst_row) in src_rows.zip(dst_rows) {
29        unsafe {
30            horiz_convolution_rows([src_row], [dst_row], &coefficients_chunks);
31        }
32    }
33}
34
35/// For safety, it is necessary to ensure the following conditions:
36/// - length of all rows in src_rows must be equal
37/// - length of all rows in dst_rows must be equal
38/// - coefficients_chunks.len() == dst_rows.0.len()
39/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
40/// - precision <= MAX_COEFS_PRECISION
41#[target_feature(enable = "sse4.1")]
42unsafe fn horiz_convolution_rows<const ROWS_COUNT: usize>(
43    src_rows: [&[F32x4]; ROWS_COUNT],
44    dst_rows: [&mut [F32x4]; ROWS_COUNT],
45    coefficients_chunks: &[CoefficientsChunk],
46) {
47    let mut rg_buf = [0f64; 2];
48    let mut ba_buf = [0f64; 2];
49
50    for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
51        let mut x: usize = coeffs_chunk.start as usize;
52        let mut rg_sums = [_mm_set1_pd(0.); ROWS_COUNT];
53        let mut ba_sums = [_mm_set1_pd(0.); ROWS_COUNT];
54
55        for &k in coeffs_chunk.values {
56            let coeffs_f64x2 = _mm_set1_pd(k);
57
58            for r in 0..ROWS_COUNT {
59                let pixel = simd_utils::loadu_ps(src_rows[r], x);
60                let rg_f64x2 = _mm_cvtps_pd(pixel);
61                rg_sums[r] = _mm_add_pd(rg_sums[r], _mm_mul_pd(rg_f64x2, coeffs_f64x2));
62                let ba_f64x2 = _mm_cvtps_pd(_mm_movehl_ps(pixel, pixel));
63                ba_sums[r] = _mm_add_pd(ba_sums[r], _mm_mul_pd(ba_f64x2, coeffs_f64x2));
64            }
65            x += 1;
66        }
67
68        for i in 0..ROWS_COUNT {
69            _mm_storeu_pd(rg_buf.as_mut_ptr(), rg_sums[i]);
70            _mm_storeu_pd(ba_buf.as_mut_ptr(), ba_sums[i]);
71            let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
72            dst_pixel.0 = [
73                rg_buf[0] as f32,
74                rg_buf[1] as f32,
75                ba_buf[0] as f32,
76                ba_buf[1] as f32,
77            ];
78        }
79    }
80}
fast_image_resize/convolution/f32x4/sse4.rs

fast_image_resize/convolution/f32x4/
sse4.rs
