Skip to main content

quick_noise/noise/generators/value/
grid_3d.rs

1use std::array::from_fn;
2use std::mem::MaybeUninit;
3use std::ops::Range;
4
5use simply_simd::{Arch, Simd, enable_targets};
6
7use crate::GridGenerator;
8use crate::api::grid::interface::GridNoiseParams;
9use crate::noise::combiners::{Combiner, CombinerState};
10use crate::noise::generators::Value;
11use crate::noise::util::constants::{BYTE_SHUFFLE, VALUE_EXP_MASK, HASH_MASK, HASH_PRIME};
12use crate::noise::util::grid_data::{GridData, Lerp};
13use crate::noise::util::grid_helpers::*;
14
15pub struct ValueGradients3D<'a> {
16    pub tlf: &'a mut [MaybeUninit<f32>],
17    pub trf: &'a mut [MaybeUninit<f32>],
18    pub blf: &'a mut [MaybeUninit<f32>],
19    pub brf: &'a mut [MaybeUninit<f32>],
20    pub tlb: &'a mut [MaybeUninit<f32>],
21    pub trb: &'a mut [MaybeUninit<f32>],
22    pub blb: &'a mut [MaybeUninit<f32>],
23    pub brb: &'a mut [MaybeUninit<f32>],
24    pub grad_buffers: [&'a mut [MaybeUninit<f32>]; 2],
25}
26
27impl<'a> ValueGradients3D<'a> {
28    #[inline(always)]
29    pub fn new(arena: &'a mut Arena, size: usize) -> Self {
30        Self {
31            tlf: arena.allocate(size),
32            trf: arena.allocate(size),
33            blf: arena.allocate(size),
34            brf: arena.allocate(size),
35            tlb: arena.allocate(size),
36            trb: arena.allocate(size),
37            blb: arena.allocate(size),
38            brb: arena.allocate(size),
39            grad_buffers: from_fn(|_| arena.allocate(size)),
40        }
41    }
42
43    #[inline(always)]
44    pub fn swap_top_bottom(&mut self) {
45        std::mem::swap(&mut self.tlf, &mut self.blf);
46        std::mem::swap(&mut self.trf, &mut self.brf);
47        std::mem::swap(&mut self.tlb, &mut self.blb);
48        std::mem::swap(&mut self.trb, &mut self.brb);
49    }
50}
51
52const LERP: u8 = Lerp::Quintic as u8;
53#[enable_targets(A)]
54impl GridGenerator<3> for Value {
55    fn sample_grid<A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>(
56        params: GridNoiseParams<3>,
57        fractal_config: C::Config,
58        state: &mut [f32],
59        dst: &mut [f32],
60    ) {
61        // Validate and pad grid size.
62        validate_grid_size(params.grid_size, dst.len());
63        validate_state_size::<C, A, _>(params.grid_size, state.len());
64        let padded_size = pad_grid_size::<A, 3>(params.grid_size);
65
66        // Arena setup.
67        let required_cache = padded_size[0] * 17 + padded_size[1] * 3 + padded_size[2] * 3;
68        let mut cache = ArenaBuffer::<A>::with_capacity(required_cache);
69        let mut arena = Arena::with_cache(&mut cache);
70        let mut data_arena = arena.allocate_arena(padded_size.iter().fold(0, |n, x| n + 3 * x));
71        let mut trilerp_arena = arena.allocate_arena(padded_size[0] * 4);
72
73        // Allocation setup.
74        let num_blocks = TrilerpExecuter::<A, C, INIT, FINAL>::MAX_BLOCKS;
75        let bilerp_config = InterpolationConfig::new(num_blocks, params.grid_size[0]);
76        let grid_data = GridData::new::<A, LERP>(&params, &mut data_arena, &padded_size);
77        let mut trilerp_buffers = TrilerpBuffers::new(&mut trilerp_arena, padded_size[0]);
78        let mut gradients = ValueGradients3D::new(&mut arena, padded_size[0]);
79
80        // Iterate through single y chunks but full x chunks.
81        let mut z_cur_index = 0;
82        for z_it in 0..grid_data.num_loops[2] {
83            let z_next_index =
84                unsafe { grid_data.grid_indices[2].get_unchecked(z_it).assume_init() as usize };
85            let z_range = z_cur_index..z_next_index;
86
87            // Set the top gradients.
88            fill_gradients_3d::<A>(&params, &grid_data, &mut gradients, 0, z_it);
89            gradients.swap_top_bottom();
90
91            let mut y_cur_index = 0;
92            for y_it in 0..grid_data.num_loops[1] {
93                let y_next_index =
94                    unsafe { grid_data.grid_indices[1].get_unchecked(y_it).assume_init() as usize };
95                let y_range = y_cur_index..y_next_index;
96
97                // Set bottom gradients.
98                fill_gradients_3d::<A>(&params, &grid_data, &mut gradients, y_it + 1, z_it);
99
100                trilerp::<A, C, INIT, FINAL>(
101                    &mut trilerp_buffers,
102                    &bilerp_config,
103                    &fractal_config,
104                    &grid_data,
105                    &gradients,
106                    (y_range, z_range.clone()),
107                    (state, dst),
108                );
109
110                // Reuse the top and bottom gradients.
111                gradients.swap_top_bottom();
112
113                y_cur_index = y_next_index;
114            }
115            z_cur_index = z_next_index;
116        }
117    }
118}
119
120#[inline(always)]
121pub(super) fn fill_gradients_3d<'a, A: Arch>(
122    params: &GridNoiseParams<3>,
123    grid_data: &GridData<3>,
124    gradients: &mut ValueGradients3D<'a>,
125    y_it: usize,
126    z_it: usize,
127) {
128    let lanes = Simd::<f32, A>::LANES;
129    let y_start = y_it as i32 + grid_data.grid_start[1];
130    let z_start = z_it as i32 + grid_data.grid_start[2];
131    let (z1, z2) = match grid_data.octave_tiling[2] {
132        None => (
133            (z_start as u32).wrapping_mul(params.seed),
134            (z_start as u32)
135                .wrapping_mul(params.seed)
136                .wrapping_add(params.seed),
137        ),
138        Some(t) => (
139            (z_start.rem_euclid(t as i32)) as u32,
140            ((z_start + 1).rem_euclid(t as i32)) as u32,
141        ),
142    };
143    let z_vec = [Simd::splat(z1), Simd::splat(z2)];
144
145    let y_rem = grid_data.octave_tiling[1].map_or(y_start, |t| y_start.rem_euclid(t as i32));
146    let y_vec = Simd::splat((y_rem as u32).wrapping_mul(params.seed));
147
148    let shuffle_indices = Simd::<u8, A>::from_slice(&BYTE_SHUFFLE[..]);
149
150    let prime = Simd::splat(HASH_PRIME);
151    let z_shuf: [_; 2] = from_fn(|i| z_vec[i].permute_8(shuffle_indices) ^ prime);
152    let y_shuf = y_vec.permute_8(shuffle_indices) ^ prime;
153    let zy_mix: [_; 2] = from_fn(|i| z_shuf[i] * y_shuf);
154
155    // Main vectorized bit mixing loop.
156    let end_index = grid_data.num_loops[0] + 1;
157    let hash_mask: Simd<u32, A> = Simd::splat(HASH_MASK);
158    let exp_bits: Simd<u32, A> = Simd::splat(VALUE_EXP_MASK);
159    let three: Simd<f32, A> = Simd::splat(3.0);
160
161    if let Some(x_tiling) = grid_data.octave_tiling[0] {
162        let x_tiling = Simd::splat(x_tiling as f32);
163        let mut x_vec = Simd::splat(grid_data.grid_start[0]) + Simd::iota(0);
164        let x_vec_stride = Simd::splat(lanes as i32);
165        let seed_vec = Simd::splat(params.seed);
166
167        for i in (0..end_index).step_by(lanes) {
168            let x_floats = x_vec.cast_float();
169            let x_rem = x_floats - (x_floats / x_tiling).floor() * x_tiling;
170            let x_seeded = x_rem.cast_int_round().raw_cast() * seed_vec;
171            let x_shuf = x_seeded.permute_8(shuffle_indices) ^ prime;
172            let hashes: [_; 2] = from_fn(|i| zy_mix[i] + x_shuf * y_shuf);
173            let grads: [_; 2] =
174                from_fn(|i| ((hashes[i] & hash_mask) | exp_bits).raw_cast() - three);
175
176            unsafe {
177                gradients.grad_buffers[0].write_simd_aligned(i, grads[0]);
178                gradients.grad_buffers[1].write_simd_aligned(i, grads[1]);
179            };
180
181            x_vec += x_vec_stride;
182        }
183    } else {
184        let iota_vec = Simd::iota(0) * Simd::splat(params.seed);
185        let x_start_seeded = (grid_data.grid_start[0] as u32).wrapping_mul(params.seed);
186        let mut x_vec = Simd::splat(x_start_seeded) + iota_vec;
187        let x_vec_stride = Simd::splat((lanes as u32).wrapping_mul(params.seed));
188
189        for i in (0..end_index).step_by(lanes) {
190            let x_shuf = x_vec.permute_8(shuffle_indices) ^ prime;
191            let hashes: [_; 2] = from_fn(|i| zy_mix[i] + x_shuf * y_shuf);
192            let grads: [_; 2] =
193                from_fn(|i| ((hashes[i] & hash_mask) | exp_bits).raw_cast() - three);
194
195            unsafe {
196                gradients.grad_buffers[0].write_simd_aligned(i, grads[0]);
197                gradients.grad_buffers[1].write_simd_aligned(i, grads[1]);
198            };
199            x_vec += x_vec_stride;
200        }
201    }
202
203    fill_gradients_3d_set_loop::<A, true>(grid_data, gradients);
204    fill_gradients_3d_set_loop::<A, false>(grid_data, gradients);
205}
206
207#[inline(always)]
208pub(super) fn fill_gradients_3d_set_loop<'a, A: Arch, const IS_FRONT: bool>(
209    grid_data: &GridData<3>,
210    gradients: &mut ValueGradients3D<'a>,
211) {
212    let (grad_buffer, left, right) = if IS_FRONT {
213        (
214            &mut gradients.grad_buffers[0],
215            &mut gradients.blf,
216            &mut gradients.brf,
217        )
218    } else {
219        (
220            &mut gradients.grad_buffers[1],
221            &mut gradients.blb,
222            &mut gradients.brb,
223        )
224    };
225
226    let mut x_cur_index = 0;
227    for x_it in 0..grid_data.num_loops[0] {
228        // Find range of gradients to set.
229        let x_next_index = unsafe { grid_data.grid_indices[0].get_unchecked(x_it).assume_init() };
230        let mut amount = (x_next_index - x_cur_index) as isize;
231
232        unsafe {
233            let l = grad_buffer.get_unchecked(x_it).assume_init();
234            let r = grad_buffer.get_unchecked(x_it + 1).assume_init();
235
236            let mut index = x_cur_index as usize;
237            while amount > 0 {
238                left.write_simd(index, Simd::<_, A>::splat(l));
239                right.write_simd(index, Simd::<_, A>::splat(r));
240
241                amount -= Simd::<f32, A>::LANES as isize;
242                index += Simd::<f32, A>::LANES;
243            }
244        }
245
246        x_cur_index = x_next_index;
247    }
248}
249
250pub(crate) struct TrilerpBuffers<'a> {
251    tf_base: &'a mut [MaybeUninit<f32>],
252    bf_base: &'a mut [MaybeUninit<f32>],
253    top_base_dif: &'a mut [MaybeUninit<f32>],
254    bottom_base_dif: &'a mut [MaybeUninit<f32>],
255}
256
257impl<'a> TrilerpBuffers<'a> {
258    #[inline(always)]
259    pub fn new(arena: &'a mut Arena, x_size: usize) -> Self {
260        Self {
261            tf_base: arena.allocate(x_size),
262            bf_base: arena.allocate(x_size),
263            top_base_dif: arena.allocate(x_size),
264            bottom_base_dif: arena.allocate(x_size),
265        }
266    }
267}
268
269/// Handles interpolation execution state and fills
270/// the dst slice with interpolated values from gradient dot produtcts.
271pub(crate) struct TrilerpExecuter<'a, A: Arch, C: Combiner, const INIT: bool, const FINAL: bool> {
272    config: &'a InterpolationConfig<A>,
273    fractal_config: &'a C::Config,
274    grid_data: &'a GridData<'a, 3>,
275    gradients: &'a ValueGradients3D<'a>,
276    y_range: Range<usize>,
277    z_range: Range<usize>,
278    top: A::Block2<f32>,
279    dif: A::Block2<f32>,
280    weight: Simd<f32, A>,
281}
282
283/// Fills the dst slice with interpolated dot products from gradients.
284#[inline(always)]
285pub(super) fn trilerp<A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>(
286    buffers: &mut TrilerpBuffers,
287    config: &InterpolationConfig<A>,
288    fractal_config: &C::Config,
289    grid_data: &GridData<3>,
290    gradients: &ValueGradients3D,
291    ranges: (Range<usize>, Range<usize>),
292    output: (&mut [f32], &mut [f32]),
293) {
294    let mut executer = TrilerpExecuter::<A, C, INIT, FINAL> {
295        config,
296        fractal_config,
297        grid_data,
298        gradients,
299        y_range: ranges.0,
300        z_range: ranges.1,
301        top: Default::default(),
302        dif: Default::default(),
303        weight: Simd::splat(grid_data.weight),
304    };
305
306    executer.initialize_trilerp_buffers(buffers);
307
308    let (state, dst) = output;
309
310    if config.has_block_head {
311        executer.interpolate::<false, 0, FULL>(buffers, state, dst);
312    }
313
314    if config.has_block_tail {
315        std::hint::cold_path();
316        let block_tail_size = config.block_tail_size;
317        let has_partial = config.has_partial;
318        let has_block_head = config.has_block_head;
319        match TrilerpExecuter::<A, C, INIT, FINAL>::MAX_BLOCKS {
320            1 => match (block_tail_size, has_partial, has_block_head) {
321                (0, true, false) => executer.interpolate::<true, 0, SCALAR>(buffers, state, dst),
322                (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(buffers, state, dst),
323                _ => {}
324            },
325            2 => match (block_tail_size, has_partial, has_block_head) {
326                (1, false, _) => executer.interpolate::<true, 1, FULL>(buffers, state, dst),
327                (0, true, false) => executer.interpolate::<true, 0, SCALAR>(buffers, state, dst),
328                (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(buffers, state, dst),
329                (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(buffers, state, dst),
330                _ => {}
331            },
332            4 => match (block_tail_size, has_partial, has_block_head) {
333                (1, false, _) => executer.interpolate::<true, 1, FULL>(buffers, state, dst),
334                (2, false, _) => executer.interpolate::<true, 2, FULL>(buffers, state, dst),
335                (3, false, _) => executer.interpolate::<true, 3, FULL>(buffers, state, dst),
336                (0, true, false) => executer.interpolate::<true, 0, SCALAR>(buffers, state, dst),
337                (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(buffers, state, dst),
338                (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(buffers, state, dst),
339                (2, true, _) => executer.interpolate::<true, 2, PARTIAL>(buffers, state, dst),
340                (3, true, _) => executer.interpolate::<true, 3, PARTIAL>(buffers, state, dst),
341                _ => {}
342            },
343            8 => match (block_tail_size, has_partial, has_block_head) {
344                (1, false, _) => executer.interpolate::<true, 1, FULL>(buffers, state, dst),
345                (2, false, _) => executer.interpolate::<true, 2, FULL>(buffers, state, dst),
346                (3, false, _) => executer.interpolate::<true, 3, FULL>(buffers, state, dst),
347                (4, false, _) => executer.interpolate::<true, 4, FULL>(buffers, state, dst),
348                (5, false, _) => executer.interpolate::<true, 5, FULL>(buffers, state, dst),
349                (6, false, _) => executer.interpolate::<true, 6, FULL>(buffers, state, dst),
350                (7, false, _) => executer.interpolate::<true, 7, FULL>(buffers, state, dst),
351                (0, true, false) => executer.interpolate::<true, 0, SCALAR>(buffers, state, dst),
352                (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(buffers, state, dst),
353                (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(buffers, state, dst),
354                (2, true, _) => executer.interpolate::<true, 2, PARTIAL>(buffers, state, dst),
355                (3, true, _) => executer.interpolate::<true, 3, PARTIAL>(buffers, state, dst),
356                (4, true, _) => executer.interpolate::<true, 4, PARTIAL>(buffers, state, dst),
357                (5, true, _) => executer.interpolate::<true, 5, PARTIAL>(buffers, state, dst),
358                (6, true, _) => executer.interpolate::<true, 6, PARTIAL>(buffers, state, dst),
359                (7, true, _) => executer.interpolate::<true, 7, PARTIAL>(buffers, state, dst),
360                _ => {}
361            },
362            _ => {}
363        }
364    }
365}
366
367impl<'a, A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>
368    TrilerpExecuter<'a, A, C, INIT, FINAL>
369{
370    const MAX_BLOCKS: usize = A::NUM_SIMD_REG / 4;
371
372    #[inline(always)]
373    pub fn interpolate<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
374        &mut self,
375        buffers: &TrilerpBuffers,
376        state: &mut [f32],
377        dst: &mut [f32],
378    ) {
379        let range = if IS_TAIL {
380            self.config.block_tail_start..self.grid_data.grid_size[0]
381        } else {
382            0..self.config.block_tail_start
383        };
384
385        let process = |this: &mut Self, state: &mut [f32], dst: &mut [f32], x, y, index| {
386            this.process_factors::<IS_TAIL, NUM_BLOCKS, ACCESS_MODE>(x, y, index, state, dst)
387        };
388
389        let z_hop = self.grid_data.grid_size[0] * self.grid_data.grid_size[1];
390        let y_hop = self.grid_data.grid_size[0];
391        for z in self.z_range.start..self.z_range.end {
392            let z_lerp = unsafe { self.grid_data.fade_factors[2].get_unchecked(z) };
393            let z_lerp = unsafe { z_lerp.assume_init() };
394            let z_lerp = Simd::splat(z_lerp);
395
396            let mut x = range.start;
397            while x < range.end {
398                self.intialize_factors::<IS_TAIL, NUM_BLOCKS, ACCESS_MODE>(buffers, x, z_lerp);
399
400                let index = z * z_hop;
401                let mut y = self.y_range.start;
402                while y < self.y_range.end {
403                    let index = index + y * y_hop;
404                    if y + 4 > self.y_range.end {
405                        std::hint::cold_path();
406                        process(self, state, dst, x, y, index);
407                        y += 1;
408                    } else {
409                        process(self, state, dst, x, y, index);
410                        process(self, state, dst, x, y + 1, index + y_hop);
411                        process(self, state, dst, x, y + 2, index + 2 * y_hop);
412                        process(self, state, dst, x, y + 3, index + 3 * y_hop);
413                        y += 4;
414                    }
415                }
416                x += self.config.block_lanes;
417            }
418        }
419    }
420
421    #[inline(always)]
422    fn initialize_trilerp_buffers(&mut self, buffers: &mut TrilerpBuffers) {
423        for x in (0..self.grid_data.grid_size[0]).step_by(Simd::<f32, A>::LANES) {
424            unsafe {
425                let x_lerp = self.grid_data.fade_factors[0].load_simd_aligned(x);
426
427                let tlf = self.gradients.tlf.load_simd_aligned(x);
428                let trf = self.gradients.trf.load_simd_aligned(x);
429                let blf = self.gradients.blf.load_simd_aligned(x);
430                let brf = self.gradients.brf.load_simd_aligned(x);
431                let tlb = self.gradients.tlb.load_simd_aligned(x);
432                let trb = self.gradients.trb.load_simd_aligned(x);
433                let blb = self.gradients.blb.load_simd_aligned(x);
434                let brb = self.gradients.brb.load_simd_aligned(x);
435
436                let tf_base = x_lerp.mul_add(trf - tlf, tlf) * self.weight;
437                let bf_base = x_lerp.mul_add(brf - blf, blf) * self.weight;
438
439                let hi_base_dif = x_lerp.mul_add(trb - tlb, tlb).mul_sub(self.weight, tf_base);
440                let lo_base_dif = x_lerp.mul_add(brb - blb, blb).mul_sub(self.weight, bf_base);
441
442                buffers.tf_base.write_simd_aligned(x, tf_base);
443                buffers.bf_base.write_simd_aligned(x, bf_base);
444                buffers.top_base_dif.write_simd_aligned(x, hi_base_dif);
445                buffers.bottom_base_dif.write_simd_aligned(x, lo_base_dif);
446            }
447        }
448    }
449
450    #[inline(always)]
451    fn intialize_factors<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
452        &mut self,
453        buffers: &TrilerpBuffers,
454        x: usize,
455        z_lerp: Simd<f32, A>,
456    ) {
457        let num_blocks = if IS_TAIL {
458            NUM_BLOCKS
459        } else {
460            Self::MAX_BLOCKS
461        };
462
463        // These blocked loops will get entirely unrolled by the compiler.
464        for block in 0..num_blocks {
465            let index = x + Simd::<f32, A>::LANES * block;
466            self.initialize_factors_block::<FULL>(buffers, index, block, z_lerp);
467        }
468
469        if ACCESS_MODE != FULL {
470            self.initialize_factors_block::<ACCESS_MODE>(buffers, 0, NUM_BLOCKS, z_lerp);
471        }
472    }
473
474    #[inline(always)]
475    fn initialize_factors_block<const ACCESS_MODE: u8>(
476        &mut self,
477        buffers: &TrilerpBuffers,
478        index: usize,
479        block: usize,
480        z_lerp: Simd<f32, A>,
481    ) {
482        let access_mode = SimdAccessMode::from_u8(ACCESS_MODE);
483        let (tf, bf, top_dif, bottom_dif) = unsafe {
484            match access_mode {
485                // Use unaligned load to capture 'end' of the partial vector.
486                SimdAccessMode::Partial => {
487                    let index = index + self.config.partial_start;
488                    (
489                        buffers.tf_base.load_simd(index),
490                        buffers.bf_base.load_simd(index),
491                        buffers.top_base_dif.load_simd(index),
492                        buffers.bottom_base_dif.load_simd(index),
493                    )
494                }
495                // Use aligned load to capture full vector for Full mode.
496                // Use aligned load to capture padded vector for Scalar mode.
497                _ => (
498                    buffers.tf_base.load_simd_aligned(index),
499                    buffers.bf_base.load_simd_aligned(index),
500                    buffers.top_base_dif.load_simd_aligned(index),
501                    buffers.bottom_base_dif.load_simd_aligned(index),
502                ),
503            }
504        };
505
506        // Base interpolation.
507        unsafe {
508            *self.top.get_unchecked_mut(block) = z_lerp.mul_add(top_dif, tf);
509            let bottom = z_lerp.mul_add(bottom_dif, bf);
510            *self.dif.get_unchecked_mut(block) = bottom - *self.top.get_unchecked(block);
511        }
512    }
513
514    #[inline(always)]
515    fn process_factors<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
516        &mut self,
517        x: usize,
518        y: usize,
519        index: usize,
520        state: &mut [f32],
521        dst: &mut [f32],
522    ) {
523        let y_lerp = Simd::splat(unsafe {
524            self.grid_data.fade_factors[1]
525                .get_unchecked(y)
526                .assume_init()
527        });
528
529        let num_blocks = if IS_TAIL {
530            NUM_BLOCKS
531        } else {
532            Self::MAX_BLOCKS
533        };
534
535        for block in 0..num_blocks {
536            let index = index + x + block * Simd::<f32, A>::LANES;
537
538            self.process_factors_block::<FULL>(block, y_lerp, index, state, dst);
539        }
540
541        if ACCESS_MODE != FULL {
542            self.process_factors_block::<ACCESS_MODE>(NUM_BLOCKS, y_lerp, index, state, dst);
543        }
544    }
545
546    #[inline(always)]
547    fn process_factors_block<const ACCESS_MODE: u8>(
548        &mut self,
549        block: usize,
550        y_lerp: Simd<f32, A>,
551        index: usize,
552        state: &mut [f32],
553        dst: &mut [f32],
554    ) {
555        let access_mode = SimdAccessMode::from_u8(ACCESS_MODE);
556        let output = unsafe {
557            y_lerp.mul_add(
558                *self.dif.get_unchecked(block),
559                *self.top.get_unchecked(block),
560            )
561        };
562
563        // Initialize sample if it is the first and apply combiner algorithm
564        // for subsequent samples.
565        let (cur_state, mut result) = if INIT {
566            C::initialize_sample(self.fractal_config, output)
567        } else {
568            let mut cur_state = C::State::<A>::default();
569            for i in 0..C::State::<A>::STATE_SIZE {
570                let offset = i * self.grid_data.total_size;
571                let index = index + offset;
572                cur_state[i] = unsafe { load_simd_rw(state, index, self.config, access_mode) };
573            }
574
575            let cur_result = unsafe { load_simd_rw(dst, index, self.config, access_mode) };
576            C::apply_sample(self.fractal_config, cur_state, cur_result, output)
577        };
578
579        // Save changes to state.
580        if !FINAL {
581            for i in 0..C::State::<A>::STATE_SIZE {
582                let offset = i * self.grid_data.total_size;
583                let index = index + offset;
584                unsafe { write_simd_rw(state, index, cur_state[i], self.config, access_mode) };
585            }
586        }
587
588        // Finalize result if it is the last one.
589        if FINAL {
590            result = C::finalize_sample(self.fractal_config, cur_state, result);
591        }
592
593        // Save the result.
594        unsafe { write_simd_rw(dst, index, result, self.config, access_mode) };
595    }
596}