1use std::array::from_fn;
2use std::mem::MaybeUninit;
3use std::ops::Range;
4
5use simply_simd::{Arch, Simd, enable_targets};
6
7use crate::GridGenerator;
8use crate::api::grid::interface::GridNoiseParams;
9use crate::noise::combiners::{Combiner, CombinerState};
10use crate::noise::generators::Value;
11use crate::noise::util::constants::{BYTE_SHUFFLE, VALUE_EXP_MASK, HASH_MASK, HASH_PRIME};
12use crate::noise::util::grid_data::{GridData, Lerp};
13use crate::noise::util::grid_helpers::*;
14
15pub struct ValueGradients3D<'a> {
16 pub tlf: &'a mut [MaybeUninit<f32>],
17 pub trf: &'a mut [MaybeUninit<f32>],
18 pub blf: &'a mut [MaybeUninit<f32>],
19 pub brf: &'a mut [MaybeUninit<f32>],
20 pub tlb: &'a mut [MaybeUninit<f32>],
21 pub trb: &'a mut [MaybeUninit<f32>],
22 pub blb: &'a mut [MaybeUninit<f32>],
23 pub brb: &'a mut [MaybeUninit<f32>],
24 pub grad_buffers: [&'a mut [MaybeUninit<f32>]; 2],
25}
26
27impl<'a> ValueGradients3D<'a> {
28 #[inline(always)]
29 pub fn new(arena: &'a mut Arena, size: usize) -> Self {
30 Self {
31 tlf: arena.allocate(size),
32 trf: arena.allocate(size),
33 blf: arena.allocate(size),
34 brf: arena.allocate(size),
35 tlb: arena.allocate(size),
36 trb: arena.allocate(size),
37 blb: arena.allocate(size),
38 brb: arena.allocate(size),
39 grad_buffers: from_fn(|_| arena.allocate(size)),
40 }
41 }
42
43 #[inline(always)]
44 pub fn swap_top_bottom(&mut self) {
45 std::mem::swap(&mut self.tlf, &mut self.blf);
46 std::mem::swap(&mut self.trf, &mut self.brf);
47 std::mem::swap(&mut self.tlb, &mut self.blb);
48 std::mem::swap(&mut self.trb, &mut self.brb);
49 }
50}
51
52const LERP: u8 = Lerp::Quintic as u8;
53#[enable_targets(A)]
54impl GridGenerator<3> for Value {
55 fn sample_grid<A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>(
56 params: GridNoiseParams<3>,
57 fractal_config: C::Config,
58 state: &mut [f32],
59 dst: &mut [f32],
60 ) {
61 validate_grid_size(params.grid_size, dst.len());
63 validate_state_size::<C, A, _>(params.grid_size, state.len());
64 let padded_size = pad_grid_size::<A, 3>(params.grid_size);
65
66 let required_cache = padded_size[0] * 17 + padded_size[1] * 3 + padded_size[2] * 3;
68 let mut cache = ArenaBuffer::<A>::with_capacity(required_cache);
69 let mut arena = Arena::with_cache(&mut cache);
70 let mut data_arena = arena.allocate_arena(padded_size.iter().fold(0, |n, x| n + 3 * x));
71 let mut trilerp_arena = arena.allocate_arena(padded_size[0] * 4);
72
73 let num_blocks = TrilerpExecuter::<A, C, INIT, FINAL>::MAX_BLOCKS;
75 let bilerp_config = InterpolationConfig::new(num_blocks, params.grid_size[0]);
76 let grid_data = GridData::new::<A, LERP>(¶ms, &mut data_arena, &padded_size);
77 let mut trilerp_buffers = TrilerpBuffers::new(&mut trilerp_arena, padded_size[0]);
78 let mut gradients = ValueGradients3D::new(&mut arena, padded_size[0]);
79
80 let mut z_cur_index = 0;
82 for z_it in 0..grid_data.num_loops[2] {
83 let z_next_index =
84 unsafe { grid_data.grid_indices[2].get_unchecked(z_it).assume_init() as usize };
85 let z_range = z_cur_index..z_next_index;
86
87 fill_gradients_3d::<A>(¶ms, &grid_data, &mut gradients, 0, z_it);
89 gradients.swap_top_bottom();
90
91 let mut y_cur_index = 0;
92 for y_it in 0..grid_data.num_loops[1] {
93 let y_next_index =
94 unsafe { grid_data.grid_indices[1].get_unchecked(y_it).assume_init() as usize };
95 let y_range = y_cur_index..y_next_index;
96
97 fill_gradients_3d::<A>(¶ms, &grid_data, &mut gradients, y_it + 1, z_it);
99
100 trilerp::<A, C, INIT, FINAL>(
101 &mut trilerp_buffers,
102 &bilerp_config,
103 &fractal_config,
104 &grid_data,
105 &gradients,
106 (y_range, z_range.clone()),
107 (state, dst),
108 );
109
110 gradients.swap_top_bottom();
112
113 y_cur_index = y_next_index;
114 }
115 z_cur_index = z_next_index;
116 }
117 }
118}
119
120#[inline(always)]
121pub(super) fn fill_gradients_3d<'a, A: Arch>(
122 params: &GridNoiseParams<3>,
123 grid_data: &GridData<3>,
124 gradients: &mut ValueGradients3D<'a>,
125 y_it: usize,
126 z_it: usize,
127) {
128 let lanes = Simd::<f32, A>::LANES;
129 let y_start = y_it as i32 + grid_data.grid_start[1];
130 let z_start = z_it as i32 + grid_data.grid_start[2];
131 let (z1, z2) = match grid_data.octave_tiling[2] {
132 None => (
133 (z_start as u32).wrapping_mul(params.seed),
134 (z_start as u32)
135 .wrapping_mul(params.seed)
136 .wrapping_add(params.seed),
137 ),
138 Some(t) => (
139 (z_start.rem_euclid(t as i32)) as u32,
140 ((z_start + 1).rem_euclid(t as i32)) as u32,
141 ),
142 };
143 let z_vec = [Simd::splat(z1), Simd::splat(z2)];
144
145 let y_rem = grid_data.octave_tiling[1].map_or(y_start, |t| y_start.rem_euclid(t as i32));
146 let y_vec = Simd::splat((y_rem as u32).wrapping_mul(params.seed));
147
148 let shuffle_indices = Simd::<u8, A>::from_slice(&BYTE_SHUFFLE[..]);
149
150 let prime = Simd::splat(HASH_PRIME);
151 let z_shuf: [_; 2] = from_fn(|i| z_vec[i].permute_8(shuffle_indices) ^ prime);
152 let y_shuf = y_vec.permute_8(shuffle_indices) ^ prime;
153 let zy_mix: [_; 2] = from_fn(|i| z_shuf[i] * y_shuf);
154
155 let end_index = grid_data.num_loops[0] + 1;
157 let hash_mask: Simd<u32, A> = Simd::splat(HASH_MASK);
158 let exp_bits: Simd<u32, A> = Simd::splat(VALUE_EXP_MASK);
159 let three: Simd<f32, A> = Simd::splat(3.0);
160
161 if let Some(x_tiling) = grid_data.octave_tiling[0] {
162 let x_tiling = Simd::splat(x_tiling as f32);
163 let mut x_vec = Simd::splat(grid_data.grid_start[0]) + Simd::iota(0);
164 let x_vec_stride = Simd::splat(lanes as i32);
165 let seed_vec = Simd::splat(params.seed);
166
167 for i in (0..end_index).step_by(lanes) {
168 let x_floats = x_vec.cast_float();
169 let x_rem = x_floats - (x_floats / x_tiling).floor() * x_tiling;
170 let x_seeded = x_rem.cast_int_round().raw_cast() * seed_vec;
171 let x_shuf = x_seeded.permute_8(shuffle_indices) ^ prime;
172 let hashes: [_; 2] = from_fn(|i| zy_mix[i] + x_shuf * y_shuf);
173 let grads: [_; 2] =
174 from_fn(|i| ((hashes[i] & hash_mask) | exp_bits).raw_cast() - three);
175
176 unsafe {
177 gradients.grad_buffers[0].write_simd_aligned(i, grads[0]);
178 gradients.grad_buffers[1].write_simd_aligned(i, grads[1]);
179 };
180
181 x_vec += x_vec_stride;
182 }
183 } else {
184 let iota_vec = Simd::iota(0) * Simd::splat(params.seed);
185 let x_start_seeded = (grid_data.grid_start[0] as u32).wrapping_mul(params.seed);
186 let mut x_vec = Simd::splat(x_start_seeded) + iota_vec;
187 let x_vec_stride = Simd::splat((lanes as u32).wrapping_mul(params.seed));
188
189 for i in (0..end_index).step_by(lanes) {
190 let x_shuf = x_vec.permute_8(shuffle_indices) ^ prime;
191 let hashes: [_; 2] = from_fn(|i| zy_mix[i] + x_shuf * y_shuf);
192 let grads: [_; 2] =
193 from_fn(|i| ((hashes[i] & hash_mask) | exp_bits).raw_cast() - three);
194
195 unsafe {
196 gradients.grad_buffers[0].write_simd_aligned(i, grads[0]);
197 gradients.grad_buffers[1].write_simd_aligned(i, grads[1]);
198 };
199 x_vec += x_vec_stride;
200 }
201 }
202
203 fill_gradients_3d_set_loop::<A, true>(grid_data, gradients);
204 fill_gradients_3d_set_loop::<A, false>(grid_data, gradients);
205}
206
207#[inline(always)]
208pub(super) fn fill_gradients_3d_set_loop<'a, A: Arch, const IS_FRONT: bool>(
209 grid_data: &GridData<3>,
210 gradients: &mut ValueGradients3D<'a>,
211) {
212 let (grad_buffer, left, right) = if IS_FRONT {
213 (
214 &mut gradients.grad_buffers[0],
215 &mut gradients.blf,
216 &mut gradients.brf,
217 )
218 } else {
219 (
220 &mut gradients.grad_buffers[1],
221 &mut gradients.blb,
222 &mut gradients.brb,
223 )
224 };
225
226 let mut x_cur_index = 0;
227 for x_it in 0..grid_data.num_loops[0] {
228 let x_next_index = unsafe { grid_data.grid_indices[0].get_unchecked(x_it).assume_init() };
230 let mut amount = (x_next_index - x_cur_index) as isize;
231
232 unsafe {
233 let l = grad_buffer.get_unchecked(x_it).assume_init();
234 let r = grad_buffer.get_unchecked(x_it + 1).assume_init();
235
236 let mut index = x_cur_index as usize;
237 while amount > 0 {
238 left.write_simd(index, Simd::<_, A>::splat(l));
239 right.write_simd(index, Simd::<_, A>::splat(r));
240
241 amount -= Simd::<f32, A>::LANES as isize;
242 index += Simd::<f32, A>::LANES;
243 }
244 }
245
246 x_cur_index = x_next_index;
247 }
248}
249
250pub(crate) struct TrilerpBuffers<'a> {
251 tf_base: &'a mut [MaybeUninit<f32>],
252 bf_base: &'a mut [MaybeUninit<f32>],
253 top_base_dif: &'a mut [MaybeUninit<f32>],
254 bottom_base_dif: &'a mut [MaybeUninit<f32>],
255}
256
257impl<'a> TrilerpBuffers<'a> {
258 #[inline(always)]
259 pub fn new(arena: &'a mut Arena, x_size: usize) -> Self {
260 Self {
261 tf_base: arena.allocate(x_size),
262 bf_base: arena.allocate(x_size),
263 top_base_dif: arena.allocate(x_size),
264 bottom_base_dif: arena.allocate(x_size),
265 }
266 }
267}
268
269pub(crate) struct TrilerpExecuter<'a, A: Arch, C: Combiner, const INIT: bool, const FINAL: bool> {
272 config: &'a InterpolationConfig<A>,
273 fractal_config: &'a C::Config,
274 grid_data: &'a GridData<'a, 3>,
275 gradients: &'a ValueGradients3D<'a>,
276 y_range: Range<usize>,
277 z_range: Range<usize>,
278 top: A::Block2<f32>,
279 dif: A::Block2<f32>,
280 weight: Simd<f32, A>,
281}
282
283#[inline(always)]
285pub(super) fn trilerp<A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>(
286 buffers: &mut TrilerpBuffers,
287 config: &InterpolationConfig<A>,
288 fractal_config: &C::Config,
289 grid_data: &GridData<3>,
290 gradients: &ValueGradients3D,
291 ranges: (Range<usize>, Range<usize>),
292 output: (&mut [f32], &mut [f32]),
293) {
294 let mut executer = TrilerpExecuter::<A, C, INIT, FINAL> {
295 config,
296 fractal_config,
297 grid_data,
298 gradients,
299 y_range: ranges.0,
300 z_range: ranges.1,
301 top: Default::default(),
302 dif: Default::default(),
303 weight: Simd::splat(grid_data.weight),
304 };
305
306 executer.initialize_trilerp_buffers(buffers);
307
308 let (state, dst) = output;
309
310 if config.has_block_head {
311 executer.interpolate::<false, 0, FULL>(buffers, state, dst);
312 }
313
314 if config.has_block_tail {
315 std::hint::cold_path();
316 let block_tail_size = config.block_tail_size;
317 let has_partial = config.has_partial;
318 let has_block_head = config.has_block_head;
319 match TrilerpExecuter::<A, C, INIT, FINAL>::MAX_BLOCKS {
320 1 => match (block_tail_size, has_partial, has_block_head) {
321 (0, true, false) => executer.interpolate::<true, 0, SCALAR>(buffers, state, dst),
322 (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(buffers, state, dst),
323 _ => {}
324 },
325 2 => match (block_tail_size, has_partial, has_block_head) {
326 (1, false, _) => executer.interpolate::<true, 1, FULL>(buffers, state, dst),
327 (0, true, false) => executer.interpolate::<true, 0, SCALAR>(buffers, state, dst),
328 (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(buffers, state, dst),
329 (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(buffers, state, dst),
330 _ => {}
331 },
332 4 => match (block_tail_size, has_partial, has_block_head) {
333 (1, false, _) => executer.interpolate::<true, 1, FULL>(buffers, state, dst),
334 (2, false, _) => executer.interpolate::<true, 2, FULL>(buffers, state, dst),
335 (3, false, _) => executer.interpolate::<true, 3, FULL>(buffers, state, dst),
336 (0, true, false) => executer.interpolate::<true, 0, SCALAR>(buffers, state, dst),
337 (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(buffers, state, dst),
338 (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(buffers, state, dst),
339 (2, true, _) => executer.interpolate::<true, 2, PARTIAL>(buffers, state, dst),
340 (3, true, _) => executer.interpolate::<true, 3, PARTIAL>(buffers, state, dst),
341 _ => {}
342 },
343 8 => match (block_tail_size, has_partial, has_block_head) {
344 (1, false, _) => executer.interpolate::<true, 1, FULL>(buffers, state, dst),
345 (2, false, _) => executer.interpolate::<true, 2, FULL>(buffers, state, dst),
346 (3, false, _) => executer.interpolate::<true, 3, FULL>(buffers, state, dst),
347 (4, false, _) => executer.interpolate::<true, 4, FULL>(buffers, state, dst),
348 (5, false, _) => executer.interpolate::<true, 5, FULL>(buffers, state, dst),
349 (6, false, _) => executer.interpolate::<true, 6, FULL>(buffers, state, dst),
350 (7, false, _) => executer.interpolate::<true, 7, FULL>(buffers, state, dst),
351 (0, true, false) => executer.interpolate::<true, 0, SCALAR>(buffers, state, dst),
352 (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(buffers, state, dst),
353 (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(buffers, state, dst),
354 (2, true, _) => executer.interpolate::<true, 2, PARTIAL>(buffers, state, dst),
355 (3, true, _) => executer.interpolate::<true, 3, PARTIAL>(buffers, state, dst),
356 (4, true, _) => executer.interpolate::<true, 4, PARTIAL>(buffers, state, dst),
357 (5, true, _) => executer.interpolate::<true, 5, PARTIAL>(buffers, state, dst),
358 (6, true, _) => executer.interpolate::<true, 6, PARTIAL>(buffers, state, dst),
359 (7, true, _) => executer.interpolate::<true, 7, PARTIAL>(buffers, state, dst),
360 _ => {}
361 },
362 _ => {}
363 }
364 }
365}
366
367impl<'a, A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>
368 TrilerpExecuter<'a, A, C, INIT, FINAL>
369{
370 const MAX_BLOCKS: usize = A::NUM_SIMD_REG / 4;
371
372 #[inline(always)]
373 pub fn interpolate<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
374 &mut self,
375 buffers: &TrilerpBuffers,
376 state: &mut [f32],
377 dst: &mut [f32],
378 ) {
379 let range = if IS_TAIL {
380 self.config.block_tail_start..self.grid_data.grid_size[0]
381 } else {
382 0..self.config.block_tail_start
383 };
384
385 let process = |this: &mut Self, state: &mut [f32], dst: &mut [f32], x, y, index| {
386 this.process_factors::<IS_TAIL, NUM_BLOCKS, ACCESS_MODE>(x, y, index, state, dst)
387 };
388
389 let z_hop = self.grid_data.grid_size[0] * self.grid_data.grid_size[1];
390 let y_hop = self.grid_data.grid_size[0];
391 for z in self.z_range.start..self.z_range.end {
392 let z_lerp = unsafe { self.grid_data.fade_factors[2].get_unchecked(z) };
393 let z_lerp = unsafe { z_lerp.assume_init() };
394 let z_lerp = Simd::splat(z_lerp);
395
396 let mut x = range.start;
397 while x < range.end {
398 self.intialize_factors::<IS_TAIL, NUM_BLOCKS, ACCESS_MODE>(buffers, x, z_lerp);
399
400 let index = z * z_hop;
401 let mut y = self.y_range.start;
402 while y < self.y_range.end {
403 let index = index + y * y_hop;
404 if y + 4 > self.y_range.end {
405 std::hint::cold_path();
406 process(self, state, dst, x, y, index);
407 y += 1;
408 } else {
409 process(self, state, dst, x, y, index);
410 process(self, state, dst, x, y + 1, index + y_hop);
411 process(self, state, dst, x, y + 2, index + 2 * y_hop);
412 process(self, state, dst, x, y + 3, index + 3 * y_hop);
413 y += 4;
414 }
415 }
416 x += self.config.block_lanes;
417 }
418 }
419 }
420
421 #[inline(always)]
422 fn initialize_trilerp_buffers(&mut self, buffers: &mut TrilerpBuffers) {
423 for x in (0..self.grid_data.grid_size[0]).step_by(Simd::<f32, A>::LANES) {
424 unsafe {
425 let x_lerp = self.grid_data.fade_factors[0].load_simd_aligned(x);
426
427 let tlf = self.gradients.tlf.load_simd_aligned(x);
428 let trf = self.gradients.trf.load_simd_aligned(x);
429 let blf = self.gradients.blf.load_simd_aligned(x);
430 let brf = self.gradients.brf.load_simd_aligned(x);
431 let tlb = self.gradients.tlb.load_simd_aligned(x);
432 let trb = self.gradients.trb.load_simd_aligned(x);
433 let blb = self.gradients.blb.load_simd_aligned(x);
434 let brb = self.gradients.brb.load_simd_aligned(x);
435
436 let tf_base = x_lerp.mul_add(trf - tlf, tlf) * self.weight;
437 let bf_base = x_lerp.mul_add(brf - blf, blf) * self.weight;
438
439 let hi_base_dif = x_lerp.mul_add(trb - tlb, tlb).mul_sub(self.weight, tf_base);
440 let lo_base_dif = x_lerp.mul_add(brb - blb, blb).mul_sub(self.weight, bf_base);
441
442 buffers.tf_base.write_simd_aligned(x, tf_base);
443 buffers.bf_base.write_simd_aligned(x, bf_base);
444 buffers.top_base_dif.write_simd_aligned(x, hi_base_dif);
445 buffers.bottom_base_dif.write_simd_aligned(x, lo_base_dif);
446 }
447 }
448 }
449
450 #[inline(always)]
451 fn intialize_factors<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
452 &mut self,
453 buffers: &TrilerpBuffers,
454 x: usize,
455 z_lerp: Simd<f32, A>,
456 ) {
457 let num_blocks = if IS_TAIL {
458 NUM_BLOCKS
459 } else {
460 Self::MAX_BLOCKS
461 };
462
463 for block in 0..num_blocks {
465 let index = x + Simd::<f32, A>::LANES * block;
466 self.initialize_factors_block::<FULL>(buffers, index, block, z_lerp);
467 }
468
469 if ACCESS_MODE != FULL {
470 self.initialize_factors_block::<ACCESS_MODE>(buffers, 0, NUM_BLOCKS, z_lerp);
471 }
472 }
473
474 #[inline(always)]
475 fn initialize_factors_block<const ACCESS_MODE: u8>(
476 &mut self,
477 buffers: &TrilerpBuffers,
478 index: usize,
479 block: usize,
480 z_lerp: Simd<f32, A>,
481 ) {
482 let access_mode = SimdAccessMode::from_u8(ACCESS_MODE);
483 let (tf, bf, top_dif, bottom_dif) = unsafe {
484 match access_mode {
485 SimdAccessMode::Partial => {
487 let index = index + self.config.partial_start;
488 (
489 buffers.tf_base.load_simd(index),
490 buffers.bf_base.load_simd(index),
491 buffers.top_base_dif.load_simd(index),
492 buffers.bottom_base_dif.load_simd(index),
493 )
494 }
495 _ => (
498 buffers.tf_base.load_simd_aligned(index),
499 buffers.bf_base.load_simd_aligned(index),
500 buffers.top_base_dif.load_simd_aligned(index),
501 buffers.bottom_base_dif.load_simd_aligned(index),
502 ),
503 }
504 };
505
506 unsafe {
508 *self.top.get_unchecked_mut(block) = z_lerp.mul_add(top_dif, tf);
509 let bottom = z_lerp.mul_add(bottom_dif, bf);
510 *self.dif.get_unchecked_mut(block) = bottom - *self.top.get_unchecked(block);
511 }
512 }
513
514 #[inline(always)]
515 fn process_factors<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
516 &mut self,
517 x: usize,
518 y: usize,
519 index: usize,
520 state: &mut [f32],
521 dst: &mut [f32],
522 ) {
523 let y_lerp = Simd::splat(unsafe {
524 self.grid_data.fade_factors[1]
525 .get_unchecked(y)
526 .assume_init()
527 });
528
529 let num_blocks = if IS_TAIL {
530 NUM_BLOCKS
531 } else {
532 Self::MAX_BLOCKS
533 };
534
535 for block in 0..num_blocks {
536 let index = index + x + block * Simd::<f32, A>::LANES;
537
538 self.process_factors_block::<FULL>(block, y_lerp, index, state, dst);
539 }
540
541 if ACCESS_MODE != FULL {
542 self.process_factors_block::<ACCESS_MODE>(NUM_BLOCKS, y_lerp, index, state, dst);
543 }
544 }
545
546 #[inline(always)]
547 fn process_factors_block<const ACCESS_MODE: u8>(
548 &mut self,
549 block: usize,
550 y_lerp: Simd<f32, A>,
551 index: usize,
552 state: &mut [f32],
553 dst: &mut [f32],
554 ) {
555 let access_mode = SimdAccessMode::from_u8(ACCESS_MODE);
556 let output = unsafe {
557 y_lerp.mul_add(
558 *self.dif.get_unchecked(block),
559 *self.top.get_unchecked(block),
560 )
561 };
562
563 let (cur_state, mut result) = if INIT {
566 C::initialize_sample(self.fractal_config, output)
567 } else {
568 let mut cur_state = C::State::<A>::default();
569 for i in 0..C::State::<A>::STATE_SIZE {
570 let offset = i * self.grid_data.total_size;
571 let index = index + offset;
572 cur_state[i] = unsafe { load_simd_rw(state, index, self.config, access_mode) };
573 }
574
575 let cur_result = unsafe { load_simd_rw(dst, index, self.config, access_mode) };
576 C::apply_sample(self.fractal_config, cur_state, cur_result, output)
577 };
578
579 if !FINAL {
581 for i in 0..C::State::<A>::STATE_SIZE {
582 let offset = i * self.grid_data.total_size;
583 let index = index + offset;
584 unsafe { write_simd_rw(state, index, cur_state[i], self.config, access_mode) };
585 }
586 }
587
588 if FINAL {
590 result = C::finalize_sample(self.fractal_config, cur_state, result);
591 }
592
593 unsafe { write_simd_rw(dst, index, result, self.config, access_mode) };
595 }
596}