Skip to main content

quick_noise/noise/generators/value/
grid_2d.rs

1use std::mem::MaybeUninit;
2use std::ops::Range;
3
4use simply_simd::{Arch, Simd, enable_targets};
5
6use crate::GridGenerator;
7use crate::api::grid::interface::GridNoiseParams;
8use crate::noise::combiners::{Combiner, CombinerState};
9use crate::noise::generators::Value;
10use crate::noise::util::constants::{BYTE_SHUFFLE, VALUE_EXP_MASK, HASH_MASK, HASH_PRIME};
11use crate::noise::util::grid_data::{GridData, Lerp};
12use crate::noise::util::grid_helpers::*;
13
14pub struct ValueGradients2D<'a> {
15    pub tl: &'a mut [MaybeUninit<f32>],
16    pub tr: &'a mut [MaybeUninit<f32>],
17    pub bl: &'a mut [MaybeUninit<f32>],
18    pub br: &'a mut [MaybeUninit<f32>],
19}
20
21impl<'a> ValueGradients2D<'a> {
22    #[inline(always)]
23    pub fn new(arena: &'a mut Arena, size: usize) -> Self {
24        Self {
25            tl: arena.allocate(size),
26            tr: arena.allocate(size),
27            bl: arena.allocate(size),
28            br: arena.allocate(size),
29        }
30    }
31
32    #[inline(always)]
33    pub fn swap_top_bottom(&mut self) {
34        std::mem::swap(&mut self.tl, &mut self.bl);
35        std::mem::swap(&mut self.tr, &mut self.br);
36    }
37}
38
39const LERP: u8 = Lerp::Cubic as u8;
40#[enable_targets(A)]
41impl GridGenerator<2> for Value {
42    fn sample_grid<A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>(
43        params: GridNoiseParams<2>,
44        fractal_config: C::Config,
45        state: &mut [f32],
46        dst: &mut [f32],
47    ) {
48        validate_grid_size(params.grid_size, dst.len());
49        validate_state_size::<C, A, _>(params.grid_size, state.len());
50        let padded_size = pad_grid_size::<A, 2>(params.grid_size);
51
52        let required_cache = padded_size[1] * 3 + padded_size[0] * 8;
53        let mut cache = ArenaBuffer::<A>::with_capacity(required_cache);
54        let mut arena = Arena::with_cache(&mut cache);
55
56        // SIMD Slice constants.
57        let num_blocks = BilerpExecuter::<A, C, INIT, FINAL>::MAX_BLOCKS;
58        let bilerp_config = InterpolationConfig::new(num_blocks, params.grid_size[0]);
59
60        let mut sub_arena = arena.allocate_arena(padded_size[0] * 3 + padded_size[1] * 3);
61        let mut grid_data = GridData::new::<A, LERP>(&params, &mut sub_arena, &padded_size);
62
63        // Allocate scratch buffer for gradients.
64        let grad_scratch = arena.allocate(padded_size[0]);
65
66        // Initialize gradient vectors.
67        let mut gradients = ValueGradients2D::new(&mut arena, padded_size[0]);
68
69        // Set the top gradients.
70        fill_gradients_2d::<A>(
71            &params,
72            &mut grid_data,
73            grad_scratch,
74            gradients.tl,
75            gradients.tr,
76            0,
77        );
78
79        // Iterate through single y chunks but full x chunks.
80        let mut y_cur_index = 0;
81        for y_it in 0..grid_data.num_loops[1] {
82            let y_next_index =
83                unsafe { grid_data.grid_indices[1].get_unchecked(y_it).assume_init() as usize };
84
85            // Set bottom gradients.
86            fill_gradients_2d::<A>(
87                &params,
88                &mut grid_data,
89                grad_scratch,
90                gradients.bl,
91                gradients.br,
92                y_it + 1,
93            );
94
95            let y_range = y_cur_index..y_next_index;
96            bilerp::<A, C, INIT, FINAL>(
97                &bilerp_config,
98                &fractal_config,
99                &grid_data,
100                &gradients,
101                y_range,
102                (state, dst),
103            );
104
105            // Reuse the top and bottom gradients.
106            gradients.swap_top_bottom();
107
108            y_cur_index = y_next_index;
109        }
110    }
111}
112
113#[inline(always)]
114pub(super) fn fill_gradients_2d<'a, A: Arch>(
115    params: &GridNoiseParams<2>,
116    grid_data: &mut GridData<2>,
117    grad_buffer: &mut [MaybeUninit<f32>],
118    left: &'a mut [MaybeUninit<f32>],
119    right: &'a mut [MaybeUninit<f32>],
120    y_it: usize,
121) {
122    let lanes = Simd::<f32, A>::LANES;
123    let y_start = grid_data.grid_start[1] + y_it as i32;
124    let y_rem = grid_data.octave_tiling[1].map_or(y_start, |t| {
125        let t = t as i32;
126        // Assert t > 0 to avoid checks after validation.
127        unsafe { core::hint::assert_unchecked(t > 0) };
128        y_start.rem_euclid(t)
129    });
130
131    let y_vec = Simd::splat((y_rem as u32).wrapping_mul(params.seed));
132
133    let prime = Simd::splat(HASH_PRIME);
134    let shuffle_indices = unsafe { Simd::<u8, A>::from_slice_unchecked(&BYTE_SHUFFLE[..]) };
135    let y_shuf = y_vec.permute_8(shuffle_indices) ^ prime;
136    let y_shuf = y_shuf * y_shuf;
137
138    let hash_mask: Simd<u32, A> = Simd::splat(HASH_MASK);
139    let exp_bits: Simd<u32, A> = Simd::splat(VALUE_EXP_MASK);
140    let three: Simd<f32, A> = Simd::splat(3.0);
141
142    if let Some(x_tiling) = grid_data.octave_tiling[0] {
143        let x_tiling = Simd::splat(x_tiling as f32);
144        let mut x_vec = Simd::splat(grid_data.grid_start[0]) + Simd::iota(0);
145        let x_vec_stride = Simd::splat(lanes as i32);
146        let seed_vec = Simd::splat(params.seed);
147
148        let end_index = grid_data.num_loops[0] + 1;
149        for i in (0..end_index).step_by(lanes) {
150            let x_floats = x_vec.cast_float();
151            let x_rem = x_floats - (x_floats / x_tiling).floor() * x_tiling;
152            let x_seeded = x_rem.cast_int_round().raw_cast() * seed_vec;
153
154            let x_shuf = x_seeded.permute_8(shuffle_indices) ^ prime;
155            let hash = y_shuf * x_shuf;
156            let grad = ((hash & hash_mask) | exp_bits).raw_cast() - three;
157            unsafe { grad_buffer.write_simd_aligned(i, grad) };
158            x_vec += x_vec_stride;
159        }
160    } else {
161        let iota_vec = Simd::iota(0) * Simd::splat(params.seed);
162        let mut x_vec =
163            Simd::splat((grid_data.grid_start[0] as u32).wrapping_mul(params.seed)) + iota_vec;
164        let x_vec_stride = Simd::splat((lanes as u32).wrapping_mul(params.seed));
165
166        // Main vectorized bit mixing loop.
167        let end_index = grid_data.num_loops[0] + 1;
168        for i in (0..end_index).step_by(lanes) {
169            let x_shuf = x_vec.permute_8(shuffle_indices) ^ prime;
170            let hash = y_shuf * x_shuf;
171            let grad = ((hash & hash_mask) | exp_bits).raw_cast() - three;
172            unsafe { grad_buffer.write_simd_aligned(i, grad) };
173            x_vec += x_vec_stride;
174        }
175    }
176
177    // Loop through the x chunks.
178    let mut x_cur_index = 0;
179    for x_it in 0..grid_data.num_loops[0] {
180        // Find range of gradients to set.
181        let x_next_index = unsafe { grid_data.grid_indices[0].get_unchecked(x_it).assume_init() };
182        let mut amount = (x_next_index - x_cur_index) as isize;
183
184        unsafe {
185            let l = grad_buffer.get_unchecked(x_it).assume_init();
186            let r = grad_buffer.get_unchecked(x_it + 1).assume_init();
187
188            let mut index = x_cur_index as usize;
189            while amount > 0 {
190                left.write_simd(index, Simd::<f32, A>::splat(l));
191                right.write_simd(index, Simd::<f32, A>::splat(r));
192                amount -= lanes as isize;
193                index += lanes;
194            }
195        }
196
197        x_cur_index = x_next_index;
198    }
199}
200
201/// Handles interpolation execution state and fills
202/// the dst slice with interpolated values from gradient dot produtcts.
203pub(crate) struct BilerpExecuter<'a, A: Arch, C: Combiner, const INIT: bool, const FINAL: bool> {
204    config: &'a InterpolationConfig<A>,
205    fractal_config: &'a C::Config,
206    grid_data: &'a GridData<'a, 2>,
207    gradients: &'a ValueGradients2D<'a>,
208    y_range: Range<usize>,
209    top: A::Block2<f32>,
210    dif: A::Block2<f32>,
211    weight: Simd<f32, A>,
212}
213
214/// Fills the dst slice with interpolated dot products from gradients.
215#[inline(always)]
216pub(super) fn bilerp<A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>(
217    config: &InterpolationConfig<A>,
218    fractal_config: &C::Config,
219    grid_data: &GridData<2>,
220    gradients: &ValueGradients2D,
221    y_range: Range<usize>,
222    output: (&mut [f32], &mut [f32]),
223) {
224    let mut executer = BilerpExecuter::<A, C, INIT, FINAL> {
225        config,
226        fractal_config,
227        grid_data,
228        gradients,
229        y_range,
230        top: Default::default(),
231        dif: Default::default(),
232        weight: Simd::splat(grid_data.weight),
233    };
234
235    let (state, dst) = output;
236
237    if config.has_block_head {
238        executer.interpolate::<false, 0, FULL>(state, dst);
239    }
240
241    if config.has_block_tail {
242        std::hint::cold_path();
243        let block_tail_size = config.block_tail_size;
244        let has_partial = config.has_partial;
245        let has_block_head = config.has_block_head;
246        match BilerpExecuter::<A, C, INIT, FINAL>::MAX_BLOCKS {
247            1 => match (block_tail_size, has_partial, has_block_head) {
248                (0, true, false) => executer.interpolate::<true, 0, SCALAR>(state, dst),
249                (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(state, dst),
250                _ => {}
251            },
252            2 => match (block_tail_size, has_partial, has_block_head) {
253                (1, false, _) => executer.interpolate::<true, 1, FULL>(state, dst),
254                (0, true, false) => executer.interpolate::<true, 0, SCALAR>(state, dst),
255                (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(state, dst),
256                (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(state, dst),
257                _ => {}
258            },
259            4 => match (block_tail_size, has_partial, has_block_head) {
260                (1, false, _) => executer.interpolate::<true, 1, FULL>(state, dst),
261                (2, false, _) => executer.interpolate::<true, 2, FULL>(state, dst),
262                (3, false, _) => executer.interpolate::<true, 3, FULL>(state, dst),
263                (0, true, false) => executer.interpolate::<true, 0, SCALAR>(state, dst),
264                (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(state, dst),
265                (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(state, dst),
266                (2, true, _) => executer.interpolate::<true, 2, PARTIAL>(state, dst),
267                (3, true, _) => executer.interpolate::<true, 3, PARTIAL>(state, dst),
268                _ => {}
269            },
270            8 => match (block_tail_size, has_partial, has_block_head) {
271                (1, false, _) => executer.interpolate::<true, 1, FULL>(state, dst),
272                (2, false, _) => executer.interpolate::<true, 2, FULL>(state, dst),
273                (3, false, _) => executer.interpolate::<true, 3, FULL>(state, dst),
274                (4, false, _) => executer.interpolate::<true, 4, FULL>(state, dst),
275                (5, false, _) => executer.interpolate::<true, 5, FULL>(state, dst),
276                (6, false, _) => executer.interpolate::<true, 6, FULL>(state, dst),
277                (7, false, _) => executer.interpolate::<true, 7, FULL>(state, dst),
278                (0, true, false) => executer.interpolate::<true, 0, SCALAR>(state, dst),
279                (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(state, dst),
280                (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(state, dst),
281                (2, true, _) => executer.interpolate::<true, 2, PARTIAL>(state, dst),
282                (3, true, _) => executer.interpolate::<true, 3, PARTIAL>(state, dst),
283                (4, true, _) => executer.interpolate::<true, 4, PARTIAL>(state, dst),
284                (5, true, _) => executer.interpolate::<true, 5, PARTIAL>(state, dst),
285                (6, true, _) => executer.interpolate::<true, 6, PARTIAL>(state, dst),
286                (7, true, _) => executer.interpolate::<true, 7, PARTIAL>(state, dst),
287                _ => {}
288            },
289            _ => {}
290        }
291    }
292}
293
294impl<'a, A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>
295    BilerpExecuter<'a, A, C, INIT, FINAL>
296{
297    const MAX_BLOCKS: usize = A::NUM_SIMD_REG / 4;
298
299    #[inline(always)]
300    pub fn interpolate<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
301        &mut self,
302        state: &mut [f32],
303        dst: &mut [f32],
304    ) {
305        let range = if IS_TAIL {
306            self.config.block_tail_start..self.grid_data.grid_size[0]
307        } else {
308            0..self.config.block_tail_start
309        };
310
311        let process = |this: &mut Self, state: &mut [f32], dst: &mut [f32], x, y, index| {
312            this.process_factors::<IS_TAIL, NUM_BLOCKS, ACCESS_MODE>(x, y, index, state, dst)
313        };
314
315        let mut x = range.start;
316        while x < range.end {
317            self.initialize_factors::<IS_TAIL, NUM_BLOCKS, ACCESS_MODE>(x);
318
319            let mut y = self.y_range.start;
320            let y_step = self.grid_data.grid_size[0];
321            let mut idx = y * y_step;
322            while y < self.y_range.end {
323                if y + 4 > self.y_range.end {
324                    // Slow single y iteration path.
325                    std::hint::cold_path();
326                    process(self, state, dst, x, y, idx);
327                    y += 1;
328                    idx += y_step;
329                } else {
330                    // Fast unrolled 4x y iteration path.
331                    process(self, state, dst, x, y, idx);
332                    process(self, state, dst, x, y + 1, idx + y_step);
333                    process(self, state, dst, x, y + 2, idx + 2 * y_step);
334                    process(self, state, dst, x, y + 3, idx + 3 * y_step);
335                    y += 4;
336                    idx += y_step * 4;
337                }
338            }
339            x += self.config.block_lanes;
340        }
341    }
342
343    #[inline(always)]
344    fn initialize_factors<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
345        &mut self,
346        x: usize,
347    ) {
348        let num_blocks = if IS_TAIL {
349            NUM_BLOCKS
350        } else {
351            Self::MAX_BLOCKS
352        };
353
354        // These blocked loops will get entirely unrolled by the compiler.
355        for block in 0..num_blocks {
356            // Load gradients into registers.
357            let index = x + Simd::<f32, A>::LANES * block;
358            self.initialize_factors_block::<FULL>(index, block);
359        }
360
361        if ACCESS_MODE != FULL {
362            self.initialize_factors_block::<ACCESS_MODE>(0, NUM_BLOCKS);
363        }
364    }
365
366    #[inline(always)]
367    fn initialize_factors_block<const ACCESS_MODE: u8>(&mut self, index: usize, block: usize) {
368        let (x_lerp, tl, tr, bl, br) = unsafe {
369            (
370                self.grid_data.fade_factors[0].ld_buf::<ACCESS_MODE>(index, self.config),
371                self.gradients.tl.ld_buf::<ACCESS_MODE>(index, self.config),
372                self.gradients.tr.ld_buf::<ACCESS_MODE>(index, self.config),
373                self.gradients.bl.ld_buf::<ACCESS_MODE>(index, self.config),
374                self.gradients.br.ld_buf::<ACCESS_MODE>(index, self.config),
375            )
376        };
377
378        // Base interpolation.
379        unsafe {
380            *self.top.get_unchecked_mut(block) = x_lerp.mul_add(tr - tl, tl) * self.weight;
381            let bottom = x_lerp.mul_add(br - bl, bl) * self.weight;
382            *self.dif.get_unchecked_mut(block) = bottom - *self.top.get_unchecked(block);
383        }
384    }
385
386    #[inline(always)]
387    fn process_factors<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
388        &mut self,
389        x: usize,
390        y: usize,
391        index: usize,
392        state: &mut [f32],
393        dst: &mut [f32],
394    ) {
395        let y_lerp = Simd::splat(unsafe {
396            self.grid_data.fade_factors[1]
397                .get_unchecked(y)
398                .assume_init()
399        });
400
401        let num_blocks = if IS_TAIL {
402            NUM_BLOCKS
403        } else {
404            Self::MAX_BLOCKS
405        };
406
407        for block in 0..num_blocks {
408            let index = index + x + block * Simd::<f32, A>::LANES;
409
410            self.process_factors_block::<FULL>(block, y_lerp, index, state, dst);
411        }
412
413        if ACCESS_MODE != FULL {
414            self.process_factors_block::<ACCESS_MODE>(NUM_BLOCKS, y_lerp, index, state, dst);
415        }
416    }
417
418    #[inline(always)]
419    fn process_factors_block<const ACCESS_MODE: u8>(
420        &mut self,
421        block: usize,
422        y_lerp: Simd<f32, A>,
423        index: usize,
424        state: &mut [f32],
425        dst: &mut [f32],
426    ) {
427        let access_mode = SimdAccessMode::from_u8(ACCESS_MODE);
428        let output = unsafe {
429            y_lerp.mul_add(
430                *self.dif.get_unchecked(block),
431                *self.top.get_unchecked(block),
432            )
433        };
434
435        // Initialize sample if it is the first and apply combiner algorithm
436        // for subsequent samples.
437        let (cur_state, mut result) = if INIT {
438            C::initialize_sample(self.fractal_config, output)
439        } else {
440            let mut cur_state = C::State::<A>::default();
441            for i in 0..C::State::<A>::STATE_SIZE {
442                let offset = i * self.grid_data.total_size;
443                let index = index + offset;
444                cur_state[i] = unsafe { load_simd_rw(state, index, self.config, access_mode) };
445            }
446
447            let cur_result = unsafe { load_simd_rw(dst, index, self.config, access_mode) };
448            C::apply_sample(self.fractal_config, cur_state, cur_result, output)
449        };
450
451        // Save changes to state.
452        if !FINAL {
453            for i in 0..C::State::<A>::STATE_SIZE {
454                let offset = i * self.grid_data.total_size;
455                let index = index + offset;
456                unsafe { write_simd_rw(state, index, cur_state[i], self.config, access_mode) };
457            }
458        }
459
460        // Finalize result if it is the last one.
461        if FINAL {
462            result = C::finalize_sample(self.fractal_config, cur_state, result);
463        }
464
465        // Save the result.
466        unsafe { write_simd_rw(dst, index, result, self.config, access_mode) };
467    }
468}