1use std::mem::MaybeUninit;
2use std::ops::Range;
3
4use simply_simd::{Arch, Simd, enable_targets};
5
6use crate::GridGenerator;
7use crate::api::grid::interface::GridNoiseParams;
8use crate::noise::combiners::{Combiner, CombinerState};
9use crate::noise::generators::Value;
10use crate::noise::util::constants::{BYTE_SHUFFLE, VALUE_EXP_MASK, HASH_MASK, HASH_PRIME};
11use crate::noise::util::grid_data::{GridData, Lerp};
12use crate::noise::util::grid_helpers::*;
13
14pub struct ValueGradients2D<'a> {
15 pub tl: &'a mut [MaybeUninit<f32>],
16 pub tr: &'a mut [MaybeUninit<f32>],
17 pub bl: &'a mut [MaybeUninit<f32>],
18 pub br: &'a mut [MaybeUninit<f32>],
19}
20
21impl<'a> ValueGradients2D<'a> {
22 #[inline(always)]
23 pub fn new(arena: &'a mut Arena, size: usize) -> Self {
24 Self {
25 tl: arena.allocate(size),
26 tr: arena.allocate(size),
27 bl: arena.allocate(size),
28 br: arena.allocate(size),
29 }
30 }
31
32 #[inline(always)]
33 pub fn swap_top_bottom(&mut self) {
34 std::mem::swap(&mut self.tl, &mut self.bl);
35 std::mem::swap(&mut self.tr, &mut self.br);
36 }
37}
38
39const LERP: u8 = Lerp::Cubic as u8;
40#[enable_targets(A)]
41impl GridGenerator<2> for Value {
42 fn sample_grid<A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>(
43 params: GridNoiseParams<2>,
44 fractal_config: C::Config,
45 state: &mut [f32],
46 dst: &mut [f32],
47 ) {
48 validate_grid_size(params.grid_size, dst.len());
49 validate_state_size::<C, A, _>(params.grid_size, state.len());
50 let padded_size = pad_grid_size::<A, 2>(params.grid_size);
51
52 let required_cache = padded_size[1] * 3 + padded_size[0] * 8;
53 let mut cache = ArenaBuffer::<A>::with_capacity(required_cache);
54 let mut arena = Arena::with_cache(&mut cache);
55
56 let num_blocks = BilerpExecuter::<A, C, INIT, FINAL>::MAX_BLOCKS;
58 let bilerp_config = InterpolationConfig::new(num_blocks, params.grid_size[0]);
59
60 let mut sub_arena = arena.allocate_arena(padded_size[0] * 3 + padded_size[1] * 3);
61 let mut grid_data = GridData::new::<A, LERP>(¶ms, &mut sub_arena, &padded_size);
62
63 let grad_scratch = arena.allocate(padded_size[0]);
65
66 let mut gradients = ValueGradients2D::new(&mut arena, padded_size[0]);
68
69 fill_gradients_2d::<A>(
71 ¶ms,
72 &mut grid_data,
73 grad_scratch,
74 gradients.tl,
75 gradients.tr,
76 0,
77 );
78
79 let mut y_cur_index = 0;
81 for y_it in 0..grid_data.num_loops[1] {
82 let y_next_index =
83 unsafe { grid_data.grid_indices[1].get_unchecked(y_it).assume_init() as usize };
84
85 fill_gradients_2d::<A>(
87 ¶ms,
88 &mut grid_data,
89 grad_scratch,
90 gradients.bl,
91 gradients.br,
92 y_it + 1,
93 );
94
95 let y_range = y_cur_index..y_next_index;
96 bilerp::<A, C, INIT, FINAL>(
97 &bilerp_config,
98 &fractal_config,
99 &grid_data,
100 &gradients,
101 y_range,
102 (state, dst),
103 );
104
105 gradients.swap_top_bottom();
107
108 y_cur_index = y_next_index;
109 }
110 }
111}
112
113#[inline(always)]
114pub(super) fn fill_gradients_2d<'a, A: Arch>(
115 params: &GridNoiseParams<2>,
116 grid_data: &mut GridData<2>,
117 grad_buffer: &mut [MaybeUninit<f32>],
118 left: &'a mut [MaybeUninit<f32>],
119 right: &'a mut [MaybeUninit<f32>],
120 y_it: usize,
121) {
122 let lanes = Simd::<f32, A>::LANES;
123 let y_start = grid_data.grid_start[1] + y_it as i32;
124 let y_rem = grid_data.octave_tiling[1].map_or(y_start, |t| {
125 let t = t as i32;
126 unsafe { core::hint::assert_unchecked(t > 0) };
128 y_start.rem_euclid(t)
129 });
130
131 let y_vec = Simd::splat((y_rem as u32).wrapping_mul(params.seed));
132
133 let prime = Simd::splat(HASH_PRIME);
134 let shuffle_indices = unsafe { Simd::<u8, A>::from_slice_unchecked(&BYTE_SHUFFLE[..]) };
135 let y_shuf = y_vec.permute_8(shuffle_indices) ^ prime;
136 let y_shuf = y_shuf * y_shuf;
137
138 let hash_mask: Simd<u32, A> = Simd::splat(HASH_MASK);
139 let exp_bits: Simd<u32, A> = Simd::splat(VALUE_EXP_MASK);
140 let three: Simd<f32, A> = Simd::splat(3.0);
141
142 if let Some(x_tiling) = grid_data.octave_tiling[0] {
143 let x_tiling = Simd::splat(x_tiling as f32);
144 let mut x_vec = Simd::splat(grid_data.grid_start[0]) + Simd::iota(0);
145 let x_vec_stride = Simd::splat(lanes as i32);
146 let seed_vec = Simd::splat(params.seed);
147
148 let end_index = grid_data.num_loops[0] + 1;
149 for i in (0..end_index).step_by(lanes) {
150 let x_floats = x_vec.cast_float();
151 let x_rem = x_floats - (x_floats / x_tiling).floor() * x_tiling;
152 let x_seeded = x_rem.cast_int_round().raw_cast() * seed_vec;
153
154 let x_shuf = x_seeded.permute_8(shuffle_indices) ^ prime;
155 let hash = y_shuf * x_shuf;
156 let grad = ((hash & hash_mask) | exp_bits).raw_cast() - three;
157 unsafe { grad_buffer.write_simd_aligned(i, grad) };
158 x_vec += x_vec_stride;
159 }
160 } else {
161 let iota_vec = Simd::iota(0) * Simd::splat(params.seed);
162 let mut x_vec =
163 Simd::splat((grid_data.grid_start[0] as u32).wrapping_mul(params.seed)) + iota_vec;
164 let x_vec_stride = Simd::splat((lanes as u32).wrapping_mul(params.seed));
165
166 let end_index = grid_data.num_loops[0] + 1;
168 for i in (0..end_index).step_by(lanes) {
169 let x_shuf = x_vec.permute_8(shuffle_indices) ^ prime;
170 let hash = y_shuf * x_shuf;
171 let grad = ((hash & hash_mask) | exp_bits).raw_cast() - three;
172 unsafe { grad_buffer.write_simd_aligned(i, grad) };
173 x_vec += x_vec_stride;
174 }
175 }
176
177 let mut x_cur_index = 0;
179 for x_it in 0..grid_data.num_loops[0] {
180 let x_next_index = unsafe { grid_data.grid_indices[0].get_unchecked(x_it).assume_init() };
182 let mut amount = (x_next_index - x_cur_index) as isize;
183
184 unsafe {
185 let l = grad_buffer.get_unchecked(x_it).assume_init();
186 let r = grad_buffer.get_unchecked(x_it + 1).assume_init();
187
188 let mut index = x_cur_index as usize;
189 while amount > 0 {
190 left.write_simd(index, Simd::<f32, A>::splat(l));
191 right.write_simd(index, Simd::<f32, A>::splat(r));
192 amount -= lanes as isize;
193 index += lanes;
194 }
195 }
196
197 x_cur_index = x_next_index;
198 }
199}
200
201pub(crate) struct BilerpExecuter<'a, A: Arch, C: Combiner, const INIT: bool, const FINAL: bool> {
204 config: &'a InterpolationConfig<A>,
205 fractal_config: &'a C::Config,
206 grid_data: &'a GridData<'a, 2>,
207 gradients: &'a ValueGradients2D<'a>,
208 y_range: Range<usize>,
209 top: A::Block2<f32>,
210 dif: A::Block2<f32>,
211 weight: Simd<f32, A>,
212}
213
214#[inline(always)]
216pub(super) fn bilerp<A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>(
217 config: &InterpolationConfig<A>,
218 fractal_config: &C::Config,
219 grid_data: &GridData<2>,
220 gradients: &ValueGradients2D,
221 y_range: Range<usize>,
222 output: (&mut [f32], &mut [f32]),
223) {
224 let mut executer = BilerpExecuter::<A, C, INIT, FINAL> {
225 config,
226 fractal_config,
227 grid_data,
228 gradients,
229 y_range,
230 top: Default::default(),
231 dif: Default::default(),
232 weight: Simd::splat(grid_data.weight),
233 };
234
235 let (state, dst) = output;
236
237 if config.has_block_head {
238 executer.interpolate::<false, 0, FULL>(state, dst);
239 }
240
241 if config.has_block_tail {
242 std::hint::cold_path();
243 let block_tail_size = config.block_tail_size;
244 let has_partial = config.has_partial;
245 let has_block_head = config.has_block_head;
246 match BilerpExecuter::<A, C, INIT, FINAL>::MAX_BLOCKS {
247 1 => match (block_tail_size, has_partial, has_block_head) {
248 (0, true, false) => executer.interpolate::<true, 0, SCALAR>(state, dst),
249 (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(state, dst),
250 _ => {}
251 },
252 2 => match (block_tail_size, has_partial, has_block_head) {
253 (1, false, _) => executer.interpolate::<true, 1, FULL>(state, dst),
254 (0, true, false) => executer.interpolate::<true, 0, SCALAR>(state, dst),
255 (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(state, dst),
256 (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(state, dst),
257 _ => {}
258 },
259 4 => match (block_tail_size, has_partial, has_block_head) {
260 (1, false, _) => executer.interpolate::<true, 1, FULL>(state, dst),
261 (2, false, _) => executer.interpolate::<true, 2, FULL>(state, dst),
262 (3, false, _) => executer.interpolate::<true, 3, FULL>(state, dst),
263 (0, true, false) => executer.interpolate::<true, 0, SCALAR>(state, dst),
264 (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(state, dst),
265 (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(state, dst),
266 (2, true, _) => executer.interpolate::<true, 2, PARTIAL>(state, dst),
267 (3, true, _) => executer.interpolate::<true, 3, PARTIAL>(state, dst),
268 _ => {}
269 },
270 8 => match (block_tail_size, has_partial, has_block_head) {
271 (1, false, _) => executer.interpolate::<true, 1, FULL>(state, dst),
272 (2, false, _) => executer.interpolate::<true, 2, FULL>(state, dst),
273 (3, false, _) => executer.interpolate::<true, 3, FULL>(state, dst),
274 (4, false, _) => executer.interpolate::<true, 4, FULL>(state, dst),
275 (5, false, _) => executer.interpolate::<true, 5, FULL>(state, dst),
276 (6, false, _) => executer.interpolate::<true, 6, FULL>(state, dst),
277 (7, false, _) => executer.interpolate::<true, 7, FULL>(state, dst),
278 (0, true, false) => executer.interpolate::<true, 0, SCALAR>(state, dst),
279 (0, true, true) => executer.interpolate::<true, 0, PARTIAL>(state, dst),
280 (1, true, _) => executer.interpolate::<true, 1, PARTIAL>(state, dst),
281 (2, true, _) => executer.interpolate::<true, 2, PARTIAL>(state, dst),
282 (3, true, _) => executer.interpolate::<true, 3, PARTIAL>(state, dst),
283 (4, true, _) => executer.interpolate::<true, 4, PARTIAL>(state, dst),
284 (5, true, _) => executer.interpolate::<true, 5, PARTIAL>(state, dst),
285 (6, true, _) => executer.interpolate::<true, 6, PARTIAL>(state, dst),
286 (7, true, _) => executer.interpolate::<true, 7, PARTIAL>(state, dst),
287 _ => {}
288 },
289 _ => {}
290 }
291 }
292}
293
294impl<'a, A: Arch, C: Combiner, const INIT: bool, const FINAL: bool>
295 BilerpExecuter<'a, A, C, INIT, FINAL>
296{
297 const MAX_BLOCKS: usize = A::NUM_SIMD_REG / 4;
298
299 #[inline(always)]
300 pub fn interpolate<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
301 &mut self,
302 state: &mut [f32],
303 dst: &mut [f32],
304 ) {
305 let range = if IS_TAIL {
306 self.config.block_tail_start..self.grid_data.grid_size[0]
307 } else {
308 0..self.config.block_tail_start
309 };
310
311 let process = |this: &mut Self, state: &mut [f32], dst: &mut [f32], x, y, index| {
312 this.process_factors::<IS_TAIL, NUM_BLOCKS, ACCESS_MODE>(x, y, index, state, dst)
313 };
314
315 let mut x = range.start;
316 while x < range.end {
317 self.initialize_factors::<IS_TAIL, NUM_BLOCKS, ACCESS_MODE>(x);
318
319 let mut y = self.y_range.start;
320 let y_step = self.grid_data.grid_size[0];
321 let mut idx = y * y_step;
322 while y < self.y_range.end {
323 if y + 4 > self.y_range.end {
324 std::hint::cold_path();
326 process(self, state, dst, x, y, idx);
327 y += 1;
328 idx += y_step;
329 } else {
330 process(self, state, dst, x, y, idx);
332 process(self, state, dst, x, y + 1, idx + y_step);
333 process(self, state, dst, x, y + 2, idx + 2 * y_step);
334 process(self, state, dst, x, y + 3, idx + 3 * y_step);
335 y += 4;
336 idx += y_step * 4;
337 }
338 }
339 x += self.config.block_lanes;
340 }
341 }
342
343 #[inline(always)]
344 fn initialize_factors<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
345 &mut self,
346 x: usize,
347 ) {
348 let num_blocks = if IS_TAIL {
349 NUM_BLOCKS
350 } else {
351 Self::MAX_BLOCKS
352 };
353
354 for block in 0..num_blocks {
356 let index = x + Simd::<f32, A>::LANES * block;
358 self.initialize_factors_block::<FULL>(index, block);
359 }
360
361 if ACCESS_MODE != FULL {
362 self.initialize_factors_block::<ACCESS_MODE>(0, NUM_BLOCKS);
363 }
364 }
365
366 #[inline(always)]
367 fn initialize_factors_block<const ACCESS_MODE: u8>(&mut self, index: usize, block: usize) {
368 let (x_lerp, tl, tr, bl, br) = unsafe {
369 (
370 self.grid_data.fade_factors[0].ld_buf::<ACCESS_MODE>(index, self.config),
371 self.gradients.tl.ld_buf::<ACCESS_MODE>(index, self.config),
372 self.gradients.tr.ld_buf::<ACCESS_MODE>(index, self.config),
373 self.gradients.bl.ld_buf::<ACCESS_MODE>(index, self.config),
374 self.gradients.br.ld_buf::<ACCESS_MODE>(index, self.config),
375 )
376 };
377
378 unsafe {
380 *self.top.get_unchecked_mut(block) = x_lerp.mul_add(tr - tl, tl) * self.weight;
381 let bottom = x_lerp.mul_add(br - bl, bl) * self.weight;
382 *self.dif.get_unchecked_mut(block) = bottom - *self.top.get_unchecked(block);
383 }
384 }
385
386 #[inline(always)]
387 fn process_factors<const IS_TAIL: bool, const NUM_BLOCKS: usize, const ACCESS_MODE: u8>(
388 &mut self,
389 x: usize,
390 y: usize,
391 index: usize,
392 state: &mut [f32],
393 dst: &mut [f32],
394 ) {
395 let y_lerp = Simd::splat(unsafe {
396 self.grid_data.fade_factors[1]
397 .get_unchecked(y)
398 .assume_init()
399 });
400
401 let num_blocks = if IS_TAIL {
402 NUM_BLOCKS
403 } else {
404 Self::MAX_BLOCKS
405 };
406
407 for block in 0..num_blocks {
408 let index = index + x + block * Simd::<f32, A>::LANES;
409
410 self.process_factors_block::<FULL>(block, y_lerp, index, state, dst);
411 }
412
413 if ACCESS_MODE != FULL {
414 self.process_factors_block::<ACCESS_MODE>(NUM_BLOCKS, y_lerp, index, state, dst);
415 }
416 }
417
418 #[inline(always)]
419 fn process_factors_block<const ACCESS_MODE: u8>(
420 &mut self,
421 block: usize,
422 y_lerp: Simd<f32, A>,
423 index: usize,
424 state: &mut [f32],
425 dst: &mut [f32],
426 ) {
427 let access_mode = SimdAccessMode::from_u8(ACCESS_MODE);
428 let output = unsafe {
429 y_lerp.mul_add(
430 *self.dif.get_unchecked(block),
431 *self.top.get_unchecked(block),
432 )
433 };
434
435 let (cur_state, mut result) = if INIT {
438 C::initialize_sample(self.fractal_config, output)
439 } else {
440 let mut cur_state = C::State::<A>::default();
441 for i in 0..C::State::<A>::STATE_SIZE {
442 let offset = i * self.grid_data.total_size;
443 let index = index + offset;
444 cur_state[i] = unsafe { load_simd_rw(state, index, self.config, access_mode) };
445 }
446
447 let cur_result = unsafe { load_simd_rw(dst, index, self.config, access_mode) };
448 C::apply_sample(self.fractal_config, cur_state, cur_result, output)
449 };
450
451 if !FINAL {
453 for i in 0..C::State::<A>::STATE_SIZE {
454 let offset = i * self.grid_data.total_size;
455 let index = index + offset;
456 unsafe { write_simd_rw(state, index, cur_state[i], self.config, access_mode) };
457 }
458 }
459
460 if FINAL {
462 result = C::finalize_sample(self.fractal_config, cur_state, result);
463 }
464
465 unsafe { write_simd_rw(dst, index, result, self.config, access_mode) };
467 }
468}