1use std::marker::PhantomData;
2
3use simply_simd::{Arch, Mask, Simd};
4
5use crate::Grid;
6
7const MAX_LANES: usize = 32;
10
11impl<A: Arch> Grid<2, A> {
12 #[inline(always)]
15 pub fn x_iter(&self) -> RowIter<A> {
16 let pos = self.config.position;
17 let dim = self.config.grid_size;
18 RowIter::new(dim[0], dim[1], pos[0] as f32)
19 }
20
21 #[inline(always)]
24 pub fn y_iter(&self) -> SliceIter<A> {
25 let pos = self.config.position;
26 let dim = self.config.grid_size;
27 SliceIter::new(dim[0], dim[1], 1, pos[1] as f32)
28 }
29}
30
31impl<A: Arch> Grid<3, A> {
32 #[inline(always)]
35 pub fn x_iter(&self) -> RowIter<A> {
36 let pos = self.config.position;
37 let dim = self.config.grid_size;
38 RowIter::new(dim[0], dim[1] * dim[2], pos[0] as f32)
39 }
40
41 #[inline(always)]
44 pub fn y_iter(&self) -> SliceIter<A> {
45 let pos = self.config.position;
46 let dim = self.config.grid_size;
47 SliceIter::new(dim[0], dim[1], dim[2], pos[1] as f32)
48 }
49
50 #[inline(always)]
53 pub fn z_iter(&self) -> SliceIter<A> {
54 let pos = self.config.position;
55 let dim = self.config.grid_size;
56 SliceIter::new(dim[0] * dim[1], dim[2], 1, pos[2] as f32)
57 }
58}
59
60#[derive(Clone, Copy)]
61struct RowScalar {
62 row_size: usize,
63 left_in_row: usize,
64 rows_left: usize,
65 cur: f32,
66 start: f32,
67}
68
69#[cold]
71#[inline(never)]
72fn row_scalar_fill(s: &mut RowScalar, buf: &mut [f32; MAX_LANES], lanes: usize) -> bool {
73 if s.left_in_row == 0 && s.rows_left == 0 {
74 return false;
75 }
76
77 for slot in buf[..lanes].iter_mut() {
78 *slot = if s.left_in_row > 0 {
79 let next = s.cur;
80 s.cur += 1.0;
81 s.left_in_row -= 1;
82 next
83 } else if s.rows_left > 0 {
84 s.cur = s.start + 1.0;
85 s.rows_left -= 1;
86 s.left_in_row = s.row_size - 1;
87 s.start
88 } else {
89 0.0
90 };
91 }
92 true
93}
94
95#[derive(Clone, Copy)]
96struct SliceScalar {
97 row_size: usize,
98 slice_size: usize,
99 left_in_row: usize,
100 left_in_slice: usize,
101 slices_left: usize,
102 cur_val: f32,
103 start_val: f32,
104}
105
106#[cold]
108#[inline(never)]
109fn slice_scalar_fill(s: &mut SliceScalar, buf: &mut [f32; MAX_LANES], lanes: usize) -> bool {
110 if s.left_in_row == 0 && s.left_in_slice == 0 && s.slices_left == 0 {
111 return false;
112 }
113
114 for slot in buf[..lanes].iter_mut() {
115 *slot = if s.left_in_row > 0 {
116 s.left_in_row -= 1;
117 s.cur_val
118 } else if s.left_in_slice > 0 {
119 s.cur_val += 1.0;
120 s.left_in_row = s.row_size - 1;
121 s.left_in_slice -= 1;
122 s.cur_val
123 } else if s.slices_left > 0 {
124 s.cur_val = s.start_val;
125 s.left_in_row = s.row_size - 1;
126 s.left_in_slice = s.slice_size - 1;
127 s.slices_left -= 1;
128 s.start_val
129 } else {
130 0.0
131 };
132 }
133 true
134}
135
136#[derive(Debug)]
140pub struct RowIter<A: Arch> {
141 row_size: usize,
142 left_in_row: usize,
143 rows_left: usize,
144 cur_vec: Simd<f32, A>,
145 start_vec: Simd<f32, A>,
146 cur_scalar: f32,
148 start_scalar: f32,
149 _arch: PhantomData<A>,
150}
151
152impl<A: Arch> RowIter<A> {
153 #[inline(always)]
154 fn new(row_size: usize, num_rows: usize, start_val: f32) -> Self {
155 Self {
156 row_size,
157 left_in_row: row_size,
158 rows_left: num_rows - 1,
159 cur_vec: Simd::iota(start_val),
160 start_vec: Simd::iota(start_val),
161 cur_scalar: start_val,
162 start_scalar: start_val,
163 _arch: PhantomData::<A>,
164 }
165 }
166
167 #[inline(always)]
168 fn next_scalar(&mut self) -> Option<Simd<f32, A>> {
169 let mut s = RowScalar {
172 row_size: self.row_size,
173 left_in_row: self.left_in_row,
174 rows_left: self.rows_left,
175 cur: self.cur_scalar,
176 start: self.start_scalar,
177 };
178 let mut buf = [0.0f32; MAX_LANES];
179
180 if !row_scalar_fill(&mut s, &mut buf, Simd::<f32, A>::LANES) {
181 return None;
182 }
183
184 self.left_in_row = s.left_in_row;
185 self.rows_left = s.rows_left;
186 self.cur_scalar = s.cur;
187 Some(Simd::from_slice(&buf[..Simd::<f32, A>::LANES]))
188 }
189}
190
191impl<A: Arch> Iterator for RowIter<A> {
192 type Item = Simd<f32, A>;
193
194 #[inline(always)]
195 fn next(&mut self) -> Option<Simd<f32, A>> {
196 let lanes = Simd::<f32, A>::LANES;
197
198 if self.row_size < lanes {
200 std::hint::cold_path();
201 return self.next_scalar();
202 }
203
204 if self.left_in_row >= lanes {
206 let next = self.cur_vec;
207 self.cur_vec += Simd::splat(lanes as f32);
208 self.left_in_row -= lanes;
209 return Some(next);
210 }
211
212 if self.rows_left > 0 {
214 let mask = Mask::first_n_true(self.left_in_row as u32);
215 let old = self.cur_vec;
216 let next = self.start_vec - Simd::splat(self.left_in_row as f32);
217 self.left_in_row += self.row_size - lanes;
218 self.rows_left -= 1;
219 self.cur_vec = next + Simd::splat(lanes as f32);
220 return Some(mask.select(old, next));
221 }
222
223 if self.left_in_row > 0 {
225 let mask = Mask::first_n_true(self.left_in_row as u32);
226 let vec = self.cur_vec;
227 self.left_in_row = 0;
228 return Some(mask.select(vec, Simd::zero()));
229 }
230
231 None
233 }
234
235 #[inline(always)]
236 fn size_hint(&self) -> (usize, Option<usize>) {
237 let left = self.left_in_row + self.rows_left * self.row_size;
238 let chunks_left = left.div_ceil(Simd::<f32, A>::LANES);
239 (chunks_left, Some(chunks_left))
240 }
241}
242
243#[derive(Debug)]
247pub struct SliceIter<A: Arch> {
248 row_size: usize,
249 slice_size: usize,
250 left_in_row: usize,
251 left_in_slice: usize,
252 slices_left: usize,
253 cur_val: f32,
254 start_val: f32,
255 _arch: PhantomData<A>,
256}
257
258impl<A: Arch> SliceIter<A> {
259 #[inline(always)]
260 pub fn new(row_size: usize, slice_size: usize, num_slices: usize, start_val: f32) -> Self {
261 Self {
262 row_size,
263 slice_size,
264 left_in_row: row_size,
265 left_in_slice: slice_size - 1,
266 slices_left: num_slices - 1,
267 cur_val: start_val,
268 start_val,
269 _arch: PhantomData::<A>,
270 }
271 }
272
273 #[inline(always)]
274 fn next_scalar(&mut self) -> Option<Simd<f32, A>> {
275 let mut s = SliceScalar {
276 row_size: self.row_size,
277 slice_size: self.slice_size,
278 left_in_row: self.left_in_row,
279 left_in_slice: self.left_in_slice,
280 slices_left: self.slices_left,
281 cur_val: self.cur_val,
282 start_val: self.start_val,
283 };
284 let mut buf = [0.0f32; MAX_LANES];
285
286 if !slice_scalar_fill(&mut s, &mut buf, Simd::<f32, A>::LANES) {
287 return None;
288 }
289
290 self.left_in_row = s.left_in_row;
291 self.left_in_slice = s.left_in_slice;
292 self.slices_left = s.slices_left;
293 self.cur_val = s.cur_val;
294 Some(Simd::from_slice(&buf[..Simd::<f32, A>::LANES]))
295 }
296}
297
298impl<A: Arch> Iterator for SliceIter<A> {
299 type Item = Simd<f32, A>;
300
301 #[inline(always)]
302 fn next(&mut self) -> Option<Simd<f32, A>> {
303 let lanes = Simd::<f32, A>::LANES;
304
305 if self.row_size < lanes {
307 std::hint::cold_path();
308 return self.next_scalar();
309 }
310
311 if self.left_in_row >= lanes {
313 self.left_in_row -= lanes;
314 return Some(Simd::splat(self.cur_val));
315 }
316
317 if self.left_in_slice > 0 {
319 let old = Simd::splat(self.cur_val);
320 self.cur_val += 1.0;
321 let new = Simd::splat(self.cur_val);
322
323 let mask = Mask::first_n_true(self.left_in_row as u32);
324 self.left_in_row += self.row_size - lanes;
325 self.left_in_slice -= 1;
326 return Some(mask.select(old, new));
327 }
328
329 if self.slices_left > 0 {
331 let old = Simd::splat(self.cur_val);
332 let new = Simd::splat(self.start_val);
333 self.cur_val = self.start_val;
334
335 let mask = Mask::first_n_true(self.left_in_row as u32);
336 self.left_in_row += self.row_size - lanes;
337 self.left_in_slice = self.slice_size - 1;
338 self.slices_left -= 1;
339 return Some(mask.select(old, new));
340 }
341
342 if self.left_in_row > 0 {
344 let old = Simd::splat(self.cur_val);
345 let mask = Mask::first_n_true(self.left_in_row as u32);
346 self.left_in_row = 0;
347 return Some(mask.select(old, Simd::zero()));
348 }
349
350 None
352 }
353
354 #[inline(always)]
355 fn size_hint(&self) -> (usize, Option<usize>) {
356 let left_in_slice = self.left_in_row + self.left_in_slice * self.row_size;
357 let left_after_slice = self.slices_left * self.row_size * self.slice_size;
358 let left = left_in_slice + left_after_slice;
359 let chunks_left = left.div_ceil(Simd::<f32, A>::LANES);
360 (chunks_left, Some(chunks_left))
361 }
362}