1use crate::prelude::*;
4
5pub struct ShlsSlice {
12 pub shls_slice: Vec<[usize; 2]>,
13}
14
15#[duplicate_item(
16 T N;
17 [Vec<[usize; 2]>] [];
18 [&Vec<[usize; 2]>] [];
19 [&[[usize; 2]]] [];
20 [[[usize; 2]; N]] [const N: usize];
21 [&[[usize; 2]; N]] [const N: usize];
22)]
23impl<N> From<T> for ShlsSlice {
24 fn from(shls_slice: T) -> Self {
25 let shls_slice = shls_slice.to_vec();
26 Self { shls_slice }
27 }
28}
29
30#[duplicate_item(
31 T N;
32 [Vec<[isize; 2]>] [];
33 [&Vec<[isize; 2]>] [];
34 [&[[isize; 2]]] [];
35 [[[isize; 2]; N]] [const N: usize];
36 [&[[isize; 2]; N]] [const N: usize];
37 [Vec<[i32; 2]>] [];
38 [&Vec<[i32; 2]>] [];
39 [&[[i32; 2]]] [];
40 [[[i32; 2]; N]] [const N: usize];
41 [&[[i32; 2]; N]] [const N: usize];
42 [Vec<[i64; 2]>] [];
43 [&Vec<[i64; 2]>] [];
44 [&[[i64; 2]]] [];
45 [[[i64; 2]; N]] [const N: usize];
46 [&[[i64; 2]; N]] [const N: usize];
47)]
48impl<N> From<T> for ShlsSlice {
49 fn from(shls_slice: T) -> Self {
50 let shls_slice = shls_slice.iter().map(|arr| [arr[0] as usize, arr[1] as usize]).collect();
51 Self { shls_slice }
52 }
53}
54
55impl From<Option<&[[usize; 2]]>> for ShlsSlice {
57 fn from(shls_slice: Option<&[[usize; 2]]>) -> Self {
58 match shls_slice {
59 Some(slice) => slice.to_vec().into(),
60 None => ShlsSlice { shls_slice: vec![] },
61 }
62 }
63}
64
65#[duplicate_item(
66 T N;
67 [Vec<Vec<usize>>] [];
68 [&Vec<Vec<usize>>] [];
69 [&[Vec<usize>]] [];
70 [[Vec<usize>; N]] [const N: usize];
71 [&[Vec<usize>; N]] [const N: usize];
72)]
73impl<N> From<T> for ShlsSlice {
74 fn from(shls_slice: T) -> Self {
75 let shls_slice: Vec<[usize; 2]> = shls_slice
77 .iter()
78 .map(|vec| {
79 if vec.len() == 2 {
80 [vec[0], vec[1]]
81 } else {
82 panic!("Each inner vector must have exactly two elements.");
83 }
84 })
85 .collect();
86 Self { shls_slice }
87 }
88}
89
90impl AsRef<[[usize; 2]]> for ShlsSlice {
91 fn as_ref(&self) -> &[[usize; 2]] {
92 self.shls_slice.as_slice()
93 }
94}
95
96pub fn blocksize_partition(arr: &[usize], block_size: usize) -> Vec<usize> {
121 assert!(block_size > 0, "Block size must be greater than zero");
122 assert!(!arr.is_empty(), "Array must not be empty");
123 let n = arr.len() - 1;
124 if n == 0 {
125 return vec![0];
126 }
127
128 let mut partition = vec![0];
129 let mut p0 = 0;
130 for i in 1..n {
131 if arr[i + 1] - arr[p0] > block_size {
132 partition.push(i);
133 p0 = i;
134 }
135 }
136 partition.push(n);
137 partition
138}
139
140pub fn balance_partition(cint_data: &CInt, block_size: usize, start_id: Option<usize>, end_id: Option<usize>) -> Vec<[usize; 3]> {
164 let shl_start = start_id.unwrap_or(0);
165 let shl_end = end_id.unwrap_or(cint_data.nbas());
166 let ao_loc = cint_data.ao_loc();
167 let ao_loc_sliced = &ao_loc[shl_start..=shl_end];
168 let block_partition = blocksize_partition(ao_loc_sliced, block_size);
169 let mut shl_partition = vec![];
170 for i in 0..(block_partition.len() - 1) {
171 let start = block_partition[i] + shl_start;
172 let end = block_partition[i + 1] + shl_start;
173 let step = ao_loc[end] - ao_loc[start];
174 shl_partition.push([start, end, step]);
175 }
176 shl_partition
177}
178
179pub fn gaussian_int(n: f64, alpha: f64) -> f64 {
189 let n1 = (n + 1.0) / 2.0;
190 libm::tgamma(n1) / (2.0 * alpha.powf(n1))
191}
192
193#[allow(clippy::mut_from_ref)]
198pub(crate) unsafe fn cast_mut_slice<T>(slc: &[T]) -> &mut [T] {
199 let len = slc.len();
200 let ptr = slc.as_ptr() as *mut T;
201 unsafe { std::slice::from_raw_parts_mut(ptr, len) }
202}
203
204pub unsafe fn as_chunks_unchecked_mut<const N: usize, T>(slc: &mut [T]) -> &mut [[T; N]] {
214 let new_len = slc.len() / N;
215 core::slice::from_raw_parts_mut(slc.as_mut_ptr().cast(), new_len)
216}
217
218#[inline(always)]
223pub(crate) fn unravel_s2_indices(x: usize) -> [usize; 2] {
224 let j = (((x * 2 + 1) as f64).sqrt() - 0.5) as usize;
225 let i = x - j * (j + 1) / 2;
226 [i, j]
227}
228
229#[inline(always)]
230pub(crate) fn get_f_index_3d(indices: &[usize; 3], shape: &[usize; 3]) -> usize {
231 indices[0] + shape[0] * (indices[1] + shape[1] * (indices[2]))
232}
233
234#[inline(always)]
235pub(crate) fn get_f_index_4d(indices: &[usize; 4], shape: &[usize; 4]) -> usize {
236 indices[0] + shape[0] * (indices[1] + shape[1] * (indices[2] + shape[2] * (indices[3])))
237}
238
239#[inline(always)]
240pub(crate) fn get_f_index_5d(indices: &[usize; 5], shape: &[usize; 5]) -> usize {
241 indices[0] + shape[0] * (indices[1] + shape[1] * (indices[2] + shape[2] * (indices[3] + shape[3] * (indices[4]))))
242}
243
244#[inline(always)]
245pub(crate) fn get_f_index_3d_s2ij(indices: &[usize; 3], shape: &[usize; 2]) -> usize {
246 indices[0] + indices[1] * (indices[1] + 1) / 2 + shape[0] * (indices[2])
247}
248
249#[inline(always)]
250pub(crate) fn get_f_index_4d_s2ij(indices: &[usize; 4], shape: &[usize; 3]) -> usize {
251 indices[0] + indices[1] * (indices[1] + 1) / 2 + shape[0] * (indices[2] + shape[1] * (indices[3]))
252}
253
254#[inline(always)]
255pub(crate) fn get_f_index_5d_s2ij(indices: &[usize; 5], shape: &[usize; 4]) -> usize {
256 indices[0] + indices[1] * (indices[1] + 1) / 2 + shape[0] * (indices[2] + shape[1] * (indices[3] + shape[2] * (indices[4])))
257}
258
259#[inline(always)]
260pub(crate) fn get_f_index_5d_s2kl(indices: &[usize; 5], shape: &[usize; 4]) -> usize {
261 let [i0, i1, i2, i3, i4] = indices;
262 let [s0, s1, s2, _] = shape;
263 let indices_kl = i2 + i3 * (i3 + 1) / 2;
264 i0 + s0 * (i1 + s1 * (indices_kl + s2 * i4))
265}
266
267#[inline(always)]
268pub(crate) fn get_f_index_5d_s4(indices: &[usize; 5], shape: &[usize; 3]) -> usize {
269 let [i0, i1, i2, i3, i4] = indices;
270 let [s0, s1, _] = shape;
271 let indices_ij = i0 + i1 * (i1 + 1) / 2;
272 let indices_kl = i2 + i3 * (i3 + 1) / 2;
273 indices_ij + s0 * (indices_kl + s1 * i4)
274}
275
276#[inline(always)]
277pub(crate) fn get_f_index_5d_s8(indices: &[usize; 5], shape: &[usize; 2]) -> usize {
278 let [i0, i1, i2, i3, i4] = indices;
279 let [s0, _] = shape;
280 let indices_ij = i0 + i1 * (i1 + 1) / 2;
281 let indices_kl = i2 + i3 * (i3 + 1) / 2;
282 indices_ij + indices_kl * (indices_kl + 1) / 2 + s0 * i4
283}
284
285#[inline]
290pub(crate) fn copy_f_3d_s1<T>(out: &mut [T], out_offsets: &[usize; 3], out_shape: &[usize; 3], buf: &[T], buf_shape: &[usize; 3])
291where
292 T: Copy,
293{
294 for c in 0..buf_shape[2] {
295 for j in 0..buf_shape[1] {
296 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + c];
297 let buf_indices = [0, j, c];
298 let out_start = get_f_index_3d(&out_indices, out_shape);
299 let buf_start = get_f_index_3d(&buf_indices, buf_shape);
300 let out_stop = out_start + buf_shape[0];
301 let buf_stop = buf_start + buf_shape[0];
302 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
303 }
304 }
305}
306
307#[inline]
308pub(crate) fn copy_f_4d_s1<T>(out: &mut [T], out_offsets: &[usize; 4], out_shape: &[usize; 4], buf: &[T], buf_shape: &[usize; 4])
309where
310 T: Copy,
311{
312 for c in 0..buf_shape[3] {
313 for k in 0..buf_shape[2] {
314 for j in 0..buf_shape[1] {
315 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + k, out_offsets[3] + c];
316 let buf_indices = [0, j, k, c];
317 let out_start = get_f_index_4d(&out_indices, out_shape);
318 let buf_start = get_f_index_4d(&buf_indices, buf_shape);
319 let out_stop = out_start + buf_shape[0];
320 let buf_stop = buf_start + buf_shape[0];
321 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
322 }
323 }
324 }
325}
326
327#[inline]
328pub(crate) fn copy_f_5d_s1<T>(out: &mut [T], out_offsets: &[usize; 5], out_shape: &[usize; 5], buf: &[T], buf_shape: &[usize; 5])
329where
330 T: Copy,
331{
332 for c in 0..buf_shape[4] {
333 for l in 0..buf_shape[3] {
334 for k in 0..buf_shape[2] {
335 for j in 0..buf_shape[1] {
336 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + k, out_offsets[3] + l, out_offsets[4] + c];
337 let buf_indices = [0, j, k, l, c];
338 let out_start = get_f_index_5d(&out_indices, out_shape);
339 let buf_start = get_f_index_5d(&buf_indices, buf_shape);
340 let out_stop = out_start + buf_shape[0];
341 let buf_stop = buf_start + buf_shape[0];
342 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
343 }
344 }
345 }
346 }
347}
348
349#[inline]
350pub(crate) fn copy_f_3d_s2ij<T>(out: &mut [T], out_offsets: &[usize; 3], out_shape: &[usize; 2], buf: &[T], buf_shape: &[usize; 3])
351where
352 T: Copy,
353{
354 if out_offsets[0] != out_offsets[1] {
355 for c in 0..buf_shape[2] {
357 for j in 0..buf_shape[1] {
358 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + c];
359 let buf_indices = [0, j, c];
360 let out_start = get_f_index_3d_s2ij(&out_indices, out_shape);
361 let buf_start = get_f_index_3d(&buf_indices, buf_shape);
362 let out_stop = out_start + buf_shape[0];
363 let buf_stop = buf_start + buf_shape[0];
364 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
365 }
366 }
367 } else {
368 for c in 0..buf_shape[2] {
370 for j in 0..buf_shape[1] {
371 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + c];
372 let buf_indices = [0, j, c];
373 let out_start = get_f_index_3d_s2ij(&out_indices, out_shape);
374 let buf_start = get_f_index_3d(&buf_indices, buf_shape);
375 let out_stop = out_start + j + 1;
376 let buf_stop = buf_start + j + 1;
377 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
378 }
379 }
380 }
381}
382
383#[inline]
384pub(crate) fn copy_f_4d_s2ij<T>(out: &mut [T], out_offsets: &[usize; 4], out_shape: &[usize; 3], buf: &[T], buf_shape: &[usize; 4])
385where
386 T: Copy,
387{
388 if out_offsets[0] != out_offsets[1] {
389 for c in 0..buf_shape[3] {
391 for k in 0..buf_shape[2] {
392 for j in 0..buf_shape[1] {
393 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + k, out_offsets[3] + c];
394 let buf_indices = [0, j, k, c];
395 let out_start = get_f_index_4d_s2ij(&out_indices, out_shape);
396 let buf_start = get_f_index_4d(&buf_indices, buf_shape);
397 let out_stop = out_start + buf_shape[0];
398 let buf_stop = buf_start + buf_shape[0];
399 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
400 }
401 }
402 }
403 } else {
404 for c in 0..buf_shape[3] {
406 for k in 0..buf_shape[2] {
407 for j in 0..buf_shape[1] {
408 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + k, out_offsets[3] + c];
409 let buf_indices = [0, j, k, c];
410 let out_start = get_f_index_4d_s2ij(&out_indices, out_shape);
411 let buf_start = get_f_index_4d(&buf_indices, buf_shape);
412 let out_stop = out_start + j + 1;
413 let buf_stop = buf_start + j + 1;
414 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
415 }
416 }
417 }
418 }
419}
420
421#[inline]
422pub(crate) fn copy_f_5d_s2ij<T>(out: &mut [T], out_offsets: &[usize; 5], out_shape: &[usize; 4], buf: &[T], buf_shape: &[usize; 5])
423where
424 T: Copy,
425{
426 if out_offsets[0] != out_offsets[1] {
427 for c in 0..buf_shape[4] {
429 for l in 0..buf_shape[3] {
430 for k in 0..buf_shape[2] {
431 for j in 0..buf_shape[1] {
432 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + k, out_offsets[3] + l, out_offsets[4] + c];
433 let buf_indices = [0, j, k, l, c];
434 let out_start = get_f_index_5d_s2ij(&out_indices, out_shape);
435 let buf_start = get_f_index_5d(&buf_indices, buf_shape);
436 let out_stop = out_start + buf_shape[0];
437 let buf_stop = buf_start + buf_shape[0];
438 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
439 }
440 }
441 }
442 }
443 } else {
444 for c in 0..buf_shape[4] {
446 for l in 0..buf_shape[3] {
447 for k in 0..buf_shape[2] {
448 for j in 0..buf_shape[1] {
449 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + k, out_offsets[3] + l, out_offsets[4] + c];
450 let buf_indices = [0, j, k, l, c];
451 let out_start = get_f_index_5d_s2ij(&out_indices, out_shape);
452 let buf_start = get_f_index_5d(&buf_indices, buf_shape);
453 let out_stop = out_start + j + 1;
454 let buf_stop = buf_start + j + 1;
455 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
456 }
457 }
458 }
459 }
460 }
461}
462
463#[inline]
464pub(crate) fn copy_f_5d_s2kl<T>(out: &mut [T], out_offsets: &[usize; 5], out_shape: &[usize; 4], buf: &[T], buf_shape: &[usize; 5])
465where
466 T: Copy,
467{
468 let is_diag_kl = out_offsets[2] == out_offsets[3];
469 for c in 0..buf_shape[4] {
470 for l in 0..buf_shape[3] {
471 let k_max = if is_diag_kl { l + 1 } else { buf_shape[2] };
472 for k in 0..k_max {
473 for j in 0..buf_shape[1] {
474 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + k, out_offsets[3] + l, out_offsets[4] + c];
475 let buf_indices = [0, j, k, l, c];
476 let out_start = get_f_index_5d_s2kl(&out_indices, out_shape);
477 let buf_start = get_f_index_5d(&buf_indices, buf_shape);
478 let out_stop = out_start + buf_shape[0];
479 let buf_stop = buf_start + buf_shape[0];
480 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
481 }
482 }
483 }
484 }
485}
486
487#[inline]
488pub(crate) fn copy_f_5d_s4<T>(out: &mut [T], out_offsets: &[usize; 5], out_shape: &[usize; 3], buf: &[T], buf_shape: &[usize; 5])
489where
490 T: Copy,
491{
492 let is_diag_ij = out_offsets[0] == out_offsets[1];
493 let is_diag_kl = out_offsets[2] == out_offsets[3];
494 for c in 0..buf_shape[4] {
495 for l in 0..buf_shape[3] {
496 let k_max = if is_diag_kl { l + 1 } else { buf_shape[2] };
497 for k in 0..k_max {
498 for j in 0..buf_shape[1] {
499 let i_max = if is_diag_ij { j + 1 } else { buf_shape[0] };
500 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + k, out_offsets[3] + l, out_offsets[4] + c];
501 let buf_indices = [0, j, k, l, c];
502 let out_start = get_f_index_5d_s4(&out_indices, out_shape);
503 let buf_start = get_f_index_5d(&buf_indices, buf_shape);
504 let out_stop = out_start + i_max;
505 let buf_stop = buf_start + i_max;
506 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
507 }
508 }
509 }
510 }
511}
512
513#[inline]
514pub(crate) fn copy_f_5d_s8<T>(out: &mut [T], out_offsets: &[usize; 5], out_shape: &[usize; 2], buf: &[T], buf_shape: &[usize; 5])
515where
516 T: Copy,
517{
518 let is_diag_ij = out_offsets[0] == out_offsets[1];
519 let is_diag_kl = out_offsets[2] == out_offsets[3];
520 let is_diag_jl = out_offsets[1] == out_offsets[3];
521 let is_i_gt_k = out_offsets[0] > out_offsets[2];
522 let is_diag_ik = out_offsets[0] == out_offsets[2];
523 for c in 0..buf_shape[4] {
524 for l in 0..buf_shape[3] {
525 let k_max = if is_diag_kl { l + 1 } else { buf_shape[2] };
526 for k in 0..k_max {
527 let j_max = if is_diag_jl { l + 1 } else { buf_shape[1] };
528 for j in 0..j_max {
529 if is_diag_jl && j == l && is_i_gt_k {
530 continue;
531 }
532 let i_max = if is_diag_ij { j + 1 } else { buf_shape[0] };
533 let i_max = if is_diag_jl && j == l && is_diag_ik { i_max.min(k + 1) } else { i_max };
534
535 let out_indices = [out_offsets[0], out_offsets[1] + j, out_offsets[2] + k, out_offsets[3] + l, out_offsets[4] + c];
536 let buf_indices = [0, j, k, l, c];
537 let out_start = get_f_index_5d_s8(&out_indices, out_shape);
538 let buf_start = get_f_index_5d(&buf_indices, buf_shape);
539 let out_stop = out_start + i_max;
540 let buf_stop = buf_start + i_max;
541 out[out_start..out_stop].copy_from_slice(&buf[buf_start..buf_stop]);
542 }
543 }
544 }
545 }
546}
547
548#[inline(always)]
553pub(crate) fn get_c_index_3d(indices: &[usize; 3], shape: &[usize; 3]) -> usize {
554 let [i0, i1, i2] = indices;
555 let [_, s1, s2] = shape;
556 i2 + s2 * (i1 + s1 * i0)
557}
558
559#[inline(always)]
560pub(crate) fn get_c_index_4d(indices: &[usize; 4], shape: &[usize; 4]) -> usize {
561 let [i0, i1, i2, i3] = indices;
562 let [_, s1, s2, s3] = shape;
563 i3 + s3 * (i2 + s2 * (i1 + s1 * i0))
564}
565
566#[inline(always)]
567pub(crate) fn get_c_index_5d(indices: &[usize; 5], shape: &[usize; 5]) -> usize {
568 let [i0, i1, i2, i3, i4] = indices;
569 let [_, s1, s2, s3, s4] = shape;
570 i4 + s4 * (i3 + s3 * (i2 + s2 * (i1 + s1 * i0)))
571}
572
573#[inline(always)]
574pub(crate) fn get_c_index_3d_s2ij(indices: &[usize; 3], shape: &[usize; 2]) -> usize {
575 let [i0, i1, i2] = indices;
576 let [_, s1] = shape;
577 i2 + i1 * (i1 + 1) / 2 + i0 * s1
578}
579
580#[inline(always)]
581pub(crate) fn get_c_index_4d_s2ij(indices: &[usize; 4], shape: &[usize; 3]) -> usize {
582 let [i0, i1, i2, i3] = indices;
583 let [_, s1, s2] = shape;
584 i3 + s2 * (i2 + i1 * (i1 + 1) / 2 + i0 * s1)
585}
586
587#[inline(always)]
588pub(crate) fn get_c_index_5d_s2ij(indices: &[usize; 5], shape: &[usize; 4]) -> usize {
589 let [i0, i1, i2, i3, i4] = indices;
590 let [_, s1, s2, s3] = shape;
591 i4 + s3 * (i3 + s2 * (i2 + i1 * (i1 + 1) / 2 + i0 * s1))
592}
593
594#[inline(always)]
595pub(crate) fn get_c_index_5d_s2kl(indices: &[usize; 5], shape: &[usize; 4]) -> usize {
596 let [i0, i1, i2, i3, i4] = indices;
597 let [_, s1, s2, s3] = shape;
598 i4 + i3 * (i3 + 1) / 2 + s3 * (i2 + s2 * (i1 + i0 * s1))
599}
600
601pub(crate) fn get_c_index_5d_s4(indices: &[usize; 5], shape: &[usize; 3]) -> usize {
602 let [i0, i1, i2, i3, i4] = indices;
603 let [_, s1, s2] = shape;
604 let indices_ij = i2 + i1 * (i1 + 1) / 2;
605 let indices_kl = i4 + i3 * (i3 + 1) / 2;
606 indices_kl + s2 * (indices_ij + i0 * s1)
607}
608
609pub(crate) fn copy_c_3d_s1<T>(out: &mut [T], out_offsets: &[usize; 3], out_shape: &[usize; 3], buf: &[T], buf_shape: &[usize; 3])
614where
615 T: Copy,
616{
617 let buf_stride = buf_shape[0];
618 for c in 0..buf_shape[2] {
619 for i in 0..buf_shape[0] {
620 let out_indices = [out_offsets[0] + c, out_offsets[1] + i, out_offsets[2]];
621 let buf_indices = [i, 0, c];
622 let out_idx = get_c_index_3d(&out_indices, out_shape);
623 let buf_idx = get_f_index_3d(&buf_indices, buf_shape);
624 for j in 0..buf_shape[1] {
625 out[out_idx + j] = buf[buf_idx + j * buf_stride];
626 }
627 }
628 }
629}
630
631pub(crate) fn copy_c_4d_s1<T>(out: &mut [T], out_offsets: &[usize; 4], out_shape: &[usize; 4], buf: &[T], buf_shape: &[usize; 4])
632where
633 T: Copy,
634{
635 let buf_stride = buf_shape[0] * buf_shape[1];
636 for c in 0..buf_shape[3] {
637 for i in 0..buf_shape[0] {
638 for j in 0..buf_shape[1] {
639 let out_indices = [out_offsets[0] + c, out_offsets[1] + i, out_offsets[2] + j, out_offsets[3]];
640 let buf_indices = [i, j, 0, c];
641 let out_idx = get_c_index_4d(&out_indices, out_shape);
642 let buf_idx = get_f_index_4d(&buf_indices, buf_shape);
643 for k in 0..buf_shape[2] {
644 out[out_idx + k] = buf[buf_idx + k * buf_stride];
645 }
646 }
647 }
648 }
649}
650
651pub(crate) fn copy_c_5d_s1<T>(out: &mut [T], out_offsets: &[usize; 5], out_shape: &[usize; 5], buf: &[T], buf_shape: &[usize; 5])
652where
653 T: Copy,
654{
655 let buf_stride = buf_shape[0] * buf_shape[1] * buf_shape[2];
656 for c in 0..buf_shape[4] {
657 for i in 0..buf_shape[0] {
658 for j in 0..buf_shape[1] {
659 for k in 0..buf_shape[2] {
660 let out_indices = [out_offsets[0] + c, out_offsets[1] + i, out_offsets[2] + j, out_offsets[3] + k, out_offsets[4]];
661 let buf_indices = [i, j, k, 0, c];
662 let out_idx = get_c_index_5d(&out_indices, out_shape);
663 let buf_idx = get_f_index_5d(&buf_indices, buf_shape);
664 for l in 0..buf_shape[3] {
665 out[out_idx + l] = buf[buf_idx + l * buf_stride];
666 }
667 }
668 }
669 }
670 }
671}
672
673#[inline]
674pub(crate) fn copy_c_3d_s2ij<T>(out: &mut [T], out_offsets: &[usize; 3], out_shape: &[usize; 2], buf: &[T], buf_shape: &[usize; 3])
675where
676 T: Copy,
677{
678 let buf_stride = buf_shape[0];
679 let is_diag_ij = out_offsets[1] == out_offsets[2];
680 for c in 0..buf_shape[2] {
681 for i in 0..buf_shape[0] {
682 let out_indices = [out_offsets[0] + c, out_offsets[1] + i, out_offsets[2]];
683 let buf_indices = [i, 0, c];
684 let out_idx = get_c_index_3d_s2ij(&out_indices, out_shape);
685 let buf_idx = get_f_index_3d(&buf_indices, buf_shape);
686 let j_max = if is_diag_ij { i + 1 } else { buf_shape[1] };
687 for j in 0..j_max {
688 out[out_idx + j] = buf[buf_idx + j * buf_stride];
689 }
690 }
691 }
692}
693
694#[inline]
695pub(crate) fn copy_c_4d_s2ij<T>(out: &mut [T], out_offsets: &[usize; 4], out_shape: &[usize; 3], buf: &[T], buf_shape: &[usize; 4])
696where
697 T: Copy,
698{
699 let buf_stride = buf_shape[0] * buf_shape[1];
700 let is_diag_ij = out_offsets[1] == out_offsets[2];
701 for c in 0..buf_shape[3] {
702 for i in 0..buf_shape[0] {
703 let j_max = if is_diag_ij { i + 1 } else { buf_shape[1] };
704 for j in 0..j_max {
705 let out_indices = [out_offsets[0] + c, out_offsets[1] + i, out_offsets[2] + j, out_offsets[3]];
706 let buf_indices = [i, j, 0, c];
707 let out_idx = get_c_index_4d_s2ij(&out_indices, out_shape);
708 let buf_idx = get_f_index_4d(&buf_indices, buf_shape);
709 for k in 0..buf_shape[2] {
710 out[out_idx + k] = buf[buf_idx + k * buf_stride];
711 }
712 }
713 }
714 }
715}
716
717#[inline]
718pub(crate) fn copy_c_5d_s2ij<T>(out: &mut [T], out_offsets: &[usize; 5], out_shape: &[usize; 4], buf: &[T], buf_shape: &[usize; 5])
719where
720 T: Copy,
721{
722 let buf_stride = buf_shape[0] * buf_shape[1] * buf_shape[2];
723 let is_diag_ij = out_offsets[1] == out_offsets[2];
724 for c in 0..buf_shape[4] {
725 for i in 0..buf_shape[0] {
726 let j_max = if is_diag_ij { i + 1 } else { buf_shape[1] };
727 for j in 0..j_max {
728 for k in 0..buf_shape[2] {
729 let out_indices = [out_offsets[0] + c, out_offsets[1] + i, out_offsets[2] + j, out_offsets[3] + k, out_offsets[4]];
730 let buf_indices = [i, j, k, 0, c];
731 let out_idx = get_c_index_5d_s2ij(&out_indices, out_shape);
732 let buf_idx = get_f_index_5d(&buf_indices, buf_shape);
733 for l in 0..buf_shape[3] {
734 out[out_idx + l] = buf[buf_idx + l * buf_stride];
735 }
736 }
737 }
738 }
739 }
740}
741
742#[inline]
743pub(crate) fn copy_c_5d_s2kl<T>(out: &mut [T], out_offsets: &[usize; 5], out_shape: &[usize; 4], buf: &[T], buf_shape: &[usize; 5])
744where
745 T: Copy,
746{
747 let buf_stride = buf_shape[0] * buf_shape[1] * buf_shape[2];
748 let is_diag_kl = out_offsets[3] == out_offsets[4];
749 for c in 0..buf_shape[4] {
750 for i in 0..buf_shape[0] {
751 for j in 0..buf_shape[1] {
752 for k in 0..buf_shape[2] {
753 let out_indices = [out_offsets[0] + c, out_offsets[1] + i, out_offsets[2] + j, out_offsets[3] + k, out_offsets[4]];
754 let buf_indices = [i, j, k, 0, c];
755 let out_idx = get_c_index_5d_s2kl(&out_indices, out_shape);
756 let buf_idx = get_f_index_5d(&buf_indices, buf_shape);
757 let l_max = if is_diag_kl { k + 1 } else { buf_shape[3] };
758 for l in 0..l_max {
759 out[out_idx + l] = buf[buf_idx + l * buf_stride];
760 }
761 }
762 }
763 }
764 }
765}
766
767#[inline]
768pub(crate) fn copy_c_5d_s4<T>(out: &mut [T], out_offsets: &[usize; 5], out_shape: &[usize; 3], buf: &[T], buf_shape: &[usize; 5])
769where
770 T: Copy,
771{
772 let buf_stride = buf_shape[0] * buf_shape[1] * buf_shape[2];
773 let is_diag_ij = out_offsets[1] == out_offsets[2];
774 let is_diag_kl = out_offsets[3] == out_offsets[4];
775 for c in 0..buf_shape[4] {
776 for i in 0..buf_shape[0] {
777 let j_max = if is_diag_ij { i + 1 } else { buf_shape[1] };
778 for j in 0..j_max {
779 for k in 0..buf_shape[2] {
780 let out_indices = [out_offsets[0] + c, out_offsets[1] + i, out_offsets[2] + j, out_offsets[3] + k, out_offsets[4]];
781 let buf_indices = [i, j, k, 0, c];
782 let out_idx = get_c_index_5d_s4(&out_indices, out_shape);
783 let buf_idx = get_f_index_5d(&buf_indices, buf_shape);
784 let l_max = if is_diag_kl { k + 1 } else { buf_shape[3] };
785 for l in 0..l_max {
786 out[out_idx + l] = buf[buf_idx + l * buf_stride];
787 }
788 }
789 }
790 }
791 }
792}
793
794#[allow(clippy::uninit_vec)]
808#[inline]
809pub unsafe fn unaligned_uninitialized_vec<T>(size: usize) -> Vec<T> {
810 let mut v: Vec<T> = vec![];
811 v.try_reserve_exact(size).unwrap();
812 unsafe { v.set_len(size) };
813 v
814}
815
816#[inline]
823pub fn aligned_alloc(numbytes: usize, alignment: usize) -> Option<NonNull<()>> {
824 extern crate alloc;
825
826 if numbytes == 0 {
827 return None;
828 }
829 let layout = alloc::alloc::Layout::from_size_align(numbytes, alignment).unwrap();
830 NonNull::new(unsafe { alloc::alloc::alloc(layout) }).map(|p| p.cast::<()>())
831}
832
833#[inline]
843pub unsafe fn aligned_uninitialized_vec<T>(size: usize) -> Vec<T> {
844 const MIN_ALIGN: usize = 64;
846 const ALIGNMENT: usize = 64;
848
849 if size == 0 {
850 vec![]
851 } else if size < MIN_ALIGN {
852 unsafe { unaligned_uninitialized_vec(size) }
853 } else {
854 let sizeof = core::mem::size_of::<T>();
855 let pointer = aligned_alloc(size * sizeof, ALIGNMENT);
856 if let Some(pointer) = pointer {
857 let mut v = unsafe { Vec::from_raw_parts(pointer.as_ptr() as *mut T, size, size) };
858 unsafe { v.set_len(size) };
859 v
860 } else {
861 panic!("Allocation failed (probably due to out-of-memory, of size {size})")
862 }
863 }
864}
865
866