1use crate::kernel::{
4 ensure_same_shape, for_each_inner_block_preordered, pointwise_plan, same_contiguous_layout,
5 sequential_contiguous_layout, total_len,
6};
7use crate::map_view::{copy_map_into, map_into, zip_map2_into};
8use crate::reduce_view::reduce;
9use crate::simd;
10use crate::view::{StridedView, StridedViewMut};
11use crate::{Exec, Result, StridedError};
12use num_traits::{One, Zero};
13use std::mem::MaybeUninit;
14use std::ops::{Add, Mul};
15use strided_view::{ElementOp, ElementOpApply};
16
17use crate::cost::KernelClass;
18use crate::fuse::compute_costs;
19use crate::threading::{
20 for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded, SendPtr,
21};
22#[inline(always)]
34unsafe fn inner_loop_add<D: Copy + Add<S, Output = D>, S: Copy, Op: ElementOp<S>>(
35 dp: *mut D,
36 ds: isize,
37 sp: *const S,
38 ss: isize,
39 len: usize,
40) {
41 if ds == 1 && ss == 1 {
42 let dst = std::slice::from_raw_parts_mut(dp, len);
43 let src = std::slice::from_raw_parts(sp, len);
44 simd::dispatch_if_large(len, || {
45 for i in 0..len {
46 dst[i] = dst[i] + Op::apply(src[i]);
47 }
48 });
49 } else {
50 let mut dp = dp;
51 let mut sp = sp;
52 for _ in 0..len {
53 *dp = *dp + Op::apply(*sp);
54 dp = dp.wrapping_offset(ds);
55 sp = sp.wrapping_offset(ss);
56 }
57 }
58}
59
60#[inline(always)]
62unsafe fn inner_loop_mul<D: Copy + Mul<S, Output = D>, S: Copy, Op: ElementOp<S>>(
63 dp: *mut D,
64 ds: isize,
65 sp: *const S,
66 ss: isize,
67 len: usize,
68) {
69 if ds == 1 && ss == 1 {
70 let dst = std::slice::from_raw_parts_mut(dp, len);
71 let src = std::slice::from_raw_parts(sp, len);
72 simd::dispatch_if_large(len, || {
73 for i in 0..len {
74 dst[i] = dst[i] * Op::apply(src[i]);
75 }
76 });
77 } else {
78 let mut dp = dp;
79 let mut sp = sp;
80 for _ in 0..len {
81 *dp = *dp * Op::apply(*sp);
82 dp = dp.wrapping_offset(ds);
83 sp = sp.wrapping_offset(ss);
84 }
85 }
86}
87
88#[inline(always)]
90unsafe fn inner_loop_axpy<
91 D: Copy + Add<D, Output = D>,
92 S: Copy,
93 A: Copy + Mul<S, Output = D>,
94 Op: ElementOp<S>,
95>(
96 dp: *mut D,
97 ds: isize,
98 sp: *const S,
99 ss: isize,
100 len: usize,
101 alpha: A,
102) {
103 if ds == 1 && ss == 1 {
104 let dst = std::slice::from_raw_parts_mut(dp, len);
105 let src = std::slice::from_raw_parts(sp, len);
106 simd::dispatch_if_large(len, || {
107 for i in 0..len {
108 dst[i] = alpha * Op::apply(src[i]) + dst[i];
109 }
110 });
111 } else {
112 let mut dp = dp;
113 let mut sp = sp;
114 for _ in 0..len {
115 *dp = alpha * Op::apply(*sp) + *dp;
116 dp = dp.wrapping_offset(ds);
117 sp = sp.wrapping_offset(ss);
118 }
119 }
120}
121
122#[inline(always)]
124unsafe fn inner_loop_fma<
125 D: Copy + Add<D, Output = D>,
126 A: Copy + Mul<B, Output = D>,
127 B: Copy,
128 OpA: ElementOp<A>,
129 OpB: ElementOp<B>,
130>(
131 dp: *mut D,
132 ds: isize,
133 ap: *const A,
134 a_s: isize,
135 bp: *const B,
136 b_s: isize,
137 len: usize,
138) {
139 if ds == 1 && a_s == 1 && b_s == 1 {
140 let dst = std::slice::from_raw_parts_mut(dp, len);
141 let sa = std::slice::from_raw_parts(ap, len);
142 let sb = std::slice::from_raw_parts(bp, len);
143 simd::dispatch_if_large(len, || {
144 for i in 0..len {
145 dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
146 }
147 });
148 } else {
149 let mut dp = dp;
150 let mut ap = ap;
151 let mut bp = bp;
152 for _ in 0..len {
153 *dp = *dp + OpA::apply(*ap) * OpB::apply(*bp);
154 dp = dp.wrapping_offset(ds);
155 ap = ap.wrapping_offset(a_s);
156 bp = bp.wrapping_offset(b_s);
157 }
158 }
159}
160
161#[inline(always)]
163unsafe fn inner_loop_dot<
164 A: Copy + Mul<B, Output = R>,
165 B: Copy,
166 R: Copy + Add<R, Output = R>,
167 OpA: ElementOp<A>,
168 OpB: ElementOp<B>,
169>(
170 ap: *const A,
171 a_s: isize,
172 bp: *const B,
173 b_s: isize,
174 len: usize,
175 mut acc: R,
176) -> R {
177 if a_s == 1 && b_s == 1 {
178 let sa = std::slice::from_raw_parts(ap, len);
179 let sb = std::slice::from_raw_parts(bp, len);
180 simd::dispatch_if_large(len, || {
181 for i in 0..len {
182 acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
183 }
184 });
185 } else {
186 let mut ap = ap;
187 let mut bp = bp;
188 for _ in 0..len {
189 acc = acc + OpA::apply(*ap) * OpB::apply(*bp);
190 ap = ap.wrapping_offset(a_s);
191 bp = bp.wrapping_offset(b_s);
192 }
193 }
194 acc
195}
196
197pub fn copy_into<T: Copy + Send + Sync, Op: ElementOp<T>>(
210 dest: &mut StridedViewMut<T>,
211 src: &StridedView<T, Op>,
212 exec: Exec<'_>,
213) -> Result<()> {
214 ensure_same_shape(dest.dims(), src.dims())?;
215
216 let dst_ptr = dest.as_mut_ptr();
217 let src_ptr = src.ptr();
218 let dst_dims = dest.dims();
219 let dst_strides = dest.strides();
220 let src_strides = src.strides();
221
222 if sequential_contiguous_layout(
223 KernelClass::Copy,
224 dst_dims,
225 &[dst_strides, src_strides],
226 &exec,
227 )?
228 .is_some()
229 {
230 let len = total_len(dst_dims)?;
231 if Op::IS_IDENTITY {
232 debug_assert!(
233 {
234 let nbytes = len
235 .checked_mul(std::mem::size_of::<T>())
236 .expect("copy size must not overflow");
237 let dst_start = dst_ptr as usize;
238 let src_start = src_ptr as usize;
239 let dst_end = dst_start.saturating_add(nbytes);
240 let src_end = src_start.saturating_add(nbytes);
241 dst_end <= src_start || src_end <= dst_start
242 },
243 "overlapping src/dest is not supported"
244 );
245 unsafe { std::ptr::copy_nonoverlapping(src_ptr, dst_ptr, len) };
246 } else {
247 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
248 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
249 simd::dispatch_if_large(len, || {
250 for i in 0..len {
251 dst[i] = Op::apply(src[i]);
252 }
253 });
254 }
255 return Ok(());
256 }
257
258 copy_map_into(dest, src, |x| x, exec)
259}
260
261pub fn copy_into_uninit<T: Copy + Send + Sync + 'static>(
288 dest: &mut StridedViewMut<MaybeUninit<T>>,
289 src: &StridedView<T>,
290 exec: Exec<'_>,
291) -> Result<()> {
292 ensure_same_shape(dest.dims(), src.dims())?;
293 if !crate::layout_check::is_injective_layout(dest.dims(), dest.strides()) {
294 return Err(StridedError::NonInjectiveOutputLayout);
295 }
296 if src.is_empty() {
297 return Ok(());
298 }
299 let _len = src
300 .dims()
301 .iter()
302 .try_fold(1usize, |n, &dim| n.checked_mul(dim))
303 .ok_or(StridedError::OffsetOverflow)?;
304 let native_float = std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>()
308 || std::any::TypeId::of::<T>() == std::any::TypeId::of::<f64>();
309 if !native_float
314 || same_contiguous_layout(dest.dims(), &[dest.strides(), src.strides()]).is_some()
315 {
316 return copy_map_into(dest, src, MaybeUninit::new, exec);
317 }
318 crate::threading::copy_permuted_uninit(dest, src, exec)
322}
323
324pub fn copy_into_col_major<T: Copy + Send + Sync + 'static>(
341 dst: &mut StridedViewMut<T>,
342 src: &StridedView<T>,
343 exec: Exec<'_>,
344) -> Result<()> {
345 strided_perm::copy_into_col_major(dst, src, exec)
346}
347
348pub fn add_acc<
360 D: Copy + Add<S, Output = D> + Send + Sync,
361 S: Copy + Send + Sync,
362 Op: ElementOp<S>,
363>(
364 dest: &mut StridedViewMut<D>,
365 src: &StridedView<S, Op>,
366 exec: Exec<'_>,
367) -> Result<()> {
368 ensure_same_shape(dest.dims(), src.dims())?;
369
370 let dst_ptr = dest.as_mut_ptr();
371 let src_ptr = src.ptr();
372 let dst_dims = dest.dims();
373 let dst_strides = dest.strides();
374 let src_strides = src.strides();
375
376 if sequential_contiguous_layout(
377 KernelClass::Map,
378 dst_dims,
379 &[dst_strides, src_strides],
380 &exec,
381 )?
382 .is_some()
383 {
384 let len = total_len(dst_dims)?;
385 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
386 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
387 simd::dispatch_if_large(len, || {
388 for i in 0..len {
389 dst[i] = dst[i] + Op::apply(src[i]);
390 }
391 });
392 return Ok(());
393 }
394
395 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
396 let total = total_len(dst_dims)?;
397 let (fused_dims, ordered_strides, plan) = pointwise_plan(
398 dst_dims,
399 &strides_list,
400 Some(0),
401 &[std::mem::size_of::<D>(), std::mem::size_of::<S>()],
402 total,
403 );
404
405 let total = total_len(&fused_dims)?;
406 if is_parallel(KernelClass::Map, total, &exec) {
407 let dst_send = SendPtr(dst_ptr);
408 let src_send = SendPtr(src_ptr as *mut S);
409
410 let costs = compute_costs(&ordered_strides);
411 let initial_offsets: crate::inline::OperandVec<isize> =
412 strides_list.iter().map(|_| 0isize).collect();
413 return mapreduce_threaded(
414 &fused_dims,
415 &plan.block,
416 &ordered_strides,
417 &initial_offsets,
418 &costs,
419 KernelClass::Map,
420 exec,
421 &|dims, blocks, strides_list, offsets| {
422 for_each_inner_block_with_offsets(
423 dims,
424 blocks,
425 strides_list,
426 offsets,
427 |offsets, len, strides| {
428 unsafe {
429 inner_loop_add::<D, S, Op>(
430 dst_send.as_ptr().offset(offsets[0]),
431 strides[0],
432 src_send.as_const().offset(offsets[1]),
433 strides[1],
434 len,
435 )
436 };
437 Ok(())
438 },
439 )
440 },
441 );
442 }
443
444 let initial_offsets: crate::inline::OperandVec<isize> =
445 ordered_strides.iter().map(|_| 0isize).collect();
446 for_each_inner_block_preordered(
447 &fused_dims,
448 &plan.block,
449 &ordered_strides,
450 &initial_offsets,
451 |offsets, len, strides| {
452 unsafe {
453 inner_loop_add::<D, S, Op>(
454 dst_ptr.offset(offsets[0]),
455 strides[0],
456 src_ptr.offset(offsets[1]),
457 strides[1],
458 len,
459 )
460 };
461 Ok(())
462 },
463 )
464}
465
466pub fn mul_acc<
478 D: Copy + Mul<S, Output = D> + Send + Sync,
479 S: Copy + Send + Sync,
480 Op: ElementOp<S>,
481>(
482 dest: &mut StridedViewMut<D>,
483 src: &StridedView<S, Op>,
484 exec: Exec<'_>,
485) -> Result<()> {
486 ensure_same_shape(dest.dims(), src.dims())?;
487
488 let dst_ptr = dest.as_mut_ptr();
489 let src_ptr = src.ptr();
490 let dst_dims = dest.dims();
491 let dst_strides = dest.strides();
492 let src_strides = src.strides();
493
494 if sequential_contiguous_layout(
495 KernelClass::Map,
496 dst_dims,
497 &[dst_strides, src_strides],
498 &exec,
499 )?
500 .is_some()
501 {
502 let len = total_len(dst_dims)?;
503 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
504 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
505 simd::dispatch_if_large(len, || {
506 for i in 0..len {
507 dst[i] = dst[i] * Op::apply(src[i]);
508 }
509 });
510 return Ok(());
511 }
512
513 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
514 let total = total_len(dst_dims)?;
515 let (fused_dims, ordered_strides, plan) = pointwise_plan(
516 dst_dims,
517 &strides_list,
518 Some(0),
519 &[std::mem::size_of::<D>(), std::mem::size_of::<S>()],
520 total,
521 );
522
523 let total = total_len(&fused_dims)?;
524 if is_parallel(KernelClass::Map, total, &exec) {
525 let dst_send = SendPtr(dst_ptr);
526 let src_send = SendPtr(src_ptr as *mut S);
527
528 let costs = compute_costs(&ordered_strides);
529 let initial_offsets: crate::inline::OperandVec<isize> =
530 strides_list.iter().map(|_| 0isize).collect();
531 return mapreduce_threaded(
532 &fused_dims,
533 &plan.block,
534 &ordered_strides,
535 &initial_offsets,
536 &costs,
537 KernelClass::Map,
538 exec,
539 &|dims, blocks, strides_list, offsets| {
540 for_each_inner_block_with_offsets(
541 dims,
542 blocks,
543 strides_list,
544 offsets,
545 |offsets, len, strides| {
546 unsafe {
547 inner_loop_mul::<D, S, Op>(
548 dst_send.as_ptr().offset(offsets[0]),
549 strides[0],
550 src_send.as_const().offset(offsets[1]),
551 strides[1],
552 len,
553 )
554 };
555 Ok(())
556 },
557 )
558 },
559 );
560 }
561
562 let initial_offsets: crate::inline::OperandVec<isize> =
563 ordered_strides.iter().map(|_| 0isize).collect();
564 for_each_inner_block_preordered(
565 &fused_dims,
566 &plan.block,
567 &ordered_strides,
568 &initial_offsets,
569 |offsets, len, strides| {
570 unsafe {
571 inner_loop_mul::<D, S, Op>(
572 dst_ptr.offset(offsets[0]),
573 strides[0],
574 src_ptr.offset(offsets[1]),
575 strides[1],
576 len,
577 )
578 };
579 Ok(())
580 },
581 )
582}
583
584pub fn axpy_acc<D, S, A, Op>(
596 dest: &mut StridedViewMut<D>,
597 src: &StridedView<S, Op>,
598 alpha: A,
599 exec: Exec<'_>,
600) -> Result<()>
601where
602 A: Copy + Mul<S, Output = D> + Sync,
603 D: Copy + Add<D, Output = D> + Send + Sync,
604 S: Copy + Send + Sync,
605 Op: ElementOp<S>,
606{
607 ensure_same_shape(dest.dims(), src.dims())?;
608
609 let dst_ptr = dest.as_mut_ptr();
610 let src_ptr = src.ptr();
611 let dst_dims = dest.dims();
612 let dst_strides = dest.strides();
613 let src_strides = src.strides();
614
615 if sequential_contiguous_layout(
616 KernelClass::Map,
617 dst_dims,
618 &[dst_strides, src_strides],
619 &exec,
620 )?
621 .is_some()
622 {
623 let len = total_len(dst_dims)?;
624 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
625 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
626 simd::dispatch_if_large(len, || {
627 for i in 0..len {
628 dst[i] = alpha * Op::apply(src[i]) + dst[i];
629 }
630 });
631 return Ok(());
632 }
633
634 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
635 let total = total_len(dst_dims)?;
636 let (fused_dims, ordered_strides, plan) = pointwise_plan(
637 dst_dims,
638 &strides_list,
639 Some(0),
640 &[std::mem::size_of::<D>(), std::mem::size_of::<S>()],
641 total,
642 );
643
644 let total = total_len(&fused_dims)?;
645 if is_parallel(KernelClass::Map, total, &exec) {
646 let dst_send = SendPtr(dst_ptr);
647 let src_send = SendPtr(src_ptr as *mut S);
648
649 let costs = compute_costs(&ordered_strides);
650 let initial_offsets: crate::inline::OperandVec<isize> =
651 strides_list.iter().map(|_| 0isize).collect();
652 return mapreduce_threaded(
653 &fused_dims,
654 &plan.block,
655 &ordered_strides,
656 &initial_offsets,
657 &costs,
658 KernelClass::Map,
659 exec,
660 &|dims, blocks, strides_list, offsets| {
661 for_each_inner_block_with_offsets(
662 dims,
663 blocks,
664 strides_list,
665 offsets,
666 |offsets, len, strides| {
667 unsafe {
668 inner_loop_axpy::<D, S, A, Op>(
669 dst_send.as_ptr().offset(offsets[0]),
670 strides[0],
671 src_send.as_const().offset(offsets[1]),
672 strides[1],
673 len,
674 alpha,
675 )
676 };
677 Ok(())
678 },
679 )
680 },
681 );
682 }
683
684 let initial_offsets: crate::inline::OperandVec<isize> =
685 ordered_strides.iter().map(|_| 0isize).collect();
686 for_each_inner_block_preordered(
687 &fused_dims,
688 &plan.block,
689 &ordered_strides,
690 &initial_offsets,
691 |offsets, len, strides| {
692 unsafe {
693 inner_loop_axpy::<D, S, A, Op>(
694 dst_ptr.offset(offsets[0]),
695 strides[0],
696 src_ptr.offset(offsets[1]),
697 strides[1],
698 len,
699 alpha,
700 )
701 };
702 Ok(())
703 },
704 )
705}
706
707pub fn fma_acc<D, A, B, OpA, OpB>(
719 dest: &mut StridedViewMut<D>,
720 a: &StridedView<A, OpA>,
721 b: &StridedView<B, OpB>,
722 exec: Exec<'_>,
723) -> Result<()>
724where
725 A: Copy + Mul<B, Output = D> + Send + Sync,
726 B: Copy + Send + Sync,
727 D: Copy + Add<D, Output = D> + Send + Sync,
728 OpA: ElementOp<A>,
729 OpB: ElementOp<B>,
730{
731 ensure_same_shape(dest.dims(), a.dims())?;
732 ensure_same_shape(dest.dims(), b.dims())?;
733
734 let dst_ptr = dest.as_mut_ptr();
735 let a_ptr = a.ptr();
736 let b_ptr = b.ptr();
737 let dst_dims = dest.dims();
738 let dst_strides = dest.strides();
739 let a_strides = a.strides();
740 let b_strides = b.strides();
741
742 if sequential_contiguous_layout(
743 KernelClass::Map,
744 dst_dims,
745 &[dst_strides, a_strides, b_strides],
746 &exec,
747 )?
748 .is_some()
749 {
750 let len = total_len(dst_dims)?;
751 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
752 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
753 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
754 simd::dispatch_if_large(len, || {
755 for i in 0..len {
756 dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
757 }
758 });
759 return Ok(());
760 }
761
762 let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
763 let total = total_len(dst_dims)?;
764 let (fused_dims, ordered_strides, plan) = pointwise_plan(
765 dst_dims,
766 &strides_list,
767 Some(0),
768 &[
769 std::mem::size_of::<D>(),
770 std::mem::size_of::<A>(),
771 std::mem::size_of::<B>(),
772 ],
773 total,
774 );
775
776 let total = total_len(&fused_dims)?;
777 if is_parallel(KernelClass::Map, total, &exec) {
778 let dst_send = SendPtr(dst_ptr);
779 let a_send = SendPtr(a_ptr as *mut A);
780 let b_send = SendPtr(b_ptr as *mut B);
781
782 let costs = compute_costs(&ordered_strides);
783 let initial_offsets: crate::inline::OperandVec<isize> =
784 strides_list.iter().map(|_| 0isize).collect();
785 return mapreduce_threaded(
786 &fused_dims,
787 &plan.block,
788 &ordered_strides,
789 &initial_offsets,
790 &costs,
791 KernelClass::Map,
792 exec,
793 &|dims, blocks, strides_list, offsets| {
794 for_each_inner_block_with_offsets(
795 dims,
796 blocks,
797 strides_list,
798 offsets,
799 |offsets, len, strides| {
800 unsafe {
801 inner_loop_fma::<D, A, B, OpA, OpB>(
802 dst_send.as_ptr().offset(offsets[0]),
803 strides[0],
804 a_send.as_const().offset(offsets[1]),
805 strides[1],
806 b_send.as_const().offset(offsets[2]),
807 strides[2],
808 len,
809 )
810 };
811 Ok(())
812 },
813 )
814 },
815 );
816 }
817
818 let initial_offsets: crate::inline::OperandVec<isize> =
819 ordered_strides.iter().map(|_| 0isize).collect();
820 for_each_inner_block_preordered(
821 &fused_dims,
822 &plan.block,
823 &ordered_strides,
824 &initial_offsets,
825 |offsets, len, strides| {
826 unsafe {
827 inner_loop_fma::<D, A, B, OpA, OpB>(
828 dst_ptr.offset(offsets[0]),
829 strides[0],
830 a_ptr.offset(offsets[1]),
831 strides[1],
832 b_ptr.offset(offsets[2]),
833 strides[2],
834 len,
835 )
836 };
837 Ok(())
838 },
839 )
840}
841
842pub fn sum<
871 T: Copy + Zero + Add<Output = T> + Send + Sync + simd::MaybeSimdOps,
872 Op: ElementOp<T>,
873>(
874 src: &StridedView<T, Op>,
875 exec: Exec<'_>,
876) -> Result<T> {
877 if total_len(src.dims())? == 0 {
880 return Ok(T::zero());
881 }
882 if Op::IS_IDENTITY && same_contiguous_layout(src.dims(), &[src.strides()]).is_some() {
884 let len = total_len(src.dims())?;
885 let src_slice = unsafe { std::slice::from_raw_parts(src.ptr(), len) };
886
887 if T::try_simd_sum(&[]).is_some() && is_parallel(KernelClass::Reduce, len, &exec) {
890 return Ok(crate::threading::parallel_map_reduce(
891 0..len,
892 crate::cost::for_len_tree(exec, KernelClass::Reduce, len),
893 &|range| T::try_simd_sum(&src_slice[range]).unwrap(),
894 &|left, right| left + right,
895 ));
896 }
897
898 if let Some(result) = T::try_simd_sum(src_slice) {
899 return Ok(result);
900 }
901 }
902 reduce(src, |x| x, |a, b| a + b, T::zero(), exec)
903}
904
905pub fn dot<A, B, R, OpA, OpB>(a: &StridedView<A, OpA>, b: &StridedView<B, OpB>) -> Result<R>
936where
937 A: Copy + Mul<B, Output = R> + Send + Sync + 'static,
938 B: Copy + Send + Sync + 'static,
939 R: Copy + Zero + Add<Output = R> + Send + Sync + simd::MaybeSimdOps + 'static,
940 OpA: ElementOp<A>,
941 OpB: ElementOp<B>,
942{
943 ensure_same_shape(a.dims(), b.dims())?;
944
945 let a_ptr = a.ptr();
946 let b_ptr = b.ptr();
947 let a_strides = a.strides();
948 let b_strides = b.strides();
949 let a_dims = a.dims();
950 let len = total_len(a_dims)?;
951
952 if len == 0 {
956 return Ok(R::zero());
957 }
958
959 if same_contiguous_layout(a_dims, &[a_strides, b_strides]).is_some() {
960 if OpA::IS_IDENTITY
962 && OpB::IS_IDENTITY
963 && std::any::TypeId::of::<A>() == std::any::TypeId::of::<R>()
964 && std::any::TypeId::of::<B>() == std::any::TypeId::of::<R>()
965 {
966 let sa = unsafe { std::slice::from_raw_parts(a_ptr as *const R, len) };
967 let sb = unsafe { std::slice::from_raw_parts(b_ptr as *const R, len) };
968 if let Some(result) = R::try_simd_dot(sa, sb) {
969 return Ok(result);
970 }
971 }
972
973 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
975 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
976 let mut acc = R::zero();
977 simd::dispatch_if_large(len, || {
978 for i in 0..len {
979 acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
980 }
981 });
982 return Ok(acc);
983 }
984
985 let strides_list: [&[isize]; 2] = [a_strides, b_strides];
986 let total = total_len(a_dims)?;
988 let (fused_dims, ordered_strides, plan) = crate::kernel::fold_plan(
990 a_dims,
991 &strides_list,
992 0,
993 &[std::mem::size_of::<A>(), std::mem::size_of::<B>()],
994 total,
995 );
996
997 let mut acc = R::zero();
998 let initial_offsets: crate::inline::OperandVec<isize> =
999 ordered_strides.iter().map(|_| 0isize).collect();
1000 for_each_inner_block_preordered(
1001 &fused_dims,
1002 &plan.block,
1003 &ordered_strides,
1004 &initial_offsets,
1005 |offsets, len, strides| {
1006 acc = unsafe {
1007 inner_loop_dot::<A, B, R, OpA, OpB>(
1008 a_ptr.offset(offsets[0]),
1009 strides[0],
1010 b_ptr.offset(offsets[1]),
1011 strides[1],
1012 len,
1013 acc,
1014 )
1015 };
1016 Ok(())
1017 },
1018 )?;
1019
1020 Ok(acc)
1021}
1022
1023pub fn symmetrize_into<T>(
1041 dest: &mut StridedViewMut<T>,
1042 src: &StridedView<T>,
1043 exec: Exec<'_>,
1044) -> Result<()>
1045where
1046 T: Copy
1047 + Add<Output = T>
1048 + Mul<Output = T>
1049 + num_traits::FromPrimitive
1050 + std::ops::Div<Output = T>
1051 + Send
1052 + Sync,
1053{
1054 if src.ndim() != 2 {
1055 return Err(StridedError::RankMismatch(src.ndim(), 2));
1056 }
1057 let rows = src.dims()[0];
1058 let cols = src.dims()[1];
1059 if rows != cols {
1060 return Err(StridedError::NonSquare { rows, cols });
1061 }
1062
1063 let src_t = src.permute(&[1, 0])?;
1064 let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1065
1066 zip_map2_into(dest, src, &src_t, |a, b| (a + b) * half, exec)
1067}
1068
1069pub fn symmetrize_conj_into<T>(
1087 dest: &mut StridedViewMut<T>,
1088 src: &StridedView<T>,
1089 exec: Exec<'_>,
1090) -> Result<()>
1091where
1092 T: Copy
1093 + ElementOpApply
1094 + Add<Output = T>
1095 + Mul<Output = T>
1096 + num_traits::FromPrimitive
1097 + std::ops::Div<Output = T>
1098 + Send
1099 + Sync,
1100{
1101 if src.ndim() != 2 {
1102 return Err(StridedError::RankMismatch(src.ndim(), 2));
1103 }
1104 let rows = src.dims()[0];
1105 let cols = src.dims()[1];
1106 if rows != cols {
1107 return Err(StridedError::NonSquare { rows, cols });
1108 }
1109
1110 let src_adj = src.adjoint_2d()?;
1112 let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1113
1114 zip_map2_into(dest, src, &src_adj, |a, b| (a + b) * half, exec)
1115}
1116
1117pub fn copy_scale_into<D, S, A, Op>(
1130 dest: &mut StridedViewMut<D>,
1131 src: &StridedView<S, Op>,
1132 scale: A,
1133 exec: Exec<'_>,
1134) -> Result<()>
1135where
1136 A: Copy + Mul<S, Output = D> + Sync,
1137 D: Copy + Send + Sync,
1138 S: Copy + Send + Sync,
1139 Op: ElementOp<S>,
1140{
1141 map_into(dest, src, |x| scale * x, exec)
1142}
1143
1144pub fn copy_conj_into<T: ElementOpApply + Send + Sync + 'static>(
1174 dest: &mut StridedViewMut<T>,
1175 src: &StridedView<T>,
1176 exec: Exec<'_>,
1177) -> Result<()> {
1178 if conj_perm_engine_route::<T>(dest.dims(), dest.strides(), src.strides()) {
1179 return crate::threading::copy_conj_permuted(dest, src, exec);
1180 }
1181 let src_conj = src.conj();
1182 copy_into(dest, &src_conj, exec)
1183}
1184
1185const CONJ_PERM_ENGINE_MAX_BYTES: usize = 24 * 1024 * 1024;
1201
1202fn conj_perm_engine_route<T: 'static>(
1209 dims: &[usize],
1210 dst_strides: &[isize],
1211 src_strides: &[isize],
1212) -> bool {
1213 if !element_conj_is_applied::<T>() {
1214 return false;
1215 }
1216 if same_contiguous_layout(dims, &[dst_strides, src_strides]).is_some() {
1219 return false;
1220 }
1221 total_len(dims).is_ok_and(|len| {
1223 len.checked_mul(std::mem::size_of::<T>())
1224 .is_some_and(|bytes| bytes < CONJ_PERM_ENGINE_MAX_BYTES)
1225 })
1226}
1227
1228#[inline]
1231fn element_conj_is_applied<T: 'static>() -> bool {
1232 use std::any::TypeId;
1233
1234 let id = TypeId::of::<T>();
1235 id == TypeId::of::<num_complex::Complex32>() || id == TypeId::of::<num_complex::Complex64>()
1236}
1237
1238#[inline]
1239fn element_transpose_is_identity<T: 'static>() -> bool {
1240 use std::any::TypeId;
1241
1242 macro_rules! matches_type {
1243 ($($ty:ty),* $(,)?) => {{
1244 let id = TypeId::of::<T>();
1245 false $(|| id == TypeId::of::<$ty>())*
1246 }};
1247 }
1248
1249 matches_type!(
1250 f32,
1251 f64,
1252 i8,
1253 i16,
1254 i32,
1255 i64,
1256 i128,
1257 isize,
1258 u8,
1259 u16,
1260 u32,
1261 u64,
1262 u128,
1263 usize,
1264 num_complex::Complex32,
1265 num_complex::Complex64,
1266 )
1267}
1268
1269#[inline]
1270fn element_zero_is_all_bits_zero<T: 'static>() -> bool {
1271 use std::any::TypeId;
1272
1273 macro_rules! matches_type {
1274 ($($ty:ty),* $(,)?) => {{
1275 let id = TypeId::of::<T>();
1276 false $(|| id == TypeId::of::<$ty>())*
1277 }};
1278 }
1279
1280 matches_type!(
1281 f32,
1282 f64,
1283 i8,
1284 i16,
1285 i32,
1286 i64,
1287 i128,
1288 isize,
1289 u8,
1290 u16,
1291 u32,
1292 u64,
1293 u128,
1294 usize,
1295 num_complex::Complex32,
1296 num_complex::Complex64,
1297 )
1298}
1299
1300#[inline]
1301unsafe fn fill_2d<T: Copy + Send + Sync>(
1302 dst: *mut T,
1303 dim0: usize,
1304 dim1: usize,
1305 dst_stride0: isize,
1306 dst_stride1: isize,
1307 value: T,
1308 exec: Exec<'_>,
1309) {
1310 let total = dim0.saturating_mul(dim1);
1311 let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
1312 if !exec.is_serial() {
1313 let dst_send = SendPtr(dst);
1314 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1315 crate::threading::parallel_for_each(0..dim1, exec, &|columns| {
1316 for j in columns {
1317 let dst = dst_send.as_ptr();
1318 unsafe {
1319 let base = j as isize * dst_stride1;
1320 for i in 0..dim0 {
1321 *dst.offset(base + i as isize * dst_stride0) = value;
1322 }
1323 }
1324 }
1325 });
1326 } else {
1327 crate::threading::parallel_for_each(0..dim0, exec, &|rows| {
1328 for i in rows {
1329 let dst = dst_send.as_ptr();
1330 unsafe {
1331 let base = i as isize * dst_stride0;
1332 for j in 0..dim1 {
1333 *dst.offset(base + j as isize * dst_stride1) = value;
1334 }
1335 }
1336 }
1337 });
1338 }
1339 return;
1340 }
1341
1342 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1343 for j in 0..dim1 {
1344 let base = j as isize * dst_stride1;
1345 for i in 0..dim0 {
1346 *dst.offset(base + i as isize * dst_stride0) = value;
1347 }
1348 }
1349 } else {
1350 for i in 0..dim0 {
1351 let base = i as isize * dst_stride0;
1352 for j in 0..dim1 {
1353 *dst.offset(base + j as isize * dst_stride1) = value;
1354 }
1355 }
1356 }
1357}
1358
1359#[inline]
1360unsafe fn fill_contiguous<T>(dst: *mut T, len: usize, value: T)
1361where
1362 T: Copy + Zero + PartialEq + Send + Sync + 'static,
1363{
1364 if element_zero_is_all_bits_zero::<T>() && value == T::zero() {
1365 std::ptr::write_bytes(dst, 0, len);
1366 return;
1367 }
1368
1369 let dst = std::slice::from_raw_parts_mut(dst, len);
1370 dst.fill(value);
1371}
1372
1373#[allow(clippy::too_many_arguments)]
1376#[inline(always)]
1377unsafe fn transpose_scale_4x4_f64(
1378 dst: *mut f64,
1379 dst_stride0: isize,
1380 dst_stride1: isize,
1381 src: *const f64,
1382 src_stride0: isize,
1383 src_stride1: isize,
1384 i: usize,
1385 j: usize,
1386 scale: f64,
1387) {
1388 let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1389
1390 let s00 = *src_base;
1391 let s10 = *src_base.offset(src_stride0);
1392 let s20 = *src_base.offset(2 * src_stride0);
1393 let s30 = *src_base.offset(3 * src_stride0);
1394
1395 let src_col1 = src_base.offset(src_stride1);
1396 let s01 = *src_col1;
1397 let s11 = *src_col1.offset(src_stride0);
1398 let s21 = *src_col1.offset(2 * src_stride0);
1399 let s31 = *src_col1.offset(3 * src_stride0);
1400
1401 let src_col2 = src_base.offset(2 * src_stride1);
1402 let s02 = *src_col2;
1403 let s12 = *src_col2.offset(src_stride0);
1404 let s22 = *src_col2.offset(2 * src_stride0);
1405 let s32 = *src_col2.offset(3 * src_stride0);
1406
1407 let src_col3 = src_base.offset(3 * src_stride1);
1408 let s03 = *src_col3;
1409 let s13 = *src_col3.offset(src_stride0);
1410 let s23 = *src_col3.offset(2 * src_stride0);
1411 let s33 = *src_col3.offset(3 * src_stride0);
1412
1413 let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1414 *dst_row0 = scale * s00;
1415 *dst_row0.offset(dst_stride0) = scale * s01;
1416 *dst_row0.offset(2 * dst_stride0) = scale * s02;
1417 *dst_row0.offset(3 * dst_stride0) = scale * s03;
1418
1419 let dst_row1 = dst_row0.offset(dst_stride1);
1420 *dst_row1 = scale * s10;
1421 *dst_row1.offset(dst_stride0) = scale * s11;
1422 *dst_row1.offset(2 * dst_stride0) = scale * s12;
1423 *dst_row1.offset(3 * dst_stride0) = scale * s13;
1424
1425 let dst_row2 = dst_row0.offset(2 * dst_stride1);
1426 *dst_row2 = scale * s20;
1427 *dst_row2.offset(dst_stride0) = scale * s21;
1428 *dst_row2.offset(2 * dst_stride0) = scale * s22;
1429 *dst_row2.offset(3 * dst_stride0) = scale * s23;
1430
1431 let dst_row3 = dst_row0.offset(3 * dst_stride1);
1432 *dst_row3 = scale * s30;
1433 *dst_row3.offset(dst_stride0) = scale * s31;
1434 *dst_row3.offset(2 * dst_stride0) = scale * s32;
1435 *dst_row3.offset(3 * dst_stride0) = scale * s33;
1436}
1437
1438#[inline]
1439#[allow(clippy::too_many_arguments)]
1442unsafe fn copy_transpose_scale_2d_f64_tiled_raw(
1443 dst: *mut f64,
1444 dst_stride0: isize,
1445 dst_stride1: isize,
1446 src: *const f64,
1447 src_stride0: isize,
1448 src_stride1: isize,
1449 src_rows: usize,
1450 src_cols: usize,
1451 scale: f64,
1452 exec: Exec<'_>,
1453) {
1454 const TILE: usize = 4;
1455 let row_full = src_rows / TILE * TILE;
1456 let col_full = src_cols / TILE * TILE;
1457
1458 let total = src_rows.saturating_mul(src_cols);
1459 let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1460 if !exec.is_serial() {
1461 let dst_send = SendPtr(dst);
1462 let src_send = SendPtr(src as *mut f64);
1463 let row_tiles = row_full / TILE;
1464 crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1465 for tile_i in tiles {
1466 let i = tile_i * TILE;
1467 let dst = dst_send.as_ptr();
1468 let src = src_send.as_const();
1469 unsafe {
1470 let mut j = 0;
1471 while j < col_full {
1472 transpose_scale_4x4_f64(
1473 dst,
1474 dst_stride0,
1475 dst_stride1,
1476 src,
1477 src_stride0,
1478 src_stride1,
1479 i,
1480 j,
1481 scale,
1482 );
1483 j += TILE;
1484 }
1485 for j in col_full..src_cols {
1486 for ii in i..i + TILE {
1487 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1488 scale
1489 * *src.offset(
1490 ii as isize * src_stride0 + j as isize * src_stride1,
1491 );
1492 }
1493 }
1494 }
1495 }
1496 });
1497 for i in row_full..src_rows {
1498 for j in 0..src_cols {
1499 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1500 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1501 }
1502 }
1503 return;
1504 }
1505
1506 let mut i = 0;
1507 while i < row_full {
1508 let mut j = 0;
1509 while j < col_full {
1510 transpose_scale_4x4_f64(
1511 dst,
1512 dst_stride0,
1513 dst_stride1,
1514 src,
1515 src_stride0,
1516 src_stride1,
1517 i,
1518 j,
1519 scale,
1520 );
1521 j += TILE;
1522 }
1523 for j in col_full..src_cols {
1524 for ii in i..i + TILE {
1525 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1526 scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1527 }
1528 }
1529 i += TILE;
1530 }
1531 for i in row_full..src_rows {
1532 for j in 0..src_cols {
1533 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1534 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1535 }
1536 }
1537}
1538
1539#[inline]
1540#[cfg(test)]
1541unsafe fn try_copy_transpose_scale_2d_f64_tiled(
1542 dest: &mut StridedViewMut<f64>,
1543 src: &StridedView<f64>,
1544 scale: f64,
1545 exec: Exec<'_>,
1546) -> bool {
1547 if src.ndim() != 2 || dest.ndim() != 2 {
1548 return false;
1549 }
1550 let src_dims = src.dims();
1551 if dest.dims() != [src_dims[1], src_dims[0]] {
1552 return false;
1553 }
1554 if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1555 return false;
1556 }
1557
1558 copy_transpose_scale_2d_f64_tiled_raw(
1559 dest.as_mut_ptr(),
1560 dest.strides()[0],
1561 dest.strides()[1],
1562 src.ptr(),
1563 src.strides()[0],
1564 src.strides()[1],
1565 src_dims[0],
1566 src_dims[1],
1567 scale,
1568 exec,
1569 );
1570 true
1571}
1572
1573#[inline]
1574unsafe fn try_copy_transpose_scale_2d_f64_tiled_typed<T>(
1575 dest: &mut StridedViewMut<T>,
1576 src: &StridedView<T>,
1577 scale: T,
1578 exec: Exec<'_>,
1579) -> bool
1580where
1581 T: Copy + 'static,
1582{
1583 if std::any::TypeId::of::<T>() != std::any::TypeId::of::<f64>() {
1584 return false;
1585 }
1586
1587 let scale = *(&scale as *const T).cast::<f64>();
1588 if src.ndim() != 2 || dest.ndim() != 2 {
1589 return false;
1590 }
1591 let src_dims = src.dims();
1592 if dest.dims() != [src_dims[1], src_dims[0]] {
1593 return false;
1594 }
1595 if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1596 return false;
1597 }
1598
1599 copy_transpose_scale_2d_f64_tiled_raw(
1600 dest.as_mut_ptr().cast::<f64>(),
1601 dest.strides()[0],
1602 dest.strides()[1],
1603 src.ptr().cast::<f64>(),
1604 src.strides()[0],
1605 src.strides()[1],
1606 src_dims[0],
1607 src_dims[1],
1608 scale,
1609 exec,
1610 );
1611 true
1612}
1613
1614#[allow(clippy::too_many_arguments)]
1616#[inline(always)]
1617unsafe fn transpose_scale_4x4_identity<T>(
1618 dst: *mut T,
1619 dst_stride0: isize,
1620 dst_stride1: isize,
1621 src: *const T,
1622 src_stride0: isize,
1623 src_stride1: isize,
1624 i: usize,
1625 j: usize,
1626 scale: T,
1627) where
1628 T: Copy + Mul<Output = T>,
1629{
1630 let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1631
1632 let s00 = *src_base;
1633 let s10 = *src_base.offset(src_stride0);
1634 let s20 = *src_base.offset(2 * src_stride0);
1635 let s30 = *src_base.offset(3 * src_stride0);
1636
1637 let src_col1 = src_base.offset(src_stride1);
1638 let s01 = *src_col1;
1639 let s11 = *src_col1.offset(src_stride0);
1640 let s21 = *src_col1.offset(2 * src_stride0);
1641 let s31 = *src_col1.offset(3 * src_stride0);
1642
1643 let src_col2 = src_base.offset(2 * src_stride1);
1644 let s02 = *src_col2;
1645 let s12 = *src_col2.offset(src_stride0);
1646 let s22 = *src_col2.offset(2 * src_stride0);
1647 let s32 = *src_col2.offset(3 * src_stride0);
1648
1649 let src_col3 = src_base.offset(3 * src_stride1);
1650 let s03 = *src_col3;
1651 let s13 = *src_col3.offset(src_stride0);
1652 let s23 = *src_col3.offset(2 * src_stride0);
1653 let s33 = *src_col3.offset(3 * src_stride0);
1654
1655 let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1656 *dst_row0 = scale * s00;
1657 *dst_row0.offset(dst_stride0) = scale * s01;
1658 *dst_row0.offset(2 * dst_stride0) = scale * s02;
1659 *dst_row0.offset(3 * dst_stride0) = scale * s03;
1660
1661 let dst_row1 = dst_row0.offset(dst_stride1);
1662 *dst_row1 = scale * s10;
1663 *dst_row1.offset(dst_stride0) = scale * s11;
1664 *dst_row1.offset(2 * dst_stride0) = scale * s12;
1665 *dst_row1.offset(3 * dst_stride0) = scale * s13;
1666
1667 let dst_row2 = dst_row0.offset(2 * dst_stride1);
1668 *dst_row2 = scale * s20;
1669 *dst_row2.offset(dst_stride0) = scale * s21;
1670 *dst_row2.offset(2 * dst_stride0) = scale * s22;
1671 *dst_row2.offset(3 * dst_stride0) = scale * s23;
1672
1673 let dst_row3 = dst_row0.offset(3 * dst_stride1);
1674 *dst_row3 = scale * s30;
1675 *dst_row3.offset(dst_stride0) = scale * s31;
1676 *dst_row3.offset(2 * dst_stride0) = scale * s32;
1677 *dst_row3.offset(3 * dst_stride0) = scale * s33;
1678}
1679
1680#[inline]
1681#[allow(clippy::too_many_arguments)]
1683unsafe fn copy_transpose_scale_2d_identity_tiled_raw<T>(
1684 dst: *mut T,
1685 dst_stride0: isize,
1686 dst_stride1: isize,
1687 src: *const T,
1688 src_stride0: isize,
1689 src_stride1: isize,
1690 src_rows: usize,
1691 src_cols: usize,
1692 scale: T,
1693 exec: Exec<'_>,
1694) where
1695 T: Copy + Mul<Output = T> + Send + Sync,
1696{
1697 const TILE: usize = 4;
1698 let row_full = src_rows / TILE * TILE;
1699 let col_full = src_cols / TILE * TILE;
1700
1701 let total = src_rows.saturating_mul(src_cols);
1702 let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1703 if !exec.is_serial() {
1704 let dst_send = SendPtr(dst);
1705 let src_send = SendPtr(src as *mut T);
1706 let row_tiles = row_full / TILE;
1707 crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1708 for tile_i in tiles {
1709 let i = tile_i * TILE;
1710 let dst = dst_send.as_ptr();
1711 let src = src_send.as_const();
1712 unsafe {
1713 let mut j = 0;
1714 while j < col_full {
1715 transpose_scale_4x4_identity(
1716 dst,
1717 dst_stride0,
1718 dst_stride1,
1719 src,
1720 src_stride0,
1721 src_stride1,
1722 i,
1723 j,
1724 scale,
1725 );
1726 j += TILE;
1727 }
1728 for j in col_full..src_cols {
1729 for ii in i..i + TILE {
1730 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1731 scale
1732 * *src.offset(
1733 ii as isize * src_stride0 + j as isize * src_stride1,
1734 );
1735 }
1736 }
1737 }
1738 }
1739 });
1740 for i in row_full..src_rows {
1741 for j in 0..src_cols {
1742 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1743 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1744 }
1745 }
1746 return;
1747 }
1748
1749 let mut i = 0;
1750 while i < row_full {
1751 let mut j = 0;
1752 while j < col_full {
1753 transpose_scale_4x4_identity(
1754 dst,
1755 dst_stride0,
1756 dst_stride1,
1757 src,
1758 src_stride0,
1759 src_stride1,
1760 i,
1761 j,
1762 scale,
1763 );
1764 j += TILE;
1765 }
1766 for j in col_full..src_cols {
1767 for ii in i..i + TILE {
1768 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1769 scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1770 }
1771 }
1772 i += TILE;
1773 }
1774 for i in row_full..src_rows {
1775 for j in 0..src_cols {
1776 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1777 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1778 }
1779 }
1780}
1781
1782#[inline]
1783unsafe fn try_copy_transpose_scale_2d_identity_tiled<T>(
1784 dest: &mut StridedViewMut<T>,
1785 src: &StridedView<T>,
1786 scale: T,
1787 exec: Exec<'_>,
1788) -> bool
1789where
1790 T: Copy + Mul<Output = T> + Send + Sync,
1791{
1792 if src.ndim() != 2 || dest.ndim() != 2 {
1793 return false;
1794 }
1795 let src_dims = src.dims();
1796 if dest.dims() != [src_dims[1], src_dims[0]] {
1797 return false;
1798 }
1799 if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1800 return false;
1801 }
1802
1803 copy_transpose_scale_2d_identity_tiled_raw(
1804 dest.as_mut_ptr(),
1805 dest.strides()[0],
1806 dest.strides()[1],
1807 src.ptr(),
1808 src.strides()[0],
1809 src.strides()[1],
1810 src_dims[0],
1811 src_dims[1],
1812 scale,
1813 exec,
1814 );
1815 true
1816}
1817
1818#[inline]
1819#[allow(clippy::too_many_arguments)]
1821unsafe fn copy_transpose_scale_2d_loop<T>(
1822 dst: *mut T,
1823 dst_stride0: isize,
1824 dst_stride1: isize,
1825 src: *const T,
1826 src_stride0: isize,
1827 src_stride1: isize,
1828 src_rows: usize,
1829 src_cols: usize,
1830 scale: T,
1831 exec: Exec<'_>,
1832) where
1833 T: Copy + ElementOpApply + Mul<Output = T> + Send + Sync,
1834{
1835 let total = src_rows.saturating_mul(src_cols);
1836 let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1837 if !exec.is_serial() {
1838 let dst_send = SendPtr(dst);
1839 let src_send = SendPtr(src as *mut T);
1840 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1841 crate::threading::parallel_for_each(0..src_rows, exec, &|rows| {
1842 for i in rows {
1843 let dst = dst_send.as_ptr();
1844 let src = src_send.as_const();
1845 unsafe {
1846 for j in 0..src_cols {
1847 let value = (*src
1848 .offset(i as isize * src_stride0 + j as isize * src_stride1))
1849 .transpose();
1850 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1851 scale * value;
1852 }
1853 }
1854 }
1855 });
1856 } else {
1857 crate::threading::parallel_for_each(0..src_cols, exec, &|columns| {
1858 for j in columns {
1859 let dst = dst_send.as_ptr();
1860 let src = src_send.as_const();
1861 unsafe {
1862 for i in 0..src_rows {
1863 let value = (*src
1864 .offset(i as isize * src_stride0 + j as isize * src_stride1))
1865 .transpose();
1866 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1867 scale * value;
1868 }
1869 }
1870 }
1871 });
1872 }
1873 return;
1874 }
1875
1876 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1877 for i in 0..src_rows {
1878 for j in 0..src_cols {
1879 let value =
1880 (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1881 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1882 }
1883 }
1884 } else {
1885 for j in 0..src_cols {
1886 for i in 0..src_rows {
1887 let value =
1888 (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1889 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1890 }
1891 }
1892 }
1893}
1894
1895pub fn copy_transpose_scale_into<T>(
1909 dest: &mut StridedViewMut<T>,
1910 src: &StridedView<T>,
1911 scale: T,
1912 exec: Exec<'_>,
1913) -> Result<()>
1914where
1915 T: Copy + ElementOpApply + Mul<Output = T> + Zero + One + PartialEq + Send + Sync + 'static,
1916{
1917 let mut exec = exec;
1918 if src.ndim() != 2 || dest.ndim() != 2 {
1919 return Err(StridedError::RankMismatch(src.ndim(), 2));
1920 }
1921
1922 let src_dims = src.dims();
1923 let expected_dims = [src_dims[1], src_dims[0]];
1924 ensure_same_shape(dest.dims(), &expected_dims)?;
1925 total_len(src_dims)?;
1928
1929 if scale == T::zero() {
1930 unsafe {
1931 if same_contiguous_layout(dest.dims(), &[dest.strides()]).is_some() {
1932 fill_contiguous(dest.as_mut_ptr(), total_len(dest.dims())?, T::zero());
1933 } else {
1934 fill_2d(
1935 dest.as_mut_ptr(),
1936 dest.dims()[0],
1937 dest.dims()[1],
1938 dest.strides()[0],
1939 dest.strides()[1],
1940 T::zero(),
1941 exec,
1942 );
1943 }
1944 }
1945 return Ok(());
1946 }
1947
1948 let transpose_is_identity = element_transpose_is_identity::<T>();
1949
1950 unsafe {
1951 if transpose_is_identity
1952 && try_copy_transpose_scale_2d_f64_tiled_typed(dest, src, scale, exec.reborrow())
1953 {
1954 return Ok(());
1955 }
1956 if transpose_is_identity
1957 && try_copy_transpose_scale_2d_identity_tiled(dest, src, scale, exec.reborrow())
1958 {
1959 return Ok(());
1960 }
1961 }
1962
1963 if scale == T::one() && transpose_is_identity {
1964 let src_t = src.permute(&[1, 0])?;
1965 return crate::threading::copy_permuted(dest, &src_t, exec);
1966 }
1967
1968 unsafe {
1969 copy_transpose_scale_2d_loop(
1970 dest.as_mut_ptr(),
1971 dest.strides()[0],
1972 dest.strides()[1],
1973 src.ptr(),
1974 src.strides()[0],
1975 src.strides()[1],
1976 src_dims[0],
1977 src_dims[1],
1978 scale,
1979 exec,
1980 );
1981 }
1982 Ok(())
1983}
1984
1985#[cfg(test)]
1986#[path = "ops_view/tests/tiled_tests.rs"]
1987mod tiled_tests;
1988
1989#[cfg(test)]
1990#[path = "ops_view/tests/conj_route_tests.rs"]
1991mod conj_route_tests;