1use crate::kernel::{
4 build_plan_fused, build_plan_fused_small, ensure_same_shape, for_each_inner_block_preordered,
5 same_contiguous_layout, sequential_contiguous_layout, total_len, SMALL_TENSOR_THRESHOLD,
6};
7use crate::map_view::{copy_map_into, map_into, zip_map2_into};
8use crate::reduce_view::reduce;
9use crate::simd;
10use crate::view::{StridedView, StridedViewMut};
11use crate::{Exec, Result, StridedError};
12use num_traits::{One, Zero};
13use std::mem::MaybeUninit;
14use std::ops::{Add, Mul};
15use strided_view::{ElementOp, ElementOpApply};
16
17use crate::cost::KernelClass;
18use crate::fuse::compute_costs;
19use crate::threading::{
20 for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded, SendPtr,
21};
22#[inline(always)]
34unsafe fn inner_loop_add<D: Copy + Add<S, Output = D>, S: Copy, Op: ElementOp<S>>(
35 dp: *mut D,
36 ds: isize,
37 sp: *const S,
38 ss: isize,
39 len: usize,
40) {
41 if ds == 1 && ss == 1 {
42 let dst = std::slice::from_raw_parts_mut(dp, len);
43 let src = std::slice::from_raw_parts(sp, len);
44 simd::dispatch_if_large(len, || {
45 for i in 0..len {
46 dst[i] = dst[i] + Op::apply(src[i]);
47 }
48 });
49 } else {
50 let mut dp = dp;
51 let mut sp = sp;
52 for _ in 0..len {
53 *dp = *dp + Op::apply(*sp);
54 dp = dp.wrapping_offset(ds);
55 sp = sp.wrapping_offset(ss);
56 }
57 }
58}
59
60#[inline(always)]
62unsafe fn inner_loop_mul<D: Copy + Mul<S, Output = D>, S: Copy, Op: ElementOp<S>>(
63 dp: *mut D,
64 ds: isize,
65 sp: *const S,
66 ss: isize,
67 len: usize,
68) {
69 if ds == 1 && ss == 1 {
70 let dst = std::slice::from_raw_parts_mut(dp, len);
71 let src = std::slice::from_raw_parts(sp, len);
72 simd::dispatch_if_large(len, || {
73 for i in 0..len {
74 dst[i] = dst[i] * Op::apply(src[i]);
75 }
76 });
77 } else {
78 let mut dp = dp;
79 let mut sp = sp;
80 for _ in 0..len {
81 *dp = *dp * Op::apply(*sp);
82 dp = dp.wrapping_offset(ds);
83 sp = sp.wrapping_offset(ss);
84 }
85 }
86}
87
88#[inline(always)]
90unsafe fn inner_loop_axpy<
91 D: Copy + Add<D, Output = D>,
92 S: Copy,
93 A: Copy + Mul<S, Output = D>,
94 Op: ElementOp<S>,
95>(
96 dp: *mut D,
97 ds: isize,
98 sp: *const S,
99 ss: isize,
100 len: usize,
101 alpha: A,
102) {
103 if ds == 1 && ss == 1 {
104 let dst = std::slice::from_raw_parts_mut(dp, len);
105 let src = std::slice::from_raw_parts(sp, len);
106 simd::dispatch_if_large(len, || {
107 for i in 0..len {
108 dst[i] = alpha * Op::apply(src[i]) + dst[i];
109 }
110 });
111 } else {
112 let mut dp = dp;
113 let mut sp = sp;
114 for _ in 0..len {
115 *dp = alpha * Op::apply(*sp) + *dp;
116 dp = dp.wrapping_offset(ds);
117 sp = sp.wrapping_offset(ss);
118 }
119 }
120}
121
122#[inline(always)]
124unsafe fn inner_loop_fma<
125 D: Copy + Add<D, Output = D>,
126 A: Copy + Mul<B, Output = D>,
127 B: Copy,
128 OpA: ElementOp<A>,
129 OpB: ElementOp<B>,
130>(
131 dp: *mut D,
132 ds: isize,
133 ap: *const A,
134 a_s: isize,
135 bp: *const B,
136 b_s: isize,
137 len: usize,
138) {
139 if ds == 1 && a_s == 1 && b_s == 1 {
140 let dst = std::slice::from_raw_parts_mut(dp, len);
141 let sa = std::slice::from_raw_parts(ap, len);
142 let sb = std::slice::from_raw_parts(bp, len);
143 simd::dispatch_if_large(len, || {
144 for i in 0..len {
145 dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
146 }
147 });
148 } else {
149 let mut dp = dp;
150 let mut ap = ap;
151 let mut bp = bp;
152 for _ in 0..len {
153 *dp = *dp + OpA::apply(*ap) * OpB::apply(*bp);
154 dp = dp.wrapping_offset(ds);
155 ap = ap.wrapping_offset(a_s);
156 bp = bp.wrapping_offset(b_s);
157 }
158 }
159}
160
161#[inline(always)]
163unsafe fn inner_loop_dot<
164 A: Copy + Mul<B, Output = R>,
165 B: Copy,
166 R: Copy + Add<R, Output = R>,
167 OpA: ElementOp<A>,
168 OpB: ElementOp<B>,
169>(
170 ap: *const A,
171 a_s: isize,
172 bp: *const B,
173 b_s: isize,
174 len: usize,
175 mut acc: R,
176) -> R {
177 if a_s == 1 && b_s == 1 {
178 let sa = std::slice::from_raw_parts(ap, len);
179 let sb = std::slice::from_raw_parts(bp, len);
180 simd::dispatch_if_large(len, || {
181 for i in 0..len {
182 acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
183 }
184 });
185 } else {
186 let mut ap = ap;
187 let mut bp = bp;
188 for _ in 0..len {
189 acc = acc + OpA::apply(*ap) * OpB::apply(*bp);
190 ap = ap.wrapping_offset(a_s);
191 bp = bp.wrapping_offset(b_s);
192 }
193 }
194 acc
195}
196
197pub fn copy_into<T: Copy + Send + Sync, Op: ElementOp<T>>(
210 dest: &mut StridedViewMut<T>,
211 src: &StridedView<T, Op>,
212 exec: Exec<'_>,
213) -> Result<()> {
214 ensure_same_shape(dest.dims(), src.dims())?;
215
216 let dst_ptr = dest.as_mut_ptr();
217 let src_ptr = src.ptr();
218 let dst_dims = dest.dims();
219 let dst_strides = dest.strides();
220 let src_strides = src.strides();
221
222 if sequential_contiguous_layout(
223 KernelClass::Copy,
224 dst_dims,
225 &[dst_strides, src_strides],
226 &exec,
227 )?
228 .is_some()
229 {
230 let len = total_len(dst_dims)?;
231 if Op::IS_IDENTITY {
232 debug_assert!(
233 {
234 let nbytes = len
235 .checked_mul(std::mem::size_of::<T>())
236 .expect("copy size must not overflow");
237 let dst_start = dst_ptr as usize;
238 let src_start = src_ptr as usize;
239 let dst_end = dst_start.saturating_add(nbytes);
240 let src_end = src_start.saturating_add(nbytes);
241 dst_end <= src_start || src_end <= dst_start
242 },
243 "overlapping src/dest is not supported"
244 );
245 unsafe { std::ptr::copy_nonoverlapping(src_ptr, dst_ptr, len) };
246 } else {
247 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
248 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
249 simd::dispatch_if_large(len, || {
250 for i in 0..len {
251 dst[i] = Op::apply(src[i]);
252 }
253 });
254 }
255 return Ok(());
256 }
257
258 copy_map_into(dest, src, |x| x, exec)
259}
260
261pub fn copy_into_uninit<T: Copy + Send + Sync + 'static>(
288 dest: &mut StridedViewMut<MaybeUninit<T>>,
289 src: &StridedView<T>,
290 exec: Exec<'_>,
291) -> Result<()> {
292 ensure_same_shape(dest.dims(), src.dims())?;
293 if !crate::layout_check::is_injective_layout(dest.dims(), dest.strides()) {
294 return Err(StridedError::NonInjectiveOutputLayout);
295 }
296 if src.is_empty() {
297 return Ok(());
298 }
299 let _len = src
300 .dims()
301 .iter()
302 .try_fold(1usize, |n, &dim| n.checked_mul(dim))
303 .ok_or(StridedError::OffsetOverflow)?;
304 let native_float = std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>()
308 || std::any::TypeId::of::<T>() == std::any::TypeId::of::<f64>();
309 if !native_float
314 || same_contiguous_layout(dest.dims(), &[dest.strides(), src.strides()]).is_some()
315 {
316 return copy_map_into(dest, src, MaybeUninit::new, exec);
317 }
318 crate::threading::copy_permuted_uninit(dest, src, exec)
322}
323
324pub fn copy_into_col_major<T: Copy + Send + Sync + 'static>(
338 dst: &mut StridedViewMut<T>,
339 src: &StridedView<T>,
340) -> Result<()> {
341 strided_perm::copy_into_col_major(dst, src, Exec::serial())
342}
343
344pub fn add_acc<
356 D: Copy + Add<S, Output = D> + Send + Sync,
357 S: Copy + Send + Sync,
358 Op: ElementOp<S>,
359>(
360 dest: &mut StridedViewMut<D>,
361 src: &StridedView<S, Op>,
362 exec: Exec<'_>,
363) -> Result<()> {
364 ensure_same_shape(dest.dims(), src.dims())?;
365
366 let dst_ptr = dest.as_mut_ptr();
367 let src_ptr = src.ptr();
368 let dst_dims = dest.dims();
369 let dst_strides = dest.strides();
370 let src_strides = src.strides();
371
372 if sequential_contiguous_layout(
373 KernelClass::Map,
374 dst_dims,
375 &[dst_strides, src_strides],
376 &exec,
377 )?
378 .is_some()
379 {
380 let len = total_len(dst_dims)?;
381 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
382 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
383 simd::dispatch_if_large(len, || {
384 for i in 0..len {
385 dst[i] = dst[i] + Op::apply(src[i]);
386 }
387 });
388 return Ok(());
389 }
390
391 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
392 let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
393
394 let total = total_len(dst_dims)?;
395 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
396 build_plan_fused_small(dst_dims, &strides_list)
397 } else {
398 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
399 };
400
401 let total = total_len(&fused_dims)?;
402 if is_parallel(KernelClass::Map, total, &exec) {
403 let dst_send = SendPtr(dst_ptr);
404 let src_send = SendPtr(src_ptr as *mut S);
405
406 let costs = compute_costs(&ordered_strides);
407 let initial_offsets: crate::inline::OperandVec<isize> =
408 strides_list.iter().map(|_| 0isize).collect();
409 return mapreduce_threaded(
410 &fused_dims,
411 &plan.block,
412 &ordered_strides,
413 &initial_offsets,
414 &costs,
415 KernelClass::Map,
416 exec,
417 &|dims, blocks, strides_list, offsets| {
418 for_each_inner_block_with_offsets(
419 dims,
420 blocks,
421 strides_list,
422 offsets,
423 |offsets, len, strides| {
424 unsafe {
425 inner_loop_add::<D, S, Op>(
426 dst_send.as_ptr().offset(offsets[0]),
427 strides[0],
428 src_send.as_const().offset(offsets[1]),
429 strides[1],
430 len,
431 )
432 };
433 Ok(())
434 },
435 )
436 },
437 );
438 }
439
440 let initial_offsets: crate::inline::OperandVec<isize> =
441 ordered_strides.iter().map(|_| 0isize).collect();
442 for_each_inner_block_preordered(
443 &fused_dims,
444 &plan.block,
445 &ordered_strides,
446 &initial_offsets,
447 |offsets, len, strides| {
448 unsafe {
449 inner_loop_add::<D, S, Op>(
450 dst_ptr.offset(offsets[0]),
451 strides[0],
452 src_ptr.offset(offsets[1]),
453 strides[1],
454 len,
455 )
456 };
457 Ok(())
458 },
459 )
460}
461
462pub fn mul_acc<
474 D: Copy + Mul<S, Output = D> + Send + Sync,
475 S: Copy + Send + Sync,
476 Op: ElementOp<S>,
477>(
478 dest: &mut StridedViewMut<D>,
479 src: &StridedView<S, Op>,
480 exec: Exec<'_>,
481) -> Result<()> {
482 ensure_same_shape(dest.dims(), src.dims())?;
483
484 let dst_ptr = dest.as_mut_ptr();
485 let src_ptr = src.ptr();
486 let dst_dims = dest.dims();
487 let dst_strides = dest.strides();
488 let src_strides = src.strides();
489
490 if sequential_contiguous_layout(
491 KernelClass::Map,
492 dst_dims,
493 &[dst_strides, src_strides],
494 &exec,
495 )?
496 .is_some()
497 {
498 let len = total_len(dst_dims)?;
499 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
500 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
501 simd::dispatch_if_large(len, || {
502 for i in 0..len {
503 dst[i] = dst[i] * Op::apply(src[i]);
504 }
505 });
506 return Ok(());
507 }
508
509 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
510 let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
511
512 let total = total_len(dst_dims)?;
513 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
514 build_plan_fused_small(dst_dims, &strides_list)
515 } else {
516 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
517 };
518
519 let total = total_len(&fused_dims)?;
520 if is_parallel(KernelClass::Map, total, &exec) {
521 let dst_send = SendPtr(dst_ptr);
522 let src_send = SendPtr(src_ptr as *mut S);
523
524 let costs = compute_costs(&ordered_strides);
525 let initial_offsets: crate::inline::OperandVec<isize> =
526 strides_list.iter().map(|_| 0isize).collect();
527 return mapreduce_threaded(
528 &fused_dims,
529 &plan.block,
530 &ordered_strides,
531 &initial_offsets,
532 &costs,
533 KernelClass::Map,
534 exec,
535 &|dims, blocks, strides_list, offsets| {
536 for_each_inner_block_with_offsets(
537 dims,
538 blocks,
539 strides_list,
540 offsets,
541 |offsets, len, strides| {
542 unsafe {
543 inner_loop_mul::<D, S, Op>(
544 dst_send.as_ptr().offset(offsets[0]),
545 strides[0],
546 src_send.as_const().offset(offsets[1]),
547 strides[1],
548 len,
549 )
550 };
551 Ok(())
552 },
553 )
554 },
555 );
556 }
557
558 let initial_offsets: crate::inline::OperandVec<isize> =
559 ordered_strides.iter().map(|_| 0isize).collect();
560 for_each_inner_block_preordered(
561 &fused_dims,
562 &plan.block,
563 &ordered_strides,
564 &initial_offsets,
565 |offsets, len, strides| {
566 unsafe {
567 inner_loop_mul::<D, S, Op>(
568 dst_ptr.offset(offsets[0]),
569 strides[0],
570 src_ptr.offset(offsets[1]),
571 strides[1],
572 len,
573 )
574 };
575 Ok(())
576 },
577 )
578}
579
580pub fn axpy_acc<D, S, A, Op>(
592 dest: &mut StridedViewMut<D>,
593 src: &StridedView<S, Op>,
594 alpha: A,
595 exec: Exec<'_>,
596) -> Result<()>
597where
598 A: Copy + Mul<S, Output = D> + Sync,
599 D: Copy + Add<D, Output = D> + Send + Sync,
600 S: Copy + Send + Sync,
601 Op: ElementOp<S>,
602{
603 ensure_same_shape(dest.dims(), src.dims())?;
604
605 let dst_ptr = dest.as_mut_ptr();
606 let src_ptr = src.ptr();
607 let dst_dims = dest.dims();
608 let dst_strides = dest.strides();
609 let src_strides = src.strides();
610
611 if sequential_contiguous_layout(
612 KernelClass::Map,
613 dst_dims,
614 &[dst_strides, src_strides],
615 &exec,
616 )?
617 .is_some()
618 {
619 let len = total_len(dst_dims)?;
620 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
621 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
622 simd::dispatch_if_large(len, || {
623 for i in 0..len {
624 dst[i] = alpha * Op::apply(src[i]) + dst[i];
625 }
626 });
627 return Ok(());
628 }
629
630 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
631 let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
632
633 let total = total_len(dst_dims)?;
634 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
635 build_plan_fused_small(dst_dims, &strides_list)
636 } else {
637 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
638 };
639
640 let total = total_len(&fused_dims)?;
641 if is_parallel(KernelClass::Map, total, &exec) {
642 let dst_send = SendPtr(dst_ptr);
643 let src_send = SendPtr(src_ptr as *mut S);
644
645 let costs = compute_costs(&ordered_strides);
646 let initial_offsets: crate::inline::OperandVec<isize> =
647 strides_list.iter().map(|_| 0isize).collect();
648 return mapreduce_threaded(
649 &fused_dims,
650 &plan.block,
651 &ordered_strides,
652 &initial_offsets,
653 &costs,
654 KernelClass::Map,
655 exec,
656 &|dims, blocks, strides_list, offsets| {
657 for_each_inner_block_with_offsets(
658 dims,
659 blocks,
660 strides_list,
661 offsets,
662 |offsets, len, strides| {
663 unsafe {
664 inner_loop_axpy::<D, S, A, Op>(
665 dst_send.as_ptr().offset(offsets[0]),
666 strides[0],
667 src_send.as_const().offset(offsets[1]),
668 strides[1],
669 len,
670 alpha,
671 )
672 };
673 Ok(())
674 },
675 )
676 },
677 );
678 }
679
680 let initial_offsets: crate::inline::OperandVec<isize> =
681 ordered_strides.iter().map(|_| 0isize).collect();
682 for_each_inner_block_preordered(
683 &fused_dims,
684 &plan.block,
685 &ordered_strides,
686 &initial_offsets,
687 |offsets, len, strides| {
688 unsafe {
689 inner_loop_axpy::<D, S, A, Op>(
690 dst_ptr.offset(offsets[0]),
691 strides[0],
692 src_ptr.offset(offsets[1]),
693 strides[1],
694 len,
695 alpha,
696 )
697 };
698 Ok(())
699 },
700 )
701}
702
703pub fn fma_acc<D, A, B, OpA, OpB>(
715 dest: &mut StridedViewMut<D>,
716 a: &StridedView<A, OpA>,
717 b: &StridedView<B, OpB>,
718 exec: Exec<'_>,
719) -> Result<()>
720where
721 A: Copy + Mul<B, Output = D> + Send + Sync,
722 B: Copy + Send + Sync,
723 D: Copy + Add<D, Output = D> + Send + Sync,
724 OpA: ElementOp<A>,
725 OpB: ElementOp<B>,
726{
727 ensure_same_shape(dest.dims(), a.dims())?;
728 ensure_same_shape(dest.dims(), b.dims())?;
729
730 let dst_ptr = dest.as_mut_ptr();
731 let a_ptr = a.ptr();
732 let b_ptr = b.ptr();
733 let dst_dims = dest.dims();
734 let dst_strides = dest.strides();
735 let a_strides = a.strides();
736 let b_strides = b.strides();
737
738 if sequential_contiguous_layout(
739 KernelClass::Map,
740 dst_dims,
741 &[dst_strides, a_strides, b_strides],
742 &exec,
743 )?
744 .is_some()
745 {
746 let len = total_len(dst_dims)?;
747 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
748 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
749 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
750 simd::dispatch_if_large(len, || {
751 for i in 0..len {
752 dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
753 }
754 });
755 return Ok(());
756 }
757
758 let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
759 let elem_size = std::mem::size_of::<D>()
760 .max(std::mem::size_of::<A>())
761 .max(std::mem::size_of::<B>());
762
763 let total = total_len(dst_dims)?;
764 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
765 build_plan_fused_small(dst_dims, &strides_list)
766 } else {
767 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
768 };
769
770 let total = total_len(&fused_dims)?;
771 if is_parallel(KernelClass::Map, total, &exec) {
772 let dst_send = SendPtr(dst_ptr);
773 let a_send = SendPtr(a_ptr as *mut A);
774 let b_send = SendPtr(b_ptr as *mut B);
775
776 let costs = compute_costs(&ordered_strides);
777 let initial_offsets: crate::inline::OperandVec<isize> =
778 strides_list.iter().map(|_| 0isize).collect();
779 return mapreduce_threaded(
780 &fused_dims,
781 &plan.block,
782 &ordered_strides,
783 &initial_offsets,
784 &costs,
785 KernelClass::Map,
786 exec,
787 &|dims, blocks, strides_list, offsets| {
788 for_each_inner_block_with_offsets(
789 dims,
790 blocks,
791 strides_list,
792 offsets,
793 |offsets, len, strides| {
794 unsafe {
795 inner_loop_fma::<D, A, B, OpA, OpB>(
796 dst_send.as_ptr().offset(offsets[0]),
797 strides[0],
798 a_send.as_const().offset(offsets[1]),
799 strides[1],
800 b_send.as_const().offset(offsets[2]),
801 strides[2],
802 len,
803 )
804 };
805 Ok(())
806 },
807 )
808 },
809 );
810 }
811
812 let initial_offsets: crate::inline::OperandVec<isize> =
813 ordered_strides.iter().map(|_| 0isize).collect();
814 for_each_inner_block_preordered(
815 &fused_dims,
816 &plan.block,
817 &ordered_strides,
818 &initial_offsets,
819 |offsets, len, strides| {
820 unsafe {
821 inner_loop_fma::<D, A, B, OpA, OpB>(
822 dst_ptr.offset(offsets[0]),
823 strides[0],
824 a_ptr.offset(offsets[1]),
825 strides[1],
826 b_ptr.offset(offsets[2]),
827 strides[2],
828 len,
829 )
830 };
831 Ok(())
832 },
833 )
834}
835
836pub fn sum<
865 T: Copy + Zero + Add<Output = T> + Send + Sync + simd::MaybeSimdOps,
866 Op: ElementOp<T>,
867>(
868 src: &StridedView<T, Op>,
869 exec: Exec<'_>,
870) -> Result<T> {
871 if total_len(src.dims())? == 0 {
874 return Ok(T::zero());
875 }
876 if Op::IS_IDENTITY && same_contiguous_layout(src.dims(), &[src.strides()]).is_some() {
878 let len = total_len(src.dims())?;
879 let src_slice = unsafe { std::slice::from_raw_parts(src.ptr(), len) };
880
881 if T::try_simd_sum(&[]).is_some() && is_parallel(KernelClass::Reduce, len, &exec) {
884 return Ok(crate::threading::parallel_map_reduce(
885 0..len,
886 crate::cost::for_len_tree(exec, KernelClass::Reduce, len),
887 &|range| T::try_simd_sum(&src_slice[range]).unwrap(),
888 &|left, right| left + right,
889 ));
890 }
891
892 if let Some(result) = T::try_simd_sum(src_slice) {
893 return Ok(result);
894 }
895 }
896 reduce(src, |x| x, |a, b| a + b, T::zero(), exec)
897}
898
899pub fn dot<A, B, R, OpA, OpB>(a: &StridedView<A, OpA>, b: &StridedView<B, OpB>) -> Result<R>
930where
931 A: Copy + Mul<B, Output = R> + Send + Sync + 'static,
932 B: Copy + Send + Sync + 'static,
933 R: Copy + Zero + Add<Output = R> + Send + Sync + simd::MaybeSimdOps + 'static,
934 OpA: ElementOp<A>,
935 OpB: ElementOp<B>,
936{
937 ensure_same_shape(a.dims(), b.dims())?;
938
939 let a_ptr = a.ptr();
940 let b_ptr = b.ptr();
941 let a_strides = a.strides();
942 let b_strides = b.strides();
943 let a_dims = a.dims();
944 let len = total_len(a_dims)?;
945
946 if len == 0 {
950 return Ok(R::zero());
951 }
952
953 if same_contiguous_layout(a_dims, &[a_strides, b_strides]).is_some() {
954 if OpA::IS_IDENTITY
956 && OpB::IS_IDENTITY
957 && std::any::TypeId::of::<A>() == std::any::TypeId::of::<R>()
958 && std::any::TypeId::of::<B>() == std::any::TypeId::of::<R>()
959 {
960 let sa = unsafe { std::slice::from_raw_parts(a_ptr as *const R, len) };
961 let sb = unsafe { std::slice::from_raw_parts(b_ptr as *const R, len) };
962 if let Some(result) = R::try_simd_dot(sa, sb) {
963 return Ok(result);
964 }
965 }
966
967 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
969 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
970 let mut acc = R::zero();
971 simd::dispatch_if_large(len, || {
972 for i in 0..len {
973 acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
974 }
975 });
976 return Ok(acc);
977 }
978
979 let strides_list: [&[isize]; 2] = [a_strides, b_strides];
980 let elem_size = std::mem::size_of::<A>()
981 .max(std::mem::size_of::<B>())
982 .max(std::mem::size_of::<R>());
983
984 let total = total_len(a_dims)?;
985 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
986 build_plan_fused_small(a_dims, &strides_list)
987 } else {
988 build_plan_fused(a_dims, &strides_list, None, elem_size)
989 };
990
991 let mut acc = R::zero();
992 let initial_offsets: crate::inline::OperandVec<isize> =
993 ordered_strides.iter().map(|_| 0isize).collect();
994 for_each_inner_block_preordered(
995 &fused_dims,
996 &plan.block,
997 &ordered_strides,
998 &initial_offsets,
999 |offsets, len, strides| {
1000 acc = unsafe {
1001 inner_loop_dot::<A, B, R, OpA, OpB>(
1002 a_ptr.offset(offsets[0]),
1003 strides[0],
1004 b_ptr.offset(offsets[1]),
1005 strides[1],
1006 len,
1007 acc,
1008 )
1009 };
1010 Ok(())
1011 },
1012 )?;
1013
1014 Ok(acc)
1015}
1016
1017pub fn symmetrize_into<T>(
1035 dest: &mut StridedViewMut<T>,
1036 src: &StridedView<T>,
1037 exec: Exec<'_>,
1038) -> Result<()>
1039where
1040 T: Copy
1041 + Add<Output = T>
1042 + Mul<Output = T>
1043 + num_traits::FromPrimitive
1044 + std::ops::Div<Output = T>
1045 + Send
1046 + Sync,
1047{
1048 if src.ndim() != 2 {
1049 return Err(StridedError::RankMismatch(src.ndim(), 2));
1050 }
1051 let rows = src.dims()[0];
1052 let cols = src.dims()[1];
1053 if rows != cols {
1054 return Err(StridedError::NonSquare { rows, cols });
1055 }
1056
1057 let src_t = src.permute(&[1, 0])?;
1058 let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1059
1060 zip_map2_into(dest, src, &src_t, |a, b| (a + b) * half, exec)
1061}
1062
1063pub fn symmetrize_conj_into<T>(
1081 dest: &mut StridedViewMut<T>,
1082 src: &StridedView<T>,
1083 exec: Exec<'_>,
1084) -> Result<()>
1085where
1086 T: Copy
1087 + ElementOpApply
1088 + Add<Output = T>
1089 + Mul<Output = T>
1090 + num_traits::FromPrimitive
1091 + std::ops::Div<Output = T>
1092 + Send
1093 + Sync,
1094{
1095 if src.ndim() != 2 {
1096 return Err(StridedError::RankMismatch(src.ndim(), 2));
1097 }
1098 let rows = src.dims()[0];
1099 let cols = src.dims()[1];
1100 if rows != cols {
1101 return Err(StridedError::NonSquare { rows, cols });
1102 }
1103
1104 let src_adj = src.adjoint_2d()?;
1106 let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1107
1108 zip_map2_into(dest, src, &src_adj, |a, b| (a + b) * half, exec)
1109}
1110
1111pub fn copy_scale_into<D, S, A, Op>(
1124 dest: &mut StridedViewMut<D>,
1125 src: &StridedView<S, Op>,
1126 scale: A,
1127 exec: Exec<'_>,
1128) -> Result<()>
1129where
1130 A: Copy + Mul<S, Output = D> + Sync,
1131 D: Copy + Send + Sync,
1132 S: Copy + Send + Sync,
1133 Op: ElementOp<S>,
1134{
1135 map_into(dest, src, |x| scale * x, exec)
1136}
1137
1138pub fn copy_conj_into<T: Copy + ElementOpApply + Send + Sync>(
1150 dest: &mut StridedViewMut<T>,
1151 src: &StridedView<T>,
1152 exec: Exec<'_>,
1153) -> Result<()> {
1154 let src_conj = src.conj();
1155 copy_into(dest, &src_conj, exec)
1156}
1157
1158#[inline]
1159fn element_transpose_is_identity<T: 'static>() -> bool {
1160 use std::any::TypeId;
1161
1162 macro_rules! matches_type {
1163 ($($ty:ty),* $(,)?) => {{
1164 let id = TypeId::of::<T>();
1165 false $(|| id == TypeId::of::<$ty>())*
1166 }};
1167 }
1168
1169 matches_type!(
1170 f32,
1171 f64,
1172 i8,
1173 i16,
1174 i32,
1175 i64,
1176 i128,
1177 isize,
1178 u8,
1179 u16,
1180 u32,
1181 u64,
1182 u128,
1183 usize,
1184 num_complex::Complex32,
1185 num_complex::Complex64,
1186 )
1187}
1188
1189#[inline]
1190fn element_zero_is_all_bits_zero<T: 'static>() -> bool {
1191 use std::any::TypeId;
1192
1193 macro_rules! matches_type {
1194 ($($ty:ty),* $(,)?) => {{
1195 let id = TypeId::of::<T>();
1196 false $(|| id == TypeId::of::<$ty>())*
1197 }};
1198 }
1199
1200 matches_type!(
1201 f32,
1202 f64,
1203 i8,
1204 i16,
1205 i32,
1206 i64,
1207 i128,
1208 isize,
1209 u8,
1210 u16,
1211 u32,
1212 u64,
1213 u128,
1214 usize,
1215 num_complex::Complex32,
1216 num_complex::Complex64,
1217 )
1218}
1219
1220#[inline]
1221unsafe fn fill_2d<T: Copy + Send + Sync>(
1222 dst: *mut T,
1223 dim0: usize,
1224 dim1: usize,
1225 dst_stride0: isize,
1226 dst_stride1: isize,
1227 value: T,
1228 exec: Exec<'_>,
1229) {
1230 let total = dim0.saturating_mul(dim1);
1231 let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
1232 if !exec.is_serial() {
1233 let dst_send = SendPtr(dst);
1234 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1235 crate::threading::parallel_for_each(0..dim1, exec, &|columns| {
1236 for j in columns {
1237 let dst = dst_send.as_ptr();
1238 unsafe {
1239 let base = j as isize * dst_stride1;
1240 for i in 0..dim0 {
1241 *dst.offset(base + i as isize * dst_stride0) = value;
1242 }
1243 }
1244 }
1245 });
1246 } else {
1247 crate::threading::parallel_for_each(0..dim0, exec, &|rows| {
1248 for i in rows {
1249 let dst = dst_send.as_ptr();
1250 unsafe {
1251 let base = i as isize * dst_stride0;
1252 for j in 0..dim1 {
1253 *dst.offset(base + j as isize * dst_stride1) = value;
1254 }
1255 }
1256 }
1257 });
1258 }
1259 return;
1260 }
1261
1262 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1263 for j in 0..dim1 {
1264 let base = j as isize * dst_stride1;
1265 for i in 0..dim0 {
1266 *dst.offset(base + i as isize * dst_stride0) = value;
1267 }
1268 }
1269 } else {
1270 for i in 0..dim0 {
1271 let base = i as isize * dst_stride0;
1272 for j in 0..dim1 {
1273 *dst.offset(base + j as isize * dst_stride1) = value;
1274 }
1275 }
1276 }
1277}
1278
1279#[inline]
1280unsafe fn fill_contiguous<T>(dst: *mut T, len: usize, value: T)
1281where
1282 T: Copy + Zero + PartialEq + Send + Sync + 'static,
1283{
1284 if element_zero_is_all_bits_zero::<T>() && value == T::zero() {
1285 std::ptr::write_bytes(dst, 0, len);
1286 return;
1287 }
1288
1289 let dst = std::slice::from_raw_parts_mut(dst, len);
1290 dst.fill(value);
1291}
1292
1293#[allow(clippy::too_many_arguments)]
1296#[inline(always)]
1297unsafe fn transpose_scale_4x4_f64(
1298 dst: *mut f64,
1299 dst_stride0: isize,
1300 dst_stride1: isize,
1301 src: *const f64,
1302 src_stride0: isize,
1303 src_stride1: isize,
1304 i: usize,
1305 j: usize,
1306 scale: f64,
1307) {
1308 let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1309
1310 let s00 = *src_base;
1311 let s10 = *src_base.offset(src_stride0);
1312 let s20 = *src_base.offset(2 * src_stride0);
1313 let s30 = *src_base.offset(3 * src_stride0);
1314
1315 let src_col1 = src_base.offset(src_stride1);
1316 let s01 = *src_col1;
1317 let s11 = *src_col1.offset(src_stride0);
1318 let s21 = *src_col1.offset(2 * src_stride0);
1319 let s31 = *src_col1.offset(3 * src_stride0);
1320
1321 let src_col2 = src_base.offset(2 * src_stride1);
1322 let s02 = *src_col2;
1323 let s12 = *src_col2.offset(src_stride0);
1324 let s22 = *src_col2.offset(2 * src_stride0);
1325 let s32 = *src_col2.offset(3 * src_stride0);
1326
1327 let src_col3 = src_base.offset(3 * src_stride1);
1328 let s03 = *src_col3;
1329 let s13 = *src_col3.offset(src_stride0);
1330 let s23 = *src_col3.offset(2 * src_stride0);
1331 let s33 = *src_col3.offset(3 * src_stride0);
1332
1333 let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1334 *dst_row0 = scale * s00;
1335 *dst_row0.offset(dst_stride0) = scale * s01;
1336 *dst_row0.offset(2 * dst_stride0) = scale * s02;
1337 *dst_row0.offset(3 * dst_stride0) = scale * s03;
1338
1339 let dst_row1 = dst_row0.offset(dst_stride1);
1340 *dst_row1 = scale * s10;
1341 *dst_row1.offset(dst_stride0) = scale * s11;
1342 *dst_row1.offset(2 * dst_stride0) = scale * s12;
1343 *dst_row1.offset(3 * dst_stride0) = scale * s13;
1344
1345 let dst_row2 = dst_row0.offset(2 * dst_stride1);
1346 *dst_row2 = scale * s20;
1347 *dst_row2.offset(dst_stride0) = scale * s21;
1348 *dst_row2.offset(2 * dst_stride0) = scale * s22;
1349 *dst_row2.offset(3 * dst_stride0) = scale * s23;
1350
1351 let dst_row3 = dst_row0.offset(3 * dst_stride1);
1352 *dst_row3 = scale * s30;
1353 *dst_row3.offset(dst_stride0) = scale * s31;
1354 *dst_row3.offset(2 * dst_stride0) = scale * s32;
1355 *dst_row3.offset(3 * dst_stride0) = scale * s33;
1356}
1357
1358#[inline]
1359#[allow(clippy::too_many_arguments)]
1362unsafe fn copy_transpose_scale_2d_f64_tiled_raw(
1363 dst: *mut f64,
1364 dst_stride0: isize,
1365 dst_stride1: isize,
1366 src: *const f64,
1367 src_stride0: isize,
1368 src_stride1: isize,
1369 src_rows: usize,
1370 src_cols: usize,
1371 scale: f64,
1372 exec: Exec<'_>,
1373) {
1374 const TILE: usize = 4;
1375 let row_full = src_rows / TILE * TILE;
1376 let col_full = src_cols / TILE * TILE;
1377
1378 let total = src_rows.saturating_mul(src_cols);
1379 let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1380 if !exec.is_serial() {
1381 let dst_send = SendPtr(dst);
1382 let src_send = SendPtr(src as *mut f64);
1383 let row_tiles = row_full / TILE;
1384 crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1385 for tile_i in tiles {
1386 let i = tile_i * TILE;
1387 let dst = dst_send.as_ptr();
1388 let src = src_send.as_const();
1389 unsafe {
1390 let mut j = 0;
1391 while j < col_full {
1392 transpose_scale_4x4_f64(
1393 dst,
1394 dst_stride0,
1395 dst_stride1,
1396 src,
1397 src_stride0,
1398 src_stride1,
1399 i,
1400 j,
1401 scale,
1402 );
1403 j += TILE;
1404 }
1405 for j in col_full..src_cols {
1406 for ii in i..i + TILE {
1407 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1408 scale
1409 * *src.offset(
1410 ii as isize * src_stride0 + j as isize * src_stride1,
1411 );
1412 }
1413 }
1414 }
1415 }
1416 });
1417 for i in row_full..src_rows {
1418 for j in 0..src_cols {
1419 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1420 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1421 }
1422 }
1423 return;
1424 }
1425
1426 let mut i = 0;
1427 while i < row_full {
1428 let mut j = 0;
1429 while j < col_full {
1430 transpose_scale_4x4_f64(
1431 dst,
1432 dst_stride0,
1433 dst_stride1,
1434 src,
1435 src_stride0,
1436 src_stride1,
1437 i,
1438 j,
1439 scale,
1440 );
1441 j += TILE;
1442 }
1443 for j in col_full..src_cols {
1444 for ii in i..i + TILE {
1445 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1446 scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1447 }
1448 }
1449 i += TILE;
1450 }
1451 for i in row_full..src_rows {
1452 for j in 0..src_cols {
1453 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1454 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1455 }
1456 }
1457}
1458
1459#[inline]
1460#[cfg(test)]
1461unsafe fn try_copy_transpose_scale_2d_f64_tiled(
1462 dest: &mut StridedViewMut<f64>,
1463 src: &StridedView<f64>,
1464 scale: f64,
1465 exec: Exec<'_>,
1466) -> bool {
1467 if src.ndim() != 2 || dest.ndim() != 2 {
1468 return false;
1469 }
1470 let src_dims = src.dims();
1471 if dest.dims() != [src_dims[1], src_dims[0]] {
1472 return false;
1473 }
1474 if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1475 return false;
1476 }
1477
1478 copy_transpose_scale_2d_f64_tiled_raw(
1479 dest.as_mut_ptr(),
1480 dest.strides()[0],
1481 dest.strides()[1],
1482 src.ptr(),
1483 src.strides()[0],
1484 src.strides()[1],
1485 src_dims[0],
1486 src_dims[1],
1487 scale,
1488 exec,
1489 );
1490 true
1491}
1492
1493#[inline]
1494unsafe fn try_copy_transpose_scale_2d_f64_tiled_typed<T>(
1495 dest: &mut StridedViewMut<T>,
1496 src: &StridedView<T>,
1497 scale: T,
1498 exec: Exec<'_>,
1499) -> bool
1500where
1501 T: Copy + 'static,
1502{
1503 if std::any::TypeId::of::<T>() != std::any::TypeId::of::<f64>() {
1504 return false;
1505 }
1506
1507 let scale = *(&scale as *const T).cast::<f64>();
1508 if src.ndim() != 2 || dest.ndim() != 2 {
1509 return false;
1510 }
1511 let src_dims = src.dims();
1512 if dest.dims() != [src_dims[1], src_dims[0]] {
1513 return false;
1514 }
1515 if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1516 return false;
1517 }
1518
1519 copy_transpose_scale_2d_f64_tiled_raw(
1520 dest.as_mut_ptr().cast::<f64>(),
1521 dest.strides()[0],
1522 dest.strides()[1],
1523 src.ptr().cast::<f64>(),
1524 src.strides()[0],
1525 src.strides()[1],
1526 src_dims[0],
1527 src_dims[1],
1528 scale,
1529 exec,
1530 );
1531 true
1532}
1533
1534#[allow(clippy::too_many_arguments)]
1536#[inline(always)]
1537unsafe fn transpose_scale_4x4_identity<T>(
1538 dst: *mut T,
1539 dst_stride0: isize,
1540 dst_stride1: isize,
1541 src: *const T,
1542 src_stride0: isize,
1543 src_stride1: isize,
1544 i: usize,
1545 j: usize,
1546 scale: T,
1547) where
1548 T: Copy + Mul<Output = T>,
1549{
1550 let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1551
1552 let s00 = *src_base;
1553 let s10 = *src_base.offset(src_stride0);
1554 let s20 = *src_base.offset(2 * src_stride0);
1555 let s30 = *src_base.offset(3 * src_stride0);
1556
1557 let src_col1 = src_base.offset(src_stride1);
1558 let s01 = *src_col1;
1559 let s11 = *src_col1.offset(src_stride0);
1560 let s21 = *src_col1.offset(2 * src_stride0);
1561 let s31 = *src_col1.offset(3 * src_stride0);
1562
1563 let src_col2 = src_base.offset(2 * src_stride1);
1564 let s02 = *src_col2;
1565 let s12 = *src_col2.offset(src_stride0);
1566 let s22 = *src_col2.offset(2 * src_stride0);
1567 let s32 = *src_col2.offset(3 * src_stride0);
1568
1569 let src_col3 = src_base.offset(3 * src_stride1);
1570 let s03 = *src_col3;
1571 let s13 = *src_col3.offset(src_stride0);
1572 let s23 = *src_col3.offset(2 * src_stride0);
1573 let s33 = *src_col3.offset(3 * src_stride0);
1574
1575 let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1576 *dst_row0 = scale * s00;
1577 *dst_row0.offset(dst_stride0) = scale * s01;
1578 *dst_row0.offset(2 * dst_stride0) = scale * s02;
1579 *dst_row0.offset(3 * dst_stride0) = scale * s03;
1580
1581 let dst_row1 = dst_row0.offset(dst_stride1);
1582 *dst_row1 = scale * s10;
1583 *dst_row1.offset(dst_stride0) = scale * s11;
1584 *dst_row1.offset(2 * dst_stride0) = scale * s12;
1585 *dst_row1.offset(3 * dst_stride0) = scale * s13;
1586
1587 let dst_row2 = dst_row0.offset(2 * dst_stride1);
1588 *dst_row2 = scale * s20;
1589 *dst_row2.offset(dst_stride0) = scale * s21;
1590 *dst_row2.offset(2 * dst_stride0) = scale * s22;
1591 *dst_row2.offset(3 * dst_stride0) = scale * s23;
1592
1593 let dst_row3 = dst_row0.offset(3 * dst_stride1);
1594 *dst_row3 = scale * s30;
1595 *dst_row3.offset(dst_stride0) = scale * s31;
1596 *dst_row3.offset(2 * dst_stride0) = scale * s32;
1597 *dst_row3.offset(3 * dst_stride0) = scale * s33;
1598}
1599
1600#[inline]
1601#[allow(clippy::too_many_arguments)]
1603unsafe fn copy_transpose_scale_2d_identity_tiled_raw<T>(
1604 dst: *mut T,
1605 dst_stride0: isize,
1606 dst_stride1: isize,
1607 src: *const T,
1608 src_stride0: isize,
1609 src_stride1: isize,
1610 src_rows: usize,
1611 src_cols: usize,
1612 scale: T,
1613 exec: Exec<'_>,
1614) where
1615 T: Copy + Mul<Output = T> + Send + Sync,
1616{
1617 const TILE: usize = 4;
1618 let row_full = src_rows / TILE * TILE;
1619 let col_full = src_cols / TILE * TILE;
1620
1621 let total = src_rows.saturating_mul(src_cols);
1622 let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1623 if !exec.is_serial() {
1624 let dst_send = SendPtr(dst);
1625 let src_send = SendPtr(src as *mut T);
1626 let row_tiles = row_full / TILE;
1627 crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1628 for tile_i in tiles {
1629 let i = tile_i * TILE;
1630 let dst = dst_send.as_ptr();
1631 let src = src_send.as_const();
1632 unsafe {
1633 let mut j = 0;
1634 while j < col_full {
1635 transpose_scale_4x4_identity(
1636 dst,
1637 dst_stride0,
1638 dst_stride1,
1639 src,
1640 src_stride0,
1641 src_stride1,
1642 i,
1643 j,
1644 scale,
1645 );
1646 j += TILE;
1647 }
1648 for j in col_full..src_cols {
1649 for ii in i..i + TILE {
1650 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1651 scale
1652 * *src.offset(
1653 ii as isize * src_stride0 + j as isize * src_stride1,
1654 );
1655 }
1656 }
1657 }
1658 }
1659 });
1660 for i in row_full..src_rows {
1661 for j in 0..src_cols {
1662 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1663 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1664 }
1665 }
1666 return;
1667 }
1668
1669 let mut i = 0;
1670 while i < row_full {
1671 let mut j = 0;
1672 while j < col_full {
1673 transpose_scale_4x4_identity(
1674 dst,
1675 dst_stride0,
1676 dst_stride1,
1677 src,
1678 src_stride0,
1679 src_stride1,
1680 i,
1681 j,
1682 scale,
1683 );
1684 j += TILE;
1685 }
1686 for j in col_full..src_cols {
1687 for ii in i..i + TILE {
1688 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1689 scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1690 }
1691 }
1692 i += TILE;
1693 }
1694 for i in row_full..src_rows {
1695 for j in 0..src_cols {
1696 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1697 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1698 }
1699 }
1700}
1701
1702#[inline]
1703unsafe fn try_copy_transpose_scale_2d_identity_tiled<T>(
1704 dest: &mut StridedViewMut<T>,
1705 src: &StridedView<T>,
1706 scale: T,
1707 exec: Exec<'_>,
1708) -> bool
1709where
1710 T: Copy + Mul<Output = T> + Send + Sync,
1711{
1712 if src.ndim() != 2 || dest.ndim() != 2 {
1713 return false;
1714 }
1715 let src_dims = src.dims();
1716 if dest.dims() != [src_dims[1], src_dims[0]] {
1717 return false;
1718 }
1719 if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1720 return false;
1721 }
1722
1723 copy_transpose_scale_2d_identity_tiled_raw(
1724 dest.as_mut_ptr(),
1725 dest.strides()[0],
1726 dest.strides()[1],
1727 src.ptr(),
1728 src.strides()[0],
1729 src.strides()[1],
1730 src_dims[0],
1731 src_dims[1],
1732 scale,
1733 exec,
1734 );
1735 true
1736}
1737
1738#[inline]
1739#[allow(clippy::too_many_arguments)]
1741unsafe fn copy_transpose_scale_2d_loop<T>(
1742 dst: *mut T,
1743 dst_stride0: isize,
1744 dst_stride1: isize,
1745 src: *const T,
1746 src_stride0: isize,
1747 src_stride1: isize,
1748 src_rows: usize,
1749 src_cols: usize,
1750 scale: T,
1751 exec: Exec<'_>,
1752) where
1753 T: Copy + ElementOpApply + Mul<Output = T> + Send + Sync,
1754{
1755 let total = src_rows.saturating_mul(src_cols);
1756 let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1757 if !exec.is_serial() {
1758 let dst_send = SendPtr(dst);
1759 let src_send = SendPtr(src as *mut T);
1760 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1761 crate::threading::parallel_for_each(0..src_rows, exec, &|rows| {
1762 for i in rows {
1763 let dst = dst_send.as_ptr();
1764 let src = src_send.as_const();
1765 unsafe {
1766 for j in 0..src_cols {
1767 let value = (*src
1768 .offset(i as isize * src_stride0 + j as isize * src_stride1))
1769 .transpose();
1770 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1771 scale * value;
1772 }
1773 }
1774 }
1775 });
1776 } else {
1777 crate::threading::parallel_for_each(0..src_cols, exec, &|columns| {
1778 for j in columns {
1779 let dst = dst_send.as_ptr();
1780 let src = src_send.as_const();
1781 unsafe {
1782 for i in 0..src_rows {
1783 let value = (*src
1784 .offset(i as isize * src_stride0 + j as isize * src_stride1))
1785 .transpose();
1786 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1787 scale * value;
1788 }
1789 }
1790 }
1791 });
1792 }
1793 return;
1794 }
1795
1796 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1797 for i in 0..src_rows {
1798 for j in 0..src_cols {
1799 let value =
1800 (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1801 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1802 }
1803 }
1804 } else {
1805 for j in 0..src_cols {
1806 for i in 0..src_rows {
1807 let value =
1808 (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1809 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1810 }
1811 }
1812 }
1813}
1814
1815pub fn copy_transpose_scale_into<T>(
1829 dest: &mut StridedViewMut<T>,
1830 src: &StridedView<T>,
1831 scale: T,
1832 exec: Exec<'_>,
1833) -> Result<()>
1834where
1835 T: Copy + ElementOpApply + Mul<Output = T> + Zero + One + PartialEq + Send + Sync + 'static,
1836{
1837 let mut exec = exec;
1838 if src.ndim() != 2 || dest.ndim() != 2 {
1839 return Err(StridedError::RankMismatch(src.ndim(), 2));
1840 }
1841
1842 let src_dims = src.dims();
1843 let expected_dims = [src_dims[1], src_dims[0]];
1844 ensure_same_shape(dest.dims(), &expected_dims)?;
1845 total_len(src_dims)?;
1848
1849 if scale == T::zero() {
1850 unsafe {
1851 if same_contiguous_layout(dest.dims(), &[dest.strides()]).is_some() {
1852 fill_contiguous(dest.as_mut_ptr(), total_len(dest.dims())?, T::zero());
1853 } else {
1854 fill_2d(
1855 dest.as_mut_ptr(),
1856 dest.dims()[0],
1857 dest.dims()[1],
1858 dest.strides()[0],
1859 dest.strides()[1],
1860 T::zero(),
1861 exec,
1862 );
1863 }
1864 }
1865 return Ok(());
1866 }
1867
1868 let transpose_is_identity = element_transpose_is_identity::<T>();
1869
1870 unsafe {
1871 if transpose_is_identity
1872 && try_copy_transpose_scale_2d_f64_tiled_typed(dest, src, scale, exec.reborrow())
1873 {
1874 return Ok(());
1875 }
1876 if transpose_is_identity
1877 && try_copy_transpose_scale_2d_identity_tiled(dest, src, scale, exec.reborrow())
1878 {
1879 return Ok(());
1880 }
1881 }
1882
1883 if scale == T::one() && transpose_is_identity {
1884 let src_t = src.permute(&[1, 0])?;
1885 return crate::threading::copy_permuted(dest, &src_t, exec);
1886 }
1887
1888 unsafe {
1889 copy_transpose_scale_2d_loop(
1890 dest.as_mut_ptr(),
1891 dest.strides()[0],
1892 dest.strides()[1],
1893 src.ptr(),
1894 src.strides()[0],
1895 src.strides()[1],
1896 src_dims[0],
1897 src_dims[1],
1898 scale,
1899 exec,
1900 );
1901 }
1902 Ok(())
1903}
1904
1905#[cfg(test)]
1906#[path = "ops_view/tests/tiled_tests.rs"]
1907mod tiled_tests;