1use crate::kernel::{
4 build_plan_fused, build_plan_fused_small, ensure_same_shape, for_each_inner_block_preordered,
5 same_contiguous_layout, sequential_contiguous_layout, total_len, SMALL_TENSOR_THRESHOLD,
6};
7use crate::map_view::{copy_map_into, map_into, zip_map2_into};
8use crate::reduce_view::reduce;
9use crate::simd;
10use crate::view::{StridedView, StridedViewMut};
11use crate::{Exec, Result, StridedError};
12use num_traits::{One, Zero};
13use std::mem::MaybeUninit;
14use std::ops::{Add, Mul};
15use strided_view::{ElementOp, ElementOpApply};
16
17use crate::cost::KernelClass;
18use crate::fuse::compute_costs;
19use crate::threading::{
20 for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded, SendPtr,
21};
22#[inline(always)]
34unsafe fn inner_loop_add<D: Copy + Add<S, Output = D>, S: Copy, Op: ElementOp<S>>(
35 dp: *mut D,
36 ds: isize,
37 sp: *const S,
38 ss: isize,
39 len: usize,
40) {
41 if ds == 1 && ss == 1 {
42 let dst = std::slice::from_raw_parts_mut(dp, len);
43 let src = std::slice::from_raw_parts(sp, len);
44 simd::dispatch_if_large(len, || {
45 for i in 0..len {
46 dst[i] = dst[i] + Op::apply(src[i]);
47 }
48 });
49 } else {
50 let mut dp = dp;
51 let mut sp = sp;
52 for _ in 0..len {
53 *dp = *dp + Op::apply(*sp);
54 dp = dp.wrapping_offset(ds);
55 sp = sp.wrapping_offset(ss);
56 }
57 }
58}
59
60#[inline(always)]
62unsafe fn inner_loop_mul<D: Copy + Mul<S, Output = D>, S: Copy, Op: ElementOp<S>>(
63 dp: *mut D,
64 ds: isize,
65 sp: *const S,
66 ss: isize,
67 len: usize,
68) {
69 if ds == 1 && ss == 1 {
70 let dst = std::slice::from_raw_parts_mut(dp, len);
71 let src = std::slice::from_raw_parts(sp, len);
72 simd::dispatch_if_large(len, || {
73 for i in 0..len {
74 dst[i] = dst[i] * Op::apply(src[i]);
75 }
76 });
77 } else {
78 let mut dp = dp;
79 let mut sp = sp;
80 for _ in 0..len {
81 *dp = *dp * Op::apply(*sp);
82 dp = dp.wrapping_offset(ds);
83 sp = sp.wrapping_offset(ss);
84 }
85 }
86}
87
88#[inline(always)]
90unsafe fn inner_loop_axpy<
91 D: Copy + Add<D, Output = D>,
92 S: Copy,
93 A: Copy + Mul<S, Output = D>,
94 Op: ElementOp<S>,
95>(
96 dp: *mut D,
97 ds: isize,
98 sp: *const S,
99 ss: isize,
100 len: usize,
101 alpha: A,
102) {
103 if ds == 1 && ss == 1 {
104 let dst = std::slice::from_raw_parts_mut(dp, len);
105 let src = std::slice::from_raw_parts(sp, len);
106 simd::dispatch_if_large(len, || {
107 for i in 0..len {
108 dst[i] = alpha * Op::apply(src[i]) + dst[i];
109 }
110 });
111 } else {
112 let mut dp = dp;
113 let mut sp = sp;
114 for _ in 0..len {
115 *dp = alpha * Op::apply(*sp) + *dp;
116 dp = dp.wrapping_offset(ds);
117 sp = sp.wrapping_offset(ss);
118 }
119 }
120}
121
122#[inline(always)]
124unsafe fn inner_loop_fma<
125 D: Copy + Add<D, Output = D>,
126 A: Copy + Mul<B, Output = D>,
127 B: Copy,
128 OpA: ElementOp<A>,
129 OpB: ElementOp<B>,
130>(
131 dp: *mut D,
132 ds: isize,
133 ap: *const A,
134 a_s: isize,
135 bp: *const B,
136 b_s: isize,
137 len: usize,
138) {
139 if ds == 1 && a_s == 1 && b_s == 1 {
140 let dst = std::slice::from_raw_parts_mut(dp, len);
141 let sa = std::slice::from_raw_parts(ap, len);
142 let sb = std::slice::from_raw_parts(bp, len);
143 simd::dispatch_if_large(len, || {
144 for i in 0..len {
145 dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
146 }
147 });
148 } else {
149 let mut dp = dp;
150 let mut ap = ap;
151 let mut bp = bp;
152 for _ in 0..len {
153 *dp = *dp + OpA::apply(*ap) * OpB::apply(*bp);
154 dp = dp.wrapping_offset(ds);
155 ap = ap.wrapping_offset(a_s);
156 bp = bp.wrapping_offset(b_s);
157 }
158 }
159}
160
161#[inline(always)]
163unsafe fn inner_loop_dot<
164 A: Copy + Mul<B, Output = R>,
165 B: Copy,
166 R: Copy + Add<R, Output = R>,
167 OpA: ElementOp<A>,
168 OpB: ElementOp<B>,
169>(
170 ap: *const A,
171 a_s: isize,
172 bp: *const B,
173 b_s: isize,
174 len: usize,
175 mut acc: R,
176) -> R {
177 if a_s == 1 && b_s == 1 {
178 let sa = std::slice::from_raw_parts(ap, len);
179 let sb = std::slice::from_raw_parts(bp, len);
180 simd::dispatch_if_large(len, || {
181 for i in 0..len {
182 acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
183 }
184 });
185 } else {
186 let mut ap = ap;
187 let mut bp = bp;
188 for _ in 0..len {
189 acc = acc + OpA::apply(*ap) * OpB::apply(*bp);
190 ap = ap.wrapping_offset(a_s);
191 bp = bp.wrapping_offset(b_s);
192 }
193 }
194 acc
195}
196
197pub fn copy_into<T: Copy + Send + Sync, Op: ElementOp<T>>(
210 dest: &mut StridedViewMut<T>,
211 src: &StridedView<T, Op>,
212 exec: Exec<'_>,
213) -> Result<()> {
214 ensure_same_shape(dest.dims(), src.dims())?;
215
216 let dst_ptr = dest.as_mut_ptr();
217 let src_ptr = src.ptr();
218 let dst_dims = dest.dims();
219 let dst_strides = dest.strides();
220 let src_strides = src.strides();
221
222 if sequential_contiguous_layout(
223 KernelClass::Copy,
224 dst_dims,
225 &[dst_strides, src_strides],
226 &exec,
227 )?
228 .is_some()
229 {
230 let len = total_len(dst_dims)?;
231 if Op::IS_IDENTITY {
232 debug_assert!(
233 {
234 let nbytes = len
235 .checked_mul(std::mem::size_of::<T>())
236 .expect("copy size must not overflow");
237 let dst_start = dst_ptr as usize;
238 let src_start = src_ptr as usize;
239 let dst_end = dst_start.saturating_add(nbytes);
240 let src_end = src_start.saturating_add(nbytes);
241 dst_end <= src_start || src_end <= dst_start
242 },
243 "overlapping src/dest is not supported"
244 );
245 unsafe { std::ptr::copy_nonoverlapping(src_ptr, dst_ptr, len) };
246 } else {
247 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
248 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
249 simd::dispatch_if_large(len, || {
250 for i in 0..len {
251 dst[i] = Op::apply(src[i]);
252 }
253 });
254 }
255 return Ok(());
256 }
257
258 copy_map_into(dest, src, |x| x, exec)
259}
260
261pub fn copy_into_uninit<T: Copy + Send + Sync + 'static>(
288 dest: &mut StridedViewMut<MaybeUninit<T>>,
289 src: &StridedView<T>,
290 exec: Exec<'_>,
291) -> Result<()> {
292 ensure_same_shape(dest.dims(), src.dims())?;
293 if !crate::layout_check::is_injective_layout(dest.dims(), dest.strides()) {
294 return Err(StridedError::NonInjectiveOutputLayout);
295 }
296 if src.is_empty() {
297 return Ok(());
298 }
299 let _len = src
300 .dims()
301 .iter()
302 .try_fold(1usize, |n, &dim| n.checked_mul(dim))
303 .ok_or(StridedError::OffsetOverflow)?;
304 let native_float = std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>()
308 || std::any::TypeId::of::<T>() == std::any::TypeId::of::<f64>();
309 if !native_float
314 || same_contiguous_layout(dest.dims(), &[dest.strides(), src.strides()]).is_some()
315 {
316 return copy_map_into(dest, src, MaybeUninit::new, exec);
317 }
318 crate::threading::copy_permuted_uninit(dest, src, exec)
322}
323
324pub fn copy_into_col_major<T: Copy + Send + Sync + 'static>(
338 dst: &mut StridedViewMut<T>,
339 src: &StridedView<T>,
340) -> Result<()> {
341 strided_perm::copy_into_col_major(dst, src, Exec::serial())
342}
343
344pub fn add_acc<
356 D: Copy + Add<S, Output = D> + Send + Sync,
357 S: Copy + Send + Sync,
358 Op: ElementOp<S>,
359>(
360 dest: &mut StridedViewMut<D>,
361 src: &StridedView<S, Op>,
362 exec: Exec<'_>,
363) -> Result<()> {
364 ensure_same_shape(dest.dims(), src.dims())?;
365
366 let dst_ptr = dest.as_mut_ptr();
367 let src_ptr = src.ptr();
368 let dst_dims = dest.dims();
369 let dst_strides = dest.strides();
370 let src_strides = src.strides();
371
372 if sequential_contiguous_layout(
373 KernelClass::Map,
374 dst_dims,
375 &[dst_strides, src_strides],
376 &exec,
377 )?
378 .is_some()
379 {
380 let len = total_len(dst_dims)?;
381 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
382 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
383 simd::dispatch_if_large(len, || {
384 for i in 0..len {
385 dst[i] = dst[i] + Op::apply(src[i]);
386 }
387 });
388 return Ok(());
389 }
390
391 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
392 let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
393
394 let total = total_len(dst_dims)?;
395 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
396 build_plan_fused_small(dst_dims, &strides_list)
397 } else {
398 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
399 };
400
401 let total = total_len(&fused_dims)?;
402 if is_parallel(KernelClass::Map, total, &exec) {
403 let dst_send = SendPtr(dst_ptr);
404 let src_send = SendPtr(src_ptr as *mut S);
405
406 let costs = compute_costs(&ordered_strides);
407 let initial_offsets: crate::inline::OperandVec<isize> =
408 strides_list.iter().map(|_| 0isize).collect();
409 return mapreduce_threaded(
410 &fused_dims,
411 &plan.block,
412 &ordered_strides,
413 &initial_offsets,
414 &costs,
415 KernelClass::Map,
416 exec,
417 &|dims, blocks, strides_list, offsets| {
418 for_each_inner_block_with_offsets(
419 dims,
420 blocks,
421 strides_list,
422 offsets,
423 |offsets, len, strides| {
424 unsafe {
425 inner_loop_add::<D, S, Op>(
426 dst_send.as_ptr().offset(offsets[0]),
427 strides[0],
428 src_send.as_const().offset(offsets[1]),
429 strides[1],
430 len,
431 )
432 };
433 Ok(())
434 },
435 )
436 },
437 );
438 }
439
440 let initial_offsets: crate::inline::OperandVec<isize> =
441 ordered_strides.iter().map(|_| 0isize).collect();
442 for_each_inner_block_preordered(
443 &fused_dims,
444 &plan.block,
445 &ordered_strides,
446 &initial_offsets,
447 |offsets, len, strides| {
448 unsafe {
449 inner_loop_add::<D, S, Op>(
450 dst_ptr.offset(offsets[0]),
451 strides[0],
452 src_ptr.offset(offsets[1]),
453 strides[1],
454 len,
455 )
456 };
457 Ok(())
458 },
459 )
460}
461
462pub fn mul_acc<
474 D: Copy + Mul<S, Output = D> + Send + Sync,
475 S: Copy + Send + Sync,
476 Op: ElementOp<S>,
477>(
478 dest: &mut StridedViewMut<D>,
479 src: &StridedView<S, Op>,
480 exec: Exec<'_>,
481) -> Result<()> {
482 ensure_same_shape(dest.dims(), src.dims())?;
483
484 let dst_ptr = dest.as_mut_ptr();
485 let src_ptr = src.ptr();
486 let dst_dims = dest.dims();
487 let dst_strides = dest.strides();
488 let src_strides = src.strides();
489
490 if sequential_contiguous_layout(
491 KernelClass::Map,
492 dst_dims,
493 &[dst_strides, src_strides],
494 &exec,
495 )?
496 .is_some()
497 {
498 let len = total_len(dst_dims)?;
499 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
500 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
501 simd::dispatch_if_large(len, || {
502 for i in 0..len {
503 dst[i] = dst[i] * Op::apply(src[i]);
504 }
505 });
506 return Ok(());
507 }
508
509 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
510 let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
511
512 let total = total_len(dst_dims)?;
513 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
514 build_plan_fused_small(dst_dims, &strides_list)
515 } else {
516 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
517 };
518
519 let total = total_len(&fused_dims)?;
520 if is_parallel(KernelClass::Map, total, &exec) {
521 let dst_send = SendPtr(dst_ptr);
522 let src_send = SendPtr(src_ptr as *mut S);
523
524 let costs = compute_costs(&ordered_strides);
525 let initial_offsets: crate::inline::OperandVec<isize> =
526 strides_list.iter().map(|_| 0isize).collect();
527 return mapreduce_threaded(
528 &fused_dims,
529 &plan.block,
530 &ordered_strides,
531 &initial_offsets,
532 &costs,
533 KernelClass::Map,
534 exec,
535 &|dims, blocks, strides_list, offsets| {
536 for_each_inner_block_with_offsets(
537 dims,
538 blocks,
539 strides_list,
540 offsets,
541 |offsets, len, strides| {
542 unsafe {
543 inner_loop_mul::<D, S, Op>(
544 dst_send.as_ptr().offset(offsets[0]),
545 strides[0],
546 src_send.as_const().offset(offsets[1]),
547 strides[1],
548 len,
549 )
550 };
551 Ok(())
552 },
553 )
554 },
555 );
556 }
557
558 let initial_offsets: crate::inline::OperandVec<isize> =
559 ordered_strides.iter().map(|_| 0isize).collect();
560 for_each_inner_block_preordered(
561 &fused_dims,
562 &plan.block,
563 &ordered_strides,
564 &initial_offsets,
565 |offsets, len, strides| {
566 unsafe {
567 inner_loop_mul::<D, S, Op>(
568 dst_ptr.offset(offsets[0]),
569 strides[0],
570 src_ptr.offset(offsets[1]),
571 strides[1],
572 len,
573 )
574 };
575 Ok(())
576 },
577 )
578}
579
580pub fn axpy_acc<D, S, A, Op>(
592 dest: &mut StridedViewMut<D>,
593 src: &StridedView<S, Op>,
594 alpha: A,
595 exec: Exec<'_>,
596) -> Result<()>
597where
598 A: Copy + Mul<S, Output = D> + Sync,
599 D: Copy + Add<D, Output = D> + Send + Sync,
600 S: Copy + Send + Sync,
601 Op: ElementOp<S>,
602{
603 ensure_same_shape(dest.dims(), src.dims())?;
604
605 let dst_ptr = dest.as_mut_ptr();
606 let src_ptr = src.ptr();
607 let dst_dims = dest.dims();
608 let dst_strides = dest.strides();
609 let src_strides = src.strides();
610
611 if sequential_contiguous_layout(
612 KernelClass::Map,
613 dst_dims,
614 &[dst_strides, src_strides],
615 &exec,
616 )?
617 .is_some()
618 {
619 let len = total_len(dst_dims)?;
620 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
621 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
622 simd::dispatch_if_large(len, || {
623 for i in 0..len {
624 dst[i] = alpha * Op::apply(src[i]) + dst[i];
625 }
626 });
627 return Ok(());
628 }
629
630 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
631 let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
632
633 let total = total_len(dst_dims)?;
634 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
635 build_plan_fused_small(dst_dims, &strides_list)
636 } else {
637 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
638 };
639
640 let total = total_len(&fused_dims)?;
641 if is_parallel(KernelClass::Map, total, &exec) {
642 let dst_send = SendPtr(dst_ptr);
643 let src_send = SendPtr(src_ptr as *mut S);
644
645 let costs = compute_costs(&ordered_strides);
646 let initial_offsets: crate::inline::OperandVec<isize> =
647 strides_list.iter().map(|_| 0isize).collect();
648 return mapreduce_threaded(
649 &fused_dims,
650 &plan.block,
651 &ordered_strides,
652 &initial_offsets,
653 &costs,
654 KernelClass::Map,
655 exec,
656 &|dims, blocks, strides_list, offsets| {
657 for_each_inner_block_with_offsets(
658 dims,
659 blocks,
660 strides_list,
661 offsets,
662 |offsets, len, strides| {
663 unsafe {
664 inner_loop_axpy::<D, S, A, Op>(
665 dst_send.as_ptr().offset(offsets[0]),
666 strides[0],
667 src_send.as_const().offset(offsets[1]),
668 strides[1],
669 len,
670 alpha,
671 )
672 };
673 Ok(())
674 },
675 )
676 },
677 );
678 }
679
680 let initial_offsets: crate::inline::OperandVec<isize> =
681 ordered_strides.iter().map(|_| 0isize).collect();
682 for_each_inner_block_preordered(
683 &fused_dims,
684 &plan.block,
685 &ordered_strides,
686 &initial_offsets,
687 |offsets, len, strides| {
688 unsafe {
689 inner_loop_axpy::<D, S, A, Op>(
690 dst_ptr.offset(offsets[0]),
691 strides[0],
692 src_ptr.offset(offsets[1]),
693 strides[1],
694 len,
695 alpha,
696 )
697 };
698 Ok(())
699 },
700 )
701}
702
703pub fn fma_acc<D, A, B, OpA, OpB>(
715 dest: &mut StridedViewMut<D>,
716 a: &StridedView<A, OpA>,
717 b: &StridedView<B, OpB>,
718 exec: Exec<'_>,
719) -> Result<()>
720where
721 A: Copy + Mul<B, Output = D> + Send + Sync,
722 B: Copy + Send + Sync,
723 D: Copy + Add<D, Output = D> + Send + Sync,
724 OpA: ElementOp<A>,
725 OpB: ElementOp<B>,
726{
727 ensure_same_shape(dest.dims(), a.dims())?;
728 ensure_same_shape(dest.dims(), b.dims())?;
729
730 let dst_ptr = dest.as_mut_ptr();
731 let a_ptr = a.ptr();
732 let b_ptr = b.ptr();
733 let dst_dims = dest.dims();
734 let dst_strides = dest.strides();
735 let a_strides = a.strides();
736 let b_strides = b.strides();
737
738 if sequential_contiguous_layout(
739 KernelClass::Map,
740 dst_dims,
741 &[dst_strides, a_strides, b_strides],
742 &exec,
743 )?
744 .is_some()
745 {
746 let len = total_len(dst_dims)?;
747 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
748 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
749 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
750 simd::dispatch_if_large(len, || {
751 for i in 0..len {
752 dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
753 }
754 });
755 return Ok(());
756 }
757
758 let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
759 let elem_size = std::mem::size_of::<D>()
760 .max(std::mem::size_of::<A>())
761 .max(std::mem::size_of::<B>());
762
763 let total = total_len(dst_dims)?;
764 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
765 build_plan_fused_small(dst_dims, &strides_list)
766 } else {
767 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
768 };
769
770 let total = total_len(&fused_dims)?;
771 if is_parallel(KernelClass::Map, total, &exec) {
772 let dst_send = SendPtr(dst_ptr);
773 let a_send = SendPtr(a_ptr as *mut A);
774 let b_send = SendPtr(b_ptr as *mut B);
775
776 let costs = compute_costs(&ordered_strides);
777 let initial_offsets: crate::inline::OperandVec<isize> =
778 strides_list.iter().map(|_| 0isize).collect();
779 return mapreduce_threaded(
780 &fused_dims,
781 &plan.block,
782 &ordered_strides,
783 &initial_offsets,
784 &costs,
785 KernelClass::Map,
786 exec,
787 &|dims, blocks, strides_list, offsets| {
788 for_each_inner_block_with_offsets(
789 dims,
790 blocks,
791 strides_list,
792 offsets,
793 |offsets, len, strides| {
794 unsafe {
795 inner_loop_fma::<D, A, B, OpA, OpB>(
796 dst_send.as_ptr().offset(offsets[0]),
797 strides[0],
798 a_send.as_const().offset(offsets[1]),
799 strides[1],
800 b_send.as_const().offset(offsets[2]),
801 strides[2],
802 len,
803 )
804 };
805 Ok(())
806 },
807 )
808 },
809 );
810 }
811
812 let initial_offsets: crate::inline::OperandVec<isize> =
813 ordered_strides.iter().map(|_| 0isize).collect();
814 for_each_inner_block_preordered(
815 &fused_dims,
816 &plan.block,
817 &ordered_strides,
818 &initial_offsets,
819 |offsets, len, strides| {
820 unsafe {
821 inner_loop_fma::<D, A, B, OpA, OpB>(
822 dst_ptr.offset(offsets[0]),
823 strides[0],
824 a_ptr.offset(offsets[1]),
825 strides[1],
826 b_ptr.offset(offsets[2]),
827 strides[2],
828 len,
829 )
830 };
831 Ok(())
832 },
833 )
834}
835
836pub fn sum<
865 T: Copy + Zero + Add<Output = T> + Send + Sync + simd::MaybeSimdOps,
866 Op: ElementOp<T>,
867>(
868 src: &StridedView<T, Op>,
869 exec: Exec<'_>,
870) -> Result<T> {
871 if total_len(src.dims())? == 0 {
874 return Ok(T::zero());
875 }
876 if Op::IS_IDENTITY {
878 if same_contiguous_layout(src.dims(), &[src.strides()]).is_some() {
879 let len = total_len(src.dims())?;
880 let src_slice = unsafe { std::slice::from_raw_parts(src.ptr(), len) };
881
882 if T::try_simd_sum(&[]).is_some() && is_parallel(KernelClass::Reduce, len, &exec) {
885 return Ok(crate::threading::parallel_map_reduce(
886 0..len,
887 crate::cost::for_len_tree(exec, KernelClass::Reduce, len),
888 &|range| T::try_simd_sum(&src_slice[range]).unwrap(),
889 &|left, right| left + right,
890 ));
891 }
892
893 if let Some(result) = T::try_simd_sum(src_slice) {
894 return Ok(result);
895 }
896 }
897 }
898 reduce(src, |x| x, |a, b| a + b, T::zero(), exec)
899}
900
901pub fn dot<A, B, R, OpA, OpB>(a: &StridedView<A, OpA>, b: &StridedView<B, OpB>) -> Result<R>
932where
933 A: Copy + Mul<B, Output = R> + Send + Sync + 'static,
934 B: Copy + Send + Sync + 'static,
935 R: Copy + Zero + Add<Output = R> + Send + Sync + simd::MaybeSimdOps + 'static,
936 OpA: ElementOp<A>,
937 OpB: ElementOp<B>,
938{
939 ensure_same_shape(a.dims(), b.dims())?;
940
941 let a_ptr = a.ptr();
942 let b_ptr = b.ptr();
943 let a_strides = a.strides();
944 let b_strides = b.strides();
945 let a_dims = a.dims();
946 let len = total_len(a_dims)?;
947
948 if len == 0 {
952 return Ok(R::zero());
953 }
954
955 if same_contiguous_layout(a_dims, &[a_strides, b_strides]).is_some() {
956 if OpA::IS_IDENTITY
958 && OpB::IS_IDENTITY
959 && std::any::TypeId::of::<A>() == std::any::TypeId::of::<R>()
960 && std::any::TypeId::of::<B>() == std::any::TypeId::of::<R>()
961 {
962 let sa = unsafe { std::slice::from_raw_parts(a_ptr as *const R, len) };
963 let sb = unsafe { std::slice::from_raw_parts(b_ptr as *const R, len) };
964 if let Some(result) = R::try_simd_dot(sa, sb) {
965 return Ok(result);
966 }
967 }
968
969 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
971 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
972 let mut acc = R::zero();
973 simd::dispatch_if_large(len, || {
974 for i in 0..len {
975 acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
976 }
977 });
978 return Ok(acc);
979 }
980
981 let strides_list: [&[isize]; 2] = [a_strides, b_strides];
982 let elem_size = std::mem::size_of::<A>()
983 .max(std::mem::size_of::<B>())
984 .max(std::mem::size_of::<R>());
985
986 let total = total_len(a_dims)?;
987 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
988 build_plan_fused_small(a_dims, &strides_list)
989 } else {
990 build_plan_fused(a_dims, &strides_list, None, elem_size)
991 };
992
993 let mut acc = R::zero();
994 let initial_offsets: crate::inline::OperandVec<isize> =
995 ordered_strides.iter().map(|_| 0isize).collect();
996 for_each_inner_block_preordered(
997 &fused_dims,
998 &plan.block,
999 &ordered_strides,
1000 &initial_offsets,
1001 |offsets, len, strides| {
1002 acc = unsafe {
1003 inner_loop_dot::<A, B, R, OpA, OpB>(
1004 a_ptr.offset(offsets[0]),
1005 strides[0],
1006 b_ptr.offset(offsets[1]),
1007 strides[1],
1008 len,
1009 acc,
1010 )
1011 };
1012 Ok(())
1013 },
1014 )?;
1015
1016 Ok(acc)
1017}
1018
1019pub fn symmetrize_into<T>(
1037 dest: &mut StridedViewMut<T>,
1038 src: &StridedView<T>,
1039 exec: Exec<'_>,
1040) -> Result<()>
1041where
1042 T: Copy
1043 + Add<Output = T>
1044 + Mul<Output = T>
1045 + num_traits::FromPrimitive
1046 + std::ops::Div<Output = T>
1047 + Send
1048 + Sync,
1049{
1050 if src.ndim() != 2 {
1051 return Err(StridedError::RankMismatch(src.ndim(), 2));
1052 }
1053 let rows = src.dims()[0];
1054 let cols = src.dims()[1];
1055 if rows != cols {
1056 return Err(StridedError::NonSquare { rows, cols });
1057 }
1058
1059 let src_t = src.permute(&[1, 0])?;
1060 let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1061
1062 zip_map2_into(dest, src, &src_t, |a, b| (a + b) * half, exec)
1063}
1064
1065pub fn symmetrize_conj_into<T>(
1083 dest: &mut StridedViewMut<T>,
1084 src: &StridedView<T>,
1085 exec: Exec<'_>,
1086) -> Result<()>
1087where
1088 T: Copy
1089 + ElementOpApply
1090 + Add<Output = T>
1091 + Mul<Output = T>
1092 + num_traits::FromPrimitive
1093 + std::ops::Div<Output = T>
1094 + Send
1095 + Sync,
1096{
1097 if src.ndim() != 2 {
1098 return Err(StridedError::RankMismatch(src.ndim(), 2));
1099 }
1100 let rows = src.dims()[0];
1101 let cols = src.dims()[1];
1102 if rows != cols {
1103 return Err(StridedError::NonSquare { rows, cols });
1104 }
1105
1106 let src_adj = src.adjoint_2d()?;
1108 let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1109
1110 zip_map2_into(dest, src, &src_adj, |a, b| (a + b) * half, exec)
1111}
1112
1113pub fn copy_scale_into<D, S, A, Op>(
1126 dest: &mut StridedViewMut<D>,
1127 src: &StridedView<S, Op>,
1128 scale: A,
1129 exec: Exec<'_>,
1130) -> Result<()>
1131where
1132 A: Copy + Mul<S, Output = D> + Sync,
1133 D: Copy + Send + Sync,
1134 S: Copy + Send + Sync,
1135 Op: ElementOp<S>,
1136{
1137 map_into(dest, src, |x| scale * x, exec)
1138}
1139
1140pub fn copy_conj_into<T: Copy + ElementOpApply + Send + Sync>(
1152 dest: &mut StridedViewMut<T>,
1153 src: &StridedView<T>,
1154 exec: Exec<'_>,
1155) -> Result<()> {
1156 let src_conj = src.conj();
1157 copy_into(dest, &src_conj, exec)
1158}
1159
1160#[inline]
1161fn element_transpose_is_identity<T: 'static>() -> bool {
1162 use std::any::TypeId;
1163
1164 macro_rules! matches_type {
1165 ($($ty:ty),* $(,)?) => {{
1166 let id = TypeId::of::<T>();
1167 false $(|| id == TypeId::of::<$ty>())*
1168 }};
1169 }
1170
1171 matches_type!(
1172 f32,
1173 f64,
1174 i8,
1175 i16,
1176 i32,
1177 i64,
1178 i128,
1179 isize,
1180 u8,
1181 u16,
1182 u32,
1183 u64,
1184 u128,
1185 usize,
1186 num_complex::Complex32,
1187 num_complex::Complex64,
1188 )
1189}
1190
1191#[inline]
1192fn element_zero_is_all_bits_zero<T: 'static>() -> bool {
1193 use std::any::TypeId;
1194
1195 macro_rules! matches_type {
1196 ($($ty:ty),* $(,)?) => {{
1197 let id = TypeId::of::<T>();
1198 false $(|| id == TypeId::of::<$ty>())*
1199 }};
1200 }
1201
1202 matches_type!(
1203 f32,
1204 f64,
1205 i8,
1206 i16,
1207 i32,
1208 i64,
1209 i128,
1210 isize,
1211 u8,
1212 u16,
1213 u32,
1214 u64,
1215 u128,
1216 usize,
1217 num_complex::Complex32,
1218 num_complex::Complex64,
1219 )
1220}
1221
1222#[inline]
1223unsafe fn fill_2d<T: Copy + Send + Sync>(
1224 dst: *mut T,
1225 dim0: usize,
1226 dim1: usize,
1227 dst_stride0: isize,
1228 dst_stride1: isize,
1229 value: T,
1230 exec: Exec<'_>,
1231) {
1232 let total = dim0.saturating_mul(dim1);
1233 let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
1234 if !exec.is_serial() {
1235 let dst_send = SendPtr(dst);
1236 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1237 crate::threading::parallel_for_each(0..dim1, exec, &|columns| {
1238 for j in columns {
1239 let dst = dst_send.as_ptr();
1240 unsafe {
1241 let base = j as isize * dst_stride1;
1242 for i in 0..dim0 {
1243 *dst.offset(base + i as isize * dst_stride0) = value;
1244 }
1245 }
1246 }
1247 });
1248 } else {
1249 crate::threading::parallel_for_each(0..dim0, exec, &|rows| {
1250 for i in rows {
1251 let dst = dst_send.as_ptr();
1252 unsafe {
1253 let base = i as isize * dst_stride0;
1254 for j in 0..dim1 {
1255 *dst.offset(base + j as isize * dst_stride1) = value;
1256 }
1257 }
1258 }
1259 });
1260 }
1261 return;
1262 }
1263
1264 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1265 for j in 0..dim1 {
1266 let base = j as isize * dst_stride1;
1267 for i in 0..dim0 {
1268 *dst.offset(base + i as isize * dst_stride0) = value;
1269 }
1270 }
1271 } else {
1272 for i in 0..dim0 {
1273 let base = i as isize * dst_stride0;
1274 for j in 0..dim1 {
1275 *dst.offset(base + j as isize * dst_stride1) = value;
1276 }
1277 }
1278 }
1279}
1280
1281#[inline]
1282unsafe fn fill_contiguous<T>(dst: *mut T, len: usize, value: T)
1283where
1284 T: Copy + Zero + PartialEq + Send + Sync + 'static,
1285{
1286 if element_zero_is_all_bits_zero::<T>() && value == T::zero() {
1287 std::ptr::write_bytes(dst, 0, len);
1288 return;
1289 }
1290
1291 let dst = std::slice::from_raw_parts_mut(dst, len);
1292 dst.fill(value);
1293}
1294
1295#[inline(always)]
1296unsafe fn transpose_scale_4x4_f64(
1297 dst: *mut f64,
1298 dst_stride0: isize,
1299 dst_stride1: isize,
1300 src: *const f64,
1301 src_stride0: isize,
1302 src_stride1: isize,
1303 i: usize,
1304 j: usize,
1305 scale: f64,
1306) {
1307 let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1308
1309 let s00 = *src_base;
1310 let s10 = *src_base.offset(src_stride0);
1311 let s20 = *src_base.offset(2 * src_stride0);
1312 let s30 = *src_base.offset(3 * src_stride0);
1313
1314 let src_col1 = src_base.offset(src_stride1);
1315 let s01 = *src_col1;
1316 let s11 = *src_col1.offset(src_stride0);
1317 let s21 = *src_col1.offset(2 * src_stride0);
1318 let s31 = *src_col1.offset(3 * src_stride0);
1319
1320 let src_col2 = src_base.offset(2 * src_stride1);
1321 let s02 = *src_col2;
1322 let s12 = *src_col2.offset(src_stride0);
1323 let s22 = *src_col2.offset(2 * src_stride0);
1324 let s32 = *src_col2.offset(3 * src_stride0);
1325
1326 let src_col3 = src_base.offset(3 * src_stride1);
1327 let s03 = *src_col3;
1328 let s13 = *src_col3.offset(src_stride0);
1329 let s23 = *src_col3.offset(2 * src_stride0);
1330 let s33 = *src_col3.offset(3 * src_stride0);
1331
1332 let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1333 *dst_row0 = scale * s00;
1334 *dst_row0.offset(dst_stride0) = scale * s01;
1335 *dst_row0.offset(2 * dst_stride0) = scale * s02;
1336 *dst_row0.offset(3 * dst_stride0) = scale * s03;
1337
1338 let dst_row1 = dst_row0.offset(dst_stride1);
1339 *dst_row1 = scale * s10;
1340 *dst_row1.offset(dst_stride0) = scale * s11;
1341 *dst_row1.offset(2 * dst_stride0) = scale * s12;
1342 *dst_row1.offset(3 * dst_stride0) = scale * s13;
1343
1344 let dst_row2 = dst_row0.offset(2 * dst_stride1);
1345 *dst_row2 = scale * s20;
1346 *dst_row2.offset(dst_stride0) = scale * s21;
1347 *dst_row2.offset(2 * dst_stride0) = scale * s22;
1348 *dst_row2.offset(3 * dst_stride0) = scale * s23;
1349
1350 let dst_row3 = dst_row0.offset(3 * dst_stride1);
1351 *dst_row3 = scale * s30;
1352 *dst_row3.offset(dst_stride0) = scale * s31;
1353 *dst_row3.offset(2 * dst_stride0) = scale * s32;
1354 *dst_row3.offset(3 * dst_stride0) = scale * s33;
1355}
1356
1357#[inline]
1358#[allow(clippy::too_many_arguments)]
1359unsafe fn copy_transpose_scale_2d_f64_tiled_raw(
1360 dst: *mut f64,
1361 dst_stride0: isize,
1362 dst_stride1: isize,
1363 src: *const f64,
1364 src_stride0: isize,
1365 src_stride1: isize,
1366 src_rows: usize,
1367 src_cols: usize,
1368 scale: f64,
1369 exec: Exec<'_>,
1370) {
1371 const TILE: usize = 4;
1372 let row_full = src_rows / TILE * TILE;
1373 let col_full = src_cols / TILE * TILE;
1374
1375 let total = src_rows.saturating_mul(src_cols);
1376 let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1377 if !exec.is_serial() {
1378 let dst_send = SendPtr(dst);
1379 let src_send = SendPtr(src as *mut f64);
1380 let row_tiles = row_full / TILE;
1381 crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1382 for tile_i in tiles {
1383 let i = tile_i * TILE;
1384 let dst = dst_send.as_ptr();
1385 let src = src_send.as_const();
1386 unsafe {
1387 let mut j = 0;
1388 while j < col_full {
1389 transpose_scale_4x4_f64(
1390 dst,
1391 dst_stride0,
1392 dst_stride1,
1393 src,
1394 src_stride0,
1395 src_stride1,
1396 i,
1397 j,
1398 scale,
1399 );
1400 j += TILE;
1401 }
1402 for j in col_full..src_cols {
1403 for ii in i..i + TILE {
1404 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1405 scale
1406 * *src.offset(
1407 ii as isize * src_stride0 + j as isize * src_stride1,
1408 );
1409 }
1410 }
1411 }
1412 }
1413 });
1414 for i in row_full..src_rows {
1415 for j in 0..src_cols {
1416 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1417 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1418 }
1419 }
1420 return;
1421 }
1422
1423 let mut i = 0;
1424 while i < row_full {
1425 let mut j = 0;
1426 while j < col_full {
1427 transpose_scale_4x4_f64(
1428 dst,
1429 dst_stride0,
1430 dst_stride1,
1431 src,
1432 src_stride0,
1433 src_stride1,
1434 i,
1435 j,
1436 scale,
1437 );
1438 j += TILE;
1439 }
1440 for j in col_full..src_cols {
1441 for ii in i..i + TILE {
1442 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1443 scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1444 }
1445 }
1446 i += TILE;
1447 }
1448 for i in row_full..src_rows {
1449 for j in 0..src_cols {
1450 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1451 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1452 }
1453 }
1454}
1455
1456#[inline]
1457#[cfg(test)]
1458unsafe fn try_copy_transpose_scale_2d_f64_tiled(
1459 dest: &mut StridedViewMut<f64>,
1460 src: &StridedView<f64>,
1461 scale: f64,
1462 exec: Exec<'_>,
1463) -> bool {
1464 if src.ndim() != 2 || dest.ndim() != 2 {
1465 return false;
1466 }
1467 let src_dims = src.dims();
1468 if dest.dims() != [src_dims[1], src_dims[0]] {
1469 return false;
1470 }
1471 if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1472 return false;
1473 }
1474
1475 copy_transpose_scale_2d_f64_tiled_raw(
1476 dest.as_mut_ptr(),
1477 dest.strides()[0],
1478 dest.strides()[1],
1479 src.ptr(),
1480 src.strides()[0],
1481 src.strides()[1],
1482 src_dims[0],
1483 src_dims[1],
1484 scale,
1485 exec,
1486 );
1487 true
1488}
1489
1490#[inline]
1491unsafe fn try_copy_transpose_scale_2d_f64_tiled_typed<T>(
1492 dest: &mut StridedViewMut<T>,
1493 src: &StridedView<T>,
1494 scale: T,
1495 exec: Exec<'_>,
1496) -> bool
1497where
1498 T: Copy + 'static,
1499{
1500 if std::any::TypeId::of::<T>() != std::any::TypeId::of::<f64>() {
1501 return false;
1502 }
1503
1504 let scale = *(&scale as *const T).cast::<f64>();
1505 if src.ndim() != 2 || dest.ndim() != 2 {
1506 return false;
1507 }
1508 let src_dims = src.dims();
1509 if dest.dims() != [src_dims[1], src_dims[0]] {
1510 return false;
1511 }
1512 if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1513 return false;
1514 }
1515
1516 copy_transpose_scale_2d_f64_tiled_raw(
1517 dest.as_mut_ptr().cast::<f64>(),
1518 dest.strides()[0],
1519 dest.strides()[1],
1520 src.ptr().cast::<f64>(),
1521 src.strides()[0],
1522 src.strides()[1],
1523 src_dims[0],
1524 src_dims[1],
1525 scale,
1526 exec,
1527 );
1528 true
1529}
1530
1531#[inline(always)]
1532unsafe fn transpose_scale_4x4_identity<T>(
1533 dst: *mut T,
1534 dst_stride0: isize,
1535 dst_stride1: isize,
1536 src: *const T,
1537 src_stride0: isize,
1538 src_stride1: isize,
1539 i: usize,
1540 j: usize,
1541 scale: T,
1542) where
1543 T: Copy + Mul<Output = T>,
1544{
1545 let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1546
1547 let s00 = *src_base;
1548 let s10 = *src_base.offset(src_stride0);
1549 let s20 = *src_base.offset(2 * src_stride0);
1550 let s30 = *src_base.offset(3 * src_stride0);
1551
1552 let src_col1 = src_base.offset(src_stride1);
1553 let s01 = *src_col1;
1554 let s11 = *src_col1.offset(src_stride0);
1555 let s21 = *src_col1.offset(2 * src_stride0);
1556 let s31 = *src_col1.offset(3 * src_stride0);
1557
1558 let src_col2 = src_base.offset(2 * src_stride1);
1559 let s02 = *src_col2;
1560 let s12 = *src_col2.offset(src_stride0);
1561 let s22 = *src_col2.offset(2 * src_stride0);
1562 let s32 = *src_col2.offset(3 * src_stride0);
1563
1564 let src_col3 = src_base.offset(3 * src_stride1);
1565 let s03 = *src_col3;
1566 let s13 = *src_col3.offset(src_stride0);
1567 let s23 = *src_col3.offset(2 * src_stride0);
1568 let s33 = *src_col3.offset(3 * src_stride0);
1569
1570 let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1571 *dst_row0 = scale * s00;
1572 *dst_row0.offset(dst_stride0) = scale * s01;
1573 *dst_row0.offset(2 * dst_stride0) = scale * s02;
1574 *dst_row0.offset(3 * dst_stride0) = scale * s03;
1575
1576 let dst_row1 = dst_row0.offset(dst_stride1);
1577 *dst_row1 = scale * s10;
1578 *dst_row1.offset(dst_stride0) = scale * s11;
1579 *dst_row1.offset(2 * dst_stride0) = scale * s12;
1580 *dst_row1.offset(3 * dst_stride0) = scale * s13;
1581
1582 let dst_row2 = dst_row0.offset(2 * dst_stride1);
1583 *dst_row2 = scale * s20;
1584 *dst_row2.offset(dst_stride0) = scale * s21;
1585 *dst_row2.offset(2 * dst_stride0) = scale * s22;
1586 *dst_row2.offset(3 * dst_stride0) = scale * s23;
1587
1588 let dst_row3 = dst_row0.offset(3 * dst_stride1);
1589 *dst_row3 = scale * s30;
1590 *dst_row3.offset(dst_stride0) = scale * s31;
1591 *dst_row3.offset(2 * dst_stride0) = scale * s32;
1592 *dst_row3.offset(3 * dst_stride0) = scale * s33;
1593}
1594
1595#[inline]
1596#[allow(clippy::too_many_arguments)]
1597unsafe fn copy_transpose_scale_2d_identity_tiled_raw<T>(
1598 dst: *mut T,
1599 dst_stride0: isize,
1600 dst_stride1: isize,
1601 src: *const T,
1602 src_stride0: isize,
1603 src_stride1: isize,
1604 src_rows: usize,
1605 src_cols: usize,
1606 scale: T,
1607 exec: Exec<'_>,
1608) where
1609 T: Copy + Mul<Output = T> + Send + Sync,
1610{
1611 const TILE: usize = 4;
1612 let row_full = src_rows / TILE * TILE;
1613 let col_full = src_cols / TILE * TILE;
1614
1615 let total = src_rows.saturating_mul(src_cols);
1616 let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1617 if !exec.is_serial() {
1618 let dst_send = SendPtr(dst);
1619 let src_send = SendPtr(src as *mut T);
1620 let row_tiles = row_full / TILE;
1621 crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1622 for tile_i in tiles {
1623 let i = tile_i * TILE;
1624 let dst = dst_send.as_ptr();
1625 let src = src_send.as_const();
1626 unsafe {
1627 let mut j = 0;
1628 while j < col_full {
1629 transpose_scale_4x4_identity(
1630 dst,
1631 dst_stride0,
1632 dst_stride1,
1633 src,
1634 src_stride0,
1635 src_stride1,
1636 i,
1637 j,
1638 scale,
1639 );
1640 j += TILE;
1641 }
1642 for j in col_full..src_cols {
1643 for ii in i..i + TILE {
1644 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1645 scale
1646 * *src.offset(
1647 ii as isize * src_stride0 + j as isize * src_stride1,
1648 );
1649 }
1650 }
1651 }
1652 }
1653 });
1654 for i in row_full..src_rows {
1655 for j in 0..src_cols {
1656 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1657 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1658 }
1659 }
1660 return;
1661 }
1662
1663 let mut i = 0;
1664 while i < row_full {
1665 let mut j = 0;
1666 while j < col_full {
1667 transpose_scale_4x4_identity(
1668 dst,
1669 dst_stride0,
1670 dst_stride1,
1671 src,
1672 src_stride0,
1673 src_stride1,
1674 i,
1675 j,
1676 scale,
1677 );
1678 j += TILE;
1679 }
1680 for j in col_full..src_cols {
1681 for ii in i..i + TILE {
1682 *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1683 scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1684 }
1685 }
1686 i += TILE;
1687 }
1688 for i in row_full..src_rows {
1689 for j in 0..src_cols {
1690 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1691 scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1692 }
1693 }
1694}
1695
1696#[inline]
1697unsafe fn try_copy_transpose_scale_2d_identity_tiled<T>(
1698 dest: &mut StridedViewMut<T>,
1699 src: &StridedView<T>,
1700 scale: T,
1701 exec: Exec<'_>,
1702) -> bool
1703where
1704 T: Copy + Mul<Output = T> + Send + Sync,
1705{
1706 if src.ndim() != 2 || dest.ndim() != 2 {
1707 return false;
1708 }
1709 let src_dims = src.dims();
1710 if dest.dims() != [src_dims[1], src_dims[0]] {
1711 return false;
1712 }
1713 if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1714 return false;
1715 }
1716
1717 copy_transpose_scale_2d_identity_tiled_raw(
1718 dest.as_mut_ptr(),
1719 dest.strides()[0],
1720 dest.strides()[1],
1721 src.ptr(),
1722 src.strides()[0],
1723 src.strides()[1],
1724 src_dims[0],
1725 src_dims[1],
1726 scale,
1727 exec,
1728 );
1729 true
1730}
1731
1732#[inline]
1733#[allow(clippy::too_many_arguments)]
1734unsafe fn copy_transpose_scale_2d_loop<T>(
1735 dst: *mut T,
1736 dst_stride0: isize,
1737 dst_stride1: isize,
1738 src: *const T,
1739 src_stride0: isize,
1740 src_stride1: isize,
1741 src_rows: usize,
1742 src_cols: usize,
1743 scale: T,
1744 exec: Exec<'_>,
1745) where
1746 T: Copy + ElementOpApply + Mul<Output = T> + Send + Sync,
1747{
1748 let total = src_rows.saturating_mul(src_cols);
1749 let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1750 if !exec.is_serial() {
1751 let dst_send = SendPtr(dst);
1752 let src_send = SendPtr(src as *mut T);
1753 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1754 crate::threading::parallel_for_each(0..src_rows, exec, &|rows| {
1755 for i in rows {
1756 let dst = dst_send.as_ptr();
1757 let src = src_send.as_const();
1758 unsafe {
1759 for j in 0..src_cols {
1760 let value = (*src
1761 .offset(i as isize * src_stride0 + j as isize * src_stride1))
1762 .transpose();
1763 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1764 scale * value;
1765 }
1766 }
1767 }
1768 });
1769 } else {
1770 crate::threading::parallel_for_each(0..src_cols, exec, &|columns| {
1771 for j in columns {
1772 let dst = dst_send.as_ptr();
1773 let src = src_send.as_const();
1774 unsafe {
1775 for i in 0..src_rows {
1776 let value = (*src
1777 .offset(i as isize * src_stride0 + j as isize * src_stride1))
1778 .transpose();
1779 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1780 scale * value;
1781 }
1782 }
1783 }
1784 });
1785 }
1786 return;
1787 }
1788
1789 if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1790 for i in 0..src_rows {
1791 for j in 0..src_cols {
1792 let value =
1793 (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1794 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1795 }
1796 }
1797 } else {
1798 for j in 0..src_cols {
1799 for i in 0..src_rows {
1800 let value =
1801 (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1802 *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1803 }
1804 }
1805 }
1806}
1807
1808pub fn copy_transpose_scale_into<T>(
1822 dest: &mut StridedViewMut<T>,
1823 src: &StridedView<T>,
1824 scale: T,
1825 exec: Exec<'_>,
1826) -> Result<()>
1827where
1828 T: Copy + ElementOpApply + Mul<Output = T> + Zero + One + PartialEq + Send + Sync + 'static,
1829{
1830 let mut exec = exec;
1831 if src.ndim() != 2 || dest.ndim() != 2 {
1832 return Err(StridedError::RankMismatch(src.ndim(), 2));
1833 }
1834
1835 let src_dims = src.dims();
1836 let expected_dims = [src_dims[1], src_dims[0]];
1837 ensure_same_shape(dest.dims(), &expected_dims)?;
1838 total_len(src_dims)?;
1841
1842 if scale == T::zero() {
1843 unsafe {
1844 if same_contiguous_layout(dest.dims(), &[dest.strides()]).is_some() {
1845 fill_contiguous(dest.as_mut_ptr(), total_len(dest.dims())?, T::zero());
1846 } else {
1847 fill_2d(
1848 dest.as_mut_ptr(),
1849 dest.dims()[0],
1850 dest.dims()[1],
1851 dest.strides()[0],
1852 dest.strides()[1],
1853 T::zero(),
1854 exec,
1855 );
1856 }
1857 }
1858 return Ok(());
1859 }
1860
1861 let transpose_is_identity = element_transpose_is_identity::<T>();
1862
1863 unsafe {
1864 if transpose_is_identity
1865 && try_copy_transpose_scale_2d_f64_tiled_typed(dest, src, scale, exec.reborrow())
1866 {
1867 return Ok(());
1868 }
1869 if transpose_is_identity
1870 && try_copy_transpose_scale_2d_identity_tiled(dest, src, scale, exec.reborrow())
1871 {
1872 return Ok(());
1873 }
1874 }
1875
1876 if scale == T::one() && transpose_is_identity {
1877 let src_t = src.permute(&[1, 0])?;
1878 return crate::threading::copy_permuted(dest, &src_t, exec);
1879 }
1880
1881 unsafe {
1882 copy_transpose_scale_2d_loop(
1883 dest.as_mut_ptr(),
1884 dest.strides()[0],
1885 dest.strides()[1],
1886 src.ptr(),
1887 src.strides()[0],
1888 src.strides()[1],
1889 src_dims[0],
1890 src_dims[1],
1891 scale,
1892 exec,
1893 );
1894 }
1895 Ok(())
1896}
1897
1898#[cfg(test)]
1899#[path = "ops_view/tests/tiled_tests.rs"]
1900mod tiled_tests;