1use crate::kernel::{
10 build_plan_fused, build_plan_fused_small, ensure_same_shape, for_each_inner_block_preordered,
11 sequential_contiguous_layout, total_len, SMALL_TENSOR_THRESHOLD,
12};
13use crate::simd;
14use crate::view::{StridedView, StridedViewMut};
15use crate::{Result, StridedError};
16use core::mem::MaybeUninit;
17use std::ops::Mul;
18use strided_view::{ElementOp, Identity};
19
20use crate::cost::{for_len, KernelClass};
21use crate::fuse::compute_costs;
22use crate::threading::{for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded};
23use crate::Exec;
24
25type AxisVec<T> = crate::inline::InlineVec<T>;
26
27const TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT: usize = 8;
36
37#[derive(Clone, Copy, Debug, Default)]
43pub struct ValidatedDestinationLayout(());
44
45#[inline]
61pub fn validate_destination_layout(
62 dims: &[usize],
63 strides: &[isize],
64) -> Result<ValidatedDestinationLayout> {
65 if crate::layout_check::is_injective_layout(dims, strides) {
66 Ok(ValidatedDestinationLayout(()))
67 } else {
68 Err(StridedError::NonInjectiveOutputLayout)
69 }
70}
71
72#[inline]
73pub fn validate_destination_layout_without_alloc(
87 dims: &[usize],
88 strides: &[isize],
89) -> Result<ValidatedDestinationLayout> {
90 if crate::layout_check::is_injective_layout_without_alloc(dims, strides) {
91 Ok(ValidatedDestinationLayout(()))
92 } else {
93 Err(StridedError::NonInjectiveOutputLayout)
94 }
95}
96
97fn reachable_byte_range(
98 ptr: usize,
99 elem_size: usize,
100 dims: &[usize],
101 strides: &[isize],
102) -> Result<Option<(usize, usize)>> {
103 if elem_size == 0 || dims.contains(&0) {
104 return Ok(None);
105 }
106
107 let mut min_offset = 0isize;
108 let mut max_offset = 0isize;
109 for (&dim, &stride) in dims.iter().zip(strides) {
110 if dim <= 1 {
111 continue;
112 }
113 let extent = isize::try_from(dim - 1).map_err(|_| StridedError::OffsetOverflow)?;
114 let span = stride
115 .checked_mul(extent)
116 .ok_or(StridedError::OffsetOverflow)?;
117 if span < 0 {
118 min_offset = min_offset
119 .checked_add(span)
120 .ok_or(StridedError::OffsetOverflow)?;
121 } else {
122 max_offset = max_offset
123 .checked_add(span)
124 .ok_or(StridedError::OffsetOverflow)?;
125 }
126 }
127
128 let ptr = ptr as i128;
129 let elem_size = elem_size as i128;
130 let start = ptr
131 .checked_add(
132 (min_offset as i128)
133 .checked_mul(elem_size)
134 .ok_or(StridedError::OffsetOverflow)?,
135 )
136 .ok_or(StridedError::OffsetOverflow)?;
137 let end = ptr
138 .checked_add(
139 (max_offset as i128)
140 .checked_mul(elem_size)
141 .and_then(|offset| offset.checked_add(elem_size))
142 .ok_or(StridedError::OffsetOverflow)?,
143 )
144 .ok_or(StridedError::OffsetOverflow)?;
145 if start < 0 || end < 0 || start > usize::MAX as i128 || end > usize::MAX as i128 {
146 return Err(StridedError::OffsetOverflow);
147 }
148 Ok(Some((start as usize, end as usize)))
149}
150
151fn validate_typed_no_overlap<D, A, Op: ElementOp<A>>(
152 dest: &StridedViewMut<MaybeUninit<D>>,
153 input: &StridedView<A, Op>,
154 input_index: usize,
155) -> Result<()> {
156 let dest_range = reachable_byte_range(
157 dest.ptr() as usize,
158 core::mem::size_of::<D>(),
159 dest.dims(),
160 dest.strides(),
161 )?;
162 let input_range = reachable_byte_range(
163 input.ptr() as usize,
164 core::mem::size_of::<A>(),
165 input.dims(),
166 input.strides(),
167 )?;
168 if let (Some((dest_start, dest_end)), Some((input_start, input_end))) =
169 (dest_range, input_range)
170 {
171 if dest_start < input_end && input_start < dest_end {
172 return Err(StridedError::OverlappingInputOutput { input: input_index });
173 }
174 }
175 Ok(())
176}
177
178trait MapKernel<A, D> {
196 fn apply(&self, src: &[A], dst: &mut [D]) -> bool;
198}
199
200struct NoKernel;
201
202impl<A, D> MapKernel<A, D> for NoKernel {
203 #[inline(always)]
204 fn apply(&self, _src: &[A], _dst: &mut [D]) -> bool {
205 false
206 }
207}
208
209struct SliceKernel<K>(K);
210
211impl<A, D, K: Fn(&[A], &mut [D]) -> bool> MapKernel<A, D> for SliceKernel<K> {
212 #[inline(always)]
213 fn apply(&self, src: &[A], dst: &mut [D]) -> bool {
214 (self.0)(src, dst)
215 }
216}
217
218unsafe fn inner_loop_map1<D: Copy, A: Copy, Op: ElementOp<A>, K: MapKernel<A, D>>(
219 dp: *mut D,
220 ds: isize,
221 sp: *const A,
222 ss: isize,
223 len: usize,
224 f: &impl Fn(A) -> D,
225 kernel: &K,
226) {
227 if ds == 1 && ss == 1 {
228 let src = std::slice::from_raw_parts(sp, len);
229 let dst = std::slice::from_raw_parts_mut(dp, len);
230 if kernel.apply(src, dst) {
231 return;
232 }
233 simd::dispatch_if_large(len, || {
234 for (d, s) in dst.iter_mut().zip(src.iter()) {
235 *d = f(Op::apply(*s));
236 }
237 });
238 } else {
239 let mut dp = dp;
240 let mut sp = sp;
241 simd::dispatch_if_large(len, || {
242 for _ in 0..len {
243 *dp = f(Op::apply(*sp));
244 dp = dp.wrapping_offset(ds);
245 sp = sp.wrapping_offset(ss);
246 }
247 });
248 }
249}
250
251#[allow(clippy::too_many_arguments)]
255#[inline(always)]
256unsafe fn inner_loop_map2<D: Copy, A: Copy, B: Copy, OpA: ElementOp<A>, OpB: ElementOp<B>>(
257 dp: *mut D,
258 ds: isize,
259 ap: *const A,
260 a_s: isize,
261 bp: *const B,
262 b_s: isize,
263 len: usize,
264 f: &impl Fn(A, B) -> D,
265) {
266 if ds == 1 && a_s == 1 && b_s == 1 {
267 let src_a = std::slice::from_raw_parts(ap, len);
268 let src_b = std::slice::from_raw_parts(bp, len);
269 let dst = std::slice::from_raw_parts_mut(dp, len);
270 simd::dispatch_if_large(len, || {
271 for i in 0..len {
272 dst[i] = f(OpA::apply(src_a[i]), OpB::apply(src_b[i]));
273 }
274 });
275 } else if ds == 1 && a_s == 1 && b_s == 0 {
276 let src_a = std::slice::from_raw_parts(ap, len);
277 let b = OpB::apply(*bp);
278 let dst = std::slice::from_raw_parts_mut(dp, len);
279 simd::dispatch_if_large(len, || {
280 for i in 0..len {
281 dst[i] = f(OpA::apply(src_a[i]), b);
282 }
283 });
284 } else if ds == 1 && a_s == 0 && b_s == 1 {
285 let a = OpA::apply(*ap);
286 let src_b = std::slice::from_raw_parts(bp, len);
287 let dst = std::slice::from_raw_parts_mut(dp, len);
288 simd::dispatch_if_large(len, || {
289 for i in 0..len {
290 dst[i] = f(a, OpB::apply(src_b[i]));
291 }
292 });
293 } else if ds == 1 && a_s == 0 && b_s == 0 {
294 let a = OpA::apply(*ap);
295 let b = OpB::apply(*bp);
296 let dst = std::slice::from_raw_parts_mut(dp, len);
297 simd::dispatch_if_large(len, || {
298 for d in dst.iter_mut() {
299 *d = f(a, b);
300 }
301 });
302 } else if ds == 1 && b_s == 0 {
303 let b = OpB::apply(*bp);
304 let dst = std::slice::from_raw_parts_mut(dp, len);
305 let mut ap = ap;
306 simd::dispatch_if_large(len, || {
307 for d in dst.iter_mut() {
308 *d = f(OpA::apply(*ap), b);
309 ap = ap.wrapping_offset(a_s);
310 }
311 });
312 } else if ds == 1 && a_s == 0 {
313 let a = OpA::apply(*ap);
314 let dst = std::slice::from_raw_parts_mut(dp, len);
315 let mut bp = bp;
316 simd::dispatch_if_large(len, || {
317 for d in dst.iter_mut() {
318 *d = f(a, OpB::apply(*bp));
319 bp = bp.wrapping_offset(b_s);
320 }
321 });
322 } else {
323 let mut dp = dp;
324 let mut ap = ap;
325 let mut bp = bp;
326 simd::dispatch_if_large(len, || {
327 for _ in 0..len {
328 *dp = f(OpA::apply(*ap), OpB::apply(*bp));
329 dp = dp.wrapping_offset(ds);
330 ap = ap.wrapping_offset(a_s);
331 bp = bp.wrapping_offset(b_s);
332 }
333 });
334 }
335}
336
337trait MulOutput<D: Copy + 'static>: Copy + Send + Sync + 'static {
339 type Slot: Copy + Send + Sync + 'static;
340
341 unsafe fn write(dst: *mut Self::Slot, value: D);
342
343 unsafe fn try_contiguous<A: 'static, B: 'static>(
344 dst: *mut Self::Slot,
345 len: usize,
346 a: &[A],
347 b: &[B],
348 ) -> bool;
349}
350
351#[derive(Clone, Copy)]
352struct InitializedOutput;
353
354impl<D: Copy + Send + Sync + 'static> MulOutput<D> for InitializedOutput {
355 type Slot = D;
356
357 #[inline(always)]
358 unsafe fn write(dst: *mut D, value: D) {
359 unsafe { dst.write(value) };
360 }
361
362 #[inline(always)]
363 unsafe fn try_contiguous<A: 'static, B: 'static>(
364 dst: *mut D,
365 len: usize,
366 a: &[A],
367 b: &[B],
368 ) -> bool {
369 unsafe { simd::try_mul_contiguous_ptr(dst, len, a, b) }
370 }
371}
372
373#[derive(Clone, Copy)]
374struct UninitializedOutput;
375
376impl<D: Copy + Send + Sync + 'static> MulOutput<D> for UninitializedOutput {
377 type Slot = MaybeUninit<D>;
378
379 #[inline(always)]
380 unsafe fn write(dst: *mut MaybeUninit<D>, value: D) {
381 unsafe { dst.write(MaybeUninit::new(value)) };
382 }
383
384 #[inline(always)]
385 unsafe fn try_contiguous<A: 'static, B: 'static>(
386 dst: *mut MaybeUninit<D>,
387 len: usize,
388 a: &[A],
389 b: &[B],
390 ) -> bool {
391 unsafe { simd::try_mul_contiguous_ptr(dst.cast::<D>(), len, a, b) }
394 }
395}
396
397#[inline(always)]
398fn multiply_value<A, B, D>(lhs: A, rhs: B) -> D
399where
400 A: Copy + Mul<B, Output = D> + 'static,
401 B: Copy + 'static,
402 D: Copy + 'static,
403{
404 use core::any::TypeId;
405
406 if TypeId::of::<A>() == TypeId::of::<i32>()
407 && TypeId::of::<B>() == TypeId::of::<i32>()
408 && TypeId::of::<D>() == TypeId::of::<i32>()
409 {
410 let lhs = unsafe { *(&lhs as *const A).cast::<i32>() };
412 let rhs = unsafe { *(&rhs as *const B).cast::<i32>() };
413 let value = lhs.wrapping_mul(rhs);
414 return unsafe { core::mem::transmute_copy(&value) };
415 }
416 if TypeId::of::<A>() == TypeId::of::<i64>()
417 && TypeId::of::<B>() == TypeId::of::<i64>()
418 && TypeId::of::<D>() == TypeId::of::<i64>()
419 {
420 let lhs = unsafe { *(&lhs as *const A).cast::<i64>() };
422 let rhs = unsafe { *(&rhs as *const B).cast::<i64>() };
423 let value = lhs.wrapping_mul(rhs);
424 return unsafe { core::mem::transmute_copy(&value) };
425 }
426 lhs * rhs
427}
428
429#[inline(always)]
430unsafe fn inner_loop_mul2<
431 O: MulOutput<D>,
432 D: Copy + 'static,
433 A: Copy + Mul<B, Output = D> + 'static,
434 B: Copy + 'static,
435>(
436 dp: *mut O::Slot,
437 ds: isize,
438 ap: *const A,
439 a_s: isize,
440 bp: *const B,
441 b_s: isize,
442 len: usize,
443) {
444 if ds == 1 && a_s == 1 && b_s == 1 {
445 let src_a = std::slice::from_raw_parts(ap, len);
446 let src_b = std::slice::from_raw_parts(bp, len);
447 if len >= 64 && O::try_contiguous(dp, len, src_a, src_b) {
448 return;
449 }
450 simd::dispatch_if_large(len, || {
451 for i in 0..len {
452 O::write(dp.add(i), multiply_value(src_a[i], src_b[i]));
453 }
454 });
455 } else if ds == 1 && a_s == 1 && b_s == 0 {
456 let src_a = std::slice::from_raw_parts(ap, len);
457 let b = *bp;
458 simd::dispatch_if_large(len, || {
459 #[allow(clippy::needless_range_loop)]
462 for i in 0..len {
463 O::write(dp.add(i), multiply_value(src_a[i], b));
464 }
465 });
466 } else if ds == 1 && a_s == 0 && b_s == 1 {
467 let a = *ap;
468 let src_b = std::slice::from_raw_parts(bp, len);
469 simd::dispatch_if_large(len, || {
470 #[allow(clippy::needless_range_loop)]
473 for i in 0..len {
474 O::write(dp.add(i), multiply_value(a, src_b[i]));
475 }
476 });
477 } else if ds == 1 && a_s == 0 && b_s == 0 {
478 let a = *ap;
479 let b = *bp;
480 simd::dispatch_if_large(len, || {
481 for i in 0..len {
482 O::write(dp.add(i), multiply_value(a, b));
483 }
484 });
485 } else if ds == 1 && b_s == 0 {
486 let b = *bp;
487 let mut ap = ap;
488 simd::dispatch_if_large(len, || {
489 for i in 0..len {
490 O::write(dp.add(i), multiply_value(*ap, b));
491 ap = ap.wrapping_offset(a_s);
492 }
493 });
494 } else if ds == 1 && a_s == 0 {
495 let a = *ap;
496 let mut bp = bp;
497 simd::dispatch_if_large(len, || {
498 for i in 0..len {
499 O::write(dp.add(i), multiply_value(a, *bp));
500 bp = bp.wrapping_offset(b_s);
501 }
502 });
503 } else {
504 let mut dp = dp;
505 let mut ap = ap;
506 let mut bp = bp;
507 simd::dispatch_if_large(len, || {
508 for _ in 0..len {
509 O::write(dp, multiply_value(*ap, *bp));
510 dp = dp.wrapping_offset(ds);
511 ap = ap.wrapping_offset(a_s);
512 bp = bp.wrapping_offset(b_s);
513 }
514 });
515 }
516}
517
518#[derive(Clone, Debug, Eq, PartialEq)]
519struct ContiguousMulRangePlan {
520 axis_order: AxisVec<usize>,
521 inner_len: usize,
522 inner_axis_count: usize,
523 row_len: usize,
524 outer_axis_start: usize,
525 fast_axis: usize,
526 a_fast_stride: isize,
527 b_fast_stride: isize,
528 a_row_stride: isize,
529 b_row_stride: isize,
530}
531
532impl ContiguousMulRangePlan {
533 fn walks_inputs_without_blocking(&self) -> bool {
543 let streams = |stride: isize| stride == 0 || stride == 1;
544 if streams(self.a_fast_stride) && streams(self.b_fast_stride) {
545 return true;
546 }
547 if transposed_scalar_tile_kind(self).is_some() {
548 return true;
549 }
550 false
551 }
552}
553
554#[derive(Clone, Copy, Debug, Eq, PartialEq)]
555enum TransposedScalarTileKind {
556 RhsScalar,
557 LhsScalar,
558}
559
560fn transposed_scalar_tile_kind(plan: &ContiguousMulRangePlan) -> Option<TransposedScalarTileKind> {
561 if plan.inner_len > TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT.saturating_mul(plan.row_len) {
562 return None;
563 }
564 let row_len = isize::try_from(plan.row_len).ok()?;
565 if plan.a_fast_stride == row_len
566 && plan.a_row_stride == 1
567 && plan.b_fast_stride == 0
568 && plan.b_row_stride == 0
569 {
570 return Some(TransposedScalarTileKind::RhsScalar);
571 }
572
573 if plan.b_fast_stride == row_len
574 && plan.b_row_stride == 1
575 && plan.a_fast_stride == 0
576 && plan.a_row_stride == 0
577 {
578 return Some(TransposedScalarTileKind::LhsScalar);
579 }
580
581 None
582}
583
584fn compact_axis_order(dims: &[usize], strides: &[isize]) -> Option<AxisVec<usize>> {
585 if dims.len() != strides.len() {
586 return None;
587 }
588
589 let mut active = AxisVec::<usize>::new();
590 let mut inactive = AxisVec::<usize>::new();
591 for (axis, (&dim, &stride)) in dims.iter().zip(strides.iter()).enumerate() {
592 if stride < 0 {
593 return None;
594 }
595 if dim > 1 {
596 active.push(axis);
597 } else {
598 inactive.push(axis);
599 }
600 }
601
602 active.sort_by(|&lhs, &rhs| strides[lhs].cmp(&strides[rhs]).then_with(|| lhs.cmp(&rhs)));
603
604 let mut expected = 1isize;
605 for &axis in &active {
606 if strides[axis] != expected {
607 return None;
608 }
609 expected = expected.saturating_mul(dims[axis] as isize);
610 }
611
612 active.extend(inactive);
613 Some(active)
614}
615
616fn can_fuse_contiguous_range_axis(dim: usize, prev_stride: isize, next_stride: isize) -> bool {
617 dim <= 1 || (prev_stride == 0 && next_stride == 0) || next_stride == prev_stride * dim as isize
618}
619
620fn contiguous_mul_range_plan(
621 dims: &[usize],
622 dst_strides: &[isize],
623 a_strides: &[isize],
624 b_strides: &[isize],
625) -> Option<ContiguousMulRangePlan> {
626 let axis_order = compact_axis_order(dims, dst_strides)?;
627 if dims.is_empty() {
628 return Some(ContiguousMulRangePlan {
629 axis_order,
630 inner_len: 1,
631 inner_axis_count: 0,
632 row_len: 1,
633 outer_axis_start: 0,
634 fast_axis: 0,
635 a_fast_stride: 0,
636 b_fast_stride: 0,
637 a_row_stride: 0,
638 b_row_stride: 0,
639 });
640 }
641
642 let first_pos = axis_order
643 .iter()
644 .position(|&axis| dims[axis] > 1)
645 .unwrap_or(0);
646 let first_axis = axis_order[first_pos];
647 let mut inner_len = dims[first_axis].max(1);
648 let mut inner_axis_count = first_pos + 1;
649 let mut prev_axis = first_axis;
650
651 for &axis in axis_order.iter().skip(first_pos + 1) {
652 if can_fuse_contiguous_range_axis(
653 dims[prev_axis],
654 dst_strides[prev_axis],
655 dst_strides[axis],
656 ) && can_fuse_contiguous_range_axis(
657 dims[prev_axis],
658 a_strides[prev_axis],
659 a_strides[axis],
660 ) && can_fuse_contiguous_range_axis(
661 dims[prev_axis],
662 b_strides[prev_axis],
663 b_strides[axis],
664 ) {
665 inner_len = inner_len.checked_mul(dims[axis].max(1))?;
666 inner_axis_count += 1;
667 prev_axis = axis;
668 } else {
669 break;
670 }
671 }
672
673 let row = axis_order
674 .iter()
675 .enumerate()
676 .skip(inner_axis_count)
677 .find(|&(_, &axis)| dims[axis] > 1);
678 let (row_len, outer_axis_start, a_row_stride, b_row_stride) =
679 if let Some((row_pos, &row_axis)) = row {
680 (
681 dims[row_axis],
682 row_pos + 1,
683 a_strides[row_axis],
684 b_strides[row_axis],
685 )
686 } else {
687 (1, axis_order.len(), 0, 0)
688 };
689
690 Some(ContiguousMulRangePlan {
691 axis_order,
692 inner_len,
693 inner_axis_count,
694 row_len,
695 outer_axis_start,
696 fast_axis: first_axis,
697 a_fast_stride: a_strides[first_axis],
698 b_fast_stride: b_strides[first_axis],
699 a_row_stride,
700 b_row_stride,
701 })
702}
703
704struct ContiguousMulOuterCursor<'a> {
705 dims: &'a [usize],
706 a_strides: &'a [isize],
707 b_strides: &'a [isize],
708 axes: AxisVec<usize>,
709 coords: AxisVec<usize>,
710 a_offset: isize,
711 b_offset: isize,
712}
713
714impl<'a> ContiguousMulOuterCursor<'a> {
715 fn new(
716 dims: &'a [usize],
717 a_strides: &'a [isize],
718 b_strides: &'a [isize],
719 plan: &ContiguousMulRangePlan,
720 outer_group: usize,
721 ) -> Self {
722 let axes: AxisVec<usize> = plan
723 .axis_order
724 .iter()
725 .skip(plan.outer_axis_start)
726 .copied()
727 .collect();
728 let mut coords = AxisVec::<usize>::with_capacity(axes.len());
729 let mut rem = outer_group;
730 let mut a_offset = 0isize;
731 let mut b_offset = 0isize;
732
733 for &axis in &axes {
734 let dim = dims[axis].max(1);
735 let coord = rem % dim;
736 rem /= dim;
737 coords.push(coord);
738 a_offset += coord as isize * a_strides[axis];
739 b_offset += coord as isize * b_strides[axis];
740 }
741
742 Self {
743 dims,
744 a_strides,
745 b_strides,
746 axes,
747 coords,
748 a_offset,
749 b_offset,
750 }
751 }
752
753 fn advance(&mut self) {
754 for (i, &axis) in self.axes.iter().enumerate() {
755 let dim = self.dims[axis].max(1);
756 if dim <= 1 {
757 continue;
758 }
759
760 if self.coords[i] + 1 < dim {
763 self.coords[i] += 1;
764 self.a_offset += self.a_strides[axis];
765 self.b_offset += self.b_strides[axis];
766 break;
767 }
768
769 let last = (dim - 1) as isize;
770 self.coords[i] = 0;
771 self.a_offset -= last * self.a_strides[axis];
772 self.b_offset -= last * self.b_strides[axis];
773 }
774 }
775}
776
777#[allow(clippy::too_many_arguments)]
780#[inline(always)]
781unsafe fn run_contiguous_mul_row_block<
782 O: MulOutput<D>,
783 D: Copy + 'static,
784 A: Copy + Mul<B, Output = D> + 'static,
785 B: Copy + 'static,
786>(
787 dst_ptr: *mut O::Slot,
788 a_ptr: *const A,
789 b_ptr: *const B,
790 plan: &ContiguousMulRangePlan,
791 base_index: usize,
792 total: usize,
793 base_a_offset: isize,
794 base_b_offset: isize,
795) {
796 let inner_len = plan.inner_len.max(1);
797 let row_len = plan.row_len.max(1);
798 let block_len = inner_len.saturating_mul(row_len);
799
800 if total.saturating_sub(base_index) >= block_len {
801 let tiled = match transposed_scalar_tile_kind(plan) {
802 Some(TransposedScalarTileKind::RhsScalar) => {
803 simd::try_mul_transposed_scalar_rhs_2d::<D, A, B>(
804 dst_ptr.add(base_index).cast::<D>(),
805 a_ptr.offset(base_a_offset),
806 b_ptr.offset(base_b_offset),
807 inner_len,
808 row_len,
809 plan.a_fast_stride,
810 plan.a_row_stride,
811 )
812 }
813 Some(TransposedScalarTileKind::LhsScalar) => {
814 simd::try_mul_transposed_scalar_lhs_2d::<D, A, B>(
815 dst_ptr.add(base_index).cast::<D>(),
816 a_ptr.offset(base_a_offset),
817 b_ptr.offset(base_b_offset),
818 inner_len,
819 row_len,
820 plan.b_fast_stride,
821 plan.b_row_stride,
822 )
823 }
824 None => false,
825 };
826 if tiled {
827 return;
828 }
829 }
830
831 let mut index = base_index;
832 let mut a_offset = base_a_offset;
833 let mut b_offset = base_b_offset;
834
835 for _ in 0..row_len {
836 if index >= total {
837 break;
838 }
839 let len = inner_len.min(total - index);
840 inner_loop_mul2::<O, D, A, B>(
841 dst_ptr.add(index),
842 1,
843 a_ptr.offset(a_offset),
844 plan.a_fast_stride,
845 b_ptr.offset(b_offset),
846 plan.b_fast_stride,
847 len,
848 );
849 index += inner_len;
850 a_offset += plan.a_row_stride;
851 b_offset += plan.b_row_stride;
852 }
853}
854
855fn strided_offset_for_contiguous_linear_index(
856 dims: &[usize],
857 strides: &[isize],
858 axis_order: &[usize],
859 mut index: usize,
860) -> isize {
861 let mut offset = 0isize;
862 for &axis in axis_order {
863 let dim = dims[axis];
864 if dim == 0 {
865 return 0;
866 }
867 let coord = index % dim;
868 index /= dim;
869 offset += coord as isize * strides[axis];
870 }
871 offset
872}
873
874#[allow(clippy::too_many_arguments)]
879fn try_contiguous_range_mul<
880 O: MulOutput<D>,
881 D: Copy + Send + Sync + 'static,
882 A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
883 B: Copy + Send + Sync + 'static,
884>(
885 dst_ptr: *mut O::Slot,
886 dims: &[usize],
887 dst_strides: &[isize],
888 a_ptr: *const A,
889 a_strides: &[isize],
890 b_ptr: *const B,
891 b_strides: &[isize],
892 exec: Exec<'_>,
893) -> bool {
894 let Ok(total) = total_len(dims) else {
897 return false;
898 };
899 if total == 0 {
900 return true;
901 }
902 let exec = for_len(exec, KernelClass::Map, total);
904 if exec.is_serial() {
905 return false;
906 }
907
908 let Some(plan) = contiguous_mul_range_plan(dims, dst_strides, a_strides, b_strides) else {
909 return false;
910 };
911 if !plan.walks_inputs_without_blocking() {
912 return false;
913 }
914
915 let inner_len = plan.inner_len.max(1);
916 let row_len = plan.row_len.max(1);
917 let block_len = inner_len.saturating_mul(row_len).max(1);
918 let outer_groups = total.div_ceil(block_len);
919
920 let nthreads = exec.budget().get();
921 if nthreads > 1 {
922 use crate::threading::{parallel_for_each, SendPtr};
923
924 let dst = SendPtr(dst_ptr);
925 let a = SendPtr(a_ptr as *mut A);
926 let b = SendPtr(b_ptr as *mut B);
927
928 if outer_groups < nthreads {
929 let chunk_len = total.div_ceil(nthreads);
930 let nchunks = total.div_ceil(chunk_len);
931
932 parallel_for_each(0..nchunks, exec, &|chunks| {
933 for chunk in chunks {
934 let start = chunk * chunk_len;
935 let end = (start + chunk_len).min(total);
936 let mut index = start;
937
938 while index < end {
939 let in_inner = index % inner_len;
940 let len = (inner_len - in_inner).min(end - index);
941 let a_offset = strided_offset_for_contiguous_linear_index(
942 dims,
943 a_strides,
944 &plan.axis_order,
945 index,
946 );
947 let b_offset = strided_offset_for_contiguous_linear_index(
948 dims,
949 b_strides,
950 &plan.axis_order,
951 index,
952 );
953
954 unsafe {
955 inner_loop_mul2::<O, D, A, B>(
956 dst.as_ptr().add(index),
957 1,
958 a.as_const().offset(a_offset),
959 plan.a_fast_stride,
960 b.as_const().offset(b_offset),
961 plan.b_fast_stride,
962 len,
963 );
964 }
965 index += len;
966 }
967 }
968 });
969
970 return true;
971 }
972
973 let groups_per_chunk = outer_groups.div_ceil(nthreads);
974 let nchunks = outer_groups.div_ceil(groups_per_chunk);
975
976 parallel_for_each(0..nchunks, exec, &|chunks| {
977 for chunk in chunks {
978 let group_start = chunk * groups_per_chunk;
979 let group_end = (group_start + groups_per_chunk).min(outer_groups);
980 let mut cursor =
981 ContiguousMulOuterCursor::new(dims, a_strides, b_strides, &plan, group_start);
982
983 for group in group_start..group_end {
984 let index = group * block_len;
985 unsafe {
986 run_contiguous_mul_row_block::<O, D, A, B>(
987 dst.as_ptr(),
988 a.as_const(),
989 b.as_const(),
990 &plan,
991 index,
992 total,
993 cursor.a_offset,
994 cursor.b_offset,
995 );
996 }
997 cursor.advance();
998 }
999 }
1000 });
1001
1002 true
1003 } else {
1004 run_contiguous_range_mul_single_thread::<O, D, A, B>(
1005 dst_ptr,
1006 dims,
1007 a_ptr,
1008 a_strides,
1009 b_ptr,
1010 b_strides,
1011 &plan,
1012 total,
1013 block_len,
1014 outer_groups,
1015 )
1016 }
1017}
1018
1019#[allow(clippy::too_many_arguments)]
1022fn run_contiguous_range_mul_single_thread<
1023 O: MulOutput<D>,
1024 D: Copy + Send + Sync + 'static,
1025 A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
1026 B: Copy + Send + Sync + 'static,
1027>(
1028 dst_ptr: *mut O::Slot,
1029 dims: &[usize],
1030 a_ptr: *const A,
1031 a_strides: &[isize],
1032 b_ptr: *const B,
1033 b_strides: &[isize],
1034 plan: &ContiguousMulRangePlan,
1035 total: usize,
1036 block_len: usize,
1037 outer_groups: usize,
1038) -> bool {
1039 let mut cursor = ContiguousMulOuterCursor::new(dims, a_strides, b_strides, plan, 0);
1040 for group in 0..outer_groups {
1041 let index = group * block_len;
1042 unsafe {
1043 run_contiguous_mul_row_block::<O, D, A, B>(
1044 dst_ptr,
1045 a_ptr,
1046 b_ptr,
1047 plan,
1048 index,
1049 total,
1050 cursor.a_offset,
1051 cursor.b_offset,
1052 );
1053 }
1054 cursor.advance();
1055 }
1056 true
1057}
1058
1059#[allow(clippy::too_many_arguments)]
1063#[inline(always)]
1064unsafe fn inner_loop_map3<
1065 D: Copy,
1066 A: Copy,
1067 B: Copy,
1068 C: Copy,
1069 OpA: ElementOp<A>,
1070 OpB: ElementOp<B>,
1071 OpC: ElementOp<C>,
1072>(
1073 dp: *mut D,
1074 ds: isize,
1075 ap: *const A,
1076 a_s: isize,
1077 bp: *const B,
1078 b_s: isize,
1079 cp: *const C,
1080 c_s: isize,
1081 len: usize,
1082 f: &impl Fn(A, B, C) -> D,
1083) {
1084 if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 {
1085 let src_a = std::slice::from_raw_parts(ap, len);
1086 let src_b = std::slice::from_raw_parts(bp, len);
1087 let src_c = std::slice::from_raw_parts(cp, len);
1088 let dst = std::slice::from_raw_parts_mut(dp, len);
1089 simd::dispatch_if_large(len, || {
1090 for (((d, &a), &b), &c) in dst.iter_mut().zip(src_a).zip(src_b).zip(src_c) {
1091 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
1092 }
1093 });
1094 } else {
1095 let mut dp = dp;
1096 let mut ap = ap;
1097 let mut bp = bp;
1098 let mut cp = cp;
1099 simd::dispatch_if_large(len, || {
1100 for _ in 0..len {
1101 *dp = f(OpA::apply(*ap), OpB::apply(*bp), OpC::apply(*cp));
1102 dp = dp.wrapping_offset(ds);
1103 ap = ap.wrapping_offset(a_s);
1104 bp = bp.wrapping_offset(b_s);
1105 cp = cp.wrapping_offset(c_s);
1106 }
1107 });
1108 }
1109}
1110
1111#[allow(clippy::too_many_arguments)]
1115#[inline(always)]
1116unsafe fn inner_loop_map4<
1117 D: Copy,
1118 A: Copy,
1119 B: Copy,
1120 C: Copy,
1121 E: Copy,
1122 OpA: ElementOp<A>,
1123 OpB: ElementOp<B>,
1124 OpC: ElementOp<C>,
1125 OpE: ElementOp<E>,
1126>(
1127 dp: *mut D,
1128 ds: isize,
1129 ap: *const A,
1130 a_s: isize,
1131 bp: *const B,
1132 b_s: isize,
1133 cp: *const C,
1134 c_s: isize,
1135 ep: *const E,
1136 e_s: isize,
1137 len: usize,
1138 f: &impl Fn(A, B, C, E) -> D,
1139) {
1140 if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 && e_s == 1 {
1141 let src_a = std::slice::from_raw_parts(ap, len);
1142 let src_b = std::slice::from_raw_parts(bp, len);
1143 let src_c = std::slice::from_raw_parts(cp, len);
1144 let src_e = std::slice::from_raw_parts(ep, len);
1145 let dst = std::slice::from_raw_parts_mut(dp, len);
1146 simd::dispatch_if_large(len, || {
1150 for ((((d, &a), &b), &c), &e) in
1151 dst.iter_mut().zip(src_a).zip(src_b).zip(src_c).zip(src_e)
1152 {
1153 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
1154 }
1155 });
1156 } else {
1157 let mut dp = dp;
1158 let mut ap = ap;
1159 let mut bp = bp;
1160 let mut cp = cp;
1161 let mut ep = ep;
1162 simd::dispatch_if_large(len, || {
1163 for _ in 0..len {
1164 *dp = f(
1165 OpA::apply(*ap),
1166 OpB::apply(*bp),
1167 OpC::apply(*cp),
1168 OpE::apply(*ep),
1169 );
1170 dp = dp.wrapping_offset(ds);
1171 ap = ap.wrapping_offset(a_s);
1172 bp = bp.wrapping_offset(b_s);
1173 cp = cp.wrapping_offset(c_s);
1174 ep = ep.wrapping_offset(e_s);
1175 }
1176 });
1177 }
1178}
1179
1180pub trait MapFn<A, D>: Sync {
1208 fn call(&self, a: A) -> D;
1210
1211 fn class(&self) -> KernelClass {
1213 KernelClass::Map
1214 }
1215}
1216
1217impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for F {
1218 #[inline(always)]
1219 fn call(&self, a: A) -> D {
1220 self(a)
1221 }
1222}
1223
1224#[derive(Debug, Clone, Copy)]
1228pub struct Classed<F> {
1229 pub f: F,
1231 pub class: KernelClass,
1233}
1234
1235impl<F> Classed<F> {
1236 pub const fn new(class: KernelClass, f: F) -> Self {
1238 Self { f, class }
1239 }
1240}
1241
1242impl KernelClass {
1243 pub const fn with<F>(self, f: F) -> Classed<F> {
1245 Classed::new(self, f)
1246 }
1247}
1248
1249impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for Classed<F> {
1250 #[inline(always)]
1251 fn call(&self, a: A) -> D {
1252 (self.f)(a)
1253 }
1254
1255 fn class(&self) -> KernelClass {
1256 self.class
1257 }
1258}
1259
1260pub fn map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1277 dest: &mut StridedViewMut<D>,
1278 src: &StridedView<A, Op>,
1279 f: impl MapFn<A, D>,
1280 exec: Exec<'_>,
1281) -> Result<()> {
1282 let class = f.class();
1283 map_parts_into::<D, A, Op, NoKernel>(
1284 dest.as_mut_ptr(),
1285 dest.dims(),
1286 dest.strides(),
1287 src.ptr(),
1288 src.dims(),
1289 src.strides(),
1290 |a| f.call(a),
1291 &NoKernel,
1292 class,
1293 exec,
1294 )
1295}
1296
1297pub(crate) fn copy_map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1302 dest: &mut StridedViewMut<D>,
1303 src: &StridedView<A, Op>,
1304 f: impl Fn(A) -> D + Sync,
1305 exec: Exec<'_>,
1306) -> Result<()> {
1307 map_parts_into::<D, A, Op, NoKernel>(
1308 dest.as_mut_ptr(),
1309 dest.dims(),
1310 dest.strides(),
1311 src.ptr(),
1312 src.dims(),
1313 src.strides(),
1314 f,
1315 &NoKernel,
1316 KernelClass::Copy,
1317 exec,
1318 )
1319}
1320
1321pub(crate) fn map_into_validated<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1322 dest: &mut StridedViewMut<D>,
1323 src: &StridedView<A, Op>,
1324 f: impl MapFn<A, D>,
1325 validated: ValidatedDestinationLayout,
1326 exec: Exec<'_>,
1327) -> Result<()> {
1328 ensure_same_shape(dest.dims(), src.dims())?;
1329 let class = f.class();
1330 map_parts_into_validated::<D, A, Op, NoKernel>(
1331 dest.as_mut_ptr(),
1332 dest.dims(),
1333 dest.strides(),
1334 src.ptr(),
1335 src.strides(),
1336 |a| f.call(a),
1337 &NoKernel,
1338 validated,
1339 class,
1340 exec,
1341 )
1342}
1343
1344pub fn map_into_uninit_with_kernel<T: Copy + Send + Sync>(
1388 dest: &mut StridedViewMut<MaybeUninit<T>>,
1389 src: &StridedView<T>,
1390 kernel: impl Fn(&[T], &mut [MaybeUninit<T>]) -> bool + Sync,
1391 scalar: impl MapFn<T, T>,
1392 exec: Exec<'_>,
1393) -> Result<()> {
1394 let class = scalar.class();
1395 map_parts_into::<MaybeUninit<T>, T, Identity, _>(
1396 dest.as_mut_ptr(),
1397 dest.dims(),
1398 dest.strides(),
1399 src.ptr(),
1400 src.dims(),
1401 src.strides(),
1402 |value| MaybeUninit::new(scalar.call(value)),
1403 &SliceKernel(kernel),
1404 class,
1405 exec,
1406 )
1407}
1408
1409#[cfg(test)]
1415pub(crate) fn map_raw_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1416 dest: &mut crate::RawStridedMut<'_, D>,
1417 src: &crate::RawStridedRef<'_, A>,
1418 f: impl Fn(A) -> D + Sync,
1419 exec: Exec<'_>,
1420) -> Result<()> {
1421 map_parts_into::<D, A, Op, NoKernel>(
1422 dest.as_mut_ptr(),
1423 dest.dims(),
1424 dest.strides(),
1425 src.ptr(),
1426 src.dims(),
1427 src.strides(),
1428 f,
1429 &NoKernel,
1430 KernelClass::Map,
1431 exec,
1432 )
1433}
1434
1435pub(crate) fn map_raw_into_validated<
1436 D: Copy + Send + Sync,
1437 A: Copy + Send + Sync,
1438 Op: ElementOp<A>,
1439>(
1440 dest: &mut crate::RawStridedMut<'_, D>,
1441 src: &crate::RawStridedRef<'_, A>,
1442 f: impl Fn(A) -> D + Sync,
1443 validated: ValidatedDestinationLayout,
1444 exec: Exec<'_>,
1445) -> Result<()> {
1446 ensure_same_shape(dest.dims(), src.dims())?;
1447 map_parts_into_validated::<D, A, Op, NoKernel>(
1448 dest.as_mut_ptr(),
1449 dest.dims(),
1450 dest.strides(),
1451 src.ptr(),
1452 src.strides(),
1453 f,
1454 &NoKernel,
1455 validated,
1456 KernelClass::Map,
1457 exec,
1458 )
1459}
1460
1461#[allow(clippy::too_many_arguments)]
1464fn map_parts_into<
1465 D: Copy + Send + Sync,
1466 A: Copy + Send + Sync,
1467 Op: ElementOp<A>,
1468 K: MapKernel<A, D> + Sync,
1469>(
1470 dst_ptr: *mut D,
1471 dst_dims: &[usize],
1472 dst_strides: &[isize],
1473 src_ptr: *const A,
1474 src_dims: &[usize],
1475 src_strides: &[isize],
1476 f: impl Fn(A) -> D + Sync,
1477 kernel: &K,
1478 class: KernelClass,
1479 exec: Exec<'_>,
1480) -> Result<()> {
1481 ensure_same_shape(dst_dims, src_dims)?;
1482 let validated = validate_destination_layout(dst_dims, dst_strides)?;
1483 map_parts_into_validated::<D, A, Op, K>(
1484 dst_ptr,
1485 dst_dims,
1486 dst_strides,
1487 src_ptr,
1488 src_strides,
1489 f,
1490 kernel,
1491 validated,
1492 class,
1493 exec,
1494 )
1495}
1496
1497#[allow(clippy::too_many_arguments)]
1498#[inline(never)]
1505fn map_contiguous<D: Copy, A: Copy, Op: ElementOp<A>>(
1506 dst: &mut [D],
1507 src: &[A],
1508 f: impl Fn(A) -> D,
1509) {
1510 simd::dispatch_if_large(dst.len(), move || {
1511 for i in 0..dst.len() {
1512 dst[i] = f(Op::apply(src[i]));
1513 }
1514 });
1515}
1516
1517#[allow(clippy::too_many_arguments)]
1520fn map_parts_into_validated<
1521 D: Copy + Send + Sync,
1522 A: Copy + Send + Sync,
1523 Op: ElementOp<A>,
1524 K: MapKernel<A, D> + Sync,
1525>(
1526 dst_ptr: *mut D,
1527 dst_dims: &[usize],
1528 dst_strides: &[isize],
1529 src_ptr: *const A,
1530 src_strides: &[isize],
1531 f: impl Fn(A) -> D + Sync,
1532 kernel: &K,
1533 _validated: ValidatedDestinationLayout,
1534 class: KernelClass,
1535 exec: Exec<'_>,
1536) -> Result<()> {
1537 if sequential_contiguous_layout(class, dst_dims, &[dst_strides, src_strides], &exec)?.is_some()
1538 {
1539 let len = total_len(dst_dims)?;
1540 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1544 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
1545 if kernel.apply(src, dst) {
1546 return Ok(());
1547 }
1548 map_contiguous::<D, A, Op>(dst, src, f);
1549 return Ok(());
1550 }
1551
1552 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
1553 let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<A>());
1554 let total = total_len(dst_dims)?;
1555
1556 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
1558 build_plan_fused_small(dst_dims, &strides_list)
1559 } else {
1560 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
1561 };
1562
1563 let total = total_len(&fused_dims)?;
1564 if is_parallel(class, total, &exec) {
1565 use crate::threading::SendPtr;
1566 let dst_send = SendPtr(dst_ptr);
1567 let src_send = SendPtr(src_ptr as *mut A);
1568
1569 let costs = compute_costs(&ordered_strides);
1570 let initial_offsets: crate::inline::OperandVec<isize> =
1571 strides_list.iter().map(|_| 0isize).collect();
1572 return mapreduce_threaded(
1573 &fused_dims,
1574 &plan.block,
1575 &ordered_strides,
1576 &initial_offsets,
1577 &costs,
1578 class,
1579 exec,
1580 &|dims, blocks, strides_list, offsets| {
1581 for_each_inner_block_with_offsets(
1582 dims,
1583 blocks,
1584 strides_list,
1585 offsets,
1586 |offsets, len, strides| {
1587 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1588 let sp = unsafe { src_send.as_const().offset(offsets[1]) };
1589 unsafe {
1590 inner_loop_map1::<D, A, Op, K>(
1591 dp, strides[0], sp, strides[1], len, &f, kernel,
1592 )
1593 };
1594 Ok(())
1595 },
1596 )
1597 },
1598 );
1599 }
1600
1601 let initial_offsets: crate::inline::OperandVec<isize> =
1602 ordered_strides.iter().map(|_| 0isize).collect();
1603 for_each_inner_block_preordered(
1604 &fused_dims,
1605 &plan.block,
1606 &ordered_strides,
1607 &initial_offsets,
1608 |offsets, len, strides| {
1609 let dp = unsafe { dst_ptr.offset(offsets[0]) };
1610 let sp = unsafe { src_ptr.offset(offsets[1]) };
1611 unsafe {
1612 inner_loop_map1::<D, A, Op, K>(dp, strides[0], sp, strides[1], len, &f, kernel)
1613 };
1614 Ok(())
1615 },
1616 )
1617}
1618
1619pub fn zip_map2_into<
1634 D: Copy + Send + Sync,
1635 A: Copy + Send + Sync,
1636 B: Copy + Send + Sync,
1637 OpA: ElementOp<A>,
1638 OpB: ElementOp<B>,
1639>(
1640 dest: &mut StridedViewMut<D>,
1641 a: &StridedView<A, OpA>,
1642 b: &StridedView<B, OpB>,
1643 f: impl Fn(A, B) -> D + Sync,
1644 exec: Exec<'_>,
1645) -> Result<()> {
1646 zip_map2_parts_into::<D, A, B, OpA, OpB>(
1647 dest.as_mut_ptr(),
1648 dest.dims(),
1649 dest.strides(),
1650 a.ptr(),
1651 a.dims(),
1652 a.strides(),
1653 b.ptr(),
1654 b.dims(),
1655 b.strides(),
1656 f,
1657 exec,
1658 )
1659}
1660
1661pub(crate) fn zip_map2_into_validated<
1662 D: Copy + Send + Sync,
1663 A: Copy + Send + Sync,
1664 B: Copy + Send + Sync,
1665 OpA: ElementOp<A>,
1666 OpB: ElementOp<B>,
1667>(
1668 dest: &mut StridedViewMut<D>,
1669 a: &StridedView<A, OpA>,
1670 b: &StridedView<B, OpB>,
1671 f: impl Fn(A, B) -> D + Sync,
1672 validated: ValidatedDestinationLayout,
1673 exec: Exec<'_>,
1674) -> Result<()> {
1675 zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1676 dest.as_mut_ptr(),
1677 dest.dims(),
1678 dest.strides(),
1679 a.ptr(),
1680 a.strides(),
1681 b.ptr(),
1682 b.strides(),
1683 f,
1684 validated,
1685 exec,
1686 )
1687}
1688
1689#[non_exhaustive]
1691#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1692pub enum CompareOp {
1693 Eq,
1694 Lt,
1695 Le,
1696 Gt,
1697 Ge,
1698}
1699
1700pub fn compare_into<T, OpA, OpB>(
1711 dest: &mut StridedViewMut<bool>,
1712 a: &StridedView<T, OpA>,
1713 b: &StridedView<T, OpB>,
1714 op: CompareOp,
1715 exec: Exec<'_>,
1716) -> Result<()>
1717where
1718 T: Copy + Send + Sync + PartialOrd,
1719 OpA: ElementOp<T>,
1720 OpB: ElementOp<T>,
1721{
1722 match op {
1723 CompareOp::Eq => zip_map2_into(dest, a, b, |lhs, rhs| lhs == rhs, exec),
1724 CompareOp::Lt => zip_map2_into(dest, a, b, |lhs, rhs| lhs < rhs, exec),
1725 CompareOp::Le => zip_map2_into(dest, a, b, |lhs, rhs| lhs <= rhs, exec),
1726 CompareOp::Gt => zip_map2_into(dest, a, b, |lhs, rhs| lhs > rhs, exec),
1727 CompareOp::Ge => zip_map2_into(dest, a, b, |lhs, rhs| lhs >= rhs, exec),
1728 }
1729}
1730
1731pub fn compare_into_uninit<T, OpA, OpB>(
1750 dest: &mut StridedViewMut<MaybeUninit<bool>>,
1751 a: &StridedView<T, OpA>,
1752 b: &StridedView<T, OpB>,
1753 op: CompareOp,
1754 exec: Exec<'_>,
1755) -> Result<()>
1756where
1757 T: Copy + Send + Sync + PartialOrd,
1758 OpA: ElementOp<T>,
1759 OpB: ElementOp<T>,
1760{
1761 ensure_same_shape(dest.dims(), a.dims())?;
1762 ensure_same_shape(dest.dims(), b.dims())?;
1763 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
1764 validate_typed_no_overlap(dest, a, 0)?;
1765 validate_typed_no_overlap(dest, b, 1)?;
1766 match op {
1767 CompareOp::Eq => zip_map2_into_validated(
1768 dest,
1769 a,
1770 b,
1771 |lhs, rhs| MaybeUninit::new(lhs == rhs),
1772 validated,
1773 exec,
1774 ),
1775 CompareOp::Lt => zip_map2_into_validated(
1776 dest,
1777 a,
1778 b,
1779 |lhs, rhs| MaybeUninit::new(lhs < rhs),
1780 validated,
1781 exec,
1782 ),
1783 CompareOp::Le => zip_map2_into_validated(
1784 dest,
1785 a,
1786 b,
1787 |lhs, rhs| MaybeUninit::new(lhs <= rhs),
1788 validated,
1789 exec,
1790 ),
1791 CompareOp::Gt => zip_map2_into_validated(
1792 dest,
1793 a,
1794 b,
1795 |lhs, rhs| MaybeUninit::new(lhs > rhs),
1796 validated,
1797 exec,
1798 ),
1799 CompareOp::Ge => zip_map2_into_validated(
1800 dest,
1801 a,
1802 b,
1803 |lhs, rhs| MaybeUninit::new(lhs >= rhs),
1804 validated,
1805 exec,
1806 ),
1807 }
1808}
1809
1810pub(crate) fn zip_map2_raw_into_validated<
1811 D: Copy + Send + Sync,
1812 A: Copy + Send + Sync,
1813 B: Copy + Send + Sync,
1814 OpA: ElementOp<A>,
1815 OpB: ElementOp<B>,
1816>(
1817 dest: &mut crate::RawStridedMut<'_, D>,
1818 a: &crate::RawStridedRef<'_, A>,
1819 b: &crate::RawStridedRef<'_, B>,
1820 f: impl Fn(A, B) -> D + Sync,
1821 validated: ValidatedDestinationLayout,
1822 exec: Exec<'_>,
1823) -> Result<()> {
1824 ensure_same_shape(dest.dims(), a.dims())?;
1825 ensure_same_shape(dest.dims(), b.dims())?;
1826 zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1827 dest.as_mut_ptr(),
1828 dest.dims(),
1829 dest.strides(),
1830 a.ptr(),
1831 a.strides(),
1832 b.ptr(),
1833 b.strides(),
1834 f,
1835 validated,
1836 exec,
1837 )
1838}
1839
1840#[allow(clippy::too_many_arguments)]
1843fn zip_map2_parts_into<
1844 D: Copy + Send + Sync,
1845 A: Copy + Send + Sync,
1846 B: Copy + Send + Sync,
1847 OpA: ElementOp<A>,
1848 OpB: ElementOp<B>,
1849>(
1850 dst_ptr: *mut D,
1851 dst_dims: &[usize],
1852 dst_strides: &[isize],
1853 a_ptr: *const A,
1854 a_dims: &[usize],
1855 a_strides: &[isize],
1856 b_ptr: *const B,
1857 b_dims: &[usize],
1858 b_strides: &[isize],
1859 f: impl Fn(A, B) -> D + Sync,
1860 exec: Exec<'_>,
1861) -> Result<()> {
1862 ensure_same_shape(dst_dims, a_dims)?;
1863 ensure_same_shape(dst_dims, b_dims)?;
1864 let validated = validate_destination_layout(dst_dims, dst_strides)?;
1865 zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1866 dst_ptr,
1867 dst_dims,
1868 dst_strides,
1869 a_ptr,
1870 a_strides,
1871 b_ptr,
1872 b_strides,
1873 f,
1874 validated,
1875 exec,
1876 )
1877}
1878
1879#[allow(clippy::too_many_arguments)]
1882fn zip_map2_parts_into_validated<
1883 D: Copy + Send + Sync,
1884 A: Copy + Send + Sync,
1885 B: Copy + Send + Sync,
1886 OpA: ElementOp<A>,
1887 OpB: ElementOp<B>,
1888>(
1889 dst_ptr: *mut D,
1890 dst_dims: &[usize],
1891 dst_strides: &[isize],
1892 a_ptr: *const A,
1893 a_strides: &[isize],
1894 b_ptr: *const B,
1895 b_strides: &[isize],
1896 f: impl Fn(A, B) -> D + Sync,
1897 _validated: ValidatedDestinationLayout,
1898 exec: Exec<'_>,
1899) -> Result<()> {
1900 if sequential_contiguous_layout(
1901 KernelClass::Map,
1902 dst_dims,
1903 &[dst_strides, a_strides, b_strides],
1904 &exec,
1905 )?
1906 .is_some()
1907 {
1908 let len = total_len(dst_dims)?;
1909 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1910 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
1911 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
1912 simd::dispatch_if_large(len, || {
1913 for i in 0..len {
1914 dst[i] = f(OpA::apply(sa[i]), OpB::apply(sb[i]));
1915 }
1916 });
1917 return Ok(());
1918 }
1919
1920 let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
1921 let elem_size = std::mem::size_of::<D>()
1922 .max(std::mem::size_of::<A>())
1923 .max(std::mem::size_of::<B>());
1924 let total = total_len(dst_dims)?;
1925
1926 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
1928 build_plan_fused_small(dst_dims, &strides_list)
1929 } else {
1930 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
1931 };
1932
1933 let total = total_len(&fused_dims)?;
1934 if is_parallel(KernelClass::Map, total, &exec) {
1935 use crate::threading::SendPtr;
1936 let dst_send = SendPtr(dst_ptr);
1937 let a_send = SendPtr(a_ptr as *mut A);
1938 let b_send = SendPtr(b_ptr as *mut B);
1939
1940 let costs = compute_costs(&ordered_strides);
1941 let initial_offsets: crate::inline::OperandVec<isize> =
1942 strides_list.iter().map(|_| 0isize).collect();
1943 return mapreduce_threaded(
1944 &fused_dims,
1945 &plan.block,
1946 &ordered_strides,
1947 &initial_offsets,
1948 &costs,
1949 KernelClass::Map,
1950 exec,
1951 &|dims, blocks, strides_list, offsets| {
1952 for_each_inner_block_with_offsets(
1953 dims,
1954 blocks,
1955 strides_list,
1956 offsets,
1957 |offsets, len, strides| {
1958 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1959 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
1960 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
1961 unsafe {
1962 inner_loop_map2::<D, A, B, OpA, OpB>(
1963 dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1964 )
1965 };
1966 Ok(())
1967 },
1968 )
1969 },
1970 );
1971 }
1972
1973 let initial_offsets: crate::inline::OperandVec<isize> =
1974 ordered_strides.iter().map(|_| 0isize).collect();
1975 for_each_inner_block_preordered(
1976 &fused_dims,
1977 &plan.block,
1978 &ordered_strides,
1979 &initial_offsets,
1980 |offsets, len, strides| {
1981 let dp = unsafe { dst_ptr.offset(offsets[0]) };
1982 let ap = unsafe { a_ptr.offset(offsets[1]) };
1983 let bp = unsafe { b_ptr.offset(offsets[2]) };
1984 unsafe {
1985 inner_loop_map2::<D, A, B, OpA, OpB>(
1986 dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1987 )
1988 };
1989 Ok(())
1990 },
1991 )
1992}
1993
1994#[allow(clippy::too_many_arguments)]
1998fn mul_identity_into_raw<
1999 O: MulOutput<D>,
2000 D: Copy + Send + Sync + 'static,
2001 A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
2002 B: Copy + Send + Sync + 'static,
2003>(
2004 dst_ptr: *mut O::Slot,
2005 dst_dims: &[usize],
2006 dst_strides: &[isize],
2007 a_ptr: *const A,
2008 a_strides: &[isize],
2009 b_ptr: *const B,
2010 b_strides: &[isize],
2011 _validated: ValidatedDestinationLayout,
2012 mut exec: Exec<'_>,
2013) -> Result<()> {
2014 debug_assert_eq!(dst_dims.len(), a_strides.len());
2015 debug_assert_eq!(dst_dims.len(), b_strides.len());
2016
2017 if sequential_contiguous_layout(
2018 KernelClass::Map,
2019 dst_dims,
2020 &[dst_strides, a_strides, b_strides],
2021 &exec,
2022 )?
2023 .is_some()
2024 {
2025 let len = total_len(dst_dims)?;
2026 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2027 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2028 if unsafe { O::try_contiguous(dst_ptr, len, sa, sb) } {
2029 return Ok(());
2030 }
2031 for i in 0..len {
2032 unsafe { O::write(dst_ptr.add(i), multiply_value(sa[i], sb[i])) };
2033 }
2034 return Ok(());
2035 }
2036
2037 let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
2038 let elem_size = std::mem::size_of::<D>()
2039 .max(std::mem::size_of::<A>())
2040 .max(std::mem::size_of::<B>());
2041 let total = total_len(dst_dims)?;
2042
2043 if try_contiguous_range_mul::<O, D, A, B>(
2044 dst_ptr,
2045 dst_dims,
2046 dst_strides,
2047 a_ptr,
2048 a_strides,
2049 b_ptr,
2050 b_strides,
2051 exec.reborrow(),
2052 ) {
2053 return Ok(());
2054 }
2055
2056 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2057 build_plan_fused_small(dst_dims, &strides_list)
2058 } else {
2059 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2060 };
2061
2062 let total = total_len(&fused_dims)?;
2063 if is_parallel(KernelClass::Map, total, &exec) {
2064 use crate::threading::SendPtr;
2065 let dst_send = SendPtr(dst_ptr);
2066 let a_send = SendPtr(a_ptr as *mut A);
2067 let b_send = SendPtr(b_ptr as *mut B);
2068
2069 let costs = compute_costs(&ordered_strides);
2070 let initial_offsets: crate::inline::OperandVec<isize> =
2071 strides_list.iter().map(|_| 0isize).collect();
2072 return mapreduce_threaded(
2073 &fused_dims,
2074 &plan.block,
2075 &ordered_strides,
2076 &initial_offsets,
2077 &costs,
2078 KernelClass::Map,
2079 exec,
2080 &|dims, blocks, strides_list, offsets| {
2081 for_each_inner_block_with_offsets(
2082 dims,
2083 blocks,
2084 strides_list,
2085 offsets,
2086 |offsets, len, strides| {
2087 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2088 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2089 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2090 unsafe {
2091 inner_loop_mul2::<O, D, A, B>(
2092 dp, strides[0], ap, strides[1], bp, strides[2], len,
2093 )
2094 };
2095 Ok(())
2096 },
2097 )
2098 },
2099 );
2100 }
2101
2102 let initial_offsets: crate::inline::OperandVec<isize> =
2103 ordered_strides.iter().map(|_| 0isize).collect();
2104 for_each_inner_block_preordered(
2105 &fused_dims,
2106 &plan.block,
2107 &ordered_strides,
2108 &initial_offsets,
2109 |offsets, len, strides| {
2110 let dp = unsafe { dst_ptr.offset(offsets[0]) };
2111 let ap = unsafe { a_ptr.offset(offsets[1]) };
2112 let bp = unsafe { b_ptr.offset(offsets[2]) };
2113 unsafe {
2114 inner_loop_mul2::<O, D, A, B>(dp, strides[0], ap, strides[1], bp, strides[2], len)
2115 };
2116 Ok(())
2117 },
2118 )
2119}
2120
2121pub fn mul_into<
2136 D: Copy + Send + Sync + 'static,
2137 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2138 B: Copy + Send + Sync + 'static,
2139 OpA: ElementOp<A>,
2140 OpB: ElementOp<B>,
2141>(
2142 dest: &mut StridedViewMut<D>,
2143 a: &StridedView<A, OpA>,
2144 b: &StridedView<B, OpB>,
2145 exec: Exec<'_>,
2146) -> Result<()> {
2147 ensure_same_shape(dest.dims(), a.dims())?;
2148 ensure_same_shape(dest.dims(), b.dims())?;
2149 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2150
2151 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2152 return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2153 dest.as_mut_ptr(),
2154 dest.dims(),
2155 dest.strides(),
2156 a.ptr(),
2157 a.strides(),
2158 b.ptr(),
2159 b.strides(),
2160 validated,
2161 exec,
2162 );
2163 }
2164
2165 zip_map2_into_validated(dest, a, b, multiply_value, validated, exec)
2166}
2167
2168pub fn mul_into_uninit<
2187 D: Copy + Send + Sync + 'static,
2188 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2189 B: Copy + Send + Sync + 'static,
2190 OpA: ElementOp<A>,
2191 OpB: ElementOp<B>,
2192>(
2193 dest: &mut StridedViewMut<MaybeUninit<D>>,
2194 a: &StridedView<A, OpA>,
2195 b: &StridedView<B, OpB>,
2196 exec: Exec<'_>,
2197) -> Result<()> {
2198 ensure_same_shape(dest.dims(), a.dims())?;
2199 ensure_same_shape(dest.dims(), b.dims())?;
2200 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2201 validate_typed_no_overlap(dest, a, 0)?;
2202 validate_typed_no_overlap(dest, b, 1)?;
2203
2204 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2205 return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2206 dest.as_mut_ptr(),
2207 dest.dims(),
2208 dest.strides(),
2209 a.ptr(),
2210 a.strides(),
2211 b.ptr(),
2212 b.strides(),
2213 validated,
2214 exec,
2215 );
2216 }
2217
2218 zip_map2_into_validated(
2219 dest,
2220 a,
2221 b,
2222 |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2223 validated,
2224 exec,
2225 )
2226}
2227
2228fn broadcast_strides_for_axes(
2229 source_dims: &[usize],
2230 source_strides: &[isize],
2231 target_dims: &[usize],
2232 axes: &[usize],
2233) -> Result<AxisVec<isize>> {
2234 if source_dims.len() != axes.len() {
2235 return Err(StridedError::RankMismatch(source_dims.len(), axes.len()));
2236 }
2237 debug_assert_eq!(source_dims.len(), source_strides.len());
2238
2239 let mut seen = AxisVec::<bool>::new();
2240 seen.resize(target_dims.len(), false);
2241 let mut strides = AxisVec::<isize>::new();
2242 strides.resize(target_dims.len(), 0);
2243 for (src_axis, &dst_axis) in axes.iter().enumerate() {
2244 if dst_axis >= target_dims.len() {
2245 return Err(StridedError::InvalidAxis {
2246 axis: dst_axis,
2247 rank: target_dims.len(),
2248 });
2249 }
2250 if seen[dst_axis] {
2251 return Err(StridedError::InvalidAxis {
2252 axis: dst_axis,
2253 rank: target_dims.len(),
2254 });
2255 }
2256 seen[dst_axis] = true;
2257
2258 let source_dim = source_dims[src_axis];
2259 let target_dim = target_dims[dst_axis];
2260 if source_dim != target_dim && source_dim != 1 {
2261 return Err(StridedError::ShapeMismatch(
2262 source_dims.to_vec(),
2263 target_dims.to_vec(),
2264 ));
2265 }
2266 if source_dim == target_dim {
2267 strides[dst_axis] = source_strides[src_axis];
2268 }
2269 }
2270
2271 Ok(strides)
2272}
2273
2274fn broadcast_view_with_strides<'a, T, Op: ElementOp<T>>(
2281 view: &StridedView<'a, T, Op>,
2282 target_dims: &[usize],
2283 strides: &[isize],
2284) -> StridedView<'a, T, Op> {
2285 unsafe { StridedView::new_unchecked(view.data(), target_dims, strides, view.offset()) }
2297}
2298
2299pub fn broadcast_mul_into<
2315 D: Copy + Send + Sync + 'static,
2316 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2317 B: Copy + Send + Sync + 'static,
2318 OpA: ElementOp<A>,
2319 OpB: ElementOp<B>,
2320>(
2321 dest: &mut StridedViewMut<D>,
2322 a: &StridedView<A, OpA>,
2323 a_axes: &[usize],
2324 b: &StridedView<B, OpB>,
2325 b_axes: &[usize],
2326 exec: Exec<'_>,
2327) -> Result<()> {
2328 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2329 let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2330 let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2331
2332 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2333 return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2334 dest.as_mut_ptr(),
2335 dest.dims(),
2336 dest.strides(),
2337 a.ptr(),
2338 &a_strides,
2339 b.ptr(),
2340 &b_strides,
2341 validated,
2342 exec,
2343 );
2344 }
2345
2346 let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2347 let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2348 zip_map2_into_validated(dest, &a, &b, multiply_value, validated, exec)
2349}
2350
2351pub fn broadcast_mul_into_uninit<
2370 D: Copy + Send + Sync + 'static,
2371 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2372 B: Copy + Send + Sync + 'static,
2373 OpA: ElementOp<A>,
2374 OpB: ElementOp<B>,
2375>(
2376 dest: &mut StridedViewMut<MaybeUninit<D>>,
2377 a: &StridedView<A, OpA>,
2378 a_axes: &[usize],
2379 b: &StridedView<B, OpB>,
2380 b_axes: &[usize],
2381 exec: Exec<'_>,
2382) -> Result<()> {
2383 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2384 let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2385 let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2386 validate_typed_no_overlap(dest, a, 0)?;
2387 validate_typed_no_overlap(dest, b, 1)?;
2388
2389 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2390 return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2391 dest.as_mut_ptr(),
2392 dest.dims(),
2393 dest.strides(),
2394 a.ptr(),
2395 &a_strides,
2396 b.ptr(),
2397 &b_strides,
2398 validated,
2399 exec,
2400 );
2401 }
2402
2403 let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2404 let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2405 zip_map2_into_validated(
2406 dest,
2407 &a,
2408 &b,
2409 |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2410 validated,
2411 exec,
2412 )
2413}
2414
2415pub fn zip_map3_into<
2427 D: Copy + Send + Sync,
2428 A: Copy + Send + Sync,
2429 B: Copy + Send + Sync,
2430 C: Copy + Send + Sync,
2431 OpA: ElementOp<A>,
2432 OpB: ElementOp<B>,
2433 OpC: ElementOp<C>,
2434>(
2435 dest: &mut StridedViewMut<D>,
2436 a: &StridedView<A, OpA>,
2437 b: &StridedView<B, OpB>,
2438 c: &StridedView<C, OpC>,
2439 f: impl Fn(A, B, C) -> D + Sync,
2440 exec: Exec<'_>,
2441) -> Result<()> {
2442 ensure_same_shape(dest.dims(), a.dims())?;
2443 ensure_same_shape(dest.dims(), b.dims())?;
2444 ensure_same_shape(dest.dims(), c.dims())?;
2445 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2446 zip_map3_into_validated(dest, a, b, c, f, validated, exec)
2447}
2448
2449pub(crate) fn zip_map3_into_validated<
2450 D: Copy + Send + Sync,
2451 A: Copy + Send + Sync,
2452 B: Copy + Send + Sync,
2453 C: Copy + Send + Sync,
2454 OpA: ElementOp<A>,
2455 OpB: ElementOp<B>,
2456 OpC: ElementOp<C>,
2457>(
2458 dest: &mut StridedViewMut<D>,
2459 a: &StridedView<A, OpA>,
2460 b: &StridedView<B, OpB>,
2461 c: &StridedView<C, OpC>,
2462 f: impl Fn(A, B, C) -> D + Sync,
2463 _validated: ValidatedDestinationLayout,
2464 exec: Exec<'_>,
2465) -> Result<()> {
2466 ensure_same_shape(dest.dims(), a.dims())?;
2467 ensure_same_shape(dest.dims(), b.dims())?;
2468 ensure_same_shape(dest.dims(), c.dims())?;
2469 let dst_ptr = dest.as_mut_ptr();
2470 let a_ptr = a.ptr();
2471 let b_ptr = b.ptr();
2472 let c_ptr = c.ptr();
2473
2474 let dst_dims = dest.dims();
2475 let dst_strides = dest.strides();
2476
2477 if sequential_contiguous_layout(
2478 KernelClass::Map,
2479 dst_dims,
2480 &[dst_strides, a.strides(), b.strides(), c.strides()],
2481 &exec,
2482 )?
2483 .is_some()
2484 {
2485 let len = total_len(dst_dims)?;
2486 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2487 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2488 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2489 let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2490 simd::dispatch_if_large(len, || {
2491 for (((d, &a), &b), &c) in dst.iter_mut().zip(sa).zip(sb).zip(sc) {
2492 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
2493 }
2494 });
2495 return Ok(());
2496 }
2497
2498 let strides_list: [&[isize]; 4] = [dst_strides, a.strides(), b.strides(), c.strides()];
2499 let elem_size = std::mem::size_of::<D>()
2500 .max(std::mem::size_of::<A>())
2501 .max(std::mem::size_of::<B>())
2502 .max(std::mem::size_of::<C>());
2503 let total = total_len(dst_dims)?;
2504
2505 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2507 build_plan_fused_small(dst_dims, &strides_list)
2508 } else {
2509 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2510 };
2511
2512 let total = total_len(&fused_dims)?;
2513 if is_parallel(KernelClass::Map, total, &exec) {
2514 use crate::threading::SendPtr;
2515 let dst_send = SendPtr(dst_ptr);
2516 let a_send = SendPtr(a_ptr as *mut A);
2517 let b_send = SendPtr(b_ptr as *mut B);
2518 let c_send = SendPtr(c_ptr as *mut C);
2519
2520 let costs = compute_costs(&ordered_strides);
2521 let initial_offsets: crate::inline::OperandVec<isize> =
2522 strides_list.iter().map(|_| 0isize).collect();
2523 return mapreduce_threaded(
2524 &fused_dims,
2525 &plan.block,
2526 &ordered_strides,
2527 &initial_offsets,
2528 &costs,
2529 KernelClass::Map,
2530 exec,
2531 &|dims, blocks, strides_list, offsets| {
2532 for_each_inner_block_with_offsets(
2533 dims,
2534 blocks,
2535 strides_list,
2536 offsets,
2537 |offsets, len, strides| {
2538 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2539 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2540 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2541 let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2542 unsafe {
2543 inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2544 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3],
2545 len, &f,
2546 )
2547 };
2548 Ok(())
2549 },
2550 )
2551 },
2552 );
2553 }
2554
2555 let initial_offsets: crate::inline::OperandVec<isize> =
2556 ordered_strides.iter().map(|_| 0isize).collect();
2557 for_each_inner_block_preordered(
2558 &fused_dims,
2559 &plan.block,
2560 &ordered_strides,
2561 &initial_offsets,
2562 |offsets, len, strides| {
2563 let dp = unsafe { dst_ptr.offset(offsets[0]) };
2564 let ap = unsafe { a_ptr.offset(offsets[1]) };
2565 let bp = unsafe { b_ptr.offset(offsets[2]) };
2566 let cp = unsafe { c_ptr.offset(offsets[3]) };
2567 unsafe {
2568 inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2569 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], len, &f,
2570 )
2571 };
2572 Ok(())
2573 },
2574 )
2575}
2576
2577pub fn zip_map4_into<
2589 D: Copy + Send + Sync,
2590 A: Copy + Send + Sync,
2591 B: Copy + Send + Sync,
2592 C: Copy + Send + Sync,
2593 E: Copy + Send + Sync,
2594 OpA: ElementOp<A>,
2595 OpB: ElementOp<B>,
2596 OpC: ElementOp<C>,
2597 OpE: ElementOp<E>,
2598>(
2599 dest: &mut StridedViewMut<D>,
2600 a: &StridedView<A, OpA>,
2601 b: &StridedView<B, OpB>,
2602 c: &StridedView<C, OpC>,
2603 e: &StridedView<E, OpE>,
2604 f: impl Fn(A, B, C, E) -> D + Sync,
2605 exec: Exec<'_>,
2606) -> Result<()> {
2607 ensure_same_shape(dest.dims(), a.dims())?;
2608 ensure_same_shape(dest.dims(), b.dims())?;
2609 ensure_same_shape(dest.dims(), c.dims())?;
2610 ensure_same_shape(dest.dims(), e.dims())?;
2611 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2612 zip_map4_into_validated(dest, a, b, c, e, f, validated, exec)
2613}
2614
2615#[allow(clippy::too_many_arguments)]
2618pub(crate) fn zip_map4_into_validated<
2619 D: Copy + Send + Sync,
2620 A: Copy + Send + Sync,
2621 B: Copy + Send + Sync,
2622 C: Copy + Send + Sync,
2623 E: Copy + Send + Sync,
2624 OpA: ElementOp<A>,
2625 OpB: ElementOp<B>,
2626 OpC: ElementOp<C>,
2627 OpE: ElementOp<E>,
2628>(
2629 dest: &mut StridedViewMut<D>,
2630 a: &StridedView<A, OpA>,
2631 b: &StridedView<B, OpB>,
2632 c: &StridedView<C, OpC>,
2633 e: &StridedView<E, OpE>,
2634 f: impl Fn(A, B, C, E) -> D + Sync,
2635 _validated: ValidatedDestinationLayout,
2636 exec: Exec<'_>,
2637) -> Result<()> {
2638 ensure_same_shape(dest.dims(), a.dims())?;
2639 ensure_same_shape(dest.dims(), b.dims())?;
2640 ensure_same_shape(dest.dims(), c.dims())?;
2641 ensure_same_shape(dest.dims(), e.dims())?;
2642 let dst_ptr = dest.as_mut_ptr();
2643 let a_ptr = a.ptr();
2644 let b_ptr = b.ptr();
2645 let c_ptr = c.ptr();
2646 let e_ptr = e.ptr();
2647
2648 let dst_dims = dest.dims();
2649 let dst_strides = dest.strides();
2650
2651 if sequential_contiguous_layout(
2652 KernelClass::Map,
2653 dst_dims,
2654 &[
2655 dst_strides,
2656 a.strides(),
2657 b.strides(),
2658 c.strides(),
2659 e.strides(),
2660 ],
2661 &exec,
2662 )?
2663 .is_some()
2664 {
2665 let len = total_len(dst_dims)?;
2666 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2667 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2668 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2669 let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2670 let se = unsafe { std::slice::from_raw_parts(e_ptr, len) };
2671 simd::dispatch_if_large(len, || {
2674 for ((((d, &a), &b), &c), &e) in dst.iter_mut().zip(sa).zip(sb).zip(sc).zip(se) {
2675 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
2676 }
2677 });
2678 return Ok(());
2679 }
2680
2681 let strides_list: [&[isize]; 5] = [
2682 dst_strides,
2683 a.strides(),
2684 b.strides(),
2685 c.strides(),
2686 e.strides(),
2687 ];
2688 let elem_size = std::mem::size_of::<D>()
2689 .max(std::mem::size_of::<A>())
2690 .max(std::mem::size_of::<B>())
2691 .max(std::mem::size_of::<C>())
2692 .max(std::mem::size_of::<E>());
2693 let total = total_len(dst_dims)?;
2694
2695 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2697 build_plan_fused_small(dst_dims, &strides_list)
2698 } else {
2699 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2700 };
2701
2702 let total = total_len(&fused_dims)?;
2703 if is_parallel(KernelClass::Map, total, &exec) {
2704 use crate::threading::SendPtr;
2705 let dst_send = SendPtr(dst_ptr);
2706 let a_send = SendPtr(a_ptr as *mut A);
2707 let b_send = SendPtr(b_ptr as *mut B);
2708 let c_send = SendPtr(c_ptr as *mut C);
2709 let e_send = SendPtr(e_ptr as *mut E);
2710
2711 let costs = compute_costs(&ordered_strides);
2712 let initial_offsets: crate::inline::OperandVec<isize> =
2713 strides_list.iter().map(|_| 0isize).collect();
2714 return mapreduce_threaded(
2715 &fused_dims,
2716 &plan.block,
2717 &ordered_strides,
2718 &initial_offsets,
2719 &costs,
2720 KernelClass::Map,
2721 exec,
2722 &|dims, blocks, strides_list, offsets| {
2723 for_each_inner_block_with_offsets(
2724 dims,
2725 blocks,
2726 strides_list,
2727 offsets,
2728 |offsets, len, strides| {
2729 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2730 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2731 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2732 let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2733 let ep = unsafe { e_send.as_const().offset(offsets[4]) };
2734 unsafe {
2735 inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2736 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep,
2737 strides[4], len, &f,
2738 )
2739 };
2740 Ok(())
2741 },
2742 )
2743 },
2744 );
2745 }
2746
2747 let initial_offsets: crate::inline::OperandVec<isize> =
2748 ordered_strides.iter().map(|_| 0isize).collect();
2749 for_each_inner_block_preordered(
2750 &fused_dims,
2751 &plan.block,
2752 &ordered_strides,
2753 &initial_offsets,
2754 |offsets, len, strides| {
2755 let dp = unsafe { dst_ptr.offset(offsets[0]) };
2756 let ap = unsafe { a_ptr.offset(offsets[1]) };
2757 let bp = unsafe { b_ptr.offset(offsets[2]) };
2758 let cp = unsafe { c_ptr.offset(offsets[3]) };
2759 let ep = unsafe { e_ptr.offset(offsets[4]) };
2760 unsafe {
2761 inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2762 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep, strides[4],
2763 len, &f,
2764 )
2765 };
2766 Ok(())
2767 },
2768 )
2769}
2770
2771#[cfg(test)]
2772#[path = "map_view/tests/scalar_branch_tests.rs"]
2773mod scalar_branch_tests;
2774
2775#[cfg(test)]
2776#[path = "map_view/tests/tests.rs"]
2777mod tests;