1use crate::kernel::{
10 ensure_same_shape, for_each_inner_block_preordered, pointwise_plan,
11 sequential_contiguous_layout, total_len,
12};
13use crate::simd;
14use crate::view::{StridedView, StridedViewMut};
15use crate::{Result, StridedError};
16use core::mem::MaybeUninit;
17use std::ops::Mul;
18use strided_view::{ElementOp, Identity};
19
20use crate::cost::{for_len, KernelClass};
21use crate::fuse::compute_costs;
22use crate::threading::{for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded};
23use crate::Exec;
24
25type AxisVec<T> = crate::inline::InlineVec<T>;
26
27const TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT: usize = 8;
36
37#[derive(Clone, Copy, Debug, Default)]
43pub struct ValidatedDestinationLayout(());
44
45#[inline]
61pub fn validate_destination_layout(
62 dims: &[usize],
63 strides: &[isize],
64) -> Result<ValidatedDestinationLayout> {
65 if crate::layout_check::is_injective_layout(dims, strides) {
66 Ok(ValidatedDestinationLayout(()))
67 } else {
68 Err(StridedError::NonInjectiveOutputLayout)
69 }
70}
71
72#[inline]
73pub fn validate_destination_layout_without_alloc(
87 dims: &[usize],
88 strides: &[isize],
89) -> Result<ValidatedDestinationLayout> {
90 if crate::layout_check::is_injective_layout_without_alloc(dims, strides) {
91 Ok(ValidatedDestinationLayout(()))
92 } else {
93 Err(StridedError::NonInjectiveOutputLayout)
94 }
95}
96
97fn reachable_byte_range(
98 ptr: usize,
99 elem_size: usize,
100 dims: &[usize],
101 strides: &[isize],
102) -> Result<Option<(usize, usize)>> {
103 if elem_size == 0 || dims.contains(&0) {
104 return Ok(None);
105 }
106
107 let mut min_offset = 0isize;
108 let mut max_offset = 0isize;
109 for (&dim, &stride) in dims.iter().zip(strides) {
110 if dim <= 1 {
111 continue;
112 }
113 let extent = isize::try_from(dim - 1).map_err(|_| StridedError::OffsetOverflow)?;
114 let span = stride
115 .checked_mul(extent)
116 .ok_or(StridedError::OffsetOverflow)?;
117 if span < 0 {
118 min_offset = min_offset
119 .checked_add(span)
120 .ok_or(StridedError::OffsetOverflow)?;
121 } else {
122 max_offset = max_offset
123 .checked_add(span)
124 .ok_or(StridedError::OffsetOverflow)?;
125 }
126 }
127
128 let ptr = ptr as i128;
129 let elem_size = elem_size as i128;
130 let start = ptr
131 .checked_add(
132 (min_offset as i128)
133 .checked_mul(elem_size)
134 .ok_or(StridedError::OffsetOverflow)?,
135 )
136 .ok_or(StridedError::OffsetOverflow)?;
137 let end = ptr
138 .checked_add(
139 (max_offset as i128)
140 .checked_mul(elem_size)
141 .and_then(|offset| offset.checked_add(elem_size))
142 .ok_or(StridedError::OffsetOverflow)?,
143 )
144 .ok_or(StridedError::OffsetOverflow)?;
145 if start < 0 || end < 0 || start > usize::MAX as i128 || end > usize::MAX as i128 {
146 return Err(StridedError::OffsetOverflow);
147 }
148 Ok(Some((start as usize, end as usize)))
149}
150
151fn validate_typed_no_overlap<D, A, Op: ElementOp<A>>(
152 dest: &StridedViewMut<MaybeUninit<D>>,
153 input: &StridedView<A, Op>,
154 input_index: usize,
155) -> Result<()> {
156 let dest_range = reachable_byte_range(
157 dest.ptr() as usize,
158 core::mem::size_of::<D>(),
159 dest.dims(),
160 dest.strides(),
161 )?;
162 let input_range = reachable_byte_range(
163 input.ptr() as usize,
164 core::mem::size_of::<A>(),
165 input.dims(),
166 input.strides(),
167 )?;
168 if let (Some((dest_start, dest_end)), Some((input_start, input_end))) =
169 (dest_range, input_range)
170 {
171 if dest_start < input_end && input_start < dest_end {
172 return Err(StridedError::OverlappingInputOutput { input: input_index });
173 }
174 }
175 Ok(())
176}
177
178trait MapKernel<A, D> {
196 fn apply(&self, src: &[A], dst: &mut [D]) -> bool;
198}
199
200struct NoKernel;
201
202impl<A, D> MapKernel<A, D> for NoKernel {
203 #[inline(always)]
204 fn apply(&self, _src: &[A], _dst: &mut [D]) -> bool {
205 false
206 }
207}
208
209struct SliceKernel<K>(K);
210
211impl<A, D, K: Fn(&[A], &mut [D]) -> bool> MapKernel<A, D> for SliceKernel<K> {
212 #[inline(always)]
213 fn apply(&self, src: &[A], dst: &mut [D]) -> bool {
214 (self.0)(src, dst)
215 }
216}
217
218unsafe fn inner_loop_map1<D: Copy, A: Copy, Op: ElementOp<A>, K: MapKernel<A, D>>(
219 dp: *mut D,
220 ds: isize,
221 sp: *const A,
222 ss: isize,
223 len: usize,
224 f: &impl Fn(A) -> D,
225 kernel: &K,
226) {
227 if ds == 1 && ss == 1 {
228 let src = std::slice::from_raw_parts(sp, len);
229 let dst = std::slice::from_raw_parts_mut(dp, len);
230 if kernel.apply(src, dst) {
231 return;
232 }
233 simd::dispatch_if_large(len, || {
234 for (d, s) in dst.iter_mut().zip(src.iter()) {
235 *d = f(Op::apply(*s));
236 }
237 });
238 } else {
239 let mut dp = dp;
240 let mut sp = sp;
241 simd::dispatch_if_large(len, || {
242 for _ in 0..len {
243 *dp = f(Op::apply(*sp));
244 dp = dp.wrapping_offset(ds);
245 sp = sp.wrapping_offset(ss);
246 }
247 });
248 }
249}
250
251#[allow(clippy::too_many_arguments)]
255#[inline(always)]
256unsafe fn inner_loop_map2<D: Copy, A: Copy, B: Copy, OpA: ElementOp<A>, OpB: ElementOp<B>>(
257 dp: *mut D,
258 ds: isize,
259 ap: *const A,
260 a_s: isize,
261 bp: *const B,
262 b_s: isize,
263 len: usize,
264 f: &impl Fn(A, B) -> D,
265) {
266 if ds == 1 && a_s == 1 && b_s == 1 {
267 let src_a = std::slice::from_raw_parts(ap, len);
268 let src_b = std::slice::from_raw_parts(bp, len);
269 let dst = std::slice::from_raw_parts_mut(dp, len);
270 simd::dispatch_if_large(len, || {
271 for i in 0..len {
272 dst[i] = f(OpA::apply(src_a[i]), OpB::apply(src_b[i]));
273 }
274 });
275 } else if ds == 1 && a_s == 1 && b_s == 0 {
276 let src_a = std::slice::from_raw_parts(ap, len);
277 let b = OpB::apply(*bp);
278 let dst = std::slice::from_raw_parts_mut(dp, len);
279 simd::dispatch_if_large(len, || {
280 for i in 0..len {
281 dst[i] = f(OpA::apply(src_a[i]), b);
282 }
283 });
284 } else if ds == 1 && a_s == 0 && b_s == 1 {
285 let a = OpA::apply(*ap);
286 let src_b = std::slice::from_raw_parts(bp, len);
287 let dst = std::slice::from_raw_parts_mut(dp, len);
288 simd::dispatch_if_large(len, || {
289 for i in 0..len {
290 dst[i] = f(a, OpB::apply(src_b[i]));
291 }
292 });
293 } else if ds == 1 && a_s == 0 && b_s == 0 {
294 let a = OpA::apply(*ap);
295 let b = OpB::apply(*bp);
296 let dst = std::slice::from_raw_parts_mut(dp, len);
297 simd::dispatch_if_large(len, || {
298 for d in dst.iter_mut() {
299 *d = f(a, b);
300 }
301 });
302 } else if ds == 1 && b_s == 0 {
303 let b = OpB::apply(*bp);
304 let dst = std::slice::from_raw_parts_mut(dp, len);
305 let mut ap = ap;
306 simd::dispatch_if_large(len, || {
307 for d in dst.iter_mut() {
308 *d = f(OpA::apply(*ap), b);
309 ap = ap.wrapping_offset(a_s);
310 }
311 });
312 } else if ds == 1 && a_s == 0 {
313 let a = OpA::apply(*ap);
314 let dst = std::slice::from_raw_parts_mut(dp, len);
315 let mut bp = bp;
316 simd::dispatch_if_large(len, || {
317 for d in dst.iter_mut() {
318 *d = f(a, OpB::apply(*bp));
319 bp = bp.wrapping_offset(b_s);
320 }
321 });
322 } else {
323 let mut dp = dp;
324 let mut ap = ap;
325 let mut bp = bp;
326 simd::dispatch_if_large(len, || {
327 for _ in 0..len {
328 *dp = f(OpA::apply(*ap), OpB::apply(*bp));
329 dp = dp.wrapping_offset(ds);
330 ap = ap.wrapping_offset(a_s);
331 bp = bp.wrapping_offset(b_s);
332 }
333 });
334 }
335}
336
337trait MulOutput<D: Copy + 'static>: Copy + Send + Sync + 'static {
339 type Slot: Copy + Send + Sync + 'static;
340
341 unsafe fn write(dst: *mut Self::Slot, value: D);
342
343 unsafe fn try_contiguous<A: 'static, B: 'static>(
344 dst: *mut Self::Slot,
345 len: usize,
346 a: &[A],
347 b: &[B],
348 ) -> bool;
349}
350
351#[derive(Clone, Copy)]
352struct InitializedOutput;
353
354impl<D: Copy + Send + Sync + 'static> MulOutput<D> for InitializedOutput {
355 type Slot = D;
356
357 #[inline(always)]
358 unsafe fn write(dst: *mut D, value: D) {
359 unsafe { dst.write(value) };
360 }
361
362 #[inline(always)]
363 unsafe fn try_contiguous<A: 'static, B: 'static>(
364 dst: *mut D,
365 len: usize,
366 a: &[A],
367 b: &[B],
368 ) -> bool {
369 unsafe { simd::try_mul_contiguous_ptr(dst, len, a, b) }
370 }
371}
372
373#[derive(Clone, Copy)]
374struct UninitializedOutput;
375
376impl<D: Copy + Send + Sync + 'static> MulOutput<D> for UninitializedOutput {
377 type Slot = MaybeUninit<D>;
378
379 #[inline(always)]
380 unsafe fn write(dst: *mut MaybeUninit<D>, value: D) {
381 unsafe { dst.write(MaybeUninit::new(value)) };
382 }
383
384 #[inline(always)]
385 unsafe fn try_contiguous<A: 'static, B: 'static>(
386 dst: *mut MaybeUninit<D>,
387 len: usize,
388 a: &[A],
389 b: &[B],
390 ) -> bool {
391 unsafe { simd::try_mul_contiguous_ptr(dst.cast::<D>(), len, a, b) }
394 }
395}
396
397#[inline(always)]
398fn multiply_value<A, B, D>(lhs: A, rhs: B) -> D
399where
400 A: Copy + Mul<B, Output = D> + 'static,
401 B: Copy + 'static,
402 D: Copy + 'static,
403{
404 use core::any::TypeId;
405
406 if TypeId::of::<A>() == TypeId::of::<i32>()
407 && TypeId::of::<B>() == TypeId::of::<i32>()
408 && TypeId::of::<D>() == TypeId::of::<i32>()
409 {
410 let lhs = unsafe { *(&lhs as *const A).cast::<i32>() };
412 let rhs = unsafe { *(&rhs as *const B).cast::<i32>() };
413 let value = lhs.wrapping_mul(rhs);
414 return unsafe { core::mem::transmute_copy(&value) };
415 }
416 if TypeId::of::<A>() == TypeId::of::<i64>()
417 && TypeId::of::<B>() == TypeId::of::<i64>()
418 && TypeId::of::<D>() == TypeId::of::<i64>()
419 {
420 let lhs = unsafe { *(&lhs as *const A).cast::<i64>() };
422 let rhs = unsafe { *(&rhs as *const B).cast::<i64>() };
423 let value = lhs.wrapping_mul(rhs);
424 return unsafe { core::mem::transmute_copy(&value) };
425 }
426 lhs * rhs
427}
428
429#[inline(always)]
430unsafe fn inner_loop_mul2<
431 O: MulOutput<D>,
432 D: Copy + 'static,
433 A: Copy + Mul<B, Output = D> + 'static,
434 B: Copy + 'static,
435>(
436 dp: *mut O::Slot,
437 ds: isize,
438 ap: *const A,
439 a_s: isize,
440 bp: *const B,
441 b_s: isize,
442 len: usize,
443) {
444 if ds == 1 && a_s == 1 && b_s == 1 {
445 let src_a = std::slice::from_raw_parts(ap, len);
446 let src_b = std::slice::from_raw_parts(bp, len);
447 if len >= 64 && O::try_contiguous(dp, len, src_a, src_b) {
448 return;
449 }
450 simd::dispatch_if_large(len, || {
451 for i in 0..len {
452 O::write(dp.add(i), multiply_value(src_a[i], src_b[i]));
453 }
454 });
455 } else if ds == 1 && a_s == 1 && b_s == 0 {
456 let src_a = std::slice::from_raw_parts(ap, len);
457 let b = *bp;
458 simd::dispatch_if_large(len, || {
459 #[allow(clippy::needless_range_loop)]
462 for i in 0..len {
463 O::write(dp.add(i), multiply_value(src_a[i], b));
464 }
465 });
466 } else if ds == 1 && a_s == 0 && b_s == 1 {
467 let a = *ap;
468 let src_b = std::slice::from_raw_parts(bp, len);
469 simd::dispatch_if_large(len, || {
470 #[allow(clippy::needless_range_loop)]
473 for i in 0..len {
474 O::write(dp.add(i), multiply_value(a, src_b[i]));
475 }
476 });
477 } else if ds == 1 && a_s == 0 && b_s == 0 {
478 let a = *ap;
479 let b = *bp;
480 simd::dispatch_if_large(len, || {
481 for i in 0..len {
482 O::write(dp.add(i), multiply_value(a, b));
483 }
484 });
485 } else if ds == 1 && b_s == 0 {
486 let b = *bp;
487 let mut ap = ap;
488 simd::dispatch_if_large(len, || {
489 for i in 0..len {
490 O::write(dp.add(i), multiply_value(*ap, b));
491 ap = ap.wrapping_offset(a_s);
492 }
493 });
494 } else if ds == 1 && a_s == 0 {
495 let a = *ap;
496 let mut bp = bp;
497 simd::dispatch_if_large(len, || {
498 for i in 0..len {
499 O::write(dp.add(i), multiply_value(a, *bp));
500 bp = bp.wrapping_offset(b_s);
501 }
502 });
503 } else {
504 let mut dp = dp;
505 let mut ap = ap;
506 let mut bp = bp;
507 simd::dispatch_if_large(len, || {
508 for _ in 0..len {
509 O::write(dp, multiply_value(*ap, *bp));
510 dp = dp.wrapping_offset(ds);
511 ap = ap.wrapping_offset(a_s);
512 bp = bp.wrapping_offset(b_s);
513 }
514 });
515 }
516}
517
518#[derive(Clone, Debug, Eq, PartialEq)]
519struct ContiguousMulRangePlan {
520 axis_order: AxisVec<usize>,
521 inner_len: usize,
522 inner_axis_count: usize,
523 row_len: usize,
524 outer_axis_start: usize,
525 fast_axis: usize,
526 a_fast_stride: isize,
527 b_fast_stride: isize,
528 a_row_stride: isize,
529 b_row_stride: isize,
530}
531
532impl ContiguousMulRangePlan {
533 fn walks_inputs_without_blocking(&self) -> bool {
543 let streams = |stride: isize| stride == 0 || stride == 1;
544 if streams(self.a_fast_stride) && streams(self.b_fast_stride) {
545 return true;
546 }
547 if transposed_scalar_tile_kind(self).is_some() {
548 return true;
549 }
550 false
551 }
552}
553
554#[derive(Clone, Copy, Debug, Eq, PartialEq)]
555enum TransposedScalarTileKind {
556 RhsScalar,
557 LhsScalar,
558}
559
560fn transposed_scalar_tile_kind(plan: &ContiguousMulRangePlan) -> Option<TransposedScalarTileKind> {
561 if plan.inner_len > TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT.saturating_mul(plan.row_len) {
562 return None;
563 }
564 let row_len = isize::try_from(plan.row_len).ok()?;
565 if plan.a_fast_stride == row_len
566 && plan.a_row_stride == 1
567 && plan.b_fast_stride == 0
568 && plan.b_row_stride == 0
569 {
570 return Some(TransposedScalarTileKind::RhsScalar);
571 }
572
573 if plan.b_fast_stride == row_len
574 && plan.b_row_stride == 1
575 && plan.a_fast_stride == 0
576 && plan.a_row_stride == 0
577 {
578 return Some(TransposedScalarTileKind::LhsScalar);
579 }
580
581 None
582}
583
584fn compact_axis_order(dims: &[usize], strides: &[isize]) -> Option<AxisVec<usize>> {
585 if dims.len() != strides.len() {
586 return None;
587 }
588
589 let mut active = AxisVec::<usize>::new();
590 let mut inactive = AxisVec::<usize>::new();
591 for (axis, (&dim, &stride)) in dims.iter().zip(strides.iter()).enumerate() {
592 if stride < 0 {
593 return None;
594 }
595 if dim > 1 {
596 active.push(axis);
597 } else {
598 inactive.push(axis);
599 }
600 }
601
602 active.sort_by(|&lhs, &rhs| strides[lhs].cmp(&strides[rhs]).then_with(|| lhs.cmp(&rhs)));
603
604 let mut expected = 1isize;
605 for &axis in &active {
606 if strides[axis] != expected {
607 return None;
608 }
609 expected = expected.saturating_mul(dims[axis] as isize);
610 }
611
612 active.extend(inactive);
613 Some(active)
614}
615
616fn can_fuse_contiguous_range_axis(dim: usize, prev_stride: isize, next_stride: isize) -> bool {
617 dim <= 1 || (prev_stride == 0 && next_stride == 0) || next_stride == prev_stride * dim as isize
618}
619
620fn contiguous_mul_range_plan(
621 dims: &[usize],
622 dst_strides: &[isize],
623 a_strides: &[isize],
624 b_strides: &[isize],
625) -> Option<ContiguousMulRangePlan> {
626 let axis_order = compact_axis_order(dims, dst_strides)?;
627 if dims.is_empty() {
628 return Some(ContiguousMulRangePlan {
629 axis_order,
630 inner_len: 1,
631 inner_axis_count: 0,
632 row_len: 1,
633 outer_axis_start: 0,
634 fast_axis: 0,
635 a_fast_stride: 0,
636 b_fast_stride: 0,
637 a_row_stride: 0,
638 b_row_stride: 0,
639 });
640 }
641
642 let first_pos = axis_order
643 .iter()
644 .position(|&axis| dims[axis] > 1)
645 .unwrap_or(0);
646 let first_axis = axis_order[first_pos];
647 let mut inner_len = dims[first_axis].max(1);
648 let mut inner_axis_count = first_pos + 1;
649 let mut prev_axis = first_axis;
650
651 for &axis in axis_order.iter().skip(first_pos + 1) {
652 if can_fuse_contiguous_range_axis(
653 dims[prev_axis],
654 dst_strides[prev_axis],
655 dst_strides[axis],
656 ) && can_fuse_contiguous_range_axis(
657 dims[prev_axis],
658 a_strides[prev_axis],
659 a_strides[axis],
660 ) && can_fuse_contiguous_range_axis(
661 dims[prev_axis],
662 b_strides[prev_axis],
663 b_strides[axis],
664 ) {
665 inner_len = inner_len.checked_mul(dims[axis].max(1))?;
666 inner_axis_count += 1;
667 prev_axis = axis;
668 } else {
669 break;
670 }
671 }
672
673 let row = axis_order
674 .iter()
675 .enumerate()
676 .skip(inner_axis_count)
677 .find(|&(_, &axis)| dims[axis] > 1);
678 let (row_len, outer_axis_start, a_row_stride, b_row_stride) =
679 if let Some((row_pos, &row_axis)) = row {
680 (
681 dims[row_axis],
682 row_pos + 1,
683 a_strides[row_axis],
684 b_strides[row_axis],
685 )
686 } else {
687 (1, axis_order.len(), 0, 0)
688 };
689
690 Some(ContiguousMulRangePlan {
691 axis_order,
692 inner_len,
693 inner_axis_count,
694 row_len,
695 outer_axis_start,
696 fast_axis: first_axis,
697 a_fast_stride: a_strides[first_axis],
698 b_fast_stride: b_strides[first_axis],
699 a_row_stride,
700 b_row_stride,
701 })
702}
703
704struct ContiguousMulOuterCursor<'a> {
705 dims: &'a [usize],
706 a_strides: &'a [isize],
707 b_strides: &'a [isize],
708 axes: AxisVec<usize>,
709 coords: AxisVec<usize>,
710 a_offset: isize,
711 b_offset: isize,
712}
713
714impl<'a> ContiguousMulOuterCursor<'a> {
715 fn new(
716 dims: &'a [usize],
717 a_strides: &'a [isize],
718 b_strides: &'a [isize],
719 plan: &ContiguousMulRangePlan,
720 outer_group: usize,
721 ) -> Self {
722 let axes: AxisVec<usize> = plan
723 .axis_order
724 .iter()
725 .skip(plan.outer_axis_start)
726 .copied()
727 .collect();
728 let mut coords = AxisVec::<usize>::with_capacity(axes.len());
729 let mut rem = outer_group;
730 let mut a_offset = 0isize;
731 let mut b_offset = 0isize;
732
733 for &axis in &axes {
734 let dim = dims[axis].max(1);
735 let coord = rem % dim;
736 rem /= dim;
737 coords.push(coord);
738 a_offset += coord as isize * a_strides[axis];
739 b_offset += coord as isize * b_strides[axis];
740 }
741
742 Self {
743 dims,
744 a_strides,
745 b_strides,
746 axes,
747 coords,
748 a_offset,
749 b_offset,
750 }
751 }
752
753 fn advance(&mut self) {
754 for (i, &axis) in self.axes.iter().enumerate() {
755 let dim = self.dims[axis].max(1);
756 if dim <= 1 {
757 continue;
758 }
759
760 if self.coords[i] + 1 < dim {
763 self.coords[i] += 1;
764 self.a_offset += self.a_strides[axis];
765 self.b_offset += self.b_strides[axis];
766 break;
767 }
768
769 let last = (dim - 1) as isize;
770 self.coords[i] = 0;
771 self.a_offset -= last * self.a_strides[axis];
772 self.b_offset -= last * self.b_strides[axis];
773 }
774 }
775}
776
777#[allow(clippy::too_many_arguments)]
780#[inline(always)]
781unsafe fn run_contiguous_mul_row_block<
782 O: MulOutput<D>,
783 D: Copy + 'static,
784 A: Copy + Mul<B, Output = D> + 'static,
785 B: Copy + 'static,
786>(
787 dst_ptr: *mut O::Slot,
788 a_ptr: *const A,
789 b_ptr: *const B,
790 plan: &ContiguousMulRangePlan,
791 base_index: usize,
792 total: usize,
793 base_a_offset: isize,
794 base_b_offset: isize,
795) {
796 let inner_len = plan.inner_len.max(1);
797 let row_len = plan.row_len.max(1);
798 let block_len = inner_len.saturating_mul(row_len);
799
800 if total.saturating_sub(base_index) >= block_len {
801 let tiled = match transposed_scalar_tile_kind(plan) {
802 Some(TransposedScalarTileKind::RhsScalar) => {
803 simd::try_mul_transposed_scalar_rhs_2d::<D, A, B>(
804 dst_ptr.add(base_index).cast::<D>(),
805 a_ptr.offset(base_a_offset),
806 b_ptr.offset(base_b_offset),
807 inner_len,
808 row_len,
809 plan.a_fast_stride,
810 plan.a_row_stride,
811 )
812 }
813 Some(TransposedScalarTileKind::LhsScalar) => {
814 simd::try_mul_transposed_scalar_lhs_2d::<D, A, B>(
815 dst_ptr.add(base_index).cast::<D>(),
816 a_ptr.offset(base_a_offset),
817 b_ptr.offset(base_b_offset),
818 inner_len,
819 row_len,
820 plan.b_fast_stride,
821 plan.b_row_stride,
822 )
823 }
824 None => false,
825 };
826 if tiled {
827 return;
828 }
829 }
830
831 let mut index = base_index;
832 let mut a_offset = base_a_offset;
833 let mut b_offset = base_b_offset;
834
835 for _ in 0..row_len {
836 if index >= total {
837 break;
838 }
839 let len = inner_len.min(total - index);
840 inner_loop_mul2::<O, D, A, B>(
841 dst_ptr.add(index),
842 1,
843 a_ptr.offset(a_offset),
844 plan.a_fast_stride,
845 b_ptr.offset(b_offset),
846 plan.b_fast_stride,
847 len,
848 );
849 index += inner_len;
850 a_offset += plan.a_row_stride;
851 b_offset += plan.b_row_stride;
852 }
853}
854
855fn strided_offset_for_contiguous_linear_index(
856 dims: &[usize],
857 strides: &[isize],
858 axis_order: &[usize],
859 mut index: usize,
860) -> isize {
861 let mut offset = 0isize;
862 for &axis in axis_order {
863 let dim = dims[axis];
864 if dim == 0 {
865 return 0;
866 }
867 let coord = index % dim;
868 index /= dim;
869 offset += coord as isize * strides[axis];
870 }
871 offset
872}
873
874#[allow(clippy::too_many_arguments)]
879fn try_contiguous_range_mul<
880 O: MulOutput<D>,
881 D: Copy + Send + Sync + 'static,
882 A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
883 B: Copy + Send + Sync + 'static,
884>(
885 dst_ptr: *mut O::Slot,
886 dims: &[usize],
887 dst_strides: &[isize],
888 a_ptr: *const A,
889 a_strides: &[isize],
890 b_ptr: *const B,
891 b_strides: &[isize],
892 exec: Exec<'_>,
893) -> bool {
894 let Ok(total) = total_len(dims) else {
897 return false;
898 };
899 if total == 0 {
900 return true;
901 }
902 let exec = for_len(exec, KernelClass::Map, total);
904 if exec.is_serial() {
905 return false;
906 }
907
908 let Some(plan) = contiguous_mul_range_plan(dims, dst_strides, a_strides, b_strides) else {
909 return false;
910 };
911 if !plan.walks_inputs_without_blocking() {
912 return false;
913 }
914
915 let inner_len = plan.inner_len.max(1);
916 let row_len = plan.row_len.max(1);
917 let block_len = inner_len.saturating_mul(row_len).max(1);
918 let outer_groups = total.div_ceil(block_len);
919
920 let nthreads = exec.budget().get();
921 if nthreads > 1 {
922 use crate::threading::{parallel_for_each, SendPtr};
923
924 let dst = SendPtr(dst_ptr);
925 let a = SendPtr(a_ptr as *mut A);
926 let b = SendPtr(b_ptr as *mut B);
927
928 if outer_groups < nthreads {
929 let chunk_len = total.div_ceil(nthreads);
930 let nchunks = total.div_ceil(chunk_len);
931
932 parallel_for_each(0..nchunks, exec, &|chunks| {
933 for chunk in chunks {
934 let start = chunk * chunk_len;
935 let end = (start + chunk_len).min(total);
936 let mut index = start;
937
938 while index < end {
939 let in_inner = index % inner_len;
940 let len = (inner_len - in_inner).min(end - index);
941 let a_offset = strided_offset_for_contiguous_linear_index(
942 dims,
943 a_strides,
944 &plan.axis_order,
945 index,
946 );
947 let b_offset = strided_offset_for_contiguous_linear_index(
948 dims,
949 b_strides,
950 &plan.axis_order,
951 index,
952 );
953
954 unsafe {
955 inner_loop_mul2::<O, D, A, B>(
956 dst.as_ptr().add(index),
957 1,
958 a.as_const().offset(a_offset),
959 plan.a_fast_stride,
960 b.as_const().offset(b_offset),
961 plan.b_fast_stride,
962 len,
963 );
964 }
965 index += len;
966 }
967 }
968 });
969
970 return true;
971 }
972
973 let groups_per_chunk = outer_groups.div_ceil(nthreads);
974 let nchunks = outer_groups.div_ceil(groups_per_chunk);
975
976 parallel_for_each(0..nchunks, exec, &|chunks| {
977 for chunk in chunks {
978 let group_start = chunk * groups_per_chunk;
979 let group_end = (group_start + groups_per_chunk).min(outer_groups);
980 let mut cursor =
981 ContiguousMulOuterCursor::new(dims, a_strides, b_strides, &plan, group_start);
982
983 for group in group_start..group_end {
984 let index = group * block_len;
985 unsafe {
986 run_contiguous_mul_row_block::<O, D, A, B>(
987 dst.as_ptr(),
988 a.as_const(),
989 b.as_const(),
990 &plan,
991 index,
992 total,
993 cursor.a_offset,
994 cursor.b_offset,
995 );
996 }
997 cursor.advance();
998 }
999 }
1000 });
1001
1002 true
1003 } else {
1004 run_contiguous_range_mul_single_thread::<O, D, A, B>(
1005 dst_ptr,
1006 dims,
1007 a_ptr,
1008 a_strides,
1009 b_ptr,
1010 b_strides,
1011 &plan,
1012 total,
1013 block_len,
1014 outer_groups,
1015 )
1016 }
1017}
1018
1019#[allow(clippy::too_many_arguments)]
1022fn run_contiguous_range_mul_single_thread<
1023 O: MulOutput<D>,
1024 D: Copy + Send + Sync + 'static,
1025 A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
1026 B: Copy + Send + Sync + 'static,
1027>(
1028 dst_ptr: *mut O::Slot,
1029 dims: &[usize],
1030 a_ptr: *const A,
1031 a_strides: &[isize],
1032 b_ptr: *const B,
1033 b_strides: &[isize],
1034 plan: &ContiguousMulRangePlan,
1035 total: usize,
1036 block_len: usize,
1037 outer_groups: usize,
1038) -> bool {
1039 let mut cursor = ContiguousMulOuterCursor::new(dims, a_strides, b_strides, plan, 0);
1040 for group in 0..outer_groups {
1041 let index = group * block_len;
1042 unsafe {
1043 run_contiguous_mul_row_block::<O, D, A, B>(
1044 dst_ptr,
1045 a_ptr,
1046 b_ptr,
1047 plan,
1048 index,
1049 total,
1050 cursor.a_offset,
1051 cursor.b_offset,
1052 );
1053 }
1054 cursor.advance();
1055 }
1056 true
1057}
1058
1059#[allow(clippy::too_many_arguments)]
1063#[inline(always)]
1064unsafe fn inner_loop_map3<
1065 D: Copy,
1066 A: Copy,
1067 B: Copy,
1068 C: Copy,
1069 OpA: ElementOp<A>,
1070 OpB: ElementOp<B>,
1071 OpC: ElementOp<C>,
1072>(
1073 dp: *mut D,
1074 ds: isize,
1075 ap: *const A,
1076 a_s: isize,
1077 bp: *const B,
1078 b_s: isize,
1079 cp: *const C,
1080 c_s: isize,
1081 len: usize,
1082 f: &impl Fn(A, B, C) -> D,
1083) {
1084 if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 {
1085 let src_a = std::slice::from_raw_parts(ap, len);
1086 let src_b = std::slice::from_raw_parts(bp, len);
1087 let src_c = std::slice::from_raw_parts(cp, len);
1088 let dst = std::slice::from_raw_parts_mut(dp, len);
1089 simd::dispatch_if_large(len, || {
1090 for (((d, &a), &b), &c) in dst.iter_mut().zip(src_a).zip(src_b).zip(src_c) {
1091 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
1092 }
1093 });
1094 } else {
1095 let mut dp = dp;
1096 let mut ap = ap;
1097 let mut bp = bp;
1098 let mut cp = cp;
1099 simd::dispatch_if_large(len, || {
1100 for _ in 0..len {
1101 *dp = f(OpA::apply(*ap), OpB::apply(*bp), OpC::apply(*cp));
1102 dp = dp.wrapping_offset(ds);
1103 ap = ap.wrapping_offset(a_s);
1104 bp = bp.wrapping_offset(b_s);
1105 cp = cp.wrapping_offset(c_s);
1106 }
1107 });
1108 }
1109}
1110
1111#[allow(clippy::too_many_arguments)]
1115#[inline(always)]
1116unsafe fn inner_loop_map4<
1117 D: Copy,
1118 A: Copy,
1119 B: Copy,
1120 C: Copy,
1121 E: Copy,
1122 OpA: ElementOp<A>,
1123 OpB: ElementOp<B>,
1124 OpC: ElementOp<C>,
1125 OpE: ElementOp<E>,
1126>(
1127 dp: *mut D,
1128 ds: isize,
1129 ap: *const A,
1130 a_s: isize,
1131 bp: *const B,
1132 b_s: isize,
1133 cp: *const C,
1134 c_s: isize,
1135 ep: *const E,
1136 e_s: isize,
1137 len: usize,
1138 f: &impl Fn(A, B, C, E) -> D,
1139) {
1140 if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 && e_s == 1 {
1141 let src_a = std::slice::from_raw_parts(ap, len);
1142 let src_b = std::slice::from_raw_parts(bp, len);
1143 let src_c = std::slice::from_raw_parts(cp, len);
1144 let src_e = std::slice::from_raw_parts(ep, len);
1145 let dst = std::slice::from_raw_parts_mut(dp, len);
1146 simd::dispatch_if_large(len, || {
1150 for ((((d, &a), &b), &c), &e) in
1151 dst.iter_mut().zip(src_a).zip(src_b).zip(src_c).zip(src_e)
1152 {
1153 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
1154 }
1155 });
1156 } else {
1157 let mut dp = dp;
1158 let mut ap = ap;
1159 let mut bp = bp;
1160 let mut cp = cp;
1161 let mut ep = ep;
1162 simd::dispatch_if_large(len, || {
1163 for _ in 0..len {
1164 *dp = f(
1165 OpA::apply(*ap),
1166 OpB::apply(*bp),
1167 OpC::apply(*cp),
1168 OpE::apply(*ep),
1169 );
1170 dp = dp.wrapping_offset(ds);
1171 ap = ap.wrapping_offset(a_s);
1172 bp = bp.wrapping_offset(b_s);
1173 cp = cp.wrapping_offset(c_s);
1174 ep = ep.wrapping_offset(e_s);
1175 }
1176 });
1177 }
1178}
1179
1180pub trait MapFn<A, D>: Sync {
1208 fn call(&self, a: A) -> D;
1210
1211 fn class(&self) -> KernelClass {
1213 KernelClass::Map
1214 }
1215}
1216
1217impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for F {
1218 #[inline(always)]
1219 fn call(&self, a: A) -> D {
1220 self(a)
1221 }
1222}
1223
1224#[derive(Debug, Clone, Copy)]
1228pub struct Classed<F> {
1229 pub f: F,
1231 pub class: KernelClass,
1233}
1234
1235impl<F> Classed<F> {
1236 pub const fn new(class: KernelClass, f: F) -> Self {
1238 Self { f, class }
1239 }
1240}
1241
1242impl KernelClass {
1243 pub const fn with<F>(self, f: F) -> Classed<F> {
1245 Classed::new(self, f)
1246 }
1247}
1248
1249impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for Classed<F> {
1250 #[inline(always)]
1251 fn call(&self, a: A) -> D {
1252 (self.f)(a)
1253 }
1254
1255 fn class(&self) -> KernelClass {
1256 self.class
1257 }
1258}
1259
1260pub fn map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1277 dest: &mut StridedViewMut<D>,
1278 src: &StridedView<A, Op>,
1279 f: impl MapFn<A, D>,
1280 exec: Exec<'_>,
1281) -> Result<()> {
1282 let class = f.class();
1283 map_parts_into::<D, A, Op, NoKernel>(
1284 dest.as_mut_ptr(),
1285 dest.dims(),
1286 dest.strides(),
1287 src.ptr(),
1288 src.dims(),
1289 src.strides(),
1290 |a| f.call(a),
1291 &NoKernel,
1292 class,
1293 exec,
1294 )
1295}
1296
1297pub(crate) fn copy_map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1302 dest: &mut StridedViewMut<D>,
1303 src: &StridedView<A, Op>,
1304 f: impl Fn(A) -> D + Sync,
1305 exec: Exec<'_>,
1306) -> Result<()> {
1307 map_parts_into::<D, A, Op, NoKernel>(
1308 dest.as_mut_ptr(),
1309 dest.dims(),
1310 dest.strides(),
1311 src.ptr(),
1312 src.dims(),
1313 src.strides(),
1314 f,
1315 &NoKernel,
1316 KernelClass::Copy,
1317 exec,
1318 )
1319}
1320
1321pub(crate) fn map_into_validated<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1322 dest: &mut StridedViewMut<D>,
1323 src: &StridedView<A, Op>,
1324 f: impl MapFn<A, D>,
1325 validated: ValidatedDestinationLayout,
1326 exec: Exec<'_>,
1327) -> Result<()> {
1328 ensure_same_shape(dest.dims(), src.dims())?;
1329 let class = f.class();
1330 map_parts_into_validated::<D, A, Op, NoKernel>(
1331 dest.as_mut_ptr(),
1332 dest.dims(),
1333 dest.strides(),
1334 src.ptr(),
1335 src.strides(),
1336 |a| f.call(a),
1337 &NoKernel,
1338 validated,
1339 class,
1340 exec,
1341 )
1342}
1343
1344pub fn map_into_uninit_with_kernel<T: Copy + Send + Sync>(
1388 dest: &mut StridedViewMut<MaybeUninit<T>>,
1389 src: &StridedView<T>,
1390 kernel: impl Fn(&[T], &mut [MaybeUninit<T>]) -> bool + Sync,
1391 scalar: impl MapFn<T, T>,
1392 exec: Exec<'_>,
1393) -> Result<()> {
1394 let class = scalar.class();
1395 map_parts_into::<MaybeUninit<T>, T, Identity, _>(
1396 dest.as_mut_ptr(),
1397 dest.dims(),
1398 dest.strides(),
1399 src.ptr(),
1400 src.dims(),
1401 src.strides(),
1402 |value| MaybeUninit::new(scalar.call(value)),
1403 &SliceKernel(kernel),
1404 class,
1405 exec,
1406 )
1407}
1408
1409#[cfg(test)]
1415pub(crate) fn map_raw_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1416 dest: &mut crate::RawStridedMut<'_, D>,
1417 src: &crate::RawStridedRef<'_, A>,
1418 f: impl Fn(A) -> D + Sync,
1419 exec: Exec<'_>,
1420) -> Result<()> {
1421 map_parts_into::<D, A, Op, NoKernel>(
1422 dest.as_mut_ptr(),
1423 dest.dims(),
1424 dest.strides(),
1425 src.ptr(),
1426 src.dims(),
1427 src.strides(),
1428 f,
1429 &NoKernel,
1430 KernelClass::Map,
1431 exec,
1432 )
1433}
1434
1435pub(crate) fn map_raw_into_validated<
1436 D: Copy + Send + Sync,
1437 A: Copy + Send + Sync,
1438 Op: ElementOp<A>,
1439>(
1440 dest: &mut crate::RawStridedMut<'_, D>,
1441 src: &crate::RawStridedRef<'_, A>,
1442 f: impl Fn(A) -> D + Sync,
1443 validated: ValidatedDestinationLayout,
1444 exec: Exec<'_>,
1445) -> Result<()> {
1446 ensure_same_shape(dest.dims(), src.dims())?;
1447 map_parts_into_validated::<D, A, Op, NoKernel>(
1448 dest.as_mut_ptr(),
1449 dest.dims(),
1450 dest.strides(),
1451 src.ptr(),
1452 src.strides(),
1453 f,
1454 &NoKernel,
1455 validated,
1456 KernelClass::Map,
1457 exec,
1458 )
1459}
1460
1461#[allow(clippy::too_many_arguments)]
1464fn map_parts_into<
1465 D: Copy + Send + Sync,
1466 A: Copy + Send + Sync,
1467 Op: ElementOp<A>,
1468 K: MapKernel<A, D> + Sync,
1469>(
1470 dst_ptr: *mut D,
1471 dst_dims: &[usize],
1472 dst_strides: &[isize],
1473 src_ptr: *const A,
1474 src_dims: &[usize],
1475 src_strides: &[isize],
1476 f: impl Fn(A) -> D + Sync,
1477 kernel: &K,
1478 class: KernelClass,
1479 exec: Exec<'_>,
1480) -> Result<()> {
1481 ensure_same_shape(dst_dims, src_dims)?;
1482 let validated = validate_destination_layout(dst_dims, dst_strides)?;
1483 map_parts_into_validated::<D, A, Op, K>(
1484 dst_ptr,
1485 dst_dims,
1486 dst_strides,
1487 src_ptr,
1488 src_strides,
1489 f,
1490 kernel,
1491 validated,
1492 class,
1493 exec,
1494 )
1495}
1496
1497#[allow(clippy::too_many_arguments)]
1498#[inline(never)]
1505fn map_contiguous<D: Copy, A: Copy, Op: ElementOp<A>>(
1506 dst: &mut [D],
1507 src: &[A],
1508 f: impl Fn(A) -> D,
1509) {
1510 simd::dispatch_if_large(dst.len(), move || {
1511 for i in 0..dst.len() {
1512 dst[i] = f(Op::apply(src[i]));
1513 }
1514 });
1515}
1516
1517#[allow(clippy::too_many_arguments)]
1520fn map_parts_into_validated<
1521 D: Copy + Send + Sync,
1522 A: Copy + Send + Sync,
1523 Op: ElementOp<A>,
1524 K: MapKernel<A, D> + Sync,
1525>(
1526 dst_ptr: *mut D,
1527 dst_dims: &[usize],
1528 dst_strides: &[isize],
1529 src_ptr: *const A,
1530 src_strides: &[isize],
1531 f: impl Fn(A) -> D + Sync,
1532 kernel: &K,
1533 _validated: ValidatedDestinationLayout,
1534 class: KernelClass,
1535 exec: Exec<'_>,
1536) -> Result<()> {
1537 if sequential_contiguous_layout(class, dst_dims, &[dst_strides, src_strides], &exec)?.is_some()
1538 {
1539 let len = total_len(dst_dims)?;
1540 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1544 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
1545 if kernel.apply(src, dst) {
1546 return Ok(());
1547 }
1548 map_contiguous::<D, A, Op>(dst, src, f);
1549 return Ok(());
1550 }
1551
1552 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
1553 let total = total_len(dst_dims)?;
1554 let (fused_dims, ordered_strides, plan) = pointwise_plan(
1555 dst_dims,
1556 &strides_list,
1557 Some(0),
1558 &[std::mem::size_of::<D>(), std::mem::size_of::<A>()],
1559 total,
1560 );
1561
1562 let total = total_len(&fused_dims)?;
1563 if is_parallel(class, total, &exec) {
1564 use crate::threading::SendPtr;
1565 let dst_send = SendPtr(dst_ptr);
1566 let src_send = SendPtr(src_ptr as *mut A);
1567
1568 let costs = compute_costs(&ordered_strides);
1569 let initial_offsets: crate::inline::OperandVec<isize> =
1570 strides_list.iter().map(|_| 0isize).collect();
1571 return mapreduce_threaded(
1572 &fused_dims,
1573 &plan.block,
1574 &ordered_strides,
1575 &initial_offsets,
1576 &costs,
1577 class,
1578 exec,
1579 &|dims, blocks, strides_list, offsets| {
1580 for_each_inner_block_with_offsets(
1581 dims,
1582 blocks,
1583 strides_list,
1584 offsets,
1585 |offsets, len, strides| {
1586 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1587 let sp = unsafe { src_send.as_const().offset(offsets[1]) };
1588 unsafe {
1589 inner_loop_map1::<D, A, Op, K>(
1590 dp, strides[0], sp, strides[1], len, &f, kernel,
1591 )
1592 };
1593 Ok(())
1594 },
1595 )
1596 },
1597 );
1598 }
1599
1600 let initial_offsets: crate::inline::OperandVec<isize> =
1601 ordered_strides.iter().map(|_| 0isize).collect();
1602 for_each_inner_block_preordered(
1603 &fused_dims,
1604 &plan.block,
1605 &ordered_strides,
1606 &initial_offsets,
1607 |offsets, len, strides| {
1608 let dp = unsafe { dst_ptr.offset(offsets[0]) };
1609 let sp = unsafe { src_ptr.offset(offsets[1]) };
1610 unsafe {
1611 inner_loop_map1::<D, A, Op, K>(dp, strides[0], sp, strides[1], len, &f, kernel)
1612 };
1613 Ok(())
1614 },
1615 )
1616}
1617
1618pub fn zip_map2_into<
1633 D: Copy + Send + Sync,
1634 A: Copy + Send + Sync,
1635 B: Copy + Send + Sync,
1636 OpA: ElementOp<A>,
1637 OpB: ElementOp<B>,
1638>(
1639 dest: &mut StridedViewMut<D>,
1640 a: &StridedView<A, OpA>,
1641 b: &StridedView<B, OpB>,
1642 f: impl Fn(A, B) -> D + Sync,
1643 exec: Exec<'_>,
1644) -> Result<()> {
1645 zip_map2_parts_into::<D, A, B, OpA, OpB>(
1646 dest.as_mut_ptr(),
1647 dest.dims(),
1648 dest.strides(),
1649 a.ptr(),
1650 a.dims(),
1651 a.strides(),
1652 b.ptr(),
1653 b.dims(),
1654 b.strides(),
1655 f,
1656 exec,
1657 )
1658}
1659
1660pub(crate) fn zip_map2_into_validated<
1661 D: Copy + Send + Sync,
1662 A: Copy + Send + Sync,
1663 B: Copy + Send + Sync,
1664 OpA: ElementOp<A>,
1665 OpB: ElementOp<B>,
1666>(
1667 dest: &mut StridedViewMut<D>,
1668 a: &StridedView<A, OpA>,
1669 b: &StridedView<B, OpB>,
1670 f: impl Fn(A, B) -> D + Sync,
1671 validated: ValidatedDestinationLayout,
1672 exec: Exec<'_>,
1673) -> Result<()> {
1674 zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1675 dest.as_mut_ptr(),
1676 dest.dims(),
1677 dest.strides(),
1678 a.ptr(),
1679 a.strides(),
1680 b.ptr(),
1681 b.strides(),
1682 f,
1683 validated,
1684 exec,
1685 )
1686}
1687
1688#[non_exhaustive]
1690#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1691pub enum CompareOp {
1692 Eq,
1693 Lt,
1694 Le,
1695 Gt,
1696 Ge,
1697}
1698
1699pub fn compare_into<T, OpA, OpB>(
1710 dest: &mut StridedViewMut<bool>,
1711 a: &StridedView<T, OpA>,
1712 b: &StridedView<T, OpB>,
1713 op: CompareOp,
1714 exec: Exec<'_>,
1715) -> Result<()>
1716where
1717 T: Copy + Send + Sync + PartialOrd,
1718 OpA: ElementOp<T>,
1719 OpB: ElementOp<T>,
1720{
1721 match op {
1722 CompareOp::Eq => zip_map2_into(dest, a, b, |lhs, rhs| lhs == rhs, exec),
1723 CompareOp::Lt => zip_map2_into(dest, a, b, |lhs, rhs| lhs < rhs, exec),
1724 CompareOp::Le => zip_map2_into(dest, a, b, |lhs, rhs| lhs <= rhs, exec),
1725 CompareOp::Gt => zip_map2_into(dest, a, b, |lhs, rhs| lhs > rhs, exec),
1726 CompareOp::Ge => zip_map2_into(dest, a, b, |lhs, rhs| lhs >= rhs, exec),
1727 }
1728}
1729
1730pub fn compare_into_uninit<T, OpA, OpB>(
1749 dest: &mut StridedViewMut<MaybeUninit<bool>>,
1750 a: &StridedView<T, OpA>,
1751 b: &StridedView<T, OpB>,
1752 op: CompareOp,
1753 exec: Exec<'_>,
1754) -> Result<()>
1755where
1756 T: Copy + Send + Sync + PartialOrd,
1757 OpA: ElementOp<T>,
1758 OpB: ElementOp<T>,
1759{
1760 ensure_same_shape(dest.dims(), a.dims())?;
1761 ensure_same_shape(dest.dims(), b.dims())?;
1762 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
1763 validate_typed_no_overlap(dest, a, 0)?;
1764 validate_typed_no_overlap(dest, b, 1)?;
1765 match op {
1766 CompareOp::Eq => zip_map2_into_validated(
1767 dest,
1768 a,
1769 b,
1770 |lhs, rhs| MaybeUninit::new(lhs == rhs),
1771 validated,
1772 exec,
1773 ),
1774 CompareOp::Lt => zip_map2_into_validated(
1775 dest,
1776 a,
1777 b,
1778 |lhs, rhs| MaybeUninit::new(lhs < rhs),
1779 validated,
1780 exec,
1781 ),
1782 CompareOp::Le => zip_map2_into_validated(
1783 dest,
1784 a,
1785 b,
1786 |lhs, rhs| MaybeUninit::new(lhs <= rhs),
1787 validated,
1788 exec,
1789 ),
1790 CompareOp::Gt => zip_map2_into_validated(
1791 dest,
1792 a,
1793 b,
1794 |lhs, rhs| MaybeUninit::new(lhs > rhs),
1795 validated,
1796 exec,
1797 ),
1798 CompareOp::Ge => zip_map2_into_validated(
1799 dest,
1800 a,
1801 b,
1802 |lhs, rhs| MaybeUninit::new(lhs >= rhs),
1803 validated,
1804 exec,
1805 ),
1806 }
1807}
1808
1809pub(crate) fn zip_map2_raw_into_validated<
1810 D: Copy + Send + Sync,
1811 A: Copy + Send + Sync,
1812 B: Copy + Send + Sync,
1813 OpA: ElementOp<A>,
1814 OpB: ElementOp<B>,
1815>(
1816 dest: &mut crate::RawStridedMut<'_, D>,
1817 a: &crate::RawStridedRef<'_, A>,
1818 b: &crate::RawStridedRef<'_, B>,
1819 f: impl Fn(A, B) -> D + Sync,
1820 validated: ValidatedDestinationLayout,
1821 exec: Exec<'_>,
1822) -> Result<()> {
1823 ensure_same_shape(dest.dims(), a.dims())?;
1824 ensure_same_shape(dest.dims(), b.dims())?;
1825 zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1826 dest.as_mut_ptr(),
1827 dest.dims(),
1828 dest.strides(),
1829 a.ptr(),
1830 a.strides(),
1831 b.ptr(),
1832 b.strides(),
1833 f,
1834 validated,
1835 exec,
1836 )
1837}
1838
1839#[allow(clippy::too_many_arguments)]
1842fn zip_map2_parts_into<
1843 D: Copy + Send + Sync,
1844 A: Copy + Send + Sync,
1845 B: Copy + Send + Sync,
1846 OpA: ElementOp<A>,
1847 OpB: ElementOp<B>,
1848>(
1849 dst_ptr: *mut D,
1850 dst_dims: &[usize],
1851 dst_strides: &[isize],
1852 a_ptr: *const A,
1853 a_dims: &[usize],
1854 a_strides: &[isize],
1855 b_ptr: *const B,
1856 b_dims: &[usize],
1857 b_strides: &[isize],
1858 f: impl Fn(A, B) -> D + Sync,
1859 exec: Exec<'_>,
1860) -> Result<()> {
1861 ensure_same_shape(dst_dims, a_dims)?;
1862 ensure_same_shape(dst_dims, b_dims)?;
1863 let validated = validate_destination_layout(dst_dims, dst_strides)?;
1864 zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1865 dst_ptr,
1866 dst_dims,
1867 dst_strides,
1868 a_ptr,
1869 a_strides,
1870 b_ptr,
1871 b_strides,
1872 f,
1873 validated,
1874 exec,
1875 )
1876}
1877
1878#[allow(clippy::too_many_arguments)]
1881fn zip_map2_parts_into_validated<
1882 D: Copy + Send + Sync,
1883 A: Copy + Send + Sync,
1884 B: Copy + Send + Sync,
1885 OpA: ElementOp<A>,
1886 OpB: ElementOp<B>,
1887>(
1888 dst_ptr: *mut D,
1889 dst_dims: &[usize],
1890 dst_strides: &[isize],
1891 a_ptr: *const A,
1892 a_strides: &[isize],
1893 b_ptr: *const B,
1894 b_strides: &[isize],
1895 f: impl Fn(A, B) -> D + Sync,
1896 _validated: ValidatedDestinationLayout,
1897 exec: Exec<'_>,
1898) -> Result<()> {
1899 if sequential_contiguous_layout(
1900 KernelClass::Map,
1901 dst_dims,
1902 &[dst_strides, a_strides, b_strides],
1903 &exec,
1904 )?
1905 .is_some()
1906 {
1907 let len = total_len(dst_dims)?;
1908 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1909 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
1910 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
1911 simd::dispatch_if_large(len, || {
1912 for i in 0..len {
1913 dst[i] = f(OpA::apply(sa[i]), OpB::apply(sb[i]));
1914 }
1915 });
1916 return Ok(());
1917 }
1918
1919 let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
1920 let total = total_len(dst_dims)?;
1921 let (fused_dims, ordered_strides, plan) = pointwise_plan(
1922 dst_dims,
1923 &strides_list,
1924 Some(0),
1925 &[
1926 std::mem::size_of::<D>(),
1927 std::mem::size_of::<A>(),
1928 std::mem::size_of::<B>(),
1929 ],
1930 total,
1931 );
1932
1933 let total = total_len(&fused_dims)?;
1934 if is_parallel(KernelClass::Map, total, &exec) {
1935 use crate::threading::SendPtr;
1936 let dst_send = SendPtr(dst_ptr);
1937 let a_send = SendPtr(a_ptr as *mut A);
1938 let b_send = SendPtr(b_ptr as *mut B);
1939
1940 let costs = compute_costs(&ordered_strides);
1941 let initial_offsets: crate::inline::OperandVec<isize> =
1942 strides_list.iter().map(|_| 0isize).collect();
1943 return mapreduce_threaded(
1944 &fused_dims,
1945 &plan.block,
1946 &ordered_strides,
1947 &initial_offsets,
1948 &costs,
1949 KernelClass::Map,
1950 exec,
1951 &|dims, blocks, strides_list, offsets| {
1952 for_each_inner_block_with_offsets(
1953 dims,
1954 blocks,
1955 strides_list,
1956 offsets,
1957 |offsets, len, strides| {
1958 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1959 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
1960 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
1961 unsafe {
1962 inner_loop_map2::<D, A, B, OpA, OpB>(
1963 dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1964 )
1965 };
1966 Ok(())
1967 },
1968 )
1969 },
1970 );
1971 }
1972
1973 let initial_offsets: crate::inline::OperandVec<isize> =
1974 ordered_strides.iter().map(|_| 0isize).collect();
1975 for_each_inner_block_preordered(
1976 &fused_dims,
1977 &plan.block,
1978 &ordered_strides,
1979 &initial_offsets,
1980 |offsets, len, strides| {
1981 let dp = unsafe { dst_ptr.offset(offsets[0]) };
1982 let ap = unsafe { a_ptr.offset(offsets[1]) };
1983 let bp = unsafe { b_ptr.offset(offsets[2]) };
1984 unsafe {
1985 inner_loop_map2::<D, A, B, OpA, OpB>(
1986 dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1987 )
1988 };
1989 Ok(())
1990 },
1991 )
1992}
1993
1994#[allow(clippy::too_many_arguments)]
1998fn mul_identity_into_raw<
1999 O: MulOutput<D>,
2000 D: Copy + Send + Sync + 'static,
2001 A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
2002 B: Copy + Send + Sync + 'static,
2003>(
2004 dst_ptr: *mut O::Slot,
2005 dst_dims: &[usize],
2006 dst_strides: &[isize],
2007 a_ptr: *const A,
2008 a_strides: &[isize],
2009 b_ptr: *const B,
2010 b_strides: &[isize],
2011 _validated: ValidatedDestinationLayout,
2012 mut exec: Exec<'_>,
2013) -> Result<()> {
2014 debug_assert_eq!(dst_dims.len(), a_strides.len());
2015 debug_assert_eq!(dst_dims.len(), b_strides.len());
2016
2017 if sequential_contiguous_layout(
2018 KernelClass::Map,
2019 dst_dims,
2020 &[dst_strides, a_strides, b_strides],
2021 &exec,
2022 )?
2023 .is_some()
2024 {
2025 let len = total_len(dst_dims)?;
2026 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2027 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2028 if unsafe { O::try_contiguous(dst_ptr, len, sa, sb) } {
2029 return Ok(());
2030 }
2031 for i in 0..len {
2032 unsafe { O::write(dst_ptr.add(i), multiply_value(sa[i], sb[i])) };
2033 }
2034 return Ok(());
2035 }
2036
2037 let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
2038 let total = total_len(dst_dims)?;
2039
2040 if try_contiguous_range_mul::<O, D, A, B>(
2041 dst_ptr,
2042 dst_dims,
2043 dst_strides,
2044 a_ptr,
2045 a_strides,
2046 b_ptr,
2047 b_strides,
2048 exec.reborrow(),
2049 ) {
2050 return Ok(());
2051 }
2052
2053 let (fused_dims, ordered_strides, plan) = pointwise_plan(
2054 dst_dims,
2055 &strides_list,
2056 Some(0),
2057 &[
2058 std::mem::size_of::<D>(),
2059 std::mem::size_of::<A>(),
2060 std::mem::size_of::<B>(),
2061 ],
2062 total,
2063 );
2064
2065 let total = total_len(&fused_dims)?;
2066 if is_parallel(KernelClass::Map, total, &exec) {
2067 use crate::threading::SendPtr;
2068 let dst_send = SendPtr(dst_ptr);
2069 let a_send = SendPtr(a_ptr as *mut A);
2070 let b_send = SendPtr(b_ptr as *mut B);
2071
2072 let costs = compute_costs(&ordered_strides);
2073 let initial_offsets: crate::inline::OperandVec<isize> =
2074 strides_list.iter().map(|_| 0isize).collect();
2075 return mapreduce_threaded(
2076 &fused_dims,
2077 &plan.block,
2078 &ordered_strides,
2079 &initial_offsets,
2080 &costs,
2081 KernelClass::Map,
2082 exec,
2083 &|dims, blocks, strides_list, offsets| {
2084 for_each_inner_block_with_offsets(
2085 dims,
2086 blocks,
2087 strides_list,
2088 offsets,
2089 |offsets, len, strides| {
2090 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2091 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2092 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2093 unsafe {
2094 inner_loop_mul2::<O, D, A, B>(
2095 dp, strides[0], ap, strides[1], bp, strides[2], len,
2096 )
2097 };
2098 Ok(())
2099 },
2100 )
2101 },
2102 );
2103 }
2104
2105 let initial_offsets: crate::inline::OperandVec<isize> =
2106 ordered_strides.iter().map(|_| 0isize).collect();
2107 for_each_inner_block_preordered(
2108 &fused_dims,
2109 &plan.block,
2110 &ordered_strides,
2111 &initial_offsets,
2112 |offsets, len, strides| {
2113 let dp = unsafe { dst_ptr.offset(offsets[0]) };
2114 let ap = unsafe { a_ptr.offset(offsets[1]) };
2115 let bp = unsafe { b_ptr.offset(offsets[2]) };
2116 unsafe {
2117 inner_loop_mul2::<O, D, A, B>(dp, strides[0], ap, strides[1], bp, strides[2], len)
2118 };
2119 Ok(())
2120 },
2121 )
2122}
2123
2124pub fn mul_into<
2139 D: Copy + Send + Sync + 'static,
2140 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2141 B: Copy + Send + Sync + 'static,
2142 OpA: ElementOp<A>,
2143 OpB: ElementOp<B>,
2144>(
2145 dest: &mut StridedViewMut<D>,
2146 a: &StridedView<A, OpA>,
2147 b: &StridedView<B, OpB>,
2148 exec: Exec<'_>,
2149) -> Result<()> {
2150 ensure_same_shape(dest.dims(), a.dims())?;
2151 ensure_same_shape(dest.dims(), b.dims())?;
2152 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2153
2154 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2155 return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2156 dest.as_mut_ptr(),
2157 dest.dims(),
2158 dest.strides(),
2159 a.ptr(),
2160 a.strides(),
2161 b.ptr(),
2162 b.strides(),
2163 validated,
2164 exec,
2165 );
2166 }
2167
2168 zip_map2_into_validated(dest, a, b, multiply_value, validated, exec)
2169}
2170
2171pub fn mul_into_uninit<
2190 D: Copy + Send + Sync + 'static,
2191 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2192 B: Copy + Send + Sync + 'static,
2193 OpA: ElementOp<A>,
2194 OpB: ElementOp<B>,
2195>(
2196 dest: &mut StridedViewMut<MaybeUninit<D>>,
2197 a: &StridedView<A, OpA>,
2198 b: &StridedView<B, OpB>,
2199 exec: Exec<'_>,
2200) -> Result<()> {
2201 ensure_same_shape(dest.dims(), a.dims())?;
2202 ensure_same_shape(dest.dims(), b.dims())?;
2203 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2204 validate_typed_no_overlap(dest, a, 0)?;
2205 validate_typed_no_overlap(dest, b, 1)?;
2206
2207 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2208 return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2209 dest.as_mut_ptr(),
2210 dest.dims(),
2211 dest.strides(),
2212 a.ptr(),
2213 a.strides(),
2214 b.ptr(),
2215 b.strides(),
2216 validated,
2217 exec,
2218 );
2219 }
2220
2221 zip_map2_into_validated(
2222 dest,
2223 a,
2224 b,
2225 |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2226 validated,
2227 exec,
2228 )
2229}
2230
2231fn broadcast_strides_for_axes(
2232 source_dims: &[usize],
2233 source_strides: &[isize],
2234 target_dims: &[usize],
2235 axes: &[usize],
2236) -> Result<AxisVec<isize>> {
2237 if source_dims.len() != axes.len() {
2238 return Err(StridedError::RankMismatch(source_dims.len(), axes.len()));
2239 }
2240 debug_assert_eq!(source_dims.len(), source_strides.len());
2241
2242 let mut seen = AxisVec::<bool>::new();
2243 seen.resize(target_dims.len(), false);
2244 let mut strides = AxisVec::<isize>::new();
2245 strides.resize(target_dims.len(), 0);
2246 for (src_axis, &dst_axis) in axes.iter().enumerate() {
2247 if dst_axis >= target_dims.len() {
2248 return Err(StridedError::InvalidAxis {
2249 axis: dst_axis,
2250 rank: target_dims.len(),
2251 });
2252 }
2253 if seen[dst_axis] {
2254 return Err(StridedError::InvalidAxis {
2255 axis: dst_axis,
2256 rank: target_dims.len(),
2257 });
2258 }
2259 seen[dst_axis] = true;
2260
2261 let source_dim = source_dims[src_axis];
2262 let target_dim = target_dims[dst_axis];
2263 if source_dim != target_dim && source_dim != 1 {
2264 return Err(StridedError::ShapeMismatch(
2265 source_dims.to_vec(),
2266 target_dims.to_vec(),
2267 ));
2268 }
2269 if source_dim == target_dim {
2270 strides[dst_axis] = source_strides[src_axis];
2271 }
2272 }
2273
2274 Ok(strides)
2275}
2276
2277fn broadcast_view_with_strides<'a, T, Op: ElementOp<T>>(
2284 view: &StridedView<'a, T, Op>,
2285 target_dims: &[usize],
2286 strides: &[isize],
2287) -> StridedView<'a, T, Op> {
2288 unsafe { StridedView::new_unchecked(view.data(), target_dims, strides, view.offset()) }
2300}
2301
2302pub fn broadcast_mul_into<
2318 D: Copy + Send + Sync + 'static,
2319 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2320 B: Copy + Send + Sync + 'static,
2321 OpA: ElementOp<A>,
2322 OpB: ElementOp<B>,
2323>(
2324 dest: &mut StridedViewMut<D>,
2325 a: &StridedView<A, OpA>,
2326 a_axes: &[usize],
2327 b: &StridedView<B, OpB>,
2328 b_axes: &[usize],
2329 exec: Exec<'_>,
2330) -> Result<()> {
2331 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2332 let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2333 let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2334
2335 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2336 return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2337 dest.as_mut_ptr(),
2338 dest.dims(),
2339 dest.strides(),
2340 a.ptr(),
2341 &a_strides,
2342 b.ptr(),
2343 &b_strides,
2344 validated,
2345 exec,
2346 );
2347 }
2348
2349 let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2350 let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2351 zip_map2_into_validated(dest, &a, &b, multiply_value, validated, exec)
2352}
2353
2354pub fn broadcast_mul_into_uninit<
2373 D: Copy + Send + Sync + 'static,
2374 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2375 B: Copy + Send + Sync + 'static,
2376 OpA: ElementOp<A>,
2377 OpB: ElementOp<B>,
2378>(
2379 dest: &mut StridedViewMut<MaybeUninit<D>>,
2380 a: &StridedView<A, OpA>,
2381 a_axes: &[usize],
2382 b: &StridedView<B, OpB>,
2383 b_axes: &[usize],
2384 exec: Exec<'_>,
2385) -> Result<()> {
2386 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2387 let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2388 let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2389 validate_typed_no_overlap(dest, a, 0)?;
2390 validate_typed_no_overlap(dest, b, 1)?;
2391
2392 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2393 return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2394 dest.as_mut_ptr(),
2395 dest.dims(),
2396 dest.strides(),
2397 a.ptr(),
2398 &a_strides,
2399 b.ptr(),
2400 &b_strides,
2401 validated,
2402 exec,
2403 );
2404 }
2405
2406 let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2407 let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2408 zip_map2_into_validated(
2409 dest,
2410 &a,
2411 &b,
2412 |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2413 validated,
2414 exec,
2415 )
2416}
2417
2418pub fn zip_map3_into<
2430 D: Copy + Send + Sync,
2431 A: Copy + Send + Sync,
2432 B: Copy + Send + Sync,
2433 C: Copy + Send + Sync,
2434 OpA: ElementOp<A>,
2435 OpB: ElementOp<B>,
2436 OpC: ElementOp<C>,
2437>(
2438 dest: &mut StridedViewMut<D>,
2439 a: &StridedView<A, OpA>,
2440 b: &StridedView<B, OpB>,
2441 c: &StridedView<C, OpC>,
2442 f: impl Fn(A, B, C) -> D + Sync,
2443 exec: Exec<'_>,
2444) -> Result<()> {
2445 ensure_same_shape(dest.dims(), a.dims())?;
2446 ensure_same_shape(dest.dims(), b.dims())?;
2447 ensure_same_shape(dest.dims(), c.dims())?;
2448 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2449 zip_map3_into_validated(dest, a, b, c, f, validated, exec)
2450}
2451
2452pub(crate) fn zip_map3_into_validated<
2453 D: Copy + Send + Sync,
2454 A: Copy + Send + Sync,
2455 B: Copy + Send + Sync,
2456 C: Copy + Send + Sync,
2457 OpA: ElementOp<A>,
2458 OpB: ElementOp<B>,
2459 OpC: ElementOp<C>,
2460>(
2461 dest: &mut StridedViewMut<D>,
2462 a: &StridedView<A, OpA>,
2463 b: &StridedView<B, OpB>,
2464 c: &StridedView<C, OpC>,
2465 f: impl Fn(A, B, C) -> D + Sync,
2466 _validated: ValidatedDestinationLayout,
2467 exec: Exec<'_>,
2468) -> Result<()> {
2469 ensure_same_shape(dest.dims(), a.dims())?;
2470 ensure_same_shape(dest.dims(), b.dims())?;
2471 ensure_same_shape(dest.dims(), c.dims())?;
2472 let dst_ptr = dest.as_mut_ptr();
2473 let a_ptr = a.ptr();
2474 let b_ptr = b.ptr();
2475 let c_ptr = c.ptr();
2476
2477 let dst_dims = dest.dims();
2478 let dst_strides = dest.strides();
2479
2480 if sequential_contiguous_layout(
2481 KernelClass::Map,
2482 dst_dims,
2483 &[dst_strides, a.strides(), b.strides(), c.strides()],
2484 &exec,
2485 )?
2486 .is_some()
2487 {
2488 let len = total_len(dst_dims)?;
2489 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2490 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2491 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2492 let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2493 simd::dispatch_if_large(len, || {
2494 for (((d, &a), &b), &c) in dst.iter_mut().zip(sa).zip(sb).zip(sc) {
2495 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
2496 }
2497 });
2498 return Ok(());
2499 }
2500
2501 let strides_list: [&[isize]; 4] = [dst_strides, a.strides(), b.strides(), c.strides()];
2502 let total = total_len(dst_dims)?;
2503 let (fused_dims, ordered_strides, plan) = pointwise_plan(
2504 dst_dims,
2505 &strides_list,
2506 Some(0),
2507 &[
2508 std::mem::size_of::<D>(),
2509 std::mem::size_of::<A>(),
2510 std::mem::size_of::<B>(),
2511 std::mem::size_of::<C>(),
2512 ],
2513 total,
2514 );
2515
2516 let total = total_len(&fused_dims)?;
2517 if is_parallel(KernelClass::Map, total, &exec) {
2518 use crate::threading::SendPtr;
2519 let dst_send = SendPtr(dst_ptr);
2520 let a_send = SendPtr(a_ptr as *mut A);
2521 let b_send = SendPtr(b_ptr as *mut B);
2522 let c_send = SendPtr(c_ptr as *mut C);
2523
2524 let costs = compute_costs(&ordered_strides);
2525 let initial_offsets: crate::inline::OperandVec<isize> =
2526 strides_list.iter().map(|_| 0isize).collect();
2527 return mapreduce_threaded(
2528 &fused_dims,
2529 &plan.block,
2530 &ordered_strides,
2531 &initial_offsets,
2532 &costs,
2533 KernelClass::Map,
2534 exec,
2535 &|dims, blocks, strides_list, offsets| {
2536 for_each_inner_block_with_offsets(
2537 dims,
2538 blocks,
2539 strides_list,
2540 offsets,
2541 |offsets, len, strides| {
2542 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2543 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2544 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2545 let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2546 unsafe {
2547 inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2548 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3],
2549 len, &f,
2550 )
2551 };
2552 Ok(())
2553 },
2554 )
2555 },
2556 );
2557 }
2558
2559 let initial_offsets: crate::inline::OperandVec<isize> =
2560 ordered_strides.iter().map(|_| 0isize).collect();
2561 for_each_inner_block_preordered(
2562 &fused_dims,
2563 &plan.block,
2564 &ordered_strides,
2565 &initial_offsets,
2566 |offsets, len, strides| {
2567 let dp = unsafe { dst_ptr.offset(offsets[0]) };
2568 let ap = unsafe { a_ptr.offset(offsets[1]) };
2569 let bp = unsafe { b_ptr.offset(offsets[2]) };
2570 let cp = unsafe { c_ptr.offset(offsets[3]) };
2571 unsafe {
2572 inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2573 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], len, &f,
2574 )
2575 };
2576 Ok(())
2577 },
2578 )
2579}
2580
2581pub fn zip_map4_into<
2593 D: Copy + Send + Sync,
2594 A: Copy + Send + Sync,
2595 B: Copy + Send + Sync,
2596 C: Copy + Send + Sync,
2597 E: Copy + Send + Sync,
2598 OpA: ElementOp<A>,
2599 OpB: ElementOp<B>,
2600 OpC: ElementOp<C>,
2601 OpE: ElementOp<E>,
2602>(
2603 dest: &mut StridedViewMut<D>,
2604 a: &StridedView<A, OpA>,
2605 b: &StridedView<B, OpB>,
2606 c: &StridedView<C, OpC>,
2607 e: &StridedView<E, OpE>,
2608 f: impl Fn(A, B, C, E) -> D + Sync,
2609 exec: Exec<'_>,
2610) -> Result<()> {
2611 ensure_same_shape(dest.dims(), a.dims())?;
2612 ensure_same_shape(dest.dims(), b.dims())?;
2613 ensure_same_shape(dest.dims(), c.dims())?;
2614 ensure_same_shape(dest.dims(), e.dims())?;
2615 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2616 zip_map4_into_validated(dest, a, b, c, e, f, validated, exec)
2617}
2618
2619#[allow(clippy::too_many_arguments)]
2622pub(crate) fn zip_map4_into_validated<
2623 D: Copy + Send + Sync,
2624 A: Copy + Send + Sync,
2625 B: Copy + Send + Sync,
2626 C: Copy + Send + Sync,
2627 E: Copy + Send + Sync,
2628 OpA: ElementOp<A>,
2629 OpB: ElementOp<B>,
2630 OpC: ElementOp<C>,
2631 OpE: ElementOp<E>,
2632>(
2633 dest: &mut StridedViewMut<D>,
2634 a: &StridedView<A, OpA>,
2635 b: &StridedView<B, OpB>,
2636 c: &StridedView<C, OpC>,
2637 e: &StridedView<E, OpE>,
2638 f: impl Fn(A, B, C, E) -> D + Sync,
2639 _validated: ValidatedDestinationLayout,
2640 exec: Exec<'_>,
2641) -> Result<()> {
2642 ensure_same_shape(dest.dims(), a.dims())?;
2643 ensure_same_shape(dest.dims(), b.dims())?;
2644 ensure_same_shape(dest.dims(), c.dims())?;
2645 ensure_same_shape(dest.dims(), e.dims())?;
2646 let dst_ptr = dest.as_mut_ptr();
2647 let a_ptr = a.ptr();
2648 let b_ptr = b.ptr();
2649 let c_ptr = c.ptr();
2650 let e_ptr = e.ptr();
2651
2652 let dst_dims = dest.dims();
2653 let dst_strides = dest.strides();
2654
2655 if sequential_contiguous_layout(
2656 KernelClass::Map,
2657 dst_dims,
2658 &[
2659 dst_strides,
2660 a.strides(),
2661 b.strides(),
2662 c.strides(),
2663 e.strides(),
2664 ],
2665 &exec,
2666 )?
2667 .is_some()
2668 {
2669 let len = total_len(dst_dims)?;
2670 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2671 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2672 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2673 let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2674 let se = unsafe { std::slice::from_raw_parts(e_ptr, len) };
2675 simd::dispatch_if_large(len, || {
2678 for ((((d, &a), &b), &c), &e) in dst.iter_mut().zip(sa).zip(sb).zip(sc).zip(se) {
2679 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
2680 }
2681 });
2682 return Ok(());
2683 }
2684
2685 let strides_list: [&[isize]; 5] = [
2686 dst_strides,
2687 a.strides(),
2688 b.strides(),
2689 c.strides(),
2690 e.strides(),
2691 ];
2692 let total = total_len(dst_dims)?;
2693 let (fused_dims, ordered_strides, plan) = pointwise_plan(
2694 dst_dims,
2695 &strides_list,
2696 Some(0),
2697 &[
2698 std::mem::size_of::<D>(),
2699 std::mem::size_of::<A>(),
2700 std::mem::size_of::<B>(),
2701 std::mem::size_of::<C>(),
2702 std::mem::size_of::<E>(),
2703 ],
2704 total,
2705 );
2706
2707 let total = total_len(&fused_dims)?;
2708 if is_parallel(KernelClass::Map, total, &exec) {
2709 use crate::threading::SendPtr;
2710 let dst_send = SendPtr(dst_ptr);
2711 let a_send = SendPtr(a_ptr as *mut A);
2712 let b_send = SendPtr(b_ptr as *mut B);
2713 let c_send = SendPtr(c_ptr as *mut C);
2714 let e_send = SendPtr(e_ptr as *mut E);
2715
2716 let costs = compute_costs(&ordered_strides);
2717 let initial_offsets: crate::inline::OperandVec<isize> =
2718 strides_list.iter().map(|_| 0isize).collect();
2719 return mapreduce_threaded(
2720 &fused_dims,
2721 &plan.block,
2722 &ordered_strides,
2723 &initial_offsets,
2724 &costs,
2725 KernelClass::Map,
2726 exec,
2727 &|dims, blocks, strides_list, offsets| {
2728 for_each_inner_block_with_offsets(
2729 dims,
2730 blocks,
2731 strides_list,
2732 offsets,
2733 |offsets, len, strides| {
2734 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2735 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2736 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2737 let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2738 let ep = unsafe { e_send.as_const().offset(offsets[4]) };
2739 unsafe {
2740 inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2741 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep,
2742 strides[4], len, &f,
2743 )
2744 };
2745 Ok(())
2746 },
2747 )
2748 },
2749 );
2750 }
2751
2752 let initial_offsets: crate::inline::OperandVec<isize> =
2753 ordered_strides.iter().map(|_| 0isize).collect();
2754 for_each_inner_block_preordered(
2755 &fused_dims,
2756 &plan.block,
2757 &ordered_strides,
2758 &initial_offsets,
2759 |offsets, len, strides| {
2760 let dp = unsafe { dst_ptr.offset(offsets[0]) };
2761 let ap = unsafe { a_ptr.offset(offsets[1]) };
2762 let bp = unsafe { b_ptr.offset(offsets[2]) };
2763 let cp = unsafe { c_ptr.offset(offsets[3]) };
2764 let ep = unsafe { e_ptr.offset(offsets[4]) };
2765 unsafe {
2766 inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2767 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep, strides[4],
2768 len, &f,
2769 )
2770 };
2771 Ok(())
2772 },
2773 )
2774}
2775
2776#[cfg(test)]
2777#[path = "map_view/tests/scalar_branch_tests.rs"]
2778mod scalar_branch_tests;
2779
2780#[cfg(test)]
2781#[path = "map_view/tests/tests.rs"]
2782mod tests;