1use crate::kernel::{
10 build_plan_fused, build_plan_fused_small, ensure_same_shape, for_each_inner_block_preordered,
11 sequential_contiguous_layout, total_len, SMALL_TENSOR_THRESHOLD,
12};
13use crate::simd;
14use crate::view::{StridedView, StridedViewMut};
15use crate::{Result, StridedError};
16use core::mem::MaybeUninit;
17use std::ops::Mul;
18use strided_view::{ElementOp, Identity};
19
20use crate::cost::{for_len, KernelClass};
21use crate::fuse::compute_costs;
22use crate::threading::{for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded};
23use crate::Exec;
24
25type AxisVec<T> = crate::inline::InlineVec<T>;
26
27const TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT: usize = 8;
36
37#[derive(Clone, Copy, Debug, Default)]
43pub struct ValidatedDestinationLayout(());
44
45#[inline]
61pub fn validate_destination_layout(
62 dims: &[usize],
63 strides: &[isize],
64) -> Result<ValidatedDestinationLayout> {
65 if crate::layout_check::is_injective_layout(dims, strides) {
66 Ok(ValidatedDestinationLayout(()))
67 } else {
68 Err(StridedError::NonInjectiveOutputLayout)
69 }
70}
71
72#[inline]
73pub fn validate_destination_layout_without_alloc(
87 dims: &[usize],
88 strides: &[isize],
89) -> Result<ValidatedDestinationLayout> {
90 if crate::layout_check::is_injective_layout_without_alloc(dims, strides) {
91 Ok(ValidatedDestinationLayout(()))
92 } else {
93 Err(StridedError::NonInjectiveOutputLayout)
94 }
95}
96
97fn reachable_byte_range(
98 ptr: usize,
99 elem_size: usize,
100 dims: &[usize],
101 strides: &[isize],
102) -> Result<Option<(usize, usize)>> {
103 if elem_size == 0 || dims.contains(&0) {
104 return Ok(None);
105 }
106
107 let mut min_offset = 0isize;
108 let mut max_offset = 0isize;
109 for (&dim, &stride) in dims.iter().zip(strides) {
110 if dim <= 1 {
111 continue;
112 }
113 let extent = isize::try_from(dim - 1).map_err(|_| StridedError::OffsetOverflow)?;
114 let span = stride
115 .checked_mul(extent)
116 .ok_or(StridedError::OffsetOverflow)?;
117 if span < 0 {
118 min_offset = min_offset
119 .checked_add(span)
120 .ok_or(StridedError::OffsetOverflow)?;
121 } else {
122 max_offset = max_offset
123 .checked_add(span)
124 .ok_or(StridedError::OffsetOverflow)?;
125 }
126 }
127
128 let ptr = ptr as i128;
129 let elem_size = elem_size as i128;
130 let start = ptr
131 .checked_add(
132 (min_offset as i128)
133 .checked_mul(elem_size)
134 .ok_or(StridedError::OffsetOverflow)?,
135 )
136 .ok_or(StridedError::OffsetOverflow)?;
137 let end = ptr
138 .checked_add(
139 (max_offset as i128)
140 .checked_mul(elem_size)
141 .and_then(|offset| offset.checked_add(elem_size))
142 .ok_or(StridedError::OffsetOverflow)?,
143 )
144 .ok_or(StridedError::OffsetOverflow)?;
145 if start < 0 || end < 0 || start > usize::MAX as i128 || end > usize::MAX as i128 {
146 return Err(StridedError::OffsetOverflow);
147 }
148 Ok(Some((start as usize, end as usize)))
149}
150
151fn validate_typed_no_overlap<D, A, Op: ElementOp<A>>(
152 dest: &StridedViewMut<MaybeUninit<D>>,
153 input: &StridedView<A, Op>,
154 input_index: usize,
155) -> Result<()> {
156 let dest_range = reachable_byte_range(
157 dest.ptr() as usize,
158 core::mem::size_of::<D>(),
159 dest.dims(),
160 dest.strides(),
161 )?;
162 let input_range = reachable_byte_range(
163 input.ptr() as usize,
164 core::mem::size_of::<A>(),
165 input.dims(),
166 input.strides(),
167 )?;
168 if let (Some((dest_start, dest_end)), Some((input_start, input_end))) =
169 (dest_range, input_range)
170 {
171 if dest_start < input_end && input_start < dest_end {
172 return Err(StridedError::OverlappingInputOutput { input: input_index });
173 }
174 }
175 Ok(())
176}
177
178trait MapKernel<A, D> {
196 fn apply(&self, src: &[A], dst: &mut [D]) -> bool;
198}
199
200struct NoKernel;
201
202impl<A, D> MapKernel<A, D> for NoKernel {
203 #[inline(always)]
204 fn apply(&self, _src: &[A], _dst: &mut [D]) -> bool {
205 false
206 }
207}
208
209struct SliceKernel<K>(K);
210
211impl<A, D, K: Fn(&[A], &mut [D]) -> bool> MapKernel<A, D> for SliceKernel<K> {
212 #[inline(always)]
213 fn apply(&self, src: &[A], dst: &mut [D]) -> bool {
214 (self.0)(src, dst)
215 }
216}
217
218unsafe fn inner_loop_map1<D: Copy, A: Copy, Op: ElementOp<A>, K: MapKernel<A, D>>(
219 dp: *mut D,
220 ds: isize,
221 sp: *const A,
222 ss: isize,
223 len: usize,
224 f: &impl Fn(A) -> D,
225 kernel: &K,
226) {
227 if ds == 1 && ss == 1 {
228 let src = std::slice::from_raw_parts(sp, len);
229 let dst = std::slice::from_raw_parts_mut(dp, len);
230 if kernel.apply(src, dst) {
231 return;
232 }
233 simd::dispatch_if_large(len, || {
234 for (d, s) in dst.iter_mut().zip(src.iter()) {
235 *d = f(Op::apply(*s));
236 }
237 });
238 } else {
239 let mut dp = dp;
240 let mut sp = sp;
241 simd::dispatch_if_large(len, || {
242 for _ in 0..len {
243 *dp = f(Op::apply(*sp));
244 dp = dp.wrapping_offset(ds);
245 sp = sp.wrapping_offset(ss);
246 }
247 });
248 }
249}
250
251#[inline(always)]
253unsafe fn inner_loop_map2<D: Copy, A: Copy, B: Copy, OpA: ElementOp<A>, OpB: ElementOp<B>>(
254 dp: *mut D,
255 ds: isize,
256 ap: *const A,
257 a_s: isize,
258 bp: *const B,
259 b_s: isize,
260 len: usize,
261 f: &impl Fn(A, B) -> D,
262) {
263 if ds == 1 && a_s == 1 && b_s == 1 {
264 let src_a = std::slice::from_raw_parts(ap, len);
265 let src_b = std::slice::from_raw_parts(bp, len);
266 let dst = std::slice::from_raw_parts_mut(dp, len);
267 simd::dispatch_if_large(len, || {
268 for i in 0..len {
269 dst[i] = f(OpA::apply(src_a[i]), OpB::apply(src_b[i]));
270 }
271 });
272 } else if ds == 1 && a_s == 1 && b_s == 0 {
273 let src_a = std::slice::from_raw_parts(ap, len);
274 let b = OpB::apply(*bp);
275 let dst = std::slice::from_raw_parts_mut(dp, len);
276 simd::dispatch_if_large(len, || {
277 for i in 0..len {
278 dst[i] = f(OpA::apply(src_a[i]), b);
279 }
280 });
281 } else if ds == 1 && a_s == 0 && b_s == 1 {
282 let a = OpA::apply(*ap);
283 let src_b = std::slice::from_raw_parts(bp, len);
284 let dst = std::slice::from_raw_parts_mut(dp, len);
285 simd::dispatch_if_large(len, || {
286 for i in 0..len {
287 dst[i] = f(a, OpB::apply(src_b[i]));
288 }
289 });
290 } else if ds == 1 && a_s == 0 && b_s == 0 {
291 let a = OpA::apply(*ap);
292 let b = OpB::apply(*bp);
293 let dst = std::slice::from_raw_parts_mut(dp, len);
294 simd::dispatch_if_large(len, || {
295 for d in dst.iter_mut() {
296 *d = f(a, b);
297 }
298 });
299 } else if ds == 1 && b_s == 0 {
300 let b = OpB::apply(*bp);
301 let dst = std::slice::from_raw_parts_mut(dp, len);
302 let mut ap = ap;
303 simd::dispatch_if_large(len, || {
304 for d in dst.iter_mut() {
305 *d = f(OpA::apply(*ap), b);
306 ap = ap.wrapping_offset(a_s);
307 }
308 });
309 } else if ds == 1 && a_s == 0 {
310 let a = OpA::apply(*ap);
311 let dst = std::slice::from_raw_parts_mut(dp, len);
312 let mut bp = bp;
313 simd::dispatch_if_large(len, || {
314 for d in dst.iter_mut() {
315 *d = f(a, OpB::apply(*bp));
316 bp = bp.wrapping_offset(b_s);
317 }
318 });
319 } else {
320 let mut dp = dp;
321 let mut ap = ap;
322 let mut bp = bp;
323 simd::dispatch_if_large(len, || {
324 for _ in 0..len {
325 *dp = f(OpA::apply(*ap), OpB::apply(*bp));
326 dp = dp.wrapping_offset(ds);
327 ap = ap.wrapping_offset(a_s);
328 bp = bp.wrapping_offset(b_s);
329 }
330 });
331 }
332}
333
334trait MulOutput<D: Copy + 'static>: Copy + Send + Sync + 'static {
336 type Slot: Copy + Send + Sync + 'static;
337
338 unsafe fn write(dst: *mut Self::Slot, value: D);
339
340 unsafe fn try_contiguous<A: 'static, B: 'static>(
341 dst: *mut Self::Slot,
342 len: usize,
343 a: &[A],
344 b: &[B],
345 ) -> bool;
346}
347
348#[derive(Clone, Copy)]
349struct InitializedOutput;
350
351impl<D: Copy + Send + Sync + 'static> MulOutput<D> for InitializedOutput {
352 type Slot = D;
353
354 #[inline(always)]
355 unsafe fn write(dst: *mut D, value: D) {
356 unsafe { dst.write(value) };
357 }
358
359 #[inline(always)]
360 unsafe fn try_contiguous<A: 'static, B: 'static>(
361 dst: *mut D,
362 len: usize,
363 a: &[A],
364 b: &[B],
365 ) -> bool {
366 unsafe { simd::try_mul_contiguous_ptr(dst, len, a, b) }
367 }
368}
369
370#[derive(Clone, Copy)]
371struct UninitializedOutput;
372
373impl<D: Copy + Send + Sync + 'static> MulOutput<D> for UninitializedOutput {
374 type Slot = MaybeUninit<D>;
375
376 #[inline(always)]
377 unsafe fn write(dst: *mut MaybeUninit<D>, value: D) {
378 unsafe { dst.write(MaybeUninit::new(value)) };
379 }
380
381 #[inline(always)]
382 unsafe fn try_contiguous<A: 'static, B: 'static>(
383 dst: *mut MaybeUninit<D>,
384 len: usize,
385 a: &[A],
386 b: &[B],
387 ) -> bool {
388 unsafe { simd::try_mul_contiguous_ptr(dst.cast::<D>(), len, a, b) }
391 }
392}
393
394#[inline(always)]
395fn multiply_value<A, B, D>(lhs: A, rhs: B) -> D
396where
397 A: Copy + Mul<B, Output = D> + 'static,
398 B: Copy + 'static,
399 D: Copy + 'static,
400{
401 use core::any::TypeId;
402
403 if TypeId::of::<A>() == TypeId::of::<i32>()
404 && TypeId::of::<B>() == TypeId::of::<i32>()
405 && TypeId::of::<D>() == TypeId::of::<i32>()
406 {
407 let lhs = unsafe { *(&lhs as *const A).cast::<i32>() };
409 let rhs = unsafe { *(&rhs as *const B).cast::<i32>() };
410 let value = lhs.wrapping_mul(rhs);
411 return unsafe { core::mem::transmute_copy(&value) };
412 }
413 if TypeId::of::<A>() == TypeId::of::<i64>()
414 && TypeId::of::<B>() == TypeId::of::<i64>()
415 && TypeId::of::<D>() == TypeId::of::<i64>()
416 {
417 let lhs = unsafe { *(&lhs as *const A).cast::<i64>() };
419 let rhs = unsafe { *(&rhs as *const B).cast::<i64>() };
420 let value = lhs.wrapping_mul(rhs);
421 return unsafe { core::mem::transmute_copy(&value) };
422 }
423 lhs * rhs
424}
425
426#[inline(always)]
427unsafe fn inner_loop_mul2<
428 O: MulOutput<D>,
429 D: Copy + 'static,
430 A: Copy + Mul<B, Output = D> + 'static,
431 B: Copy + 'static,
432>(
433 dp: *mut O::Slot,
434 ds: isize,
435 ap: *const A,
436 a_s: isize,
437 bp: *const B,
438 b_s: isize,
439 len: usize,
440) {
441 if ds == 1 && a_s == 1 && b_s == 1 {
442 let src_a = std::slice::from_raw_parts(ap, len);
443 let src_b = std::slice::from_raw_parts(bp, len);
444 if len >= 64 && O::try_contiguous(dp, len, src_a, src_b) {
445 return;
446 }
447 simd::dispatch_if_large(len, || {
448 for i in 0..len {
449 O::write(dp.add(i), multiply_value(src_a[i], src_b[i]));
450 }
451 });
452 } else if ds == 1 && a_s == 1 && b_s == 0 {
453 let src_a = std::slice::from_raw_parts(ap, len);
454 let b = *bp;
455 simd::dispatch_if_large(len, || {
456 for i in 0..len {
457 O::write(dp.add(i), multiply_value(src_a[i], b));
458 }
459 });
460 } else if ds == 1 && a_s == 0 && b_s == 1 {
461 let a = *ap;
462 let src_b = std::slice::from_raw_parts(bp, len);
463 simd::dispatch_if_large(len, || {
464 for i in 0..len {
465 O::write(dp.add(i), multiply_value(a, src_b[i]));
466 }
467 });
468 } else if ds == 1 && a_s == 0 && b_s == 0 {
469 let a = *ap;
470 let b = *bp;
471 simd::dispatch_if_large(len, || {
472 for i in 0..len {
473 O::write(dp.add(i), multiply_value(a, b));
474 }
475 });
476 } else if ds == 1 && b_s == 0 {
477 let b = *bp;
478 let mut ap = ap;
479 simd::dispatch_if_large(len, || {
480 for i in 0..len {
481 O::write(dp.add(i), multiply_value(*ap, b));
482 ap = ap.wrapping_offset(a_s);
483 }
484 });
485 } else if ds == 1 && a_s == 0 {
486 let a = *ap;
487 let mut bp = bp;
488 simd::dispatch_if_large(len, || {
489 for i in 0..len {
490 O::write(dp.add(i), multiply_value(a, *bp));
491 bp = bp.wrapping_offset(b_s);
492 }
493 });
494 } else {
495 let mut dp = dp;
496 let mut ap = ap;
497 let mut bp = bp;
498 simd::dispatch_if_large(len, || {
499 for _ in 0..len {
500 O::write(dp, multiply_value(*ap, *bp));
501 dp = dp.wrapping_offset(ds);
502 ap = ap.wrapping_offset(a_s);
503 bp = bp.wrapping_offset(b_s);
504 }
505 });
506 }
507}
508
509#[derive(Clone, Debug, Eq, PartialEq)]
510struct ContiguousMulRangePlan {
511 axis_order: AxisVec<usize>,
512 inner_len: usize,
513 inner_axis_count: usize,
514 row_len: usize,
515 outer_axis_start: usize,
516 fast_axis: usize,
517 a_fast_stride: isize,
518 b_fast_stride: isize,
519 a_row_stride: isize,
520 b_row_stride: isize,
521}
522
523impl ContiguousMulRangePlan {
524 fn walks_inputs_without_blocking(&self) -> bool {
534 let streams = |stride: isize| stride == 0 || stride == 1;
535 if streams(self.a_fast_stride) && streams(self.b_fast_stride) {
536 return true;
537 }
538 if transposed_scalar_tile_kind(self).is_some() {
539 return true;
540 }
541 false
542 }
543}
544
545#[derive(Clone, Copy, Debug, Eq, PartialEq)]
546enum TransposedScalarTileKind {
547 RhsScalar,
548 LhsScalar,
549}
550
551fn transposed_scalar_tile_kind(plan: &ContiguousMulRangePlan) -> Option<TransposedScalarTileKind> {
552 if plan.inner_len > TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT.saturating_mul(plan.row_len) {
553 return None;
554 }
555 let row_len = isize::try_from(plan.row_len).ok()?;
556 if plan.a_fast_stride == row_len
557 && plan.a_row_stride == 1
558 && plan.b_fast_stride == 0
559 && plan.b_row_stride == 0
560 {
561 return Some(TransposedScalarTileKind::RhsScalar);
562 }
563
564 if plan.b_fast_stride == row_len
565 && plan.b_row_stride == 1
566 && plan.a_fast_stride == 0
567 && plan.a_row_stride == 0
568 {
569 return Some(TransposedScalarTileKind::LhsScalar);
570 }
571
572 None
573}
574
575fn compact_axis_order(dims: &[usize], strides: &[isize]) -> Option<AxisVec<usize>> {
576 if dims.len() != strides.len() {
577 return None;
578 }
579
580 let mut active = AxisVec::<usize>::new();
581 let mut inactive = AxisVec::<usize>::new();
582 for (axis, (&dim, &stride)) in dims.iter().zip(strides.iter()).enumerate() {
583 if stride < 0 {
584 return None;
585 }
586 if dim > 1 {
587 active.push(axis);
588 } else {
589 inactive.push(axis);
590 }
591 }
592
593 active.sort_by(|&lhs, &rhs| strides[lhs].cmp(&strides[rhs]).then_with(|| lhs.cmp(&rhs)));
594
595 let mut expected = 1isize;
596 for &axis in &active {
597 if strides[axis] != expected {
598 return None;
599 }
600 expected = expected.saturating_mul(dims[axis] as isize);
601 }
602
603 active.extend(inactive);
604 Some(active)
605}
606
607fn can_fuse_contiguous_range_axis(dim: usize, prev_stride: isize, next_stride: isize) -> bool {
608 dim <= 1 || (prev_stride == 0 && next_stride == 0) || next_stride == prev_stride * dim as isize
609}
610
611fn contiguous_mul_range_plan(
612 dims: &[usize],
613 dst_strides: &[isize],
614 a_strides: &[isize],
615 b_strides: &[isize],
616) -> Option<ContiguousMulRangePlan> {
617 let axis_order = compact_axis_order(dims, dst_strides)?;
618 if dims.is_empty() {
619 return Some(ContiguousMulRangePlan {
620 axis_order,
621 inner_len: 1,
622 inner_axis_count: 0,
623 row_len: 1,
624 outer_axis_start: 0,
625 fast_axis: 0,
626 a_fast_stride: 0,
627 b_fast_stride: 0,
628 a_row_stride: 0,
629 b_row_stride: 0,
630 });
631 }
632
633 let first_pos = axis_order
634 .iter()
635 .position(|&axis| dims[axis] > 1)
636 .unwrap_or(0);
637 let first_axis = axis_order[first_pos];
638 let mut inner_len = dims[first_axis].max(1);
639 let mut inner_axis_count = first_pos + 1;
640 let mut prev_axis = first_axis;
641
642 for &axis in axis_order.iter().skip(first_pos + 1) {
643 if can_fuse_contiguous_range_axis(
644 dims[prev_axis],
645 dst_strides[prev_axis],
646 dst_strides[axis],
647 ) && can_fuse_contiguous_range_axis(
648 dims[prev_axis],
649 a_strides[prev_axis],
650 a_strides[axis],
651 ) && can_fuse_contiguous_range_axis(
652 dims[prev_axis],
653 b_strides[prev_axis],
654 b_strides[axis],
655 ) {
656 inner_len = inner_len.checked_mul(dims[axis].max(1))?;
657 inner_axis_count += 1;
658 prev_axis = axis;
659 } else {
660 break;
661 }
662 }
663
664 let row = axis_order
665 .iter()
666 .enumerate()
667 .skip(inner_axis_count)
668 .find(|&(_, &axis)| dims[axis] > 1);
669 let (row_len, outer_axis_start, a_row_stride, b_row_stride) =
670 if let Some((row_pos, &row_axis)) = row {
671 (
672 dims[row_axis],
673 row_pos + 1,
674 a_strides[row_axis],
675 b_strides[row_axis],
676 )
677 } else {
678 (1, axis_order.len(), 0, 0)
679 };
680
681 Some(ContiguousMulRangePlan {
682 axis_order,
683 inner_len,
684 inner_axis_count,
685 row_len,
686 outer_axis_start,
687 fast_axis: first_axis,
688 a_fast_stride: a_strides[first_axis],
689 b_fast_stride: b_strides[first_axis],
690 a_row_stride,
691 b_row_stride,
692 })
693}
694
695struct ContiguousMulOuterCursor<'a> {
696 dims: &'a [usize],
697 a_strides: &'a [isize],
698 b_strides: &'a [isize],
699 axes: AxisVec<usize>,
700 coords: AxisVec<usize>,
701 a_offset: isize,
702 b_offset: isize,
703}
704
705impl<'a> ContiguousMulOuterCursor<'a> {
706 fn new(
707 dims: &'a [usize],
708 a_strides: &'a [isize],
709 b_strides: &'a [isize],
710 plan: &ContiguousMulRangePlan,
711 outer_group: usize,
712 ) -> Self {
713 let axes: AxisVec<usize> = plan
714 .axis_order
715 .iter()
716 .skip(plan.outer_axis_start)
717 .copied()
718 .collect();
719 let mut coords = AxisVec::<usize>::with_capacity(axes.len());
720 let mut rem = outer_group;
721 let mut a_offset = 0isize;
722 let mut b_offset = 0isize;
723
724 for &axis in &axes {
725 let dim = dims[axis].max(1);
726 let coord = rem % dim;
727 rem /= dim;
728 coords.push(coord);
729 a_offset += coord as isize * a_strides[axis];
730 b_offset += coord as isize * b_strides[axis];
731 }
732
733 Self {
734 dims,
735 a_strides,
736 b_strides,
737 axes,
738 coords,
739 a_offset,
740 b_offset,
741 }
742 }
743
744 fn advance(&mut self) {
745 for (i, &axis) in self.axes.iter().enumerate() {
746 let dim = self.dims[axis].max(1);
747 if dim <= 1 {
748 continue;
749 }
750
751 if self.coords[i] + 1 < dim {
754 self.coords[i] += 1;
755 self.a_offset += self.a_strides[axis];
756 self.b_offset += self.b_strides[axis];
757 break;
758 }
759
760 let last = (dim - 1) as isize;
761 self.coords[i] = 0;
762 self.a_offset -= last * self.a_strides[axis];
763 self.b_offset -= last * self.b_strides[axis];
764 }
765 }
766}
767
768#[inline(always)]
769unsafe fn run_contiguous_mul_row_block<
770 O: MulOutput<D>,
771 D: Copy + 'static,
772 A: Copy + Mul<B, Output = D> + 'static,
773 B: Copy + 'static,
774>(
775 dst_ptr: *mut O::Slot,
776 a_ptr: *const A,
777 b_ptr: *const B,
778 plan: &ContiguousMulRangePlan,
779 base_index: usize,
780 total: usize,
781 base_a_offset: isize,
782 base_b_offset: isize,
783) {
784 let inner_len = plan.inner_len.max(1);
785 let row_len = plan.row_len.max(1);
786 let block_len = inner_len.saturating_mul(row_len);
787
788 if total.saturating_sub(base_index) >= block_len {
789 let tiled = match transposed_scalar_tile_kind(plan) {
790 Some(TransposedScalarTileKind::RhsScalar) => {
791 simd::try_mul_transposed_scalar_rhs_2d::<D, A, B>(
792 dst_ptr.add(base_index).cast::<D>(),
793 a_ptr.offset(base_a_offset),
794 b_ptr.offset(base_b_offset),
795 inner_len,
796 row_len,
797 plan.a_fast_stride,
798 plan.a_row_stride,
799 )
800 }
801 Some(TransposedScalarTileKind::LhsScalar) => {
802 simd::try_mul_transposed_scalar_lhs_2d::<D, A, B>(
803 dst_ptr.add(base_index).cast::<D>(),
804 a_ptr.offset(base_a_offset),
805 b_ptr.offset(base_b_offset),
806 inner_len,
807 row_len,
808 plan.b_fast_stride,
809 plan.b_row_stride,
810 )
811 }
812 None => false,
813 };
814 if tiled {
815 return;
816 }
817 }
818
819 let mut index = base_index;
820 let mut a_offset = base_a_offset;
821 let mut b_offset = base_b_offset;
822
823 for _ in 0..row_len {
824 if index >= total {
825 break;
826 }
827 let len = inner_len.min(total - index);
828 inner_loop_mul2::<O, D, A, B>(
829 dst_ptr.add(index),
830 1,
831 a_ptr.offset(a_offset),
832 plan.a_fast_stride,
833 b_ptr.offset(b_offset),
834 plan.b_fast_stride,
835 len,
836 );
837 index += inner_len;
838 a_offset += plan.a_row_stride;
839 b_offset += plan.b_row_stride;
840 }
841}
842
843fn strided_offset_for_contiguous_linear_index(
844 dims: &[usize],
845 strides: &[isize],
846 axis_order: &[usize],
847 mut index: usize,
848) -> isize {
849 let mut offset = 0isize;
850 for &axis in axis_order {
851 let dim = dims[axis];
852 if dim == 0 {
853 return 0;
854 }
855 let coord = index % dim;
856 index /= dim;
857 offset += coord as isize * strides[axis];
858 }
859 offset
860}
861
862#[allow(clippy::too_many_arguments)]
863fn try_contiguous_range_mul<
864 O: MulOutput<D>,
865 D: Copy + Send + Sync + 'static,
866 A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
867 B: Copy + Send + Sync + 'static,
868>(
869 dst_ptr: *mut O::Slot,
870 dims: &[usize],
871 dst_strides: &[isize],
872 a_ptr: *const A,
873 a_strides: &[isize],
874 b_ptr: *const B,
875 b_strides: &[isize],
876 exec: Exec<'_>,
877) -> bool {
878 let Ok(total) = total_len(dims) else {
881 return false;
882 };
883 if total == 0 {
884 return true;
885 }
886 let exec = for_len(exec, KernelClass::Map, total);
888 if exec.is_serial() {
889 return false;
890 }
891
892 let Some(plan) = contiguous_mul_range_plan(dims, dst_strides, a_strides, b_strides) else {
893 return false;
894 };
895 if !plan.walks_inputs_without_blocking() {
896 return false;
897 }
898
899 let inner_len = plan.inner_len.max(1);
900 let row_len = plan.row_len.max(1);
901 let block_len = inner_len.saturating_mul(row_len).max(1);
902 let outer_groups = total.div_ceil(block_len);
903
904 let nthreads = exec.budget().get();
905 if nthreads > 1 {
906 use crate::threading::{parallel_for_each, SendPtr};
907
908 let dst = SendPtr(dst_ptr);
909 let a = SendPtr(a_ptr as *mut A);
910 let b = SendPtr(b_ptr as *mut B);
911
912 if outer_groups < nthreads {
913 let chunk_len = total.div_ceil(nthreads);
914 let nchunks = total.div_ceil(chunk_len);
915
916 parallel_for_each(0..nchunks, exec, &|chunks| {
917 for chunk in chunks {
918 let start = chunk * chunk_len;
919 let end = (start + chunk_len).min(total);
920 let mut index = start;
921
922 while index < end {
923 let in_inner = index % inner_len;
924 let len = (inner_len - in_inner).min(end - index);
925 let a_offset = strided_offset_for_contiguous_linear_index(
926 dims,
927 a_strides,
928 &plan.axis_order,
929 index,
930 );
931 let b_offset = strided_offset_for_contiguous_linear_index(
932 dims,
933 b_strides,
934 &plan.axis_order,
935 index,
936 );
937
938 unsafe {
939 inner_loop_mul2::<O, D, A, B>(
940 dst.as_ptr().add(index),
941 1,
942 a.as_const().offset(a_offset),
943 plan.a_fast_stride,
944 b.as_const().offset(b_offset),
945 plan.b_fast_stride,
946 len,
947 );
948 }
949 index += len;
950 }
951 }
952 });
953
954 return true;
955 }
956
957 let groups_per_chunk = outer_groups.div_ceil(nthreads);
958 let nchunks = outer_groups.div_ceil(groups_per_chunk);
959
960 parallel_for_each(0..nchunks, exec, &|chunks| {
961 for chunk in chunks {
962 let group_start = chunk * groups_per_chunk;
963 let group_end = (group_start + groups_per_chunk).min(outer_groups);
964 let mut cursor =
965 ContiguousMulOuterCursor::new(dims, a_strides, b_strides, &plan, group_start);
966
967 for group in group_start..group_end {
968 let index = group * block_len;
969 unsafe {
970 run_contiguous_mul_row_block::<O, D, A, B>(
971 dst.as_ptr(),
972 a.as_const(),
973 b.as_const(),
974 &plan,
975 index,
976 total,
977 cursor.a_offset,
978 cursor.b_offset,
979 );
980 }
981 cursor.advance();
982 }
983 }
984 });
985
986 true
987 } else {
988 run_contiguous_range_mul_single_thread::<O, D, A, B>(
989 dst_ptr,
990 dims,
991 a_ptr,
992 a_strides,
993 b_ptr,
994 b_strides,
995 &plan,
996 total,
997 block_len,
998 outer_groups,
999 )
1000 }
1001}
1002
1003fn run_contiguous_range_mul_single_thread<
1004 O: MulOutput<D>,
1005 D: Copy + Send + Sync + 'static,
1006 A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
1007 B: Copy + Send + Sync + 'static,
1008>(
1009 dst_ptr: *mut O::Slot,
1010 dims: &[usize],
1011 a_ptr: *const A,
1012 a_strides: &[isize],
1013 b_ptr: *const B,
1014 b_strides: &[isize],
1015 plan: &ContiguousMulRangePlan,
1016 total: usize,
1017 block_len: usize,
1018 outer_groups: usize,
1019) -> bool {
1020 let mut cursor = ContiguousMulOuterCursor::new(dims, a_strides, b_strides, plan, 0);
1021 for group in 0..outer_groups {
1022 let index = group * block_len;
1023 unsafe {
1024 run_contiguous_mul_row_block::<O, D, A, B>(
1025 dst_ptr,
1026 a_ptr,
1027 b_ptr,
1028 plan,
1029 index,
1030 total,
1031 cursor.a_offset,
1032 cursor.b_offset,
1033 );
1034 }
1035 cursor.advance();
1036 }
1037 true
1038}
1039
1040#[inline(always)]
1042unsafe fn inner_loop_map3<
1043 D: Copy,
1044 A: Copy,
1045 B: Copy,
1046 C: Copy,
1047 OpA: ElementOp<A>,
1048 OpB: ElementOp<B>,
1049 OpC: ElementOp<C>,
1050>(
1051 dp: *mut D,
1052 ds: isize,
1053 ap: *const A,
1054 a_s: isize,
1055 bp: *const B,
1056 b_s: isize,
1057 cp: *const C,
1058 c_s: isize,
1059 len: usize,
1060 f: &impl Fn(A, B, C) -> D,
1061) {
1062 if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 {
1063 let src_a = std::slice::from_raw_parts(ap, len);
1064 let src_b = std::slice::from_raw_parts(bp, len);
1065 let src_c = std::slice::from_raw_parts(cp, len);
1066 let dst = std::slice::from_raw_parts_mut(dp, len);
1067 simd::dispatch_if_large(len, || {
1068 for (((d, &a), &b), &c) in dst.iter_mut().zip(src_a).zip(src_b).zip(src_c) {
1069 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
1070 }
1071 });
1072 } else {
1073 let mut dp = dp;
1074 let mut ap = ap;
1075 let mut bp = bp;
1076 let mut cp = cp;
1077 simd::dispatch_if_large(len, || {
1078 for _ in 0..len {
1079 *dp = f(OpA::apply(*ap), OpB::apply(*bp), OpC::apply(*cp));
1080 dp = dp.wrapping_offset(ds);
1081 ap = ap.wrapping_offset(a_s);
1082 bp = bp.wrapping_offset(b_s);
1083 cp = cp.wrapping_offset(c_s);
1084 }
1085 });
1086 }
1087}
1088
1089#[inline(always)]
1091unsafe fn inner_loop_map4<
1092 D: Copy,
1093 A: Copy,
1094 B: Copy,
1095 C: Copy,
1096 E: Copy,
1097 OpA: ElementOp<A>,
1098 OpB: ElementOp<B>,
1099 OpC: ElementOp<C>,
1100 OpE: ElementOp<E>,
1101>(
1102 dp: *mut D,
1103 ds: isize,
1104 ap: *const A,
1105 a_s: isize,
1106 bp: *const B,
1107 b_s: isize,
1108 cp: *const C,
1109 c_s: isize,
1110 ep: *const E,
1111 e_s: isize,
1112 len: usize,
1113 f: &impl Fn(A, B, C, E) -> D,
1114) {
1115 if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 && e_s == 1 {
1116 let src_a = std::slice::from_raw_parts(ap, len);
1117 let src_b = std::slice::from_raw_parts(bp, len);
1118 let src_c = std::slice::from_raw_parts(cp, len);
1119 let src_e = std::slice::from_raw_parts(ep, len);
1120 let dst = std::slice::from_raw_parts_mut(dp, len);
1121 simd::dispatch_if_large(len, || {
1125 for ((((d, &a), &b), &c), &e) in
1126 dst.iter_mut().zip(src_a).zip(src_b).zip(src_c).zip(src_e)
1127 {
1128 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
1129 }
1130 });
1131 } else {
1132 let mut dp = dp;
1133 let mut ap = ap;
1134 let mut bp = bp;
1135 let mut cp = cp;
1136 let mut ep = ep;
1137 simd::dispatch_if_large(len, || {
1138 for _ in 0..len {
1139 *dp = f(
1140 OpA::apply(*ap),
1141 OpB::apply(*bp),
1142 OpC::apply(*cp),
1143 OpE::apply(*ep),
1144 );
1145 dp = dp.wrapping_offset(ds);
1146 ap = ap.wrapping_offset(a_s);
1147 bp = bp.wrapping_offset(b_s);
1148 cp = cp.wrapping_offset(c_s);
1149 ep = ep.wrapping_offset(e_s);
1150 }
1151 });
1152 }
1153}
1154
1155pub trait MapFn<A, D>: Sync {
1183 fn call(&self, a: A) -> D;
1185
1186 fn class(&self) -> KernelClass {
1188 KernelClass::Map
1189 }
1190}
1191
1192impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for F {
1193 #[inline(always)]
1194 fn call(&self, a: A) -> D {
1195 self(a)
1196 }
1197}
1198
1199#[derive(Debug, Clone, Copy)]
1203pub struct Classed<F> {
1204 pub f: F,
1206 pub class: KernelClass,
1208}
1209
1210impl<F> Classed<F> {
1211 pub const fn new(class: KernelClass, f: F) -> Self {
1213 Self { f, class }
1214 }
1215}
1216
1217impl KernelClass {
1218 pub const fn with<F>(self, f: F) -> Classed<F> {
1220 Classed::new(self, f)
1221 }
1222}
1223
1224impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for Classed<F> {
1225 #[inline(always)]
1226 fn call(&self, a: A) -> D {
1227 (self.f)(a)
1228 }
1229
1230 fn class(&self) -> KernelClass {
1231 self.class
1232 }
1233}
1234
1235pub fn map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1252 dest: &mut StridedViewMut<D>,
1253 src: &StridedView<A, Op>,
1254 f: impl MapFn<A, D>,
1255 exec: Exec<'_>,
1256) -> Result<()> {
1257 let class = f.class();
1258 map_parts_into::<D, A, Op, NoKernel>(
1259 dest.as_mut_ptr(),
1260 dest.dims(),
1261 dest.strides(),
1262 src.ptr(),
1263 src.dims(),
1264 src.strides(),
1265 |a| f.call(a),
1266 &NoKernel,
1267 class,
1268 exec,
1269 )
1270}
1271
1272pub(crate) fn copy_map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1277 dest: &mut StridedViewMut<D>,
1278 src: &StridedView<A, Op>,
1279 f: impl Fn(A) -> D + Sync,
1280 exec: Exec<'_>,
1281) -> Result<()> {
1282 map_parts_into::<D, A, Op, NoKernel>(
1283 dest.as_mut_ptr(),
1284 dest.dims(),
1285 dest.strides(),
1286 src.ptr(),
1287 src.dims(),
1288 src.strides(),
1289 f,
1290 &NoKernel,
1291 KernelClass::Copy,
1292 exec,
1293 )
1294}
1295
1296pub(crate) fn map_into_validated<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1297 dest: &mut StridedViewMut<D>,
1298 src: &StridedView<A, Op>,
1299 f: impl MapFn<A, D>,
1300 validated: ValidatedDestinationLayout,
1301 exec: Exec<'_>,
1302) -> Result<()> {
1303 ensure_same_shape(dest.dims(), src.dims())?;
1304 let class = f.class();
1305 map_parts_into_validated::<D, A, Op, NoKernel>(
1306 dest.as_mut_ptr(),
1307 dest.dims(),
1308 dest.strides(),
1309 src.ptr(),
1310 src.strides(),
1311 |a| f.call(a),
1312 &NoKernel,
1313 validated,
1314 class,
1315 exec,
1316 )
1317}
1318
1319pub fn map_into_uninit_with_kernel<T: Copy + Send + Sync>(
1363 dest: &mut StridedViewMut<MaybeUninit<T>>,
1364 src: &StridedView<T>,
1365 kernel: impl Fn(&[T], &mut [MaybeUninit<T>]) -> bool + Sync,
1366 scalar: impl MapFn<T, T>,
1367 exec: Exec<'_>,
1368) -> Result<()> {
1369 let class = scalar.class();
1370 map_parts_into::<MaybeUninit<T>, T, Identity, _>(
1371 dest.as_mut_ptr(),
1372 dest.dims(),
1373 dest.strides(),
1374 src.ptr(),
1375 src.dims(),
1376 src.strides(),
1377 |value| MaybeUninit::new(scalar.call(value)),
1378 &SliceKernel(kernel),
1379 class,
1380 exec,
1381 )
1382}
1383
1384#[cfg(test)]
1390pub(crate) fn map_raw_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1391 dest: &mut crate::RawStridedMut<'_, D>,
1392 src: &crate::RawStridedRef<'_, A>,
1393 f: impl Fn(A) -> D + Sync,
1394 exec: Exec<'_>,
1395) -> Result<()> {
1396 map_parts_into::<D, A, Op, NoKernel>(
1397 dest.as_mut_ptr(),
1398 dest.dims(),
1399 dest.strides(),
1400 src.ptr(),
1401 src.dims(),
1402 src.strides(),
1403 f,
1404 &NoKernel,
1405 KernelClass::Map,
1406 exec,
1407 )
1408}
1409
1410pub(crate) fn map_raw_into_validated<
1411 D: Copy + Send + Sync,
1412 A: Copy + Send + Sync,
1413 Op: ElementOp<A>,
1414>(
1415 dest: &mut crate::RawStridedMut<'_, D>,
1416 src: &crate::RawStridedRef<'_, A>,
1417 f: impl Fn(A) -> D + Sync,
1418 validated: ValidatedDestinationLayout,
1419 exec: Exec<'_>,
1420) -> Result<()> {
1421 ensure_same_shape(dest.dims(), src.dims())?;
1422 map_parts_into_validated::<D, A, Op, NoKernel>(
1423 dest.as_mut_ptr(),
1424 dest.dims(),
1425 dest.strides(),
1426 src.ptr(),
1427 src.strides(),
1428 f,
1429 &NoKernel,
1430 validated,
1431 KernelClass::Map,
1432 exec,
1433 )
1434}
1435
1436#[allow(clippy::too_many_arguments)]
1437fn map_parts_into<
1438 D: Copy + Send + Sync,
1439 A: Copy + Send + Sync,
1440 Op: ElementOp<A>,
1441 K: MapKernel<A, D> + Sync,
1442>(
1443 dst_ptr: *mut D,
1444 dst_dims: &[usize],
1445 dst_strides: &[isize],
1446 src_ptr: *const A,
1447 src_dims: &[usize],
1448 src_strides: &[isize],
1449 f: impl Fn(A) -> D + Sync,
1450 kernel: &K,
1451 class: KernelClass,
1452 exec: Exec<'_>,
1453) -> Result<()> {
1454 ensure_same_shape(dst_dims, src_dims)?;
1455 let validated = validate_destination_layout(dst_dims, dst_strides)?;
1456 map_parts_into_validated::<D, A, Op, K>(
1457 dst_ptr,
1458 dst_dims,
1459 dst_strides,
1460 src_ptr,
1461 src_strides,
1462 f,
1463 kernel,
1464 validated,
1465 class,
1466 exec,
1467 )
1468}
1469
1470#[allow(clippy::too_many_arguments)]
1471#[inline(never)]
1478fn map_contiguous<D: Copy, A: Copy, Op: ElementOp<A>>(
1479 dst: &mut [D],
1480 src: &[A],
1481 f: impl Fn(A) -> D,
1482) {
1483 simd::dispatch_if_large(dst.len(), move || {
1484 for i in 0..dst.len() {
1485 dst[i] = f(Op::apply(src[i]));
1486 }
1487 });
1488}
1489
1490#[allow(clippy::too_many_arguments)]
1491fn map_parts_into_validated<
1492 D: Copy + Send + Sync,
1493 A: Copy + Send + Sync,
1494 Op: ElementOp<A>,
1495 K: MapKernel<A, D> + Sync,
1496>(
1497 dst_ptr: *mut D,
1498 dst_dims: &[usize],
1499 dst_strides: &[isize],
1500 src_ptr: *const A,
1501 src_strides: &[isize],
1502 f: impl Fn(A) -> D + Sync,
1503 kernel: &K,
1504 _validated: ValidatedDestinationLayout,
1505 class: KernelClass,
1506 exec: Exec<'_>,
1507) -> Result<()> {
1508 if sequential_contiguous_layout(class, dst_dims, &[dst_strides, src_strides], &exec)?.is_some()
1509 {
1510 let len = total_len(dst_dims)?;
1511 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1515 let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
1516 if kernel.apply(src, dst) {
1517 return Ok(());
1518 }
1519 map_contiguous::<D, A, Op>(dst, src, f);
1520 return Ok(());
1521 }
1522
1523 let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
1524 let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<A>());
1525 let total = total_len(dst_dims)?;
1526
1527 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
1529 build_plan_fused_small(dst_dims, &strides_list)
1530 } else {
1531 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
1532 };
1533
1534 let total = total_len(&fused_dims)?;
1535 if is_parallel(class, total, &exec) {
1536 use crate::threading::SendPtr;
1537 let dst_send = SendPtr(dst_ptr);
1538 let src_send = SendPtr(src_ptr as *mut A);
1539
1540 let costs = compute_costs(&ordered_strides);
1541 let initial_offsets: crate::inline::OperandVec<isize> =
1542 strides_list.iter().map(|_| 0isize).collect();
1543 return mapreduce_threaded(
1544 &fused_dims,
1545 &plan.block,
1546 &ordered_strides,
1547 &initial_offsets,
1548 &costs,
1549 class,
1550 exec,
1551 &|dims, blocks, strides_list, offsets| {
1552 for_each_inner_block_with_offsets(
1553 dims,
1554 blocks,
1555 strides_list,
1556 offsets,
1557 |offsets, len, strides| {
1558 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1559 let sp = unsafe { src_send.as_const().offset(offsets[1]) };
1560 unsafe {
1561 inner_loop_map1::<D, A, Op, K>(
1562 dp, strides[0], sp, strides[1], len, &f, kernel,
1563 )
1564 };
1565 Ok(())
1566 },
1567 )
1568 },
1569 );
1570 }
1571
1572 let initial_offsets: crate::inline::OperandVec<isize> =
1573 ordered_strides.iter().map(|_| 0isize).collect();
1574 for_each_inner_block_preordered(
1575 &fused_dims,
1576 &plan.block,
1577 &ordered_strides,
1578 &initial_offsets,
1579 |offsets, len, strides| {
1580 let dp = unsafe { dst_ptr.offset(offsets[0]) };
1581 let sp = unsafe { src_ptr.offset(offsets[1]) };
1582 unsafe {
1583 inner_loop_map1::<D, A, Op, K>(dp, strides[0], sp, strides[1], len, &f, kernel)
1584 };
1585 Ok(())
1586 },
1587 )
1588}
1589
1590pub fn zip_map2_into<
1605 D: Copy + Send + Sync,
1606 A: Copy + Send + Sync,
1607 B: Copy + Send + Sync,
1608 OpA: ElementOp<A>,
1609 OpB: ElementOp<B>,
1610>(
1611 dest: &mut StridedViewMut<D>,
1612 a: &StridedView<A, OpA>,
1613 b: &StridedView<B, OpB>,
1614 f: impl Fn(A, B) -> D + Sync,
1615 exec: Exec<'_>,
1616) -> Result<()> {
1617 zip_map2_parts_into::<D, A, B, OpA, OpB>(
1618 dest.as_mut_ptr(),
1619 dest.dims(),
1620 dest.strides(),
1621 a.ptr(),
1622 a.dims(),
1623 a.strides(),
1624 b.ptr(),
1625 b.dims(),
1626 b.strides(),
1627 f,
1628 exec,
1629 )
1630}
1631
1632pub(crate) fn zip_map2_into_validated<
1633 D: Copy + Send + Sync,
1634 A: Copy + Send + Sync,
1635 B: Copy + Send + Sync,
1636 OpA: ElementOp<A>,
1637 OpB: ElementOp<B>,
1638>(
1639 dest: &mut StridedViewMut<D>,
1640 a: &StridedView<A, OpA>,
1641 b: &StridedView<B, OpB>,
1642 f: impl Fn(A, B) -> D + Sync,
1643 validated: ValidatedDestinationLayout,
1644 exec: Exec<'_>,
1645) -> Result<()> {
1646 zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1647 dest.as_mut_ptr(),
1648 dest.dims(),
1649 dest.strides(),
1650 a.ptr(),
1651 a.strides(),
1652 b.ptr(),
1653 b.strides(),
1654 f,
1655 validated,
1656 exec,
1657 )
1658}
1659
1660#[non_exhaustive]
1662#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1663pub enum CompareOp {
1664 Eq,
1665 Lt,
1666 Le,
1667 Gt,
1668 Ge,
1669}
1670
1671pub fn compare_into<T, OpA, OpB>(
1682 dest: &mut StridedViewMut<bool>,
1683 a: &StridedView<T, OpA>,
1684 b: &StridedView<T, OpB>,
1685 op: CompareOp,
1686 exec: Exec<'_>,
1687) -> Result<()>
1688where
1689 T: Copy + Send + Sync + PartialOrd,
1690 OpA: ElementOp<T>,
1691 OpB: ElementOp<T>,
1692{
1693 match op {
1694 CompareOp::Eq => zip_map2_into(dest, a, b, |lhs, rhs| lhs == rhs, exec),
1695 CompareOp::Lt => zip_map2_into(dest, a, b, |lhs, rhs| lhs < rhs, exec),
1696 CompareOp::Le => zip_map2_into(dest, a, b, |lhs, rhs| lhs <= rhs, exec),
1697 CompareOp::Gt => zip_map2_into(dest, a, b, |lhs, rhs| lhs > rhs, exec),
1698 CompareOp::Ge => zip_map2_into(dest, a, b, |lhs, rhs| lhs >= rhs, exec),
1699 }
1700}
1701
1702pub fn compare_into_uninit<T, OpA, OpB>(
1721 dest: &mut StridedViewMut<MaybeUninit<bool>>,
1722 a: &StridedView<T, OpA>,
1723 b: &StridedView<T, OpB>,
1724 op: CompareOp,
1725 exec: Exec<'_>,
1726) -> Result<()>
1727where
1728 T: Copy + Send + Sync + PartialOrd,
1729 OpA: ElementOp<T>,
1730 OpB: ElementOp<T>,
1731{
1732 ensure_same_shape(dest.dims(), a.dims())?;
1733 ensure_same_shape(dest.dims(), b.dims())?;
1734 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
1735 validate_typed_no_overlap(dest, a, 0)?;
1736 validate_typed_no_overlap(dest, b, 1)?;
1737 match op {
1738 CompareOp::Eq => zip_map2_into_validated(
1739 dest,
1740 a,
1741 b,
1742 |lhs, rhs| MaybeUninit::new(lhs == rhs),
1743 validated,
1744 exec,
1745 ),
1746 CompareOp::Lt => zip_map2_into_validated(
1747 dest,
1748 a,
1749 b,
1750 |lhs, rhs| MaybeUninit::new(lhs < rhs),
1751 validated,
1752 exec,
1753 ),
1754 CompareOp::Le => zip_map2_into_validated(
1755 dest,
1756 a,
1757 b,
1758 |lhs, rhs| MaybeUninit::new(lhs <= rhs),
1759 validated,
1760 exec,
1761 ),
1762 CompareOp::Gt => zip_map2_into_validated(
1763 dest,
1764 a,
1765 b,
1766 |lhs, rhs| MaybeUninit::new(lhs > rhs),
1767 validated,
1768 exec,
1769 ),
1770 CompareOp::Ge => zip_map2_into_validated(
1771 dest,
1772 a,
1773 b,
1774 |lhs, rhs| MaybeUninit::new(lhs >= rhs),
1775 validated,
1776 exec,
1777 ),
1778 }
1779}
1780
1781pub(crate) fn zip_map2_raw_into_validated<
1782 D: Copy + Send + Sync,
1783 A: Copy + Send + Sync,
1784 B: Copy + Send + Sync,
1785 OpA: ElementOp<A>,
1786 OpB: ElementOp<B>,
1787>(
1788 dest: &mut crate::RawStridedMut<'_, D>,
1789 a: &crate::RawStridedRef<'_, A>,
1790 b: &crate::RawStridedRef<'_, B>,
1791 f: impl Fn(A, B) -> D + Sync,
1792 validated: ValidatedDestinationLayout,
1793 exec: Exec<'_>,
1794) -> Result<()> {
1795 ensure_same_shape(dest.dims(), a.dims())?;
1796 ensure_same_shape(dest.dims(), b.dims())?;
1797 zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1798 dest.as_mut_ptr(),
1799 dest.dims(),
1800 dest.strides(),
1801 a.ptr(),
1802 a.strides(),
1803 b.ptr(),
1804 b.strides(),
1805 f,
1806 validated,
1807 exec,
1808 )
1809}
1810
1811#[allow(clippy::too_many_arguments)]
1812fn zip_map2_parts_into<
1813 D: Copy + Send + Sync,
1814 A: Copy + Send + Sync,
1815 B: Copy + Send + Sync,
1816 OpA: ElementOp<A>,
1817 OpB: ElementOp<B>,
1818>(
1819 dst_ptr: *mut D,
1820 dst_dims: &[usize],
1821 dst_strides: &[isize],
1822 a_ptr: *const A,
1823 a_dims: &[usize],
1824 a_strides: &[isize],
1825 b_ptr: *const B,
1826 b_dims: &[usize],
1827 b_strides: &[isize],
1828 f: impl Fn(A, B) -> D + Sync,
1829 exec: Exec<'_>,
1830) -> Result<()> {
1831 ensure_same_shape(dst_dims, a_dims)?;
1832 ensure_same_shape(dst_dims, b_dims)?;
1833 let validated = validate_destination_layout(dst_dims, dst_strides)?;
1834 zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1835 dst_ptr,
1836 dst_dims,
1837 dst_strides,
1838 a_ptr,
1839 a_strides,
1840 b_ptr,
1841 b_strides,
1842 f,
1843 validated,
1844 exec,
1845 )
1846}
1847
1848#[allow(clippy::too_many_arguments)]
1849fn zip_map2_parts_into_validated<
1850 D: Copy + Send + Sync,
1851 A: Copy + Send + Sync,
1852 B: Copy + Send + Sync,
1853 OpA: ElementOp<A>,
1854 OpB: ElementOp<B>,
1855>(
1856 dst_ptr: *mut D,
1857 dst_dims: &[usize],
1858 dst_strides: &[isize],
1859 a_ptr: *const A,
1860 a_strides: &[isize],
1861 b_ptr: *const B,
1862 b_strides: &[isize],
1863 f: impl Fn(A, B) -> D + Sync,
1864 _validated: ValidatedDestinationLayout,
1865 exec: Exec<'_>,
1866) -> Result<()> {
1867 if sequential_contiguous_layout(
1868 KernelClass::Map,
1869 dst_dims,
1870 &[dst_strides, a_strides, b_strides],
1871 &exec,
1872 )?
1873 .is_some()
1874 {
1875 let len = total_len(dst_dims)?;
1876 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1877 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
1878 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
1879 simd::dispatch_if_large(len, || {
1880 for i in 0..len {
1881 dst[i] = f(OpA::apply(sa[i]), OpB::apply(sb[i]));
1882 }
1883 });
1884 return Ok(());
1885 }
1886
1887 let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
1888 let elem_size = std::mem::size_of::<D>()
1889 .max(std::mem::size_of::<A>())
1890 .max(std::mem::size_of::<B>());
1891 let total = total_len(dst_dims)?;
1892
1893 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
1895 build_plan_fused_small(dst_dims, &strides_list)
1896 } else {
1897 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
1898 };
1899
1900 let total = total_len(&fused_dims)?;
1901 if is_parallel(KernelClass::Map, total, &exec) {
1902 use crate::threading::SendPtr;
1903 let dst_send = SendPtr(dst_ptr);
1904 let a_send = SendPtr(a_ptr as *mut A);
1905 let b_send = SendPtr(b_ptr as *mut B);
1906
1907 let costs = compute_costs(&ordered_strides);
1908 let initial_offsets: crate::inline::OperandVec<isize> =
1909 strides_list.iter().map(|_| 0isize).collect();
1910 return mapreduce_threaded(
1911 &fused_dims,
1912 &plan.block,
1913 &ordered_strides,
1914 &initial_offsets,
1915 &costs,
1916 KernelClass::Map,
1917 exec,
1918 &|dims, blocks, strides_list, offsets| {
1919 for_each_inner_block_with_offsets(
1920 dims,
1921 blocks,
1922 strides_list,
1923 offsets,
1924 |offsets, len, strides| {
1925 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1926 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
1927 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
1928 unsafe {
1929 inner_loop_map2::<D, A, B, OpA, OpB>(
1930 dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1931 )
1932 };
1933 Ok(())
1934 },
1935 )
1936 },
1937 );
1938 }
1939
1940 let initial_offsets: crate::inline::OperandVec<isize> =
1941 ordered_strides.iter().map(|_| 0isize).collect();
1942 for_each_inner_block_preordered(
1943 &fused_dims,
1944 &plan.block,
1945 &ordered_strides,
1946 &initial_offsets,
1947 |offsets, len, strides| {
1948 let dp = unsafe { dst_ptr.offset(offsets[0]) };
1949 let ap = unsafe { a_ptr.offset(offsets[1]) };
1950 let bp = unsafe { b_ptr.offset(offsets[2]) };
1951 unsafe {
1952 inner_loop_map2::<D, A, B, OpA, OpB>(
1953 dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1954 )
1955 };
1956 Ok(())
1957 },
1958 )
1959}
1960
1961#[allow(clippy::too_many_arguments)]
1962fn mul_identity_into_raw<
1963 O: MulOutput<D>,
1964 D: Copy + Send + Sync + 'static,
1965 A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
1966 B: Copy + Send + Sync + 'static,
1967>(
1968 dst_ptr: *mut O::Slot,
1969 dst_dims: &[usize],
1970 dst_strides: &[isize],
1971 a_ptr: *const A,
1972 a_strides: &[isize],
1973 b_ptr: *const B,
1974 b_strides: &[isize],
1975 _validated: ValidatedDestinationLayout,
1976 mut exec: Exec<'_>,
1977) -> Result<()> {
1978 debug_assert_eq!(dst_dims.len(), a_strides.len());
1979 debug_assert_eq!(dst_dims.len(), b_strides.len());
1980
1981 if sequential_contiguous_layout(
1982 KernelClass::Map,
1983 dst_dims,
1984 &[dst_strides, a_strides, b_strides],
1985 &exec,
1986 )?
1987 .is_some()
1988 {
1989 let len = total_len(dst_dims)?;
1990 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
1991 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
1992 if unsafe { O::try_contiguous(dst_ptr, len, sa, sb) } {
1993 return Ok(());
1994 }
1995 for i in 0..len {
1996 unsafe { O::write(dst_ptr.add(i), multiply_value(sa[i], sb[i])) };
1997 }
1998 return Ok(());
1999 }
2000
2001 let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
2002 let elem_size = std::mem::size_of::<D>()
2003 .max(std::mem::size_of::<A>())
2004 .max(std::mem::size_of::<B>());
2005 let total = total_len(dst_dims)?;
2006
2007 if try_contiguous_range_mul::<O, D, A, B>(
2008 dst_ptr,
2009 dst_dims,
2010 dst_strides,
2011 a_ptr,
2012 a_strides,
2013 b_ptr,
2014 b_strides,
2015 exec.reborrow(),
2016 ) {
2017 return Ok(());
2018 }
2019
2020 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2021 build_plan_fused_small(dst_dims, &strides_list)
2022 } else {
2023 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2024 };
2025
2026 let total = total_len(&fused_dims)?;
2027 if is_parallel(KernelClass::Map, total, &exec) {
2028 use crate::threading::SendPtr;
2029 let dst_send = SendPtr(dst_ptr);
2030 let a_send = SendPtr(a_ptr as *mut A);
2031 let b_send = SendPtr(b_ptr as *mut B);
2032
2033 let costs = compute_costs(&ordered_strides);
2034 let initial_offsets: crate::inline::OperandVec<isize> =
2035 strides_list.iter().map(|_| 0isize).collect();
2036 return mapreduce_threaded(
2037 &fused_dims,
2038 &plan.block,
2039 &ordered_strides,
2040 &initial_offsets,
2041 &costs,
2042 KernelClass::Map,
2043 exec,
2044 &|dims, blocks, strides_list, offsets| {
2045 for_each_inner_block_with_offsets(
2046 dims,
2047 blocks,
2048 strides_list,
2049 offsets,
2050 |offsets, len, strides| {
2051 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2052 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2053 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2054 unsafe {
2055 inner_loop_mul2::<O, D, A, B>(
2056 dp, strides[0], ap, strides[1], bp, strides[2], len,
2057 )
2058 };
2059 Ok(())
2060 },
2061 )
2062 },
2063 );
2064 }
2065
2066 let initial_offsets: crate::inline::OperandVec<isize> =
2067 ordered_strides.iter().map(|_| 0isize).collect();
2068 for_each_inner_block_preordered(
2069 &fused_dims,
2070 &plan.block,
2071 &ordered_strides,
2072 &initial_offsets,
2073 |offsets, len, strides| {
2074 let dp = unsafe { dst_ptr.offset(offsets[0]) };
2075 let ap = unsafe { a_ptr.offset(offsets[1]) };
2076 let bp = unsafe { b_ptr.offset(offsets[2]) };
2077 unsafe {
2078 inner_loop_mul2::<O, D, A, B>(dp, strides[0], ap, strides[1], bp, strides[2], len)
2079 };
2080 Ok(())
2081 },
2082 )
2083}
2084
2085pub fn mul_into<
2100 D: Copy + Send + Sync + 'static,
2101 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2102 B: Copy + Send + Sync + 'static,
2103 OpA: ElementOp<A>,
2104 OpB: ElementOp<B>,
2105>(
2106 dest: &mut StridedViewMut<D>,
2107 a: &StridedView<A, OpA>,
2108 b: &StridedView<B, OpB>,
2109 exec: Exec<'_>,
2110) -> Result<()> {
2111 ensure_same_shape(dest.dims(), a.dims())?;
2112 ensure_same_shape(dest.dims(), b.dims())?;
2113 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2114
2115 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2116 return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2117 dest.as_mut_ptr(),
2118 dest.dims(),
2119 dest.strides(),
2120 a.ptr(),
2121 a.strides(),
2122 b.ptr(),
2123 b.strides(),
2124 validated,
2125 exec,
2126 );
2127 }
2128
2129 zip_map2_into_validated(dest, a, b, multiply_value, validated, exec)
2130}
2131
2132pub fn mul_into_uninit<
2151 D: Copy + Send + Sync + 'static,
2152 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2153 B: Copy + Send + Sync + 'static,
2154 OpA: ElementOp<A>,
2155 OpB: ElementOp<B>,
2156>(
2157 dest: &mut StridedViewMut<MaybeUninit<D>>,
2158 a: &StridedView<A, OpA>,
2159 b: &StridedView<B, OpB>,
2160 exec: Exec<'_>,
2161) -> Result<()> {
2162 ensure_same_shape(dest.dims(), a.dims())?;
2163 ensure_same_shape(dest.dims(), b.dims())?;
2164 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2165 validate_typed_no_overlap(dest, a, 0)?;
2166 validate_typed_no_overlap(dest, b, 1)?;
2167
2168 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2169 return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2170 dest.as_mut_ptr(),
2171 dest.dims(),
2172 dest.strides(),
2173 a.ptr(),
2174 a.strides(),
2175 b.ptr(),
2176 b.strides(),
2177 validated,
2178 exec,
2179 );
2180 }
2181
2182 zip_map2_into_validated(
2183 dest,
2184 a,
2185 b,
2186 |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2187 validated,
2188 exec,
2189 )
2190}
2191
2192fn broadcast_strides_for_axes(
2193 source_dims: &[usize],
2194 source_strides: &[isize],
2195 target_dims: &[usize],
2196 axes: &[usize],
2197) -> Result<AxisVec<isize>> {
2198 if source_dims.len() != axes.len() {
2199 return Err(StridedError::RankMismatch(source_dims.len(), axes.len()));
2200 }
2201 debug_assert_eq!(source_dims.len(), source_strides.len());
2202
2203 let mut seen = AxisVec::<bool>::new();
2204 seen.resize(target_dims.len(), false);
2205 let mut strides = AxisVec::<isize>::new();
2206 strides.resize(target_dims.len(), 0);
2207 for (src_axis, &dst_axis) in axes.iter().enumerate() {
2208 if dst_axis >= target_dims.len() {
2209 return Err(StridedError::InvalidAxis {
2210 axis: dst_axis,
2211 rank: target_dims.len(),
2212 });
2213 }
2214 if seen[dst_axis] {
2215 return Err(StridedError::InvalidAxis {
2216 axis: dst_axis,
2217 rank: target_dims.len(),
2218 });
2219 }
2220 seen[dst_axis] = true;
2221
2222 let source_dim = source_dims[src_axis];
2223 let target_dim = target_dims[dst_axis];
2224 if source_dim != target_dim && source_dim != 1 {
2225 return Err(StridedError::ShapeMismatch(
2226 source_dims.to_vec(),
2227 target_dims.to_vec(),
2228 ));
2229 }
2230 if source_dim == target_dim {
2231 strides[dst_axis] = source_strides[src_axis];
2232 }
2233 }
2234
2235 Ok(strides)
2236}
2237
2238fn broadcast_view_with_strides<'a, T, Op: ElementOp<T>>(
2245 view: &StridedView<'a, T, Op>,
2246 target_dims: &[usize],
2247 strides: &[isize],
2248) -> StridedView<'a, T, Op> {
2249 unsafe { StridedView::new_unchecked(view.data(), target_dims, strides, view.offset()) }
2261}
2262
2263pub fn broadcast_mul_into<
2279 D: Copy + Send + Sync + 'static,
2280 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2281 B: Copy + Send + Sync + 'static,
2282 OpA: ElementOp<A>,
2283 OpB: ElementOp<B>,
2284>(
2285 dest: &mut StridedViewMut<D>,
2286 a: &StridedView<A, OpA>,
2287 a_axes: &[usize],
2288 b: &StridedView<B, OpB>,
2289 b_axes: &[usize],
2290 exec: Exec<'_>,
2291) -> Result<()> {
2292 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2293 let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2294 let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2295
2296 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2297 return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2298 dest.as_mut_ptr(),
2299 dest.dims(),
2300 dest.strides(),
2301 a.ptr(),
2302 &a_strides,
2303 b.ptr(),
2304 &b_strides,
2305 validated,
2306 exec,
2307 );
2308 }
2309
2310 let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2311 let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2312 zip_map2_into_validated(dest, &a, &b, multiply_value, validated, exec)
2313}
2314
2315pub fn broadcast_mul_into_uninit<
2334 D: Copy + Send + Sync + 'static,
2335 A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2336 B: Copy + Send + Sync + 'static,
2337 OpA: ElementOp<A>,
2338 OpB: ElementOp<B>,
2339>(
2340 dest: &mut StridedViewMut<MaybeUninit<D>>,
2341 a: &StridedView<A, OpA>,
2342 a_axes: &[usize],
2343 b: &StridedView<B, OpB>,
2344 b_axes: &[usize],
2345 exec: Exec<'_>,
2346) -> Result<()> {
2347 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2348 let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2349 let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2350 validate_typed_no_overlap(dest, a, 0)?;
2351 validate_typed_no_overlap(dest, b, 1)?;
2352
2353 if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2354 return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2355 dest.as_mut_ptr(),
2356 dest.dims(),
2357 dest.strides(),
2358 a.ptr(),
2359 &a_strides,
2360 b.ptr(),
2361 &b_strides,
2362 validated,
2363 exec,
2364 );
2365 }
2366
2367 let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2368 let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2369 zip_map2_into_validated(
2370 dest,
2371 &a,
2372 &b,
2373 |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2374 validated,
2375 exec,
2376 )
2377}
2378
2379pub fn zip_map3_into<
2391 D: Copy + Send + Sync,
2392 A: Copy + Send + Sync,
2393 B: Copy + Send + Sync,
2394 C: Copy + Send + Sync,
2395 OpA: ElementOp<A>,
2396 OpB: ElementOp<B>,
2397 OpC: ElementOp<C>,
2398>(
2399 dest: &mut StridedViewMut<D>,
2400 a: &StridedView<A, OpA>,
2401 b: &StridedView<B, OpB>,
2402 c: &StridedView<C, OpC>,
2403 f: impl Fn(A, B, C) -> D + Sync,
2404 exec: Exec<'_>,
2405) -> Result<()> {
2406 ensure_same_shape(dest.dims(), a.dims())?;
2407 ensure_same_shape(dest.dims(), b.dims())?;
2408 ensure_same_shape(dest.dims(), c.dims())?;
2409 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2410 zip_map3_into_validated(dest, a, b, c, f, validated, exec)
2411}
2412
2413pub(crate) fn zip_map3_into_validated<
2414 D: Copy + Send + Sync,
2415 A: Copy + Send + Sync,
2416 B: Copy + Send + Sync,
2417 C: Copy + Send + Sync,
2418 OpA: ElementOp<A>,
2419 OpB: ElementOp<B>,
2420 OpC: ElementOp<C>,
2421>(
2422 dest: &mut StridedViewMut<D>,
2423 a: &StridedView<A, OpA>,
2424 b: &StridedView<B, OpB>,
2425 c: &StridedView<C, OpC>,
2426 f: impl Fn(A, B, C) -> D + Sync,
2427 _validated: ValidatedDestinationLayout,
2428 exec: Exec<'_>,
2429) -> Result<()> {
2430 ensure_same_shape(dest.dims(), a.dims())?;
2431 ensure_same_shape(dest.dims(), b.dims())?;
2432 ensure_same_shape(dest.dims(), c.dims())?;
2433 let dst_ptr = dest.as_mut_ptr();
2434 let a_ptr = a.ptr();
2435 let b_ptr = b.ptr();
2436 let c_ptr = c.ptr();
2437
2438 let dst_dims = dest.dims();
2439 let dst_strides = dest.strides();
2440
2441 if sequential_contiguous_layout(
2442 KernelClass::Map,
2443 dst_dims,
2444 &[dst_strides, a.strides(), b.strides(), c.strides()],
2445 &exec,
2446 )?
2447 .is_some()
2448 {
2449 let len = total_len(dst_dims)?;
2450 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2451 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2452 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2453 let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2454 simd::dispatch_if_large(len, || {
2455 for (((d, &a), &b), &c) in dst.iter_mut().zip(sa).zip(sb).zip(sc) {
2456 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
2457 }
2458 });
2459 return Ok(());
2460 }
2461
2462 let strides_list: [&[isize]; 4] = [dst_strides, a.strides(), b.strides(), c.strides()];
2463 let elem_size = std::mem::size_of::<D>()
2464 .max(std::mem::size_of::<A>())
2465 .max(std::mem::size_of::<B>())
2466 .max(std::mem::size_of::<C>());
2467 let total = total_len(dst_dims)?;
2468
2469 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2471 build_plan_fused_small(dst_dims, &strides_list)
2472 } else {
2473 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2474 };
2475
2476 let total = total_len(&fused_dims)?;
2477 if is_parallel(KernelClass::Map, total, &exec) {
2478 use crate::threading::SendPtr;
2479 let dst_send = SendPtr(dst_ptr);
2480 let a_send = SendPtr(a_ptr as *mut A);
2481 let b_send = SendPtr(b_ptr as *mut B);
2482 let c_send = SendPtr(c_ptr as *mut C);
2483
2484 let costs = compute_costs(&ordered_strides);
2485 let initial_offsets: crate::inline::OperandVec<isize> =
2486 strides_list.iter().map(|_| 0isize).collect();
2487 return mapreduce_threaded(
2488 &fused_dims,
2489 &plan.block,
2490 &ordered_strides,
2491 &initial_offsets,
2492 &costs,
2493 KernelClass::Map,
2494 exec,
2495 &|dims, blocks, strides_list, offsets| {
2496 for_each_inner_block_with_offsets(
2497 dims,
2498 blocks,
2499 strides_list,
2500 offsets,
2501 |offsets, len, strides| {
2502 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2503 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2504 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2505 let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2506 unsafe {
2507 inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2508 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3],
2509 len, &f,
2510 )
2511 };
2512 Ok(())
2513 },
2514 )
2515 },
2516 );
2517 }
2518
2519 let initial_offsets: crate::inline::OperandVec<isize> =
2520 ordered_strides.iter().map(|_| 0isize).collect();
2521 for_each_inner_block_preordered(
2522 &fused_dims,
2523 &plan.block,
2524 &ordered_strides,
2525 &initial_offsets,
2526 |offsets, len, strides| {
2527 let dp = unsafe { dst_ptr.offset(offsets[0]) };
2528 let ap = unsafe { a_ptr.offset(offsets[1]) };
2529 let bp = unsafe { b_ptr.offset(offsets[2]) };
2530 let cp = unsafe { c_ptr.offset(offsets[3]) };
2531 unsafe {
2532 inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2533 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], len, &f,
2534 )
2535 };
2536 Ok(())
2537 },
2538 )
2539}
2540
2541pub fn zip_map4_into<
2553 D: Copy + Send + Sync,
2554 A: Copy + Send + Sync,
2555 B: Copy + Send + Sync,
2556 C: Copy + Send + Sync,
2557 E: Copy + Send + Sync,
2558 OpA: ElementOp<A>,
2559 OpB: ElementOp<B>,
2560 OpC: ElementOp<C>,
2561 OpE: ElementOp<E>,
2562>(
2563 dest: &mut StridedViewMut<D>,
2564 a: &StridedView<A, OpA>,
2565 b: &StridedView<B, OpB>,
2566 c: &StridedView<C, OpC>,
2567 e: &StridedView<E, OpE>,
2568 f: impl Fn(A, B, C, E) -> D + Sync,
2569 exec: Exec<'_>,
2570) -> Result<()> {
2571 ensure_same_shape(dest.dims(), a.dims())?;
2572 ensure_same_shape(dest.dims(), b.dims())?;
2573 ensure_same_shape(dest.dims(), c.dims())?;
2574 ensure_same_shape(dest.dims(), e.dims())?;
2575 let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2576 zip_map4_into_validated(dest, a, b, c, e, f, validated, exec)
2577}
2578
2579#[allow(clippy::too_many_arguments)]
2580pub(crate) fn zip_map4_into_validated<
2581 D: Copy + Send + Sync,
2582 A: Copy + Send + Sync,
2583 B: Copy + Send + Sync,
2584 C: Copy + Send + Sync,
2585 E: Copy + Send + Sync,
2586 OpA: ElementOp<A>,
2587 OpB: ElementOp<B>,
2588 OpC: ElementOp<C>,
2589 OpE: ElementOp<E>,
2590>(
2591 dest: &mut StridedViewMut<D>,
2592 a: &StridedView<A, OpA>,
2593 b: &StridedView<B, OpB>,
2594 c: &StridedView<C, OpC>,
2595 e: &StridedView<E, OpE>,
2596 f: impl Fn(A, B, C, E) -> D + Sync,
2597 _validated: ValidatedDestinationLayout,
2598 exec: Exec<'_>,
2599) -> Result<()> {
2600 ensure_same_shape(dest.dims(), a.dims())?;
2601 ensure_same_shape(dest.dims(), b.dims())?;
2602 ensure_same_shape(dest.dims(), c.dims())?;
2603 ensure_same_shape(dest.dims(), e.dims())?;
2604 let dst_ptr = dest.as_mut_ptr();
2605 let a_ptr = a.ptr();
2606 let b_ptr = b.ptr();
2607 let c_ptr = c.ptr();
2608 let e_ptr = e.ptr();
2609
2610 let dst_dims = dest.dims();
2611 let dst_strides = dest.strides();
2612
2613 if sequential_contiguous_layout(
2614 KernelClass::Map,
2615 dst_dims,
2616 &[
2617 dst_strides,
2618 a.strides(),
2619 b.strides(),
2620 c.strides(),
2621 e.strides(),
2622 ],
2623 &exec,
2624 )?
2625 .is_some()
2626 {
2627 let len = total_len(dst_dims)?;
2628 let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2629 let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2630 let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2631 let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2632 let se = unsafe { std::slice::from_raw_parts(e_ptr, len) };
2633 simd::dispatch_if_large(len, || {
2636 for ((((d, &a), &b), &c), &e) in dst.iter_mut().zip(sa).zip(sb).zip(sc).zip(se) {
2637 *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
2638 }
2639 });
2640 return Ok(());
2641 }
2642
2643 let strides_list: [&[isize]; 5] = [
2644 dst_strides,
2645 a.strides(),
2646 b.strides(),
2647 c.strides(),
2648 e.strides(),
2649 ];
2650 let elem_size = std::mem::size_of::<D>()
2651 .max(std::mem::size_of::<A>())
2652 .max(std::mem::size_of::<B>())
2653 .max(std::mem::size_of::<C>())
2654 .max(std::mem::size_of::<E>());
2655 let total = total_len(dst_dims)?;
2656
2657 let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2659 build_plan_fused_small(dst_dims, &strides_list)
2660 } else {
2661 build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2662 };
2663
2664 let total = total_len(&fused_dims)?;
2665 if is_parallel(KernelClass::Map, total, &exec) {
2666 use crate::threading::SendPtr;
2667 let dst_send = SendPtr(dst_ptr);
2668 let a_send = SendPtr(a_ptr as *mut A);
2669 let b_send = SendPtr(b_ptr as *mut B);
2670 let c_send = SendPtr(c_ptr as *mut C);
2671 let e_send = SendPtr(e_ptr as *mut E);
2672
2673 let costs = compute_costs(&ordered_strides);
2674 let initial_offsets: crate::inline::OperandVec<isize> =
2675 strides_list.iter().map(|_| 0isize).collect();
2676 return mapreduce_threaded(
2677 &fused_dims,
2678 &plan.block,
2679 &ordered_strides,
2680 &initial_offsets,
2681 &costs,
2682 KernelClass::Map,
2683 exec,
2684 &|dims, blocks, strides_list, offsets| {
2685 for_each_inner_block_with_offsets(
2686 dims,
2687 blocks,
2688 strides_list,
2689 offsets,
2690 |offsets, len, strides| {
2691 let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2692 let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2693 let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2694 let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2695 let ep = unsafe { e_send.as_const().offset(offsets[4]) };
2696 unsafe {
2697 inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2698 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep,
2699 strides[4], len, &f,
2700 )
2701 };
2702 Ok(())
2703 },
2704 )
2705 },
2706 );
2707 }
2708
2709 let initial_offsets: crate::inline::OperandVec<isize> =
2710 ordered_strides.iter().map(|_| 0isize).collect();
2711 for_each_inner_block_preordered(
2712 &fused_dims,
2713 &plan.block,
2714 &ordered_strides,
2715 &initial_offsets,
2716 |offsets, len, strides| {
2717 let dp = unsafe { dst_ptr.offset(offsets[0]) };
2718 let ap = unsafe { a_ptr.offset(offsets[1]) };
2719 let bp = unsafe { b_ptr.offset(offsets[2]) };
2720 let cp = unsafe { c_ptr.offset(offsets[3]) };
2721 let ep = unsafe { e_ptr.offset(offsets[4]) };
2722 unsafe {
2723 inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2724 dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep, strides[4],
2725 len, &f,
2726 )
2727 };
2728 Ok(())
2729 },
2730 )
2731}
2732
2733#[cfg(test)]
2734#[path = "map_view/tests/scalar_branch_tests.rs"]
2735mod scalar_branch_tests;
2736
2737#[cfg(test)]
2738#[path = "map_view/tests/tests.rs"]
2739mod tests;