Skip to main content

strided_basic/
map_view.rs

1//! Map operations on dynamic-rank strided views.
2//!
3//! These are the canonical view-based map functions, equivalent to Julia's `Base.map!`.
4//! Mutable destinations must have an injective layout. Validation is
5//! conservative and bounded: layouts that are injective but cannot be proven
6//! by the bounded checker are rejected with
7//! [`StridedError::NonInjectiveOutputLayout`].
8
9use crate::kernel::{
10    build_plan_fused, build_plan_fused_small, ensure_same_shape, for_each_inner_block_preordered,
11    sequential_contiguous_layout, total_len, SMALL_TENSOR_THRESHOLD,
12};
13use crate::simd;
14use crate::view::{StridedView, StridedViewMut};
15use crate::{Result, StridedError};
16use core::mem::MaybeUninit;
17use std::ops::Mul;
18use strided_view::{ElementOp, Identity};
19
20use crate::cost::{for_len, KernelClass};
21use crate::fuse::compute_costs;
22use crate::threading::{for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded};
23use crate::Exec;
24
25type AxisVec<T> = crate::inline::InlineVec<T>;
26
27/// Largest fused inner:row aspect at which the tiled transposed-scalar kernel
28/// is selected.
29///
30/// The cache-tiled kernel wins or ties the generic blocked plan across the
31/// measured shape/dtype table (u4-tiled #210), except when the fused inner run
32/// is more than 8x the row extent: then its strided destination writes lose to
33/// the blocked plan on f64 (`4096x256` ~1.4x, `8192x256` ~1.3x at 1T), so the
34/// detector declines and the generic path runs.
35const TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT: usize = 8;
36
37/// Internal validation marker used by kernel-family execution adapters.
38///
39/// This marker is not bound to a destination. It does not authorize unchecked
40/// replay with a different geometry; callers of the unsafe execution entries
41/// must establish that association themselves.
42#[derive(Clone, Copy, Debug, Default)]
43pub struct ValidatedDestinationLayout(());
44
45/// Validate destination injectivity and return the replay marker.
46///
47/// Runs the exact (allocation-free pairwise, then spilling) injectivity check,
48/// so every injective layout the other kernel entry points accept is accepted
49/// here. Kernel families that need the marker for more than one step (for
50/// example validating shapes and then taking a specialization) should call this
51/// once and pass the marker on instead of re-running the check.
52///
53/// # Errors
54///
55/// Returns [`StridedError::NonInjectiveOutputLayout`] when the layout cannot
56/// be proven representable and injective: two distinct logical indices of
57/// `dims` name the same offset under `strides`, or the element count or the
58/// address span is too large to represent. A rejected layout need not be
59/// colliding.
60#[inline]
61pub fn validate_destination_layout(
62    dims: &[usize],
63    strides: &[isize],
64) -> Result<ValidatedDestinationLayout> {
65    if crate::layout_check::is_injective_layout(dims, strides) {
66        Ok(ValidatedDestinationLayout(()))
67    } else {
68        Err(StridedError::NonInjectiveOutputLayout)
69    }
70}
71
72#[inline]
73/// Validate injectivity without allocating a set of offsets.
74///
75/// # Examples
76///
77/// ```
78/// use strided_basic::execution::validate_destination_layout_without_alloc;
79/// let marker = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
80/// assert!(validate_destination_layout_without_alloc(&[2], &[0]).is_err());
81/// let _ = marker;
82/// ```
83///
84/// # Errors
85/// Returns `NonInjectiveOutputLayout` for invalid or unsupported output geometry.
86pub fn validate_destination_layout_without_alloc(
87    dims: &[usize],
88    strides: &[isize],
89) -> Result<ValidatedDestinationLayout> {
90    if crate::layout_check::is_injective_layout_without_alloc(dims, strides) {
91        Ok(ValidatedDestinationLayout(()))
92    } else {
93        Err(StridedError::NonInjectiveOutputLayout)
94    }
95}
96
97fn reachable_byte_range(
98    ptr: usize,
99    elem_size: usize,
100    dims: &[usize],
101    strides: &[isize],
102) -> Result<Option<(usize, usize)>> {
103    if elem_size == 0 || dims.contains(&0) {
104        return Ok(None);
105    }
106
107    let mut min_offset = 0isize;
108    let mut max_offset = 0isize;
109    for (&dim, &stride) in dims.iter().zip(strides) {
110        if dim <= 1 {
111            continue;
112        }
113        let extent = isize::try_from(dim - 1).map_err(|_| StridedError::OffsetOverflow)?;
114        let span = stride
115            .checked_mul(extent)
116            .ok_or(StridedError::OffsetOverflow)?;
117        if span < 0 {
118            min_offset = min_offset
119                .checked_add(span)
120                .ok_or(StridedError::OffsetOverflow)?;
121        } else {
122            max_offset = max_offset
123                .checked_add(span)
124                .ok_or(StridedError::OffsetOverflow)?;
125        }
126    }
127
128    let ptr = ptr as i128;
129    let elem_size = elem_size as i128;
130    let start = ptr
131        .checked_add(
132            (min_offset as i128)
133                .checked_mul(elem_size)
134                .ok_or(StridedError::OffsetOverflow)?,
135        )
136        .ok_or(StridedError::OffsetOverflow)?;
137    let end = ptr
138        .checked_add(
139            (max_offset as i128)
140                .checked_mul(elem_size)
141                .and_then(|offset| offset.checked_add(elem_size))
142                .ok_or(StridedError::OffsetOverflow)?,
143        )
144        .ok_or(StridedError::OffsetOverflow)?;
145    if start < 0 || end < 0 || start > usize::MAX as i128 || end > usize::MAX as i128 {
146        return Err(StridedError::OffsetOverflow);
147    }
148    Ok(Some((start as usize, end as usize)))
149}
150
151fn validate_typed_no_overlap<D, A, Op: ElementOp<A>>(
152    dest: &StridedViewMut<MaybeUninit<D>>,
153    input: &StridedView<A, Op>,
154    input_index: usize,
155) -> Result<()> {
156    let dest_range = reachable_byte_range(
157        dest.ptr() as usize,
158        core::mem::size_of::<D>(),
159        dest.dims(),
160        dest.strides(),
161    )?;
162    let input_range = reachable_byte_range(
163        input.ptr() as usize,
164        core::mem::size_of::<A>(),
165        input.dims(),
166        input.strides(),
167    )?;
168    if let (Some((dest_start, dest_end)), Some((input_start, input_end))) =
169        (dest_range, input_range)
170    {
171        if dest_start < input_end && input_start < dest_end {
172            return Err(StridedError::OverlappingInputOutput { input: input_index });
173        }
174    }
175    Ok(())
176}
177
178// ============================================================================
179// Stride-specialized inner loop helpers
180//
181// When all inner strides are 1 (contiguous in the innermost dimension),
182// we use slice-based iteration so LLVM can auto-vectorize effectively.
183// This is the Rust equivalent of Julia's @simd on the innermost loop.
184// ============================================================================
185
186// INVARIANT: validated layouts keep every dereferenced cursor in bounds.
187// Pointwise leaves use wrapping advances: the unused final cursor may lie
188// outside the allocation for negative or gapped strides.
189/// Unary inner loop: `dest[i] = f(Op::apply(src[i]))` for `len` elements.
190///
191/// A vectorized kernel a map operation offers its flat runs to.
192///
193/// [`NoKernel`] is what the entry points without a kernel pass, so the check it introduces is
194/// monomorphized away; [`SliceKernel`] adapts the closure a caller supplied.
195trait MapKernel<A, D> {
196    /// Writes every element of `dst` from `src`, or writes nothing and returns `false`.
197    fn apply(&self, src: &[A], dst: &mut [D]) -> bool;
198}
199
200struct NoKernel;
201
202impl<A, D> MapKernel<A, D> for NoKernel {
203    #[inline(always)]
204    fn apply(&self, _src: &[A], _dst: &mut [D]) -> bool {
205        false
206    }
207}
208
209struct SliceKernel<K>(K);
210
211impl<A, D, K: Fn(&[A], &mut [D]) -> bool> MapKernel<A, D> for SliceKernel<K> {
212    #[inline(always)]
213    fn apply(&self, src: &[A], dst: &mut [D]) -> bool {
214        (self.0)(src, dst)
215    }
216}
217
218unsafe fn inner_loop_map1<D: Copy, A: Copy, Op: ElementOp<A>, K: MapKernel<A, D>>(
219    dp: *mut D,
220    ds: isize,
221    sp: *const A,
222    ss: isize,
223    len: usize,
224    f: &impl Fn(A) -> D,
225    kernel: &K,
226) {
227    if ds == 1 && ss == 1 {
228        let src = std::slice::from_raw_parts(sp, len);
229        let dst = std::slice::from_raw_parts_mut(dp, len);
230        if kernel.apply(src, dst) {
231            return;
232        }
233        simd::dispatch_if_large(len, || {
234            for (d, s) in dst.iter_mut().zip(src.iter()) {
235                *d = f(Op::apply(*s));
236            }
237        });
238    } else {
239        let mut dp = dp;
240        let mut sp = sp;
241        simd::dispatch_if_large(len, || {
242            for _ in 0..len {
243                *dp = f(Op::apply(*sp));
244                dp = dp.wrapping_offset(ds);
245                sp = sp.wrapping_offset(ss);
246            }
247        });
248    }
249}
250
251/// Binary inner loop: `dest[i] = f(OpA::apply(a[i]), OpB::apply(b[i]))`.
252#[inline(always)]
253unsafe fn inner_loop_map2<D: Copy, A: Copy, B: Copy, OpA: ElementOp<A>, OpB: ElementOp<B>>(
254    dp: *mut D,
255    ds: isize,
256    ap: *const A,
257    a_s: isize,
258    bp: *const B,
259    b_s: isize,
260    len: usize,
261    f: &impl Fn(A, B) -> D,
262) {
263    if ds == 1 && a_s == 1 && b_s == 1 {
264        let src_a = std::slice::from_raw_parts(ap, len);
265        let src_b = std::slice::from_raw_parts(bp, len);
266        let dst = std::slice::from_raw_parts_mut(dp, len);
267        simd::dispatch_if_large(len, || {
268            for i in 0..len {
269                dst[i] = f(OpA::apply(src_a[i]), OpB::apply(src_b[i]));
270            }
271        });
272    } else if ds == 1 && a_s == 1 && b_s == 0 {
273        let src_a = std::slice::from_raw_parts(ap, len);
274        let b = OpB::apply(*bp);
275        let dst = std::slice::from_raw_parts_mut(dp, len);
276        simd::dispatch_if_large(len, || {
277            for i in 0..len {
278                dst[i] = f(OpA::apply(src_a[i]), b);
279            }
280        });
281    } else if ds == 1 && a_s == 0 && b_s == 1 {
282        let a = OpA::apply(*ap);
283        let src_b = std::slice::from_raw_parts(bp, len);
284        let dst = std::slice::from_raw_parts_mut(dp, len);
285        simd::dispatch_if_large(len, || {
286            for i in 0..len {
287                dst[i] = f(a, OpB::apply(src_b[i]));
288            }
289        });
290    } else if ds == 1 && a_s == 0 && b_s == 0 {
291        let a = OpA::apply(*ap);
292        let b = OpB::apply(*bp);
293        let dst = std::slice::from_raw_parts_mut(dp, len);
294        simd::dispatch_if_large(len, || {
295            for d in dst.iter_mut() {
296                *d = f(a, b);
297            }
298        });
299    } else if ds == 1 && b_s == 0 {
300        let b = OpB::apply(*bp);
301        let dst = std::slice::from_raw_parts_mut(dp, len);
302        let mut ap = ap;
303        simd::dispatch_if_large(len, || {
304            for d in dst.iter_mut() {
305                *d = f(OpA::apply(*ap), b);
306                ap = ap.wrapping_offset(a_s);
307            }
308        });
309    } else if ds == 1 && a_s == 0 {
310        let a = OpA::apply(*ap);
311        let dst = std::slice::from_raw_parts_mut(dp, len);
312        let mut bp = bp;
313        simd::dispatch_if_large(len, || {
314            for d in dst.iter_mut() {
315                *d = f(a, OpB::apply(*bp));
316                bp = bp.wrapping_offset(b_s);
317            }
318        });
319    } else {
320        let mut dp = dp;
321        let mut ap = ap;
322        let mut bp = bp;
323        simd::dispatch_if_large(len, || {
324            for _ in 0..len {
325                *dp = f(OpA::apply(*ap), OpB::apply(*bp));
326                dp = dp.wrapping_offset(ds);
327                ap = ap.wrapping_offset(a_s);
328                bp = bp.wrapping_offset(b_s);
329            }
330        });
331    }
332}
333
334/// Binary multiplication inner loop for identity element ops.
335trait MulOutput<D: Copy + 'static>: Copy + Send + Sync + 'static {
336    type Slot: Copy + Send + Sync + 'static;
337
338    unsafe fn write(dst: *mut Self::Slot, value: D);
339
340    unsafe fn try_contiguous<A: 'static, B: 'static>(
341        dst: *mut Self::Slot,
342        len: usize,
343        a: &[A],
344        b: &[B],
345    ) -> bool;
346}
347
348#[derive(Clone, Copy)]
349struct InitializedOutput;
350
351impl<D: Copy + Send + Sync + 'static> MulOutput<D> for InitializedOutput {
352    type Slot = D;
353
354    #[inline(always)]
355    unsafe fn write(dst: *mut D, value: D) {
356        unsafe { dst.write(value) };
357    }
358
359    #[inline(always)]
360    unsafe fn try_contiguous<A: 'static, B: 'static>(
361        dst: *mut D,
362        len: usize,
363        a: &[A],
364        b: &[B],
365    ) -> bool {
366        unsafe { simd::try_mul_contiguous_ptr(dst, len, a, b) }
367    }
368}
369
370#[derive(Clone, Copy)]
371struct UninitializedOutput;
372
373impl<D: Copy + Send + Sync + 'static> MulOutput<D> for UninitializedOutput {
374    type Slot = MaybeUninit<D>;
375
376    #[inline(always)]
377    unsafe fn write(dst: *mut MaybeUninit<D>, value: D) {
378        unsafe { dst.write(MaybeUninit::new(value)) };
379    }
380
381    #[inline(always)]
382    unsafe fn try_contiguous<A: 'static, B: 'static>(
383        dst: *mut MaybeUninit<D>,
384        len: usize,
385        a: &[A],
386        b: &[B],
387    ) -> bool {
388        // SIMD stores write complete values directly through raw pointers; no
389        // initialized reference is formed over the uninitialized destination.
390        unsafe { simd::try_mul_contiguous_ptr(dst.cast::<D>(), len, a, b) }
391    }
392}
393
394#[inline(always)]
395fn multiply_value<A, B, D>(lhs: A, rhs: B) -> D
396where
397    A: Copy + Mul<B, Output = D> + 'static,
398    B: Copy + 'static,
399    D: Copy + 'static,
400{
401    use core::any::TypeId;
402
403    if TypeId::of::<A>() == TypeId::of::<i32>()
404        && TypeId::of::<B>() == TypeId::of::<i32>()
405        && TypeId::of::<D>() == TypeId::of::<i32>()
406    {
407        // The exact TypeId checks prove identical layout and validity.
408        let lhs = unsafe { *(&lhs as *const A).cast::<i32>() };
409        let rhs = unsafe { *(&rhs as *const B).cast::<i32>() };
410        let value = lhs.wrapping_mul(rhs);
411        return unsafe { core::mem::transmute_copy(&value) };
412    }
413    if TypeId::of::<A>() == TypeId::of::<i64>()
414        && TypeId::of::<B>() == TypeId::of::<i64>()
415        && TypeId::of::<D>() == TypeId::of::<i64>()
416    {
417        // The exact TypeId checks prove identical layout and validity.
418        let lhs = unsafe { *(&lhs as *const A).cast::<i64>() };
419        let rhs = unsafe { *(&rhs as *const B).cast::<i64>() };
420        let value = lhs.wrapping_mul(rhs);
421        return unsafe { core::mem::transmute_copy(&value) };
422    }
423    lhs * rhs
424}
425
426#[inline(always)]
427unsafe fn inner_loop_mul2<
428    O: MulOutput<D>,
429    D: Copy + 'static,
430    A: Copy + Mul<B, Output = D> + 'static,
431    B: Copy + 'static,
432>(
433    dp: *mut O::Slot,
434    ds: isize,
435    ap: *const A,
436    a_s: isize,
437    bp: *const B,
438    b_s: isize,
439    len: usize,
440) {
441    if ds == 1 && a_s == 1 && b_s == 1 {
442        let src_a = std::slice::from_raw_parts(ap, len);
443        let src_b = std::slice::from_raw_parts(bp, len);
444        if len >= 64 && O::try_contiguous(dp, len, src_a, src_b) {
445            return;
446        }
447        simd::dispatch_if_large(len, || {
448            for i in 0..len {
449                O::write(dp.add(i), multiply_value(src_a[i], src_b[i]));
450            }
451        });
452    } else if ds == 1 && a_s == 1 && b_s == 0 {
453        let src_a = std::slice::from_raw_parts(ap, len);
454        let b = *bp;
455        simd::dispatch_if_large(len, || {
456            for i in 0..len {
457                O::write(dp.add(i), multiply_value(src_a[i], b));
458            }
459        });
460    } else if ds == 1 && a_s == 0 && b_s == 1 {
461        let a = *ap;
462        let src_b = std::slice::from_raw_parts(bp, len);
463        simd::dispatch_if_large(len, || {
464            for i in 0..len {
465                O::write(dp.add(i), multiply_value(a, src_b[i]));
466            }
467        });
468    } else if ds == 1 && a_s == 0 && b_s == 0 {
469        let a = *ap;
470        let b = *bp;
471        simd::dispatch_if_large(len, || {
472            for i in 0..len {
473                O::write(dp.add(i), multiply_value(a, b));
474            }
475        });
476    } else if ds == 1 && b_s == 0 {
477        let b = *bp;
478        let mut ap = ap;
479        simd::dispatch_if_large(len, || {
480            for i in 0..len {
481                O::write(dp.add(i), multiply_value(*ap, b));
482                ap = ap.wrapping_offset(a_s);
483            }
484        });
485    } else if ds == 1 && a_s == 0 {
486        let a = *ap;
487        let mut bp = bp;
488        simd::dispatch_if_large(len, || {
489            for i in 0..len {
490                O::write(dp.add(i), multiply_value(a, *bp));
491                bp = bp.wrapping_offset(b_s);
492            }
493        });
494    } else {
495        let mut dp = dp;
496        let mut ap = ap;
497        let mut bp = bp;
498        simd::dispatch_if_large(len, || {
499            for _ in 0..len {
500                O::write(dp, multiply_value(*ap, *bp));
501                dp = dp.wrapping_offset(ds);
502                ap = ap.wrapping_offset(a_s);
503                bp = bp.wrapping_offset(b_s);
504            }
505        });
506    }
507}
508
509#[derive(Clone, Debug, Eq, PartialEq)]
510struct ContiguousMulRangePlan {
511    axis_order: AxisVec<usize>,
512    inner_len: usize,
513    inner_axis_count: usize,
514    row_len: usize,
515    outer_axis_start: usize,
516    fast_axis: usize,
517    a_fast_stride: isize,
518    b_fast_stride: isize,
519    a_row_stride: isize,
520    b_row_stride: isize,
521}
522
523impl ContiguousMulRangePlan {
524    /// Whether running the plan row by row, without cache blocking, is sound
525    /// for performance.
526    ///
527    /// The destination is walked contiguously, so a unit-stride or broadcast
528    /// (stride 0) input streams along with it. An input read at any other
529    /// stride along the fast axis (a transposed operand) is swept across the
530    /// whole fast axis once per row, touching a new cache line per element and
531    /// again on the next row; the blocked kernel tiles that access. The one
532    /// exception is the transposed-scalar tile, which blocks itself.
533    fn walks_inputs_without_blocking(&self) -> bool {
534        let streams = |stride: isize| stride == 0 || stride == 1;
535        if streams(self.a_fast_stride) && streams(self.b_fast_stride) {
536            return true;
537        }
538        if transposed_scalar_tile_kind(self).is_some() {
539            return true;
540        }
541        false
542    }
543}
544
545#[derive(Clone, Copy, Debug, Eq, PartialEq)]
546enum TransposedScalarTileKind {
547    RhsScalar,
548    LhsScalar,
549}
550
551fn transposed_scalar_tile_kind(plan: &ContiguousMulRangePlan) -> Option<TransposedScalarTileKind> {
552    if plan.inner_len > TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT.saturating_mul(plan.row_len) {
553        return None;
554    }
555    let row_len = isize::try_from(plan.row_len).ok()?;
556    if plan.a_fast_stride == row_len
557        && plan.a_row_stride == 1
558        && plan.b_fast_stride == 0
559        && plan.b_row_stride == 0
560    {
561        return Some(TransposedScalarTileKind::RhsScalar);
562    }
563
564    if plan.b_fast_stride == row_len
565        && plan.b_row_stride == 1
566        && plan.a_fast_stride == 0
567        && plan.a_row_stride == 0
568    {
569        return Some(TransposedScalarTileKind::LhsScalar);
570    }
571
572    None
573}
574
575fn compact_axis_order(dims: &[usize], strides: &[isize]) -> Option<AxisVec<usize>> {
576    if dims.len() != strides.len() {
577        return None;
578    }
579
580    let mut active = AxisVec::<usize>::new();
581    let mut inactive = AxisVec::<usize>::new();
582    for (axis, (&dim, &stride)) in dims.iter().zip(strides.iter()).enumerate() {
583        if stride < 0 {
584            return None;
585        }
586        if dim > 1 {
587            active.push(axis);
588        } else {
589            inactive.push(axis);
590        }
591    }
592
593    active.sort_by(|&lhs, &rhs| strides[lhs].cmp(&strides[rhs]).then_with(|| lhs.cmp(&rhs)));
594
595    let mut expected = 1isize;
596    for &axis in &active {
597        if strides[axis] != expected {
598            return None;
599        }
600        expected = expected.saturating_mul(dims[axis] as isize);
601    }
602
603    active.extend(inactive);
604    Some(active)
605}
606
607fn can_fuse_contiguous_range_axis(dim: usize, prev_stride: isize, next_stride: isize) -> bool {
608    dim <= 1 || (prev_stride == 0 && next_stride == 0) || next_stride == prev_stride * dim as isize
609}
610
611fn contiguous_mul_range_plan(
612    dims: &[usize],
613    dst_strides: &[isize],
614    a_strides: &[isize],
615    b_strides: &[isize],
616) -> Option<ContiguousMulRangePlan> {
617    let axis_order = compact_axis_order(dims, dst_strides)?;
618    if dims.is_empty() {
619        return Some(ContiguousMulRangePlan {
620            axis_order,
621            inner_len: 1,
622            inner_axis_count: 0,
623            row_len: 1,
624            outer_axis_start: 0,
625            fast_axis: 0,
626            a_fast_stride: 0,
627            b_fast_stride: 0,
628            a_row_stride: 0,
629            b_row_stride: 0,
630        });
631    }
632
633    let first_pos = axis_order
634        .iter()
635        .position(|&axis| dims[axis] > 1)
636        .unwrap_or(0);
637    let first_axis = axis_order[first_pos];
638    let mut inner_len = dims[first_axis].max(1);
639    let mut inner_axis_count = first_pos + 1;
640    let mut prev_axis = first_axis;
641
642    for &axis in axis_order.iter().skip(first_pos + 1) {
643        if can_fuse_contiguous_range_axis(
644            dims[prev_axis],
645            dst_strides[prev_axis],
646            dst_strides[axis],
647        ) && can_fuse_contiguous_range_axis(
648            dims[prev_axis],
649            a_strides[prev_axis],
650            a_strides[axis],
651        ) && can_fuse_contiguous_range_axis(
652            dims[prev_axis],
653            b_strides[prev_axis],
654            b_strides[axis],
655        ) {
656            inner_len = inner_len.checked_mul(dims[axis].max(1))?;
657            inner_axis_count += 1;
658            prev_axis = axis;
659        } else {
660            break;
661        }
662    }
663
664    let row = axis_order
665        .iter()
666        .enumerate()
667        .skip(inner_axis_count)
668        .find(|&(_, &axis)| dims[axis] > 1);
669    let (row_len, outer_axis_start, a_row_stride, b_row_stride) =
670        if let Some((row_pos, &row_axis)) = row {
671            (
672                dims[row_axis],
673                row_pos + 1,
674                a_strides[row_axis],
675                b_strides[row_axis],
676            )
677        } else {
678            (1, axis_order.len(), 0, 0)
679        };
680
681    Some(ContiguousMulRangePlan {
682        axis_order,
683        inner_len,
684        inner_axis_count,
685        row_len,
686        outer_axis_start,
687        fast_axis: first_axis,
688        a_fast_stride: a_strides[first_axis],
689        b_fast_stride: b_strides[first_axis],
690        a_row_stride,
691        b_row_stride,
692    })
693}
694
695struct ContiguousMulOuterCursor<'a> {
696    dims: &'a [usize],
697    a_strides: &'a [isize],
698    b_strides: &'a [isize],
699    axes: AxisVec<usize>,
700    coords: AxisVec<usize>,
701    a_offset: isize,
702    b_offset: isize,
703}
704
705impl<'a> ContiguousMulOuterCursor<'a> {
706    fn new(
707        dims: &'a [usize],
708        a_strides: &'a [isize],
709        b_strides: &'a [isize],
710        plan: &ContiguousMulRangePlan,
711        outer_group: usize,
712    ) -> Self {
713        let axes: AxisVec<usize> = plan
714            .axis_order
715            .iter()
716            .skip(plan.outer_axis_start)
717            .copied()
718            .collect();
719        let mut coords = AxisVec::<usize>::with_capacity(axes.len());
720        let mut rem = outer_group;
721        let mut a_offset = 0isize;
722        let mut b_offset = 0isize;
723
724        for &axis in &axes {
725            let dim = dims[axis].max(1);
726            let coord = rem % dim;
727            rem /= dim;
728            coords.push(coord);
729            a_offset += coord as isize * a_strides[axis];
730            b_offset += coord as isize * b_strides[axis];
731        }
732
733        Self {
734            dims,
735            a_strides,
736            b_strides,
737            axes,
738            coords,
739            a_offset,
740            b_offset,
741        }
742    }
743
744    fn advance(&mut self) {
745        for (i, &axis) in self.axes.iter().enumerate() {
746            let dim = self.dims[axis].max(1);
747            if dim <= 1 {
748                continue;
749            }
750
751            // Advance only when another position along this axis follows,
752            // and rewind from the last one, so offsets stay reachable.
753            if self.coords[i] + 1 < dim {
754                self.coords[i] += 1;
755                self.a_offset += self.a_strides[axis];
756                self.b_offset += self.b_strides[axis];
757                break;
758            }
759
760            let last = (dim - 1) as isize;
761            self.coords[i] = 0;
762            self.a_offset -= last * self.a_strides[axis];
763            self.b_offset -= last * self.b_strides[axis];
764        }
765    }
766}
767
768#[inline(always)]
769unsafe fn run_contiguous_mul_row_block<
770    O: MulOutput<D>,
771    D: Copy + 'static,
772    A: Copy + Mul<B, Output = D> + 'static,
773    B: Copy + 'static,
774>(
775    dst_ptr: *mut O::Slot,
776    a_ptr: *const A,
777    b_ptr: *const B,
778    plan: &ContiguousMulRangePlan,
779    base_index: usize,
780    total: usize,
781    base_a_offset: isize,
782    base_b_offset: isize,
783) {
784    let inner_len = plan.inner_len.max(1);
785    let row_len = plan.row_len.max(1);
786    let block_len = inner_len.saturating_mul(row_len);
787
788    if total.saturating_sub(base_index) >= block_len {
789        let tiled = match transposed_scalar_tile_kind(plan) {
790            Some(TransposedScalarTileKind::RhsScalar) => {
791                simd::try_mul_transposed_scalar_rhs_2d::<D, A, B>(
792                    dst_ptr.add(base_index).cast::<D>(),
793                    a_ptr.offset(base_a_offset),
794                    b_ptr.offset(base_b_offset),
795                    inner_len,
796                    row_len,
797                    plan.a_fast_stride,
798                    plan.a_row_stride,
799                )
800            }
801            Some(TransposedScalarTileKind::LhsScalar) => {
802                simd::try_mul_transposed_scalar_lhs_2d::<D, A, B>(
803                    dst_ptr.add(base_index).cast::<D>(),
804                    a_ptr.offset(base_a_offset),
805                    b_ptr.offset(base_b_offset),
806                    inner_len,
807                    row_len,
808                    plan.b_fast_stride,
809                    plan.b_row_stride,
810                )
811            }
812            None => false,
813        };
814        if tiled {
815            return;
816        }
817    }
818
819    let mut index = base_index;
820    let mut a_offset = base_a_offset;
821    let mut b_offset = base_b_offset;
822
823    for _ in 0..row_len {
824        if index >= total {
825            break;
826        }
827        let len = inner_len.min(total - index);
828        inner_loop_mul2::<O, D, A, B>(
829            dst_ptr.add(index),
830            1,
831            a_ptr.offset(a_offset),
832            plan.a_fast_stride,
833            b_ptr.offset(b_offset),
834            plan.b_fast_stride,
835            len,
836        );
837        index += inner_len;
838        a_offset += plan.a_row_stride;
839        b_offset += plan.b_row_stride;
840    }
841}
842
843fn strided_offset_for_contiguous_linear_index(
844    dims: &[usize],
845    strides: &[isize],
846    axis_order: &[usize],
847    mut index: usize,
848) -> isize {
849    let mut offset = 0isize;
850    for &axis in axis_order {
851        let dim = dims[axis];
852        if dim == 0 {
853            return 0;
854        }
855        let coord = index % dim;
856        index /= dim;
857        offset += coord as isize * strides[axis];
858    }
859    offset
860}
861
862#[allow(clippy::too_many_arguments)]
863fn try_contiguous_range_mul<
864    O: MulOutput<D>,
865    D: Copy + Send + Sync + 'static,
866    A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
867    B: Copy + Send + Sync + 'static,
868>(
869    dst_ptr: *mut O::Slot,
870    dims: &[usize],
871    dst_strides: &[isize],
872    a_ptr: *const A,
873    a_strides: &[isize],
874    b_ptr: *const B,
875    b_strides: &[isize],
876    exec: Exec<'_>,
877) -> bool {
878    // An overflowing element count falls back to the general path, which
879    // reports it as `OffsetOverflow`.
880    let Ok(total) = total_len(dims) else {
881        return false;
882    };
883    if total == 0 {
884        return true;
885    }
886    // One flat section over the chunks: the width is the token's decision.
887    let exec = for_len(exec, KernelClass::Map, total);
888    if exec.is_serial() {
889        return false;
890    }
891
892    let Some(plan) = contiguous_mul_range_plan(dims, dst_strides, a_strides, b_strides) else {
893        return false;
894    };
895    if !plan.walks_inputs_without_blocking() {
896        return false;
897    }
898
899    let inner_len = plan.inner_len.max(1);
900    let row_len = plan.row_len.max(1);
901    let block_len = inner_len.saturating_mul(row_len).max(1);
902    let outer_groups = total.div_ceil(block_len);
903
904    let nthreads = exec.budget().get();
905    if nthreads > 1 {
906        use crate::threading::{parallel_for_each, SendPtr};
907
908        let dst = SendPtr(dst_ptr);
909        let a = SendPtr(a_ptr as *mut A);
910        let b = SendPtr(b_ptr as *mut B);
911
912        if outer_groups < nthreads {
913            let chunk_len = total.div_ceil(nthreads);
914            let nchunks = total.div_ceil(chunk_len);
915
916            parallel_for_each(0..nchunks, exec, &|chunks| {
917                for chunk in chunks {
918                    let start = chunk * chunk_len;
919                    let end = (start + chunk_len).min(total);
920                    let mut index = start;
921
922                    while index < end {
923                        let in_inner = index % inner_len;
924                        let len = (inner_len - in_inner).min(end - index);
925                        let a_offset = strided_offset_for_contiguous_linear_index(
926                            dims,
927                            a_strides,
928                            &plan.axis_order,
929                            index,
930                        );
931                        let b_offset = strided_offset_for_contiguous_linear_index(
932                            dims,
933                            b_strides,
934                            &plan.axis_order,
935                            index,
936                        );
937
938                        unsafe {
939                            inner_loop_mul2::<O, D, A, B>(
940                                dst.as_ptr().add(index),
941                                1,
942                                a.as_const().offset(a_offset),
943                                plan.a_fast_stride,
944                                b.as_const().offset(b_offset),
945                                plan.b_fast_stride,
946                                len,
947                            );
948                        }
949                        index += len;
950                    }
951                }
952            });
953
954            return true;
955        }
956
957        let groups_per_chunk = outer_groups.div_ceil(nthreads);
958        let nchunks = outer_groups.div_ceil(groups_per_chunk);
959
960        parallel_for_each(0..nchunks, exec, &|chunks| {
961            for chunk in chunks {
962                let group_start = chunk * groups_per_chunk;
963                let group_end = (group_start + groups_per_chunk).min(outer_groups);
964                let mut cursor =
965                    ContiguousMulOuterCursor::new(dims, a_strides, b_strides, &plan, group_start);
966
967                for group in group_start..group_end {
968                    let index = group * block_len;
969                    unsafe {
970                        run_contiguous_mul_row_block::<O, D, A, B>(
971                            dst.as_ptr(),
972                            a.as_const(),
973                            b.as_const(),
974                            &plan,
975                            index,
976                            total,
977                            cursor.a_offset,
978                            cursor.b_offset,
979                        );
980                    }
981                    cursor.advance();
982                }
983            }
984        });
985
986        true
987    } else {
988        run_contiguous_range_mul_single_thread::<O, D, A, B>(
989            dst_ptr,
990            dims,
991            a_ptr,
992            a_strides,
993            b_ptr,
994            b_strides,
995            &plan,
996            total,
997            block_len,
998            outer_groups,
999        )
1000    }
1001}
1002
1003fn run_contiguous_range_mul_single_thread<
1004    O: MulOutput<D>,
1005    D: Copy + Send + Sync + 'static,
1006    A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
1007    B: Copy + Send + Sync + 'static,
1008>(
1009    dst_ptr: *mut O::Slot,
1010    dims: &[usize],
1011    a_ptr: *const A,
1012    a_strides: &[isize],
1013    b_ptr: *const B,
1014    b_strides: &[isize],
1015    plan: &ContiguousMulRangePlan,
1016    total: usize,
1017    block_len: usize,
1018    outer_groups: usize,
1019) -> bool {
1020    let mut cursor = ContiguousMulOuterCursor::new(dims, a_strides, b_strides, plan, 0);
1021    for group in 0..outer_groups {
1022        let index = group * block_len;
1023        unsafe {
1024            run_contiguous_mul_row_block::<O, D, A, B>(
1025                dst_ptr,
1026                a_ptr,
1027                b_ptr,
1028                plan,
1029                index,
1030                total,
1031                cursor.a_offset,
1032                cursor.b_offset,
1033            );
1034        }
1035        cursor.advance();
1036    }
1037    true
1038}
1039
1040/// Ternary inner loop: `dest[i] = f(a[i], b[i], c[i])`.
1041#[inline(always)]
1042unsafe fn inner_loop_map3<
1043    D: Copy,
1044    A: Copy,
1045    B: Copy,
1046    C: Copy,
1047    OpA: ElementOp<A>,
1048    OpB: ElementOp<B>,
1049    OpC: ElementOp<C>,
1050>(
1051    dp: *mut D,
1052    ds: isize,
1053    ap: *const A,
1054    a_s: isize,
1055    bp: *const B,
1056    b_s: isize,
1057    cp: *const C,
1058    c_s: isize,
1059    len: usize,
1060    f: &impl Fn(A, B, C) -> D,
1061) {
1062    if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 {
1063        let src_a = std::slice::from_raw_parts(ap, len);
1064        let src_b = std::slice::from_raw_parts(bp, len);
1065        let src_c = std::slice::from_raw_parts(cp, len);
1066        let dst = std::slice::from_raw_parts_mut(dp, len);
1067        simd::dispatch_if_large(len, || {
1068            for (((d, &a), &b), &c) in dst.iter_mut().zip(src_a).zip(src_b).zip(src_c) {
1069                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
1070            }
1071        });
1072    } else {
1073        let mut dp = dp;
1074        let mut ap = ap;
1075        let mut bp = bp;
1076        let mut cp = cp;
1077        simd::dispatch_if_large(len, || {
1078            for _ in 0..len {
1079                *dp = f(OpA::apply(*ap), OpB::apply(*bp), OpC::apply(*cp));
1080                dp = dp.wrapping_offset(ds);
1081                ap = ap.wrapping_offset(a_s);
1082                bp = bp.wrapping_offset(b_s);
1083                cp = cp.wrapping_offset(c_s);
1084            }
1085        });
1086    }
1087}
1088
1089/// Quaternary inner loop: `dest[i] = f(a[i], b[i], c[i], e[i])`.
1090#[inline(always)]
1091unsafe fn inner_loop_map4<
1092    D: Copy,
1093    A: Copy,
1094    B: Copy,
1095    C: Copy,
1096    E: Copy,
1097    OpA: ElementOp<A>,
1098    OpB: ElementOp<B>,
1099    OpC: ElementOp<C>,
1100    OpE: ElementOp<E>,
1101>(
1102    dp: *mut D,
1103    ds: isize,
1104    ap: *const A,
1105    a_s: isize,
1106    bp: *const B,
1107    b_s: isize,
1108    cp: *const C,
1109    c_s: isize,
1110    ep: *const E,
1111    e_s: isize,
1112    len: usize,
1113    f: &impl Fn(A, B, C, E) -> D,
1114) {
1115    if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 && e_s == 1 {
1116        let src_a = std::slice::from_raw_parts(ap, len);
1117        let src_b = std::slice::from_raw_parts(bp, len);
1118        let src_c = std::slice::from_raw_parts(cp, len);
1119        let src_e = std::slice::from_raw_parts(ep, len);
1120        let dst = std::slice::from_raw_parts_mut(dp, len);
1121        // Zipped iterators keep the loop free of bounds checks, which keeps the
1122        // dispatched closure small enough for pulp's target-feature function to
1123        // inline (tensor4all/strided-rs#340).
1124        simd::dispatch_if_large(len, || {
1125            for ((((d, &a), &b), &c), &e) in
1126                dst.iter_mut().zip(src_a).zip(src_b).zip(src_c).zip(src_e)
1127            {
1128                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
1129            }
1130        });
1131    } else {
1132        let mut dp = dp;
1133        let mut ap = ap;
1134        let mut bp = bp;
1135        let mut cp = cp;
1136        let mut ep = ep;
1137        simd::dispatch_if_large(len, || {
1138            for _ in 0..len {
1139                *dp = f(
1140                    OpA::apply(*ap),
1141                    OpB::apply(*bp),
1142                    OpC::apply(*cp),
1143                    OpE::apply(*ep),
1144                );
1145                dp = dp.wrapping_offset(ds);
1146                ap = ap.wrapping_offset(a_s);
1147                bp = bp.wrapping_offset(b_s);
1148                cp = cp.wrapping_offset(c_s);
1149                ep = ep.wrapping_offset(e_s);
1150            }
1151        });
1152    }
1153}
1154
1155/// An element function that carries its [`KernelClass`].
1156///
1157/// [`map_into`] and its siblings gate their parallel fork on the serial time of the
1158/// region, which is the element count times the throughput of the function's class.
1159/// A plain closure or function item is a [`KernelClass::Map`] (a few arithmetic
1160/// operations per element) through the blanket implementation, so existing callers
1161/// are unchanged. A caller whose function is expensive declares it with
1162/// [`Classed`], usually through [`KernelClass::with`], so a region too short to pay
1163/// for a fork as arithmetic is still forked when each element costs a libm call.
1164///
1165/// The trait is implemented for every `Fn(A) -> D + Sync` and for [`Classed`]; it is
1166/// not meant to be implemented by hand.
1167///
1168/// # Examples
1169/// ```
1170/// use strided_basic::execution::KernelClass;
1171/// use strided_basic::{map_into, Exec, StridedView, StridedViewMut};
1172///
1173/// let src = vec![0.0f64, 1.0, 2.0];
1174/// let mut out = vec![0.0f64; 3];
1175/// let mut dest = StridedViewMut::new(&mut out, &[3], &[1], 0).unwrap();
1176/// let source = StridedView::<f64>::new(&src, &[3], &[1], 0).unwrap();
1177/// // `exp` costs about as much as ten arithmetic operations: say so.
1178/// let exp = KernelClass::Transcendental.with(|x: f64| x.exp());
1179/// map_into(&mut dest, &source, exp, Exec::serial()).unwrap();
1180/// assert_eq!(out[0], 1.0);
1181/// ```
1182pub trait MapFn<A, D>: Sync {
1183    /// Apply the function to one element.
1184    fn call(&self, a: A) -> D;
1185
1186    /// Serial cost class the parallel gate charges per element; `Map` by default.
1187    fn class(&self) -> KernelClass {
1188        KernelClass::Map
1189    }
1190}
1191
1192impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for F {
1193    #[inline(always)]
1194    fn call(&self, a: A) -> D {
1195        self(a)
1196    }
1197}
1198
1199/// An element function together with the [`KernelClass`] it is charged as.
1200///
1201/// Build one with [`Classed::new`] or [`KernelClass::with`]; see [`MapFn`].
1202#[derive(Debug, Clone, Copy)]
1203pub struct Classed<F> {
1204    /// The element function.
1205    pub f: F,
1206    /// The class the parallel gate charges per element.
1207    pub class: KernelClass,
1208}
1209
1210impl<F> Classed<F> {
1211    /// Declare `f` as a function of cost class `class`.
1212    pub const fn new(class: KernelClass, f: F) -> Self {
1213        Self { f, class }
1214    }
1215}
1216
1217impl KernelClass {
1218    /// Declare an element function of this class, for [`map_into`] and its siblings.
1219    pub const fn with<F>(self, f: F) -> Classed<F> {
1220        Classed::new(self, f)
1221    }
1222}
1223
1224impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for Classed<F> {
1225    #[inline(always)]
1226    fn call(&self, a: A) -> D {
1227        (self.f)(a)
1228    }
1229
1230    fn class(&self) -> KernelClass {
1231        self.class
1232    }
1233}
1234
1235/// Apply a function element-wise from source to destination.
1236///
1237/// The element operation `Op` is applied lazily when reading from `src`.
1238/// Source and destination may have different element types.
1239///
1240/// `f` is any `Fn(A) -> D + Sync`, gated as [`KernelClass::Map`], or a function declared
1241/// with a class of its own (see [`MapFn`]), so an expensive element function forks sooner.
1242///
1243/// # Errors
1244///
1245/// Returns [`StridedError::RankMismatch`] or
1246/// [`StridedError::ShapeMismatch`] when the views have different ranks or
1247/// extents, and [`StridedError::NonInjectiveOutputLayout`] when the
1248/// destination layout cannot be proven representable and injective: two
1249/// logical indices name one offset, or the element count or address span is
1250/// too large to represent. Nothing is written unless validation succeeds.
1251pub fn map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1252    dest: &mut StridedViewMut<D>,
1253    src: &StridedView<A, Op>,
1254    f: impl MapFn<A, D>,
1255    exec: Exec<'_>,
1256) -> Result<()> {
1257    let class = f.class();
1258    map_parts_into::<D, A, Op, NoKernel>(
1259        dest.as_mut_ptr(),
1260        dest.dims(),
1261        dest.strides(),
1262        src.ptr(),
1263        src.dims(),
1264        src.strides(),
1265        |a| f.call(a),
1266        &NoKernel,
1267        class,
1268        exec,
1269    )
1270}
1271
1272/// `map_into` for a pure copy: the same traversal, gated as [`KernelClass::Copy`].
1273///
1274/// A copy has no arithmetic per element, so a fork over it pays later than over
1275/// a general element map; asking as `Map` forked it too early.
1276pub(crate) fn copy_map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1277    dest: &mut StridedViewMut<D>,
1278    src: &StridedView<A, Op>,
1279    f: impl Fn(A) -> D + Sync,
1280    exec: Exec<'_>,
1281) -> Result<()> {
1282    map_parts_into::<D, A, Op, NoKernel>(
1283        dest.as_mut_ptr(),
1284        dest.dims(),
1285        dest.strides(),
1286        src.ptr(),
1287        src.dims(),
1288        src.strides(),
1289        f,
1290        &NoKernel,
1291        KernelClass::Copy,
1292        exec,
1293    )
1294}
1295
1296pub(crate) fn map_into_validated<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1297    dest: &mut StridedViewMut<D>,
1298    src: &StridedView<A, Op>,
1299    f: impl MapFn<A, D>,
1300    validated: ValidatedDestinationLayout,
1301    exec: Exec<'_>,
1302) -> Result<()> {
1303    ensure_same_shape(dest.dims(), src.dims())?;
1304    let class = f.class();
1305    map_parts_into_validated::<D, A, Op, NoKernel>(
1306        dest.as_mut_ptr(),
1307        dest.dims(),
1308        dest.strides(),
1309        src.ptr(),
1310        src.strides(),
1311        |a| f.call(a),
1312        &NoKernel,
1313        validated,
1314        class,
1315        exec,
1316    )
1317}
1318
1319/// Apply an element-wise function into a potentially uninitialized destination, handing a
1320/// vectorized kernel the flat runs it can take.
1321///
1322/// `kernel` is offered a flat, unit-stride run whenever this crate has one to give: the whole
1323/// buffer when the two views share a contiguous layout and the operation runs sequentially, and
1324/// one block at a time on the parallel path. It must write every element of the run or write
1325/// nothing — returning `false` makes that run fall back to `scalar`, element by element. Runs whose
1326/// strides are not both 1 never reach it. It is called from whichever tasks the budget of
1327/// `exec` selected, so it must be `Sync`.
1328///
1329/// The point of the entry point is that a caller which owns vector kernels — a SIMD math library,
1330/// say — reuses this crate's layout logic and this crate's threading decision instead of
1331/// rebuilding either. `scalar` and `kernel` must agree up to the caller's own error bound; this
1332/// crate only guarantees that every destination element is written exactly once.
1333///
1334/// `scalar` may carry a [`KernelClass`] (see [`MapFn`]); the gate charges the region as that
1335/// class, so a transcendental `scalar` forks sooner.
1336///
1337/// # Examples
1338/// ```
1339/// use std::mem::MaybeUninit;
1340/// use strided_basic::{map_into_uninit_with_kernel, Exec, StridedView, StridedViewMut};
1341///
1342/// let src = vec![1.0f64, 2.0, 3.0];
1343/// let mut out = vec![MaybeUninit::<f64>::uninit(); 3];
1344/// let mut dest = StridedViewMut::new(&mut out, &[3], &[1], 0).unwrap();
1345/// let source = StridedView::new(&src, &[3], &[1], 0).unwrap();
1346/// let kernel = |src: &[f64], dst: &mut [MaybeUninit<f64>]| {
1347///     for (slot, value) in dst.iter_mut().zip(src) {
1348///         slot.write(value * 2.0);
1349///     }
1350///     true
1351/// };
1352/// map_into_uninit_with_kernel(&mut dest, &source, kernel, |x| x * 2.0, Exec::serial()).unwrap();
1353/// // SAFETY: the kernel wrote every element.
1354/// let values: Vec<f64> = out.into_iter().map(|x| unsafe { x.assume_init() }).collect();
1355/// assert_eq!(values, [2.0, 4.0, 6.0]);
1356/// ```
1357///
1358/// # Errors
1359/// Returns [`StridedError::ShapeMismatch`] when the shapes differ and
1360/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout cannot be proven
1361/// injective.
1362pub fn map_into_uninit_with_kernel<T: Copy + Send + Sync>(
1363    dest: &mut StridedViewMut<MaybeUninit<T>>,
1364    src: &StridedView<T>,
1365    kernel: impl Fn(&[T], &mut [MaybeUninit<T>]) -> bool + Sync,
1366    scalar: impl MapFn<T, T>,
1367    exec: Exec<'_>,
1368) -> Result<()> {
1369    let class = scalar.class();
1370    map_parts_into::<MaybeUninit<T>, T, Identity, _>(
1371        dest.as_mut_ptr(),
1372        dest.dims(),
1373        dest.strides(),
1374        src.ptr(),
1375        src.dims(),
1376        src.strides(),
1377        |value| MaybeUninit::new(scalar.call(value)),
1378        &SliceKernel(kernel),
1379        class,
1380        exec,
1381    )
1382}
1383
1384/// Map a raw destination from a raw source without preparing a plan.
1385///
1386/// Only the raw-replay unit tests need this entry now; the crate's production
1387/// callers either go through the fused traversal (raw ops, `CopyPlan`) or
1388/// [`map_raw_into_validated`].
1389#[cfg(test)]
1390pub(crate) fn map_raw_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1391    dest: &mut crate::RawStridedMut<'_, D>,
1392    src: &crate::RawStridedRef<'_, A>,
1393    f: impl Fn(A) -> D + Sync,
1394    exec: Exec<'_>,
1395) -> Result<()> {
1396    map_parts_into::<D, A, Op, NoKernel>(
1397        dest.as_mut_ptr(),
1398        dest.dims(),
1399        dest.strides(),
1400        src.ptr(),
1401        src.dims(),
1402        src.strides(),
1403        f,
1404        &NoKernel,
1405        KernelClass::Map,
1406        exec,
1407    )
1408}
1409
1410pub(crate) fn map_raw_into_validated<
1411    D: Copy + Send + Sync,
1412    A: Copy + Send + Sync,
1413    Op: ElementOp<A>,
1414>(
1415    dest: &mut crate::RawStridedMut<'_, D>,
1416    src: &crate::RawStridedRef<'_, A>,
1417    f: impl Fn(A) -> D + Sync,
1418    validated: ValidatedDestinationLayout,
1419    exec: Exec<'_>,
1420) -> Result<()> {
1421    ensure_same_shape(dest.dims(), src.dims())?;
1422    map_parts_into_validated::<D, A, Op, NoKernel>(
1423        dest.as_mut_ptr(),
1424        dest.dims(),
1425        dest.strides(),
1426        src.ptr(),
1427        src.strides(),
1428        f,
1429        &NoKernel,
1430        validated,
1431        KernelClass::Map,
1432        exec,
1433    )
1434}
1435
1436#[allow(clippy::too_many_arguments)]
1437fn map_parts_into<
1438    D: Copy + Send + Sync,
1439    A: Copy + Send + Sync,
1440    Op: ElementOp<A>,
1441    K: MapKernel<A, D> + Sync,
1442>(
1443    dst_ptr: *mut D,
1444    dst_dims: &[usize],
1445    dst_strides: &[isize],
1446    src_ptr: *const A,
1447    src_dims: &[usize],
1448    src_strides: &[isize],
1449    f: impl Fn(A) -> D + Sync,
1450    kernel: &K,
1451    class: KernelClass,
1452    exec: Exec<'_>,
1453) -> Result<()> {
1454    ensure_same_shape(dst_dims, src_dims)?;
1455    let validated = validate_destination_layout(dst_dims, dst_strides)?;
1456    map_parts_into_validated::<D, A, Op, K>(
1457        dst_ptr,
1458        dst_dims,
1459        dst_strides,
1460        src_ptr,
1461        src_strides,
1462        f,
1463        kernel,
1464        validated,
1465        class,
1466        exec,
1467    )
1468}
1469
1470#[allow(clippy::too_many_arguments)]
1471/// Apply `f` (with the element operation) over a fully contiguous pair.
1472///
1473/// Kept out of line, and out of the caller's dispatch frame, so that it cannot
1474/// change the code shape of the blocked kernels below; the operation is moved in
1475/// by value because a dispatched closure that only borrows it stays scalar
1476/// (tensor4all/strided-rs#301).
1477#[inline(never)]
1478fn map_contiguous<D: Copy, A: Copy, Op: ElementOp<A>>(
1479    dst: &mut [D],
1480    src: &[A],
1481    f: impl Fn(A) -> D,
1482) {
1483    simd::dispatch_if_large(dst.len(), move || {
1484        for i in 0..dst.len() {
1485            dst[i] = f(Op::apply(src[i]));
1486        }
1487    });
1488}
1489
1490#[allow(clippy::too_many_arguments)]
1491fn map_parts_into_validated<
1492    D: Copy + Send + Sync,
1493    A: Copy + Send + Sync,
1494    Op: ElementOp<A>,
1495    K: MapKernel<A, D> + Sync,
1496>(
1497    dst_ptr: *mut D,
1498    dst_dims: &[usize],
1499    dst_strides: &[isize],
1500    src_ptr: *const A,
1501    src_strides: &[isize],
1502    f: impl Fn(A) -> D + Sync,
1503    kernel: &K,
1504    _validated: ValidatedDestinationLayout,
1505    class: KernelClass,
1506    exec: Exec<'_>,
1507) -> Result<()> {
1508    if sequential_contiguous_layout(class, dst_dims, &[dst_strides, src_strides], &exec)?.is_some()
1509    {
1510        let len = total_len(dst_dims)?;
1511        // SAFETY: the contiguous-layout check proves that `dst` and `src` address `len` elements in
1512        // the same order starting at their base pointer, so each is exactly a flat run of `len`
1513        // elements.
1514        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1515        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
1516        if kernel.apply(src, dst) {
1517            return Ok(());
1518        }
1519        map_contiguous::<D, A, Op>(dst, src, f);
1520        return Ok(());
1521    }
1522
1523    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
1524    let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<A>());
1525    let total = total_len(dst_dims)?;
1526
1527    // Small tensor fast path: skip compute_order and compute_block_sizes
1528    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
1529        build_plan_fused_small(dst_dims, &strides_list)
1530    } else {
1531        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
1532    };
1533
1534    let total = total_len(&fused_dims)?;
1535    if is_parallel(class, total, &exec) {
1536        use crate::threading::SendPtr;
1537        let dst_send = SendPtr(dst_ptr);
1538        let src_send = SendPtr(src_ptr as *mut A);
1539
1540        let costs = compute_costs(&ordered_strides);
1541        let initial_offsets: crate::inline::OperandVec<isize> =
1542            strides_list.iter().map(|_| 0isize).collect();
1543        return mapreduce_threaded(
1544            &fused_dims,
1545            &plan.block,
1546            &ordered_strides,
1547            &initial_offsets,
1548            &costs,
1549            class,
1550            exec,
1551            &|dims, blocks, strides_list, offsets| {
1552                for_each_inner_block_with_offsets(
1553                    dims,
1554                    blocks,
1555                    strides_list,
1556                    offsets,
1557                    |offsets, len, strides| {
1558                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1559                        let sp = unsafe { src_send.as_const().offset(offsets[1]) };
1560                        unsafe {
1561                            inner_loop_map1::<D, A, Op, K>(
1562                                dp, strides[0], sp, strides[1], len, &f, kernel,
1563                            )
1564                        };
1565                        Ok(())
1566                    },
1567                )
1568            },
1569        );
1570    }
1571
1572    let initial_offsets: crate::inline::OperandVec<isize> =
1573        ordered_strides.iter().map(|_| 0isize).collect();
1574    for_each_inner_block_preordered(
1575        &fused_dims,
1576        &plan.block,
1577        &ordered_strides,
1578        &initial_offsets,
1579        |offsets, len, strides| {
1580            let dp = unsafe { dst_ptr.offset(offsets[0]) };
1581            let sp = unsafe { src_ptr.offset(offsets[1]) };
1582            unsafe {
1583                inner_loop_map1::<D, A, Op, K>(dp, strides[0], sp, strides[1], len, &f, kernel)
1584            };
1585            Ok(())
1586        },
1587    )
1588}
1589
1590/// Binary element-wise operation: `dest[i] = f(a[i], b[i])`.
1591///
1592/// Source operands `a` and `b` may have different element types from each other
1593/// and from `dest`. The closure `f` handles per-element type conversion.
1594///
1595/// # Errors
1596///
1597/// Returns [`StridedError::RankMismatch`] or
1598/// [`StridedError::ShapeMismatch`] when `dest` and either source view have
1599/// different ranks or extents, and
1600/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1601/// cannot be proven representable and injective: two logical indices name
1602/// one offset, or the element count or address span is too large to
1603/// represent. Nothing is written unless validation succeeds.
1604pub fn zip_map2_into<
1605    D: Copy + Send + Sync,
1606    A: Copy + Send + Sync,
1607    B: Copy + Send + Sync,
1608    OpA: ElementOp<A>,
1609    OpB: ElementOp<B>,
1610>(
1611    dest: &mut StridedViewMut<D>,
1612    a: &StridedView<A, OpA>,
1613    b: &StridedView<B, OpB>,
1614    f: impl Fn(A, B) -> D + Sync,
1615    exec: Exec<'_>,
1616) -> Result<()> {
1617    zip_map2_parts_into::<D, A, B, OpA, OpB>(
1618        dest.as_mut_ptr(),
1619        dest.dims(),
1620        dest.strides(),
1621        a.ptr(),
1622        a.dims(),
1623        a.strides(),
1624        b.ptr(),
1625        b.dims(),
1626        b.strides(),
1627        f,
1628        exec,
1629    )
1630}
1631
1632pub(crate) fn zip_map2_into_validated<
1633    D: Copy + Send + Sync,
1634    A: Copy + Send + Sync,
1635    B: Copy + Send + Sync,
1636    OpA: ElementOp<A>,
1637    OpB: ElementOp<B>,
1638>(
1639    dest: &mut StridedViewMut<D>,
1640    a: &StridedView<A, OpA>,
1641    b: &StridedView<B, OpB>,
1642    f: impl Fn(A, B) -> D + Sync,
1643    validated: ValidatedDestinationLayout,
1644    exec: Exec<'_>,
1645) -> Result<()> {
1646    zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1647        dest.as_mut_ptr(),
1648        dest.dims(),
1649        dest.strides(),
1650        a.ptr(),
1651        a.strides(),
1652        b.ptr(),
1653        b.strides(),
1654        f,
1655        validated,
1656        exec,
1657    )
1658}
1659
1660/// Runtime comparison selected once before entering the element loop.
1661#[non_exhaustive]
1662#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1663pub enum CompareOp {
1664    Eq,
1665    Lt,
1666    Le,
1667    Gt,
1668    Ge,
1669}
1670
1671/// Compare two ordered views elementwise into a Boolean destination.
1672///
1673/// Unlike embedding a runtime operation match inside a [`zip_map2_into`]
1674/// closure, this entry point selects a fixed comparison before traversal.
1675///
1676/// # Errors
1677///
1678/// Returns [`StridedError::ShapeMismatch`] when the source and destination
1679/// shapes differ, or [`StridedError::NonInjectiveOutputLayout`] when distinct
1680/// logical destination elements may overlap.
1681pub fn compare_into<T, OpA, OpB>(
1682    dest: &mut StridedViewMut<bool>,
1683    a: &StridedView<T, OpA>,
1684    b: &StridedView<T, OpB>,
1685    op: CompareOp,
1686    exec: Exec<'_>,
1687) -> Result<()>
1688where
1689    T: Copy + Send + Sync + PartialOrd,
1690    OpA: ElementOp<T>,
1691    OpB: ElementOp<T>,
1692{
1693    match op {
1694        CompareOp::Eq => zip_map2_into(dest, a, b, |lhs, rhs| lhs == rhs, exec),
1695        CompareOp::Lt => zip_map2_into(dest, a, b, |lhs, rhs| lhs < rhs, exec),
1696        CompareOp::Le => zip_map2_into(dest, a, b, |lhs, rhs| lhs <= rhs, exec),
1697        CompareOp::Gt => zip_map2_into(dest, a, b, |lhs, rhs| lhs > rhs, exec),
1698        CompareOp::Ge => zip_map2_into(dest, a, b, |lhs, rhs| lhs >= rhs, exec),
1699    }
1700}
1701
1702/// Compare two views into a fully overwritten uninitialized Boolean output.
1703///
1704/// Dtype-independent shape, destination-injectivity, and reachable-byte
1705/// overlap validation completes before the first write. Safe Rust borrows
1706/// already prevent input/output aliasing; the explicit overlap check preserves
1707/// the contract for views produced through unsafe constructors.
1708///
1709/// `Ok(())` means every logical destination element is initialized. An error
1710/// occurs before writes. A panic during replay may leave a partially initialized
1711/// destination, which remains safe to drop as `MaybeUninit<bool>`.
1712///
1713/// # Errors
1714///
1715/// Returns [`StridedError::ShapeMismatch`] for unequal shapes,
1716/// [`StridedError::NonInjectiveOutputLayout`] for an overlapping output layout,
1717/// [`StridedError::OverlappingInputOutput`] for aliased storage, or
1718/// [`StridedError::OffsetOverflow`] when a reachable byte range is not
1719/// representable.
1720pub fn compare_into_uninit<T, OpA, OpB>(
1721    dest: &mut StridedViewMut<MaybeUninit<bool>>,
1722    a: &StridedView<T, OpA>,
1723    b: &StridedView<T, OpB>,
1724    op: CompareOp,
1725    exec: Exec<'_>,
1726) -> Result<()>
1727where
1728    T: Copy + Send + Sync + PartialOrd,
1729    OpA: ElementOp<T>,
1730    OpB: ElementOp<T>,
1731{
1732    ensure_same_shape(dest.dims(), a.dims())?;
1733    ensure_same_shape(dest.dims(), b.dims())?;
1734    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
1735    validate_typed_no_overlap(dest, a, 0)?;
1736    validate_typed_no_overlap(dest, b, 1)?;
1737    match op {
1738        CompareOp::Eq => zip_map2_into_validated(
1739            dest,
1740            a,
1741            b,
1742            |lhs, rhs| MaybeUninit::new(lhs == rhs),
1743            validated,
1744            exec,
1745        ),
1746        CompareOp::Lt => zip_map2_into_validated(
1747            dest,
1748            a,
1749            b,
1750            |lhs, rhs| MaybeUninit::new(lhs < rhs),
1751            validated,
1752            exec,
1753        ),
1754        CompareOp::Le => zip_map2_into_validated(
1755            dest,
1756            a,
1757            b,
1758            |lhs, rhs| MaybeUninit::new(lhs <= rhs),
1759            validated,
1760            exec,
1761        ),
1762        CompareOp::Gt => zip_map2_into_validated(
1763            dest,
1764            a,
1765            b,
1766            |lhs, rhs| MaybeUninit::new(lhs > rhs),
1767            validated,
1768            exec,
1769        ),
1770        CompareOp::Ge => zip_map2_into_validated(
1771            dest,
1772            a,
1773            b,
1774            |lhs, rhs| MaybeUninit::new(lhs >= rhs),
1775            validated,
1776            exec,
1777        ),
1778    }
1779}
1780
1781pub(crate) fn zip_map2_raw_into_validated<
1782    D: Copy + Send + Sync,
1783    A: Copy + Send + Sync,
1784    B: Copy + Send + Sync,
1785    OpA: ElementOp<A>,
1786    OpB: ElementOp<B>,
1787>(
1788    dest: &mut crate::RawStridedMut<'_, D>,
1789    a: &crate::RawStridedRef<'_, A>,
1790    b: &crate::RawStridedRef<'_, B>,
1791    f: impl Fn(A, B) -> D + Sync,
1792    validated: ValidatedDestinationLayout,
1793    exec: Exec<'_>,
1794) -> Result<()> {
1795    ensure_same_shape(dest.dims(), a.dims())?;
1796    ensure_same_shape(dest.dims(), b.dims())?;
1797    zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1798        dest.as_mut_ptr(),
1799        dest.dims(),
1800        dest.strides(),
1801        a.ptr(),
1802        a.strides(),
1803        b.ptr(),
1804        b.strides(),
1805        f,
1806        validated,
1807        exec,
1808    )
1809}
1810
1811#[allow(clippy::too_many_arguments)]
1812fn zip_map2_parts_into<
1813    D: Copy + Send + Sync,
1814    A: Copy + Send + Sync,
1815    B: Copy + Send + Sync,
1816    OpA: ElementOp<A>,
1817    OpB: ElementOp<B>,
1818>(
1819    dst_ptr: *mut D,
1820    dst_dims: &[usize],
1821    dst_strides: &[isize],
1822    a_ptr: *const A,
1823    a_dims: &[usize],
1824    a_strides: &[isize],
1825    b_ptr: *const B,
1826    b_dims: &[usize],
1827    b_strides: &[isize],
1828    f: impl Fn(A, B) -> D + Sync,
1829    exec: Exec<'_>,
1830) -> Result<()> {
1831    ensure_same_shape(dst_dims, a_dims)?;
1832    ensure_same_shape(dst_dims, b_dims)?;
1833    let validated = validate_destination_layout(dst_dims, dst_strides)?;
1834    zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1835        dst_ptr,
1836        dst_dims,
1837        dst_strides,
1838        a_ptr,
1839        a_strides,
1840        b_ptr,
1841        b_strides,
1842        f,
1843        validated,
1844        exec,
1845    )
1846}
1847
1848#[allow(clippy::too_many_arguments)]
1849fn zip_map2_parts_into_validated<
1850    D: Copy + Send + Sync,
1851    A: Copy + Send + Sync,
1852    B: Copy + Send + Sync,
1853    OpA: ElementOp<A>,
1854    OpB: ElementOp<B>,
1855>(
1856    dst_ptr: *mut D,
1857    dst_dims: &[usize],
1858    dst_strides: &[isize],
1859    a_ptr: *const A,
1860    a_strides: &[isize],
1861    b_ptr: *const B,
1862    b_strides: &[isize],
1863    f: impl Fn(A, B) -> D + Sync,
1864    _validated: ValidatedDestinationLayout,
1865    exec: Exec<'_>,
1866) -> Result<()> {
1867    if sequential_contiguous_layout(
1868        KernelClass::Map,
1869        dst_dims,
1870        &[dst_strides, a_strides, b_strides],
1871        &exec,
1872    )?
1873    .is_some()
1874    {
1875        let len = total_len(dst_dims)?;
1876        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1877        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
1878        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
1879        simd::dispatch_if_large(len, || {
1880            for i in 0..len {
1881                dst[i] = f(OpA::apply(sa[i]), OpB::apply(sb[i]));
1882            }
1883        });
1884        return Ok(());
1885    }
1886
1887    let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
1888    let elem_size = std::mem::size_of::<D>()
1889        .max(std::mem::size_of::<A>())
1890        .max(std::mem::size_of::<B>());
1891    let total = total_len(dst_dims)?;
1892
1893    // Small tensor fast path: skip compute_order and compute_block_sizes
1894    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
1895        build_plan_fused_small(dst_dims, &strides_list)
1896    } else {
1897        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
1898    };
1899
1900    let total = total_len(&fused_dims)?;
1901    if is_parallel(KernelClass::Map, total, &exec) {
1902        use crate::threading::SendPtr;
1903        let dst_send = SendPtr(dst_ptr);
1904        let a_send = SendPtr(a_ptr as *mut A);
1905        let b_send = SendPtr(b_ptr as *mut B);
1906
1907        let costs = compute_costs(&ordered_strides);
1908        let initial_offsets: crate::inline::OperandVec<isize> =
1909            strides_list.iter().map(|_| 0isize).collect();
1910        return mapreduce_threaded(
1911            &fused_dims,
1912            &plan.block,
1913            &ordered_strides,
1914            &initial_offsets,
1915            &costs,
1916            KernelClass::Map,
1917            exec,
1918            &|dims, blocks, strides_list, offsets| {
1919                for_each_inner_block_with_offsets(
1920                    dims,
1921                    blocks,
1922                    strides_list,
1923                    offsets,
1924                    |offsets, len, strides| {
1925                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1926                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
1927                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
1928                        unsafe {
1929                            inner_loop_map2::<D, A, B, OpA, OpB>(
1930                                dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1931                            )
1932                        };
1933                        Ok(())
1934                    },
1935                )
1936            },
1937        );
1938    }
1939
1940    let initial_offsets: crate::inline::OperandVec<isize> =
1941        ordered_strides.iter().map(|_| 0isize).collect();
1942    for_each_inner_block_preordered(
1943        &fused_dims,
1944        &plan.block,
1945        &ordered_strides,
1946        &initial_offsets,
1947        |offsets, len, strides| {
1948            let dp = unsafe { dst_ptr.offset(offsets[0]) };
1949            let ap = unsafe { a_ptr.offset(offsets[1]) };
1950            let bp = unsafe { b_ptr.offset(offsets[2]) };
1951            unsafe {
1952                inner_loop_map2::<D, A, B, OpA, OpB>(
1953                    dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1954                )
1955            };
1956            Ok(())
1957        },
1958    )
1959}
1960
1961#[allow(clippy::too_many_arguments)]
1962fn mul_identity_into_raw<
1963    O: MulOutput<D>,
1964    D: Copy + Send + Sync + 'static,
1965    A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
1966    B: Copy + Send + Sync + 'static,
1967>(
1968    dst_ptr: *mut O::Slot,
1969    dst_dims: &[usize],
1970    dst_strides: &[isize],
1971    a_ptr: *const A,
1972    a_strides: &[isize],
1973    b_ptr: *const B,
1974    b_strides: &[isize],
1975    _validated: ValidatedDestinationLayout,
1976    mut exec: Exec<'_>,
1977) -> Result<()> {
1978    debug_assert_eq!(dst_dims.len(), a_strides.len());
1979    debug_assert_eq!(dst_dims.len(), b_strides.len());
1980
1981    if sequential_contiguous_layout(
1982        KernelClass::Map,
1983        dst_dims,
1984        &[dst_strides, a_strides, b_strides],
1985        &exec,
1986    )?
1987    .is_some()
1988    {
1989        let len = total_len(dst_dims)?;
1990        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
1991        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
1992        if unsafe { O::try_contiguous(dst_ptr, len, sa, sb) } {
1993            return Ok(());
1994        }
1995        for i in 0..len {
1996            unsafe { O::write(dst_ptr.add(i), multiply_value(sa[i], sb[i])) };
1997        }
1998        return Ok(());
1999    }
2000
2001    let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
2002    let elem_size = std::mem::size_of::<D>()
2003        .max(std::mem::size_of::<A>())
2004        .max(std::mem::size_of::<B>());
2005    let total = total_len(dst_dims)?;
2006
2007    if try_contiguous_range_mul::<O, D, A, B>(
2008        dst_ptr,
2009        dst_dims,
2010        dst_strides,
2011        a_ptr,
2012        a_strides,
2013        b_ptr,
2014        b_strides,
2015        exec.reborrow(),
2016    ) {
2017        return Ok(());
2018    }
2019
2020    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2021        build_plan_fused_small(dst_dims, &strides_list)
2022    } else {
2023        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2024    };
2025
2026    let total = total_len(&fused_dims)?;
2027    if is_parallel(KernelClass::Map, total, &exec) {
2028        use crate::threading::SendPtr;
2029        let dst_send = SendPtr(dst_ptr);
2030        let a_send = SendPtr(a_ptr as *mut A);
2031        let b_send = SendPtr(b_ptr as *mut B);
2032
2033        let costs = compute_costs(&ordered_strides);
2034        let initial_offsets: crate::inline::OperandVec<isize> =
2035            strides_list.iter().map(|_| 0isize).collect();
2036        return mapreduce_threaded(
2037            &fused_dims,
2038            &plan.block,
2039            &ordered_strides,
2040            &initial_offsets,
2041            &costs,
2042            KernelClass::Map,
2043            exec,
2044            &|dims, blocks, strides_list, offsets| {
2045                for_each_inner_block_with_offsets(
2046                    dims,
2047                    blocks,
2048                    strides_list,
2049                    offsets,
2050                    |offsets, len, strides| {
2051                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2052                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2053                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2054                        unsafe {
2055                            inner_loop_mul2::<O, D, A, B>(
2056                                dp, strides[0], ap, strides[1], bp, strides[2], len,
2057                            )
2058                        };
2059                        Ok(())
2060                    },
2061                )
2062            },
2063        );
2064    }
2065
2066    let initial_offsets: crate::inline::OperandVec<isize> =
2067        ordered_strides.iter().map(|_| 0isize).collect();
2068    for_each_inner_block_preordered(
2069        &fused_dims,
2070        &plan.block,
2071        &ordered_strides,
2072        &initial_offsets,
2073        |offsets, len, strides| {
2074            let dp = unsafe { dst_ptr.offset(offsets[0]) };
2075            let ap = unsafe { a_ptr.offset(offsets[1]) };
2076            let bp = unsafe { b_ptr.offset(offsets[2]) };
2077            unsafe {
2078                inner_loop_mul2::<O, D, A, B>(dp, strides[0], ap, strides[1], bp, strides[2], len)
2079            };
2080            Ok(())
2081        },
2082    )
2083}
2084
2085/// Element-wise multiplication: `dest[i] = a[i] * b[i]`.
2086///
2087/// All views must have the same shape. Broadcast operands should be represented
2088/// as stride-0 views before calling this function.
2089///
2090/// # Errors
2091///
2092/// Returns [`StridedError::RankMismatch`] or
2093/// [`StridedError::ShapeMismatch`] when `dest` and either operand have
2094/// different ranks or extents, and
2095/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2096/// cannot be proven representable and injective: two logical indices name
2097/// one offset, or the element count or address span is too large to
2098/// represent. Nothing is written unless validation succeeds.
2099pub fn mul_into<
2100    D: Copy + Send + Sync + 'static,
2101    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2102    B: Copy + Send + Sync + 'static,
2103    OpA: ElementOp<A>,
2104    OpB: ElementOp<B>,
2105>(
2106    dest: &mut StridedViewMut<D>,
2107    a: &StridedView<A, OpA>,
2108    b: &StridedView<B, OpB>,
2109    exec: Exec<'_>,
2110) -> Result<()> {
2111    ensure_same_shape(dest.dims(), a.dims())?;
2112    ensure_same_shape(dest.dims(), b.dims())?;
2113    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2114
2115    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2116        return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2117            dest.as_mut_ptr(),
2118            dest.dims(),
2119            dest.strides(),
2120            a.ptr(),
2121            a.strides(),
2122            b.ptr(),
2123            b.strides(),
2124            validated,
2125            exec,
2126        );
2127    }
2128
2129    zip_map2_into_validated(dest, a, b, multiply_value, validated, exec)
2130}
2131
2132/// Multiply two views into a fully overwritten uninitialized output.
2133///
2134/// Shape, destination-injectivity, and reachable-byte overlap validation
2135/// completes before the first write. Safe Rust borrows already prevent
2136/// input/output aliasing; the explicit overlap check preserves the contract for
2137/// views produced through unsafe constructors.
2138///
2139/// `Ok(())` means every logical destination element is initialized. An error
2140/// occurs before writes. A panic during replay may leave a partially initialized
2141/// destination, which remains safe to drop as `MaybeUninit<D>`.
2142///
2143/// # Errors
2144///
2145/// Returns [`StridedError::ShapeMismatch`] for unequal shapes,
2146/// [`StridedError::NonInjectiveOutputLayout`] for an overlapping output layout,
2147/// [`StridedError::OverlappingInputOutput`] for aliased storage, or
2148/// [`StridedError::OffsetOverflow`] when a reachable byte range is not
2149/// representable.
2150pub fn mul_into_uninit<
2151    D: Copy + Send + Sync + 'static,
2152    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2153    B: Copy + Send + Sync + 'static,
2154    OpA: ElementOp<A>,
2155    OpB: ElementOp<B>,
2156>(
2157    dest: &mut StridedViewMut<MaybeUninit<D>>,
2158    a: &StridedView<A, OpA>,
2159    b: &StridedView<B, OpB>,
2160    exec: Exec<'_>,
2161) -> Result<()> {
2162    ensure_same_shape(dest.dims(), a.dims())?;
2163    ensure_same_shape(dest.dims(), b.dims())?;
2164    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2165    validate_typed_no_overlap(dest, a, 0)?;
2166    validate_typed_no_overlap(dest, b, 1)?;
2167
2168    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2169        return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2170            dest.as_mut_ptr(),
2171            dest.dims(),
2172            dest.strides(),
2173            a.ptr(),
2174            a.strides(),
2175            b.ptr(),
2176            b.strides(),
2177            validated,
2178            exec,
2179        );
2180    }
2181
2182    zip_map2_into_validated(
2183        dest,
2184        a,
2185        b,
2186        |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2187        validated,
2188        exec,
2189    )
2190}
2191
2192fn broadcast_strides_for_axes(
2193    source_dims: &[usize],
2194    source_strides: &[isize],
2195    target_dims: &[usize],
2196    axes: &[usize],
2197) -> Result<AxisVec<isize>> {
2198    if source_dims.len() != axes.len() {
2199        return Err(StridedError::RankMismatch(source_dims.len(), axes.len()));
2200    }
2201    debug_assert_eq!(source_dims.len(), source_strides.len());
2202
2203    let mut seen = AxisVec::<bool>::new();
2204    seen.resize(target_dims.len(), false);
2205    let mut strides = AxisVec::<isize>::new();
2206    strides.resize(target_dims.len(), 0);
2207    for (src_axis, &dst_axis) in axes.iter().enumerate() {
2208        if dst_axis >= target_dims.len() {
2209            return Err(StridedError::InvalidAxis {
2210                axis: dst_axis,
2211                rank: target_dims.len(),
2212            });
2213        }
2214        if seen[dst_axis] {
2215            return Err(StridedError::InvalidAxis {
2216                axis: dst_axis,
2217                rank: target_dims.len(),
2218            });
2219        }
2220        seen[dst_axis] = true;
2221
2222        let source_dim = source_dims[src_axis];
2223        let target_dim = target_dims[dst_axis];
2224        if source_dim != target_dim && source_dim != 1 {
2225            return Err(StridedError::ShapeMismatch(
2226                source_dims.to_vec(),
2227                target_dims.to_vec(),
2228            ));
2229        }
2230        if source_dim == target_dim {
2231            strides[dst_axis] = source_strides[src_axis];
2232        }
2233    }
2234
2235    Ok(strides)
2236}
2237
2238/// A view of `view`'s data with stride-0 broadcast axes.
2239///
2240/// The metadata is computed for this call only; [`StridedView::new_unchecked`]
2241/// copies it into owned metadata (inline up to [`INLINE_RANK`](crate::INLINE_RANK)
2242/// axes, boxed above, so a rank-9 view allocates), and the view still borrows
2243/// only the element data.
2244fn broadcast_view_with_strides<'a, T, Op: ElementOp<T>>(
2245    view: &StridedView<'a, T, Op>,
2246    target_dims: &[usize],
2247    strides: &[isize],
2248) -> StridedView<'a, T, Op> {
2249    // SAFETY: `broadcast_strides_for_axes` (:2192) keeps a source stride only on
2250    // axes where `source_dims[src_axis] == target_dims[dst_axis]`, sets every other
2251    // target axis to stride 0, rejects repeated and out-of-range target axes, and
2252    // requires `source_dims.len() == axes.len()`, so this view reaches a subset of
2253    // the offsets `view` reaches; `StridedView` construction (`new`,
2254    // strided-view/src/view.rs:508 via `validate_bounds`, :186) already proved those
2255    // in bounds, and `target_dims`/`strides` come from the caller's `dest.dims()`
2256    // and that validation. `new_unchecked` copies the metadata into owned
2257    // storage, not inline: `ViewMeta::from_slices` (strided-view/src/view.rs:128)
2258    // keeps it inline up to `INLINE_RANK` = 8 and boxes it above (strided-view/src/view.rs:140), which
2259    // is why a rank-9 call allocates here.
2260    unsafe { StridedView::new_unchecked(view.data(), target_dims, strides, view.offset()) }
2261}
2262
2263/// Broadcasted element-wise multiplication: `dest[i] = a[i] * b[i]`.
2264///
2265/// `a_axes` and `b_axes` map each source axis to an axis of `dest`. Output axes
2266/// not referenced by a source operand are treated as stride-0 broadcast axes.
2267///
2268/// # Errors
2269///
2270/// Returns [`StridedError::RankMismatch`] when an axis list does not have one
2271/// entry per source axis, [`StridedError::InvalidAxis`] when an axis lies
2272/// outside `dest` or is mapped twice, [`StridedError::ShapeMismatch`] when a
2273/// source extent is neither the destination extent nor 1, and
2274/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2275/// cannot be proven representable and injective: two logical indices name
2276/// one offset, or the element count or address span is too large to
2277/// represent. Nothing is written unless validation succeeds.
2278pub fn broadcast_mul_into<
2279    D: Copy + Send + Sync + 'static,
2280    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2281    B: Copy + Send + Sync + 'static,
2282    OpA: ElementOp<A>,
2283    OpB: ElementOp<B>,
2284>(
2285    dest: &mut StridedViewMut<D>,
2286    a: &StridedView<A, OpA>,
2287    a_axes: &[usize],
2288    b: &StridedView<B, OpB>,
2289    b_axes: &[usize],
2290    exec: Exec<'_>,
2291) -> Result<()> {
2292    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2293    let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2294    let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2295
2296    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2297        return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2298            dest.as_mut_ptr(),
2299            dest.dims(),
2300            dest.strides(),
2301            a.ptr(),
2302            &a_strides,
2303            b.ptr(),
2304            &b_strides,
2305            validated,
2306            exec,
2307        );
2308    }
2309
2310    let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2311    let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2312    zip_map2_into_validated(dest, &a, &b, multiply_value, validated, exec)
2313}
2314
2315/// Broadcast and multiply into a fully overwritten uninitialized output.
2316///
2317/// Axis mappings, shapes, destination injectivity, and reachable-byte overlap
2318/// are validated before the first write. Safe Rust borrows already prevent
2319/// input/output aliasing; the explicit overlap check preserves the contract for
2320/// views produced through unsafe constructors.
2321///
2322/// `Ok(())` means every logical destination element is initialized. An error
2323/// occurs before writes. A panic during replay may leave a partially initialized
2324/// destination, which remains safe to drop as `MaybeUninit<D>`.
2325///
2326/// # Errors
2327///
2328/// Returns a typed rank, axis, or shape error for an invalid broadcast mapping,
2329/// [`StridedError::NonInjectiveOutputLayout`] for an overlapping output layout,
2330/// [`StridedError::OverlappingInputOutput`] for aliased storage, or
2331/// [`StridedError::OffsetOverflow`] when a reachable byte range is not
2332/// representable.
2333pub fn broadcast_mul_into_uninit<
2334    D: Copy + Send + Sync + 'static,
2335    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2336    B: Copy + Send + Sync + 'static,
2337    OpA: ElementOp<A>,
2338    OpB: ElementOp<B>,
2339>(
2340    dest: &mut StridedViewMut<MaybeUninit<D>>,
2341    a: &StridedView<A, OpA>,
2342    a_axes: &[usize],
2343    b: &StridedView<B, OpB>,
2344    b_axes: &[usize],
2345    exec: Exec<'_>,
2346) -> Result<()> {
2347    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2348    let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2349    let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2350    validate_typed_no_overlap(dest, a, 0)?;
2351    validate_typed_no_overlap(dest, b, 1)?;
2352
2353    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2354        return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2355            dest.as_mut_ptr(),
2356            dest.dims(),
2357            dest.strides(),
2358            a.ptr(),
2359            &a_strides,
2360            b.ptr(),
2361            &b_strides,
2362            validated,
2363            exec,
2364        );
2365    }
2366
2367    let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2368    let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2369    zip_map2_into_validated(
2370        dest,
2371        &a,
2372        &b,
2373        |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2374        validated,
2375        exec,
2376    )
2377}
2378
2379/// Ternary element-wise operation: `dest[i] = f(a[i], b[i], c[i])`.
2380///
2381/// # Errors
2382///
2383/// Returns [`StridedError::RankMismatch`] or
2384/// [`StridedError::ShapeMismatch`] when `dest` and any of the three source
2385/// views have different ranks or extents, and
2386/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2387/// cannot be proven representable and injective: two logical indices name
2388/// one offset, or the element count or address span is too large to
2389/// represent. Nothing is written unless validation succeeds.
2390pub fn zip_map3_into<
2391    D: Copy + Send + Sync,
2392    A: Copy + Send + Sync,
2393    B: Copy + Send + Sync,
2394    C: Copy + Send + Sync,
2395    OpA: ElementOp<A>,
2396    OpB: ElementOp<B>,
2397    OpC: ElementOp<C>,
2398>(
2399    dest: &mut StridedViewMut<D>,
2400    a: &StridedView<A, OpA>,
2401    b: &StridedView<B, OpB>,
2402    c: &StridedView<C, OpC>,
2403    f: impl Fn(A, B, C) -> D + Sync,
2404    exec: Exec<'_>,
2405) -> Result<()> {
2406    ensure_same_shape(dest.dims(), a.dims())?;
2407    ensure_same_shape(dest.dims(), b.dims())?;
2408    ensure_same_shape(dest.dims(), c.dims())?;
2409    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2410    zip_map3_into_validated(dest, a, b, c, f, validated, exec)
2411}
2412
2413pub(crate) fn zip_map3_into_validated<
2414    D: Copy + Send + Sync,
2415    A: Copy + Send + Sync,
2416    B: Copy + Send + Sync,
2417    C: Copy + Send + Sync,
2418    OpA: ElementOp<A>,
2419    OpB: ElementOp<B>,
2420    OpC: ElementOp<C>,
2421>(
2422    dest: &mut StridedViewMut<D>,
2423    a: &StridedView<A, OpA>,
2424    b: &StridedView<B, OpB>,
2425    c: &StridedView<C, OpC>,
2426    f: impl Fn(A, B, C) -> D + Sync,
2427    _validated: ValidatedDestinationLayout,
2428    exec: Exec<'_>,
2429) -> Result<()> {
2430    ensure_same_shape(dest.dims(), a.dims())?;
2431    ensure_same_shape(dest.dims(), b.dims())?;
2432    ensure_same_shape(dest.dims(), c.dims())?;
2433    let dst_ptr = dest.as_mut_ptr();
2434    let a_ptr = a.ptr();
2435    let b_ptr = b.ptr();
2436    let c_ptr = c.ptr();
2437
2438    let dst_dims = dest.dims();
2439    let dst_strides = dest.strides();
2440
2441    if sequential_contiguous_layout(
2442        KernelClass::Map,
2443        dst_dims,
2444        &[dst_strides, a.strides(), b.strides(), c.strides()],
2445        &exec,
2446    )?
2447    .is_some()
2448    {
2449        let len = total_len(dst_dims)?;
2450        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2451        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2452        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2453        let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2454        simd::dispatch_if_large(len, || {
2455            for (((d, &a), &b), &c) in dst.iter_mut().zip(sa).zip(sb).zip(sc) {
2456                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
2457            }
2458        });
2459        return Ok(());
2460    }
2461
2462    let strides_list: [&[isize]; 4] = [dst_strides, a.strides(), b.strides(), c.strides()];
2463    let elem_size = std::mem::size_of::<D>()
2464        .max(std::mem::size_of::<A>())
2465        .max(std::mem::size_of::<B>())
2466        .max(std::mem::size_of::<C>());
2467    let total = total_len(dst_dims)?;
2468
2469    // Small tensor fast path: skip compute_order and compute_block_sizes
2470    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2471        build_plan_fused_small(dst_dims, &strides_list)
2472    } else {
2473        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2474    };
2475
2476    let total = total_len(&fused_dims)?;
2477    if is_parallel(KernelClass::Map, total, &exec) {
2478        use crate::threading::SendPtr;
2479        let dst_send = SendPtr(dst_ptr);
2480        let a_send = SendPtr(a_ptr as *mut A);
2481        let b_send = SendPtr(b_ptr as *mut B);
2482        let c_send = SendPtr(c_ptr as *mut C);
2483
2484        let costs = compute_costs(&ordered_strides);
2485        let initial_offsets: crate::inline::OperandVec<isize> =
2486            strides_list.iter().map(|_| 0isize).collect();
2487        return mapreduce_threaded(
2488            &fused_dims,
2489            &plan.block,
2490            &ordered_strides,
2491            &initial_offsets,
2492            &costs,
2493            KernelClass::Map,
2494            exec,
2495            &|dims, blocks, strides_list, offsets| {
2496                for_each_inner_block_with_offsets(
2497                    dims,
2498                    blocks,
2499                    strides_list,
2500                    offsets,
2501                    |offsets, len, strides| {
2502                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2503                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2504                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2505                        let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2506                        unsafe {
2507                            inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2508                                dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3],
2509                                len, &f,
2510                            )
2511                        };
2512                        Ok(())
2513                    },
2514                )
2515            },
2516        );
2517    }
2518
2519    let initial_offsets: crate::inline::OperandVec<isize> =
2520        ordered_strides.iter().map(|_| 0isize).collect();
2521    for_each_inner_block_preordered(
2522        &fused_dims,
2523        &plan.block,
2524        &ordered_strides,
2525        &initial_offsets,
2526        |offsets, len, strides| {
2527            let dp = unsafe { dst_ptr.offset(offsets[0]) };
2528            let ap = unsafe { a_ptr.offset(offsets[1]) };
2529            let bp = unsafe { b_ptr.offset(offsets[2]) };
2530            let cp = unsafe { c_ptr.offset(offsets[3]) };
2531            unsafe {
2532                inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2533                    dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], len, &f,
2534                )
2535            };
2536            Ok(())
2537        },
2538    )
2539}
2540
2541/// Quaternary element-wise operation: `dest[i] = f(a[i], b[i], c[i], e[i])`.
2542///
2543/// # Errors
2544///
2545/// Returns [`StridedError::RankMismatch`] or
2546/// [`StridedError::ShapeMismatch`] when `dest` and any of the four source
2547/// views have different ranks or extents, and
2548/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2549/// cannot be proven representable and injective: two logical indices name
2550/// one offset, or the element count or address span is too large to
2551/// represent. Nothing is written unless validation succeeds.
2552pub fn zip_map4_into<
2553    D: Copy + Send + Sync,
2554    A: Copy + Send + Sync,
2555    B: Copy + Send + Sync,
2556    C: Copy + Send + Sync,
2557    E: Copy + Send + Sync,
2558    OpA: ElementOp<A>,
2559    OpB: ElementOp<B>,
2560    OpC: ElementOp<C>,
2561    OpE: ElementOp<E>,
2562>(
2563    dest: &mut StridedViewMut<D>,
2564    a: &StridedView<A, OpA>,
2565    b: &StridedView<B, OpB>,
2566    c: &StridedView<C, OpC>,
2567    e: &StridedView<E, OpE>,
2568    f: impl Fn(A, B, C, E) -> D + Sync,
2569    exec: Exec<'_>,
2570) -> Result<()> {
2571    ensure_same_shape(dest.dims(), a.dims())?;
2572    ensure_same_shape(dest.dims(), b.dims())?;
2573    ensure_same_shape(dest.dims(), c.dims())?;
2574    ensure_same_shape(dest.dims(), e.dims())?;
2575    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2576    zip_map4_into_validated(dest, a, b, c, e, f, validated, exec)
2577}
2578
2579#[allow(clippy::too_many_arguments)]
2580pub(crate) fn zip_map4_into_validated<
2581    D: Copy + Send + Sync,
2582    A: Copy + Send + Sync,
2583    B: Copy + Send + Sync,
2584    C: Copy + Send + Sync,
2585    E: Copy + Send + Sync,
2586    OpA: ElementOp<A>,
2587    OpB: ElementOp<B>,
2588    OpC: ElementOp<C>,
2589    OpE: ElementOp<E>,
2590>(
2591    dest: &mut StridedViewMut<D>,
2592    a: &StridedView<A, OpA>,
2593    b: &StridedView<B, OpB>,
2594    c: &StridedView<C, OpC>,
2595    e: &StridedView<E, OpE>,
2596    f: impl Fn(A, B, C, E) -> D + Sync,
2597    _validated: ValidatedDestinationLayout,
2598    exec: Exec<'_>,
2599) -> Result<()> {
2600    ensure_same_shape(dest.dims(), a.dims())?;
2601    ensure_same_shape(dest.dims(), b.dims())?;
2602    ensure_same_shape(dest.dims(), c.dims())?;
2603    ensure_same_shape(dest.dims(), e.dims())?;
2604    let dst_ptr = dest.as_mut_ptr();
2605    let a_ptr = a.ptr();
2606    let b_ptr = b.ptr();
2607    let c_ptr = c.ptr();
2608    let e_ptr = e.ptr();
2609
2610    let dst_dims = dest.dims();
2611    let dst_strides = dest.strides();
2612
2613    if sequential_contiguous_layout(
2614        KernelClass::Map,
2615        dst_dims,
2616        &[
2617            dst_strides,
2618            a.strides(),
2619            b.strides(),
2620            c.strides(),
2621            e.strides(),
2622        ],
2623        &exec,
2624    )?
2625    .is_some()
2626    {
2627        let len = total_len(dst_dims)?;
2628        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2629        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2630        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2631        let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2632        let se = unsafe { std::slice::from_raw_parts(e_ptr, len) };
2633        // As in `inner_loop_map4`: zipped iterators keep the dispatched closure
2634        // small enough to inline (tensor4all/strided-rs#340).
2635        simd::dispatch_if_large(len, || {
2636            for ((((d, &a), &b), &c), &e) in dst.iter_mut().zip(sa).zip(sb).zip(sc).zip(se) {
2637                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
2638            }
2639        });
2640        return Ok(());
2641    }
2642
2643    let strides_list: [&[isize]; 5] = [
2644        dst_strides,
2645        a.strides(),
2646        b.strides(),
2647        c.strides(),
2648        e.strides(),
2649    ];
2650    let elem_size = std::mem::size_of::<D>()
2651        .max(std::mem::size_of::<A>())
2652        .max(std::mem::size_of::<B>())
2653        .max(std::mem::size_of::<C>())
2654        .max(std::mem::size_of::<E>());
2655    let total = total_len(dst_dims)?;
2656
2657    // Small tensor fast path: skip compute_order and compute_block_sizes
2658    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2659        build_plan_fused_small(dst_dims, &strides_list)
2660    } else {
2661        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2662    };
2663
2664    let total = total_len(&fused_dims)?;
2665    if is_parallel(KernelClass::Map, total, &exec) {
2666        use crate::threading::SendPtr;
2667        let dst_send = SendPtr(dst_ptr);
2668        let a_send = SendPtr(a_ptr as *mut A);
2669        let b_send = SendPtr(b_ptr as *mut B);
2670        let c_send = SendPtr(c_ptr as *mut C);
2671        let e_send = SendPtr(e_ptr as *mut E);
2672
2673        let costs = compute_costs(&ordered_strides);
2674        let initial_offsets: crate::inline::OperandVec<isize> =
2675            strides_list.iter().map(|_| 0isize).collect();
2676        return mapreduce_threaded(
2677            &fused_dims,
2678            &plan.block,
2679            &ordered_strides,
2680            &initial_offsets,
2681            &costs,
2682            KernelClass::Map,
2683            exec,
2684            &|dims, blocks, strides_list, offsets| {
2685                for_each_inner_block_with_offsets(
2686                    dims,
2687                    blocks,
2688                    strides_list,
2689                    offsets,
2690                    |offsets, len, strides| {
2691                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2692                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2693                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2694                        let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2695                        let ep = unsafe { e_send.as_const().offset(offsets[4]) };
2696                        unsafe {
2697                            inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2698                                dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep,
2699                                strides[4], len, &f,
2700                            )
2701                        };
2702                        Ok(())
2703                    },
2704                )
2705            },
2706        );
2707    }
2708
2709    let initial_offsets: crate::inline::OperandVec<isize> =
2710        ordered_strides.iter().map(|_| 0isize).collect();
2711    for_each_inner_block_preordered(
2712        &fused_dims,
2713        &plan.block,
2714        &ordered_strides,
2715        &initial_offsets,
2716        |offsets, len, strides| {
2717            let dp = unsafe { dst_ptr.offset(offsets[0]) };
2718            let ap = unsafe { a_ptr.offset(offsets[1]) };
2719            let bp = unsafe { b_ptr.offset(offsets[2]) };
2720            let cp = unsafe { c_ptr.offset(offsets[3]) };
2721            let ep = unsafe { e_ptr.offset(offsets[4]) };
2722            unsafe {
2723                inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2724                    dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep, strides[4],
2725                    len, &f,
2726                )
2727            };
2728            Ok(())
2729        },
2730    )
2731}
2732
2733#[cfg(test)]
2734#[path = "map_view/tests/scalar_branch_tests.rs"]
2735mod scalar_branch_tests;
2736
2737#[cfg(test)]
2738#[path = "map_view/tests/tests.rs"]
2739mod tests;