Skip to main content

strided_basic/
map_view.rs

1//! Map operations on dynamic-rank strided views.
2//!
3//! These are the canonical view-based map functions, equivalent to Julia's `Base.map!`.
4//! Mutable destinations must have an injective layout. Validation is
5//! conservative and bounded: layouts that are injective but cannot be proven
6//! by the bounded checker are rejected with
7//! [`StridedError::NonInjectiveOutputLayout`].
8
9use crate::kernel::{
10    build_plan_fused, build_plan_fused_small, ensure_same_shape, for_each_inner_block_preordered,
11    sequential_contiguous_layout, total_len, SMALL_TENSOR_THRESHOLD,
12};
13use crate::simd;
14use crate::view::{StridedView, StridedViewMut};
15use crate::{Result, StridedError};
16use core::mem::MaybeUninit;
17use std::ops::Mul;
18use strided_view::{ElementOp, Identity};
19
20use crate::cost::{for_len, KernelClass};
21use crate::fuse::compute_costs;
22use crate::threading::{for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded};
23use crate::Exec;
24
25type AxisVec<T> = crate::inline::InlineVec<T>;
26
27/// Largest fused inner:row aspect at which the tiled transposed-scalar kernel
28/// is selected.
29///
30/// The cache-tiled kernel wins or ties the generic blocked plan across the
31/// measured shape/dtype table (u4-tiled #210), except when the fused inner run
32/// is more than 8x the row extent: then its strided destination writes lose to
33/// the blocked plan on f64 (`4096x256` ~1.4x, `8192x256` ~1.3x at 1T), so the
34/// detector declines and the generic path runs.
35const TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT: usize = 8;
36
37/// Internal validation marker used by kernel-family execution adapters.
38///
39/// This marker is not bound to a destination. It does not authorize unchecked
40/// replay with a different geometry; callers of the unsafe execution entries
41/// must establish that association themselves.
42#[derive(Clone, Copy, Debug, Default)]
43pub struct ValidatedDestinationLayout(());
44
45/// Validate destination injectivity and return the replay marker.
46///
47/// Runs the exact (allocation-free pairwise, then spilling) injectivity check,
48/// so every injective layout the other kernel entry points accept is accepted
49/// here. Kernel families that need the marker for more than one step (for
50/// example validating shapes and then taking a specialization) should call this
51/// once and pass the marker on instead of re-running the check.
52///
53/// # Errors
54///
55/// Returns [`StridedError::NonInjectiveOutputLayout`] when the layout cannot
56/// be proven representable and injective: two distinct logical indices of
57/// `dims` name the same offset under `strides`, or the element count or the
58/// address span is too large to represent. A rejected layout need not be
59/// colliding.
60#[inline]
61pub fn validate_destination_layout(
62    dims: &[usize],
63    strides: &[isize],
64) -> Result<ValidatedDestinationLayout> {
65    if crate::layout_check::is_injective_layout(dims, strides) {
66        Ok(ValidatedDestinationLayout(()))
67    } else {
68        Err(StridedError::NonInjectiveOutputLayout)
69    }
70}
71
72#[inline]
73/// Validate injectivity without allocating a set of offsets.
74///
75/// # Examples
76///
77/// ```
78/// use strided_basic::execution::validate_destination_layout_without_alloc;
79/// let marker = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
80/// assert!(validate_destination_layout_without_alloc(&[2], &[0]).is_err());
81/// let _ = marker;
82/// ```
83///
84/// # Errors
85/// Returns `NonInjectiveOutputLayout` for invalid or unsupported output geometry.
86pub fn validate_destination_layout_without_alloc(
87    dims: &[usize],
88    strides: &[isize],
89) -> Result<ValidatedDestinationLayout> {
90    if crate::layout_check::is_injective_layout_without_alloc(dims, strides) {
91        Ok(ValidatedDestinationLayout(()))
92    } else {
93        Err(StridedError::NonInjectiveOutputLayout)
94    }
95}
96
97fn reachable_byte_range(
98    ptr: usize,
99    elem_size: usize,
100    dims: &[usize],
101    strides: &[isize],
102) -> Result<Option<(usize, usize)>> {
103    if elem_size == 0 || dims.contains(&0) {
104        return Ok(None);
105    }
106
107    let mut min_offset = 0isize;
108    let mut max_offset = 0isize;
109    for (&dim, &stride) in dims.iter().zip(strides) {
110        if dim <= 1 {
111            continue;
112        }
113        let extent = isize::try_from(dim - 1).map_err(|_| StridedError::OffsetOverflow)?;
114        let span = stride
115            .checked_mul(extent)
116            .ok_or(StridedError::OffsetOverflow)?;
117        if span < 0 {
118            min_offset = min_offset
119                .checked_add(span)
120                .ok_or(StridedError::OffsetOverflow)?;
121        } else {
122            max_offset = max_offset
123                .checked_add(span)
124                .ok_or(StridedError::OffsetOverflow)?;
125        }
126    }
127
128    let ptr = ptr as i128;
129    let elem_size = elem_size as i128;
130    let start = ptr
131        .checked_add(
132            (min_offset as i128)
133                .checked_mul(elem_size)
134                .ok_or(StridedError::OffsetOverflow)?,
135        )
136        .ok_or(StridedError::OffsetOverflow)?;
137    let end = ptr
138        .checked_add(
139            (max_offset as i128)
140                .checked_mul(elem_size)
141                .and_then(|offset| offset.checked_add(elem_size))
142                .ok_or(StridedError::OffsetOverflow)?,
143        )
144        .ok_or(StridedError::OffsetOverflow)?;
145    if start < 0 || end < 0 || start > usize::MAX as i128 || end > usize::MAX as i128 {
146        return Err(StridedError::OffsetOverflow);
147    }
148    Ok(Some((start as usize, end as usize)))
149}
150
151fn validate_typed_no_overlap<D, A, Op: ElementOp<A>>(
152    dest: &StridedViewMut<MaybeUninit<D>>,
153    input: &StridedView<A, Op>,
154    input_index: usize,
155) -> Result<()> {
156    let dest_range = reachable_byte_range(
157        dest.ptr() as usize,
158        core::mem::size_of::<D>(),
159        dest.dims(),
160        dest.strides(),
161    )?;
162    let input_range = reachable_byte_range(
163        input.ptr() as usize,
164        core::mem::size_of::<A>(),
165        input.dims(),
166        input.strides(),
167    )?;
168    if let (Some((dest_start, dest_end)), Some((input_start, input_end))) =
169        (dest_range, input_range)
170    {
171        if dest_start < input_end && input_start < dest_end {
172            return Err(StridedError::OverlappingInputOutput { input: input_index });
173        }
174    }
175    Ok(())
176}
177
178// ============================================================================
179// Stride-specialized inner loop helpers
180//
181// When all inner strides are 1 (contiguous in the innermost dimension),
182// we use slice-based iteration so LLVM can auto-vectorize effectively.
183// This is the Rust equivalent of Julia's @simd on the innermost loop.
184// ============================================================================
185
186// INVARIANT: validated layouts keep every dereferenced cursor in bounds.
187// Pointwise leaves use wrapping advances: the unused final cursor may lie
188// outside the allocation for negative or gapped strides.
189/// Unary inner loop: `dest[i] = f(Op::apply(src[i]))` for `len` elements.
190///
191/// A vectorized kernel a map operation offers its flat runs to.
192///
193/// [`NoKernel`] is what the entry points without a kernel pass, so the check it introduces is
194/// monomorphized away; [`SliceKernel`] adapts the closure a caller supplied.
195trait MapKernel<A, D> {
196    /// Writes every element of `dst` from `src`, or writes nothing and returns `false`.
197    fn apply(&self, src: &[A], dst: &mut [D]) -> bool;
198}
199
200struct NoKernel;
201
202impl<A, D> MapKernel<A, D> for NoKernel {
203    #[inline(always)]
204    fn apply(&self, _src: &[A], _dst: &mut [D]) -> bool {
205        false
206    }
207}
208
209struct SliceKernel<K>(K);
210
211impl<A, D, K: Fn(&[A], &mut [D]) -> bool> MapKernel<A, D> for SliceKernel<K> {
212    #[inline(always)]
213    fn apply(&self, src: &[A], dst: &mut [D]) -> bool {
214        (self.0)(src, dst)
215    }
216}
217
218unsafe fn inner_loop_map1<D: Copy, A: Copy, Op: ElementOp<A>, K: MapKernel<A, D>>(
219    dp: *mut D,
220    ds: isize,
221    sp: *const A,
222    ss: isize,
223    len: usize,
224    f: &impl Fn(A) -> D,
225    kernel: &K,
226) {
227    if ds == 1 && ss == 1 {
228        let src = std::slice::from_raw_parts(sp, len);
229        let dst = std::slice::from_raw_parts_mut(dp, len);
230        if kernel.apply(src, dst) {
231            return;
232        }
233        simd::dispatch_if_large(len, || {
234            for (d, s) in dst.iter_mut().zip(src.iter()) {
235                *d = f(Op::apply(*s));
236            }
237        });
238    } else {
239        let mut dp = dp;
240        let mut sp = sp;
241        simd::dispatch_if_large(len, || {
242            for _ in 0..len {
243                *dp = f(Op::apply(*sp));
244                dp = dp.wrapping_offset(ds);
245                sp = sp.wrapping_offset(ss);
246            }
247        });
248    }
249}
250
251/// Binary inner loop: `dest[i] = f(OpA::apply(a[i]), OpB::apply(b[i]))`.
252// Two operand pointers with their strides, the destination, the length and the
253// element op: the call's inputs, not a block's.
254#[allow(clippy::too_many_arguments)]
255#[inline(always)]
256unsafe fn inner_loop_map2<D: Copy, A: Copy, B: Copy, OpA: ElementOp<A>, OpB: ElementOp<B>>(
257    dp: *mut D,
258    ds: isize,
259    ap: *const A,
260    a_s: isize,
261    bp: *const B,
262    b_s: isize,
263    len: usize,
264    f: &impl Fn(A, B) -> D,
265) {
266    if ds == 1 && a_s == 1 && b_s == 1 {
267        let src_a = std::slice::from_raw_parts(ap, len);
268        let src_b = std::slice::from_raw_parts(bp, len);
269        let dst = std::slice::from_raw_parts_mut(dp, len);
270        simd::dispatch_if_large(len, || {
271            for i in 0..len {
272                dst[i] = f(OpA::apply(src_a[i]), OpB::apply(src_b[i]));
273            }
274        });
275    } else if ds == 1 && a_s == 1 && b_s == 0 {
276        let src_a = std::slice::from_raw_parts(ap, len);
277        let b = OpB::apply(*bp);
278        let dst = std::slice::from_raw_parts_mut(dp, len);
279        simd::dispatch_if_large(len, || {
280            for i in 0..len {
281                dst[i] = f(OpA::apply(src_a[i]), b);
282            }
283        });
284    } else if ds == 1 && a_s == 0 && b_s == 1 {
285        let a = OpA::apply(*ap);
286        let src_b = std::slice::from_raw_parts(bp, len);
287        let dst = std::slice::from_raw_parts_mut(dp, len);
288        simd::dispatch_if_large(len, || {
289            for i in 0..len {
290                dst[i] = f(a, OpB::apply(src_b[i]));
291            }
292        });
293    } else if ds == 1 && a_s == 0 && b_s == 0 {
294        let a = OpA::apply(*ap);
295        let b = OpB::apply(*bp);
296        let dst = std::slice::from_raw_parts_mut(dp, len);
297        simd::dispatch_if_large(len, || {
298            for d in dst.iter_mut() {
299                *d = f(a, b);
300            }
301        });
302    } else if ds == 1 && b_s == 0 {
303        let b = OpB::apply(*bp);
304        let dst = std::slice::from_raw_parts_mut(dp, len);
305        let mut ap = ap;
306        simd::dispatch_if_large(len, || {
307            for d in dst.iter_mut() {
308                *d = f(OpA::apply(*ap), b);
309                ap = ap.wrapping_offset(a_s);
310            }
311        });
312    } else if ds == 1 && a_s == 0 {
313        let a = OpA::apply(*ap);
314        let dst = std::slice::from_raw_parts_mut(dp, len);
315        let mut bp = bp;
316        simd::dispatch_if_large(len, || {
317            for d in dst.iter_mut() {
318                *d = f(a, OpB::apply(*bp));
319                bp = bp.wrapping_offset(b_s);
320            }
321        });
322    } else {
323        let mut dp = dp;
324        let mut ap = ap;
325        let mut bp = bp;
326        simd::dispatch_if_large(len, || {
327            for _ in 0..len {
328                *dp = f(OpA::apply(*ap), OpB::apply(*bp));
329                dp = dp.wrapping_offset(ds);
330                ap = ap.wrapping_offset(a_s);
331                bp = bp.wrapping_offset(b_s);
332            }
333        });
334    }
335}
336
337/// Binary multiplication inner loop for identity element ops.
338trait MulOutput<D: Copy + 'static>: Copy + Send + Sync + 'static {
339    type Slot: Copy + Send + Sync + 'static;
340
341    unsafe fn write(dst: *mut Self::Slot, value: D);
342
343    unsafe fn try_contiguous<A: 'static, B: 'static>(
344        dst: *mut Self::Slot,
345        len: usize,
346        a: &[A],
347        b: &[B],
348    ) -> bool;
349}
350
351#[derive(Clone, Copy)]
352struct InitializedOutput;
353
354impl<D: Copy + Send + Sync + 'static> MulOutput<D> for InitializedOutput {
355    type Slot = D;
356
357    #[inline(always)]
358    unsafe fn write(dst: *mut D, value: D) {
359        unsafe { dst.write(value) };
360    }
361
362    #[inline(always)]
363    unsafe fn try_contiguous<A: 'static, B: 'static>(
364        dst: *mut D,
365        len: usize,
366        a: &[A],
367        b: &[B],
368    ) -> bool {
369        unsafe { simd::try_mul_contiguous_ptr(dst, len, a, b) }
370    }
371}
372
373#[derive(Clone, Copy)]
374struct UninitializedOutput;
375
376impl<D: Copy + Send + Sync + 'static> MulOutput<D> for UninitializedOutput {
377    type Slot = MaybeUninit<D>;
378
379    #[inline(always)]
380    unsafe fn write(dst: *mut MaybeUninit<D>, value: D) {
381        unsafe { dst.write(MaybeUninit::new(value)) };
382    }
383
384    #[inline(always)]
385    unsafe fn try_contiguous<A: 'static, B: 'static>(
386        dst: *mut MaybeUninit<D>,
387        len: usize,
388        a: &[A],
389        b: &[B],
390    ) -> bool {
391        // SIMD stores write complete values directly through raw pointers; no
392        // initialized reference is formed over the uninitialized destination.
393        unsafe { simd::try_mul_contiguous_ptr(dst.cast::<D>(), len, a, b) }
394    }
395}
396
397#[inline(always)]
398fn multiply_value<A, B, D>(lhs: A, rhs: B) -> D
399where
400    A: Copy + Mul<B, Output = D> + 'static,
401    B: Copy + 'static,
402    D: Copy + 'static,
403{
404    use core::any::TypeId;
405
406    if TypeId::of::<A>() == TypeId::of::<i32>()
407        && TypeId::of::<B>() == TypeId::of::<i32>()
408        && TypeId::of::<D>() == TypeId::of::<i32>()
409    {
410        // The exact TypeId checks prove identical layout and validity.
411        let lhs = unsafe { *(&lhs as *const A).cast::<i32>() };
412        let rhs = unsafe { *(&rhs as *const B).cast::<i32>() };
413        let value = lhs.wrapping_mul(rhs);
414        return unsafe { core::mem::transmute_copy(&value) };
415    }
416    if TypeId::of::<A>() == TypeId::of::<i64>()
417        && TypeId::of::<B>() == TypeId::of::<i64>()
418        && TypeId::of::<D>() == TypeId::of::<i64>()
419    {
420        // The exact TypeId checks prove identical layout and validity.
421        let lhs = unsafe { *(&lhs as *const A).cast::<i64>() };
422        let rhs = unsafe { *(&rhs as *const B).cast::<i64>() };
423        let value = lhs.wrapping_mul(rhs);
424        return unsafe { core::mem::transmute_copy(&value) };
425    }
426    lhs * rhs
427}
428
429#[inline(always)]
430unsafe fn inner_loop_mul2<
431    O: MulOutput<D>,
432    D: Copy + 'static,
433    A: Copy + Mul<B, Output = D> + 'static,
434    B: Copy + 'static,
435>(
436    dp: *mut O::Slot,
437    ds: isize,
438    ap: *const A,
439    a_s: isize,
440    bp: *const B,
441    b_s: isize,
442    len: usize,
443) {
444    if ds == 1 && a_s == 1 && b_s == 1 {
445        let src_a = std::slice::from_raw_parts(ap, len);
446        let src_b = std::slice::from_raw_parts(bp, len);
447        if len >= 64 && O::try_contiguous(dp, len, src_a, src_b) {
448            return;
449        }
450        simd::dispatch_if_large(len, || {
451            for i in 0..len {
452                O::write(dp.add(i), multiply_value(src_a[i], src_b[i]));
453            }
454        });
455    } else if ds == 1 && a_s == 1 && b_s == 0 {
456        let src_a = std::slice::from_raw_parts(ap, len);
457        let b = *bp;
458        simd::dispatch_if_large(len, || {
459            // The stride-dispatch branches below all walk `0..len` against the
460            // destination pointer; keep this one in the same shape.
461            #[allow(clippy::needless_range_loop)]
462            for i in 0..len {
463                O::write(dp.add(i), multiply_value(src_a[i], b));
464            }
465        });
466    } else if ds == 1 && a_s == 0 && b_s == 1 {
467        let a = *ap;
468        let src_b = std::slice::from_raw_parts(bp, len);
469        simd::dispatch_if_large(len, || {
470            // Same shape as the branch above, mirrored: the walk is over the
471            // destination pointer.
472            #[allow(clippy::needless_range_loop)]
473            for i in 0..len {
474                O::write(dp.add(i), multiply_value(a, src_b[i]));
475            }
476        });
477    } else if ds == 1 && a_s == 0 && b_s == 0 {
478        let a = *ap;
479        let b = *bp;
480        simd::dispatch_if_large(len, || {
481            for i in 0..len {
482                O::write(dp.add(i), multiply_value(a, b));
483            }
484        });
485    } else if ds == 1 && b_s == 0 {
486        let b = *bp;
487        let mut ap = ap;
488        simd::dispatch_if_large(len, || {
489            for i in 0..len {
490                O::write(dp.add(i), multiply_value(*ap, b));
491                ap = ap.wrapping_offset(a_s);
492            }
493        });
494    } else if ds == 1 && a_s == 0 {
495        let a = *ap;
496        let mut bp = bp;
497        simd::dispatch_if_large(len, || {
498            for i in 0..len {
499                O::write(dp.add(i), multiply_value(a, *bp));
500                bp = bp.wrapping_offset(b_s);
501            }
502        });
503    } else {
504        let mut dp = dp;
505        let mut ap = ap;
506        let mut bp = bp;
507        simd::dispatch_if_large(len, || {
508            for _ in 0..len {
509                O::write(dp, multiply_value(*ap, *bp));
510                dp = dp.wrapping_offset(ds);
511                ap = ap.wrapping_offset(a_s);
512                bp = bp.wrapping_offset(b_s);
513            }
514        });
515    }
516}
517
518#[derive(Clone, Debug, Eq, PartialEq)]
519struct ContiguousMulRangePlan {
520    axis_order: AxisVec<usize>,
521    inner_len: usize,
522    inner_axis_count: usize,
523    row_len: usize,
524    outer_axis_start: usize,
525    fast_axis: usize,
526    a_fast_stride: isize,
527    b_fast_stride: isize,
528    a_row_stride: isize,
529    b_row_stride: isize,
530}
531
532impl ContiguousMulRangePlan {
533    /// Whether running the plan row by row, without cache blocking, is sound
534    /// for performance.
535    ///
536    /// The destination is walked contiguously, so a unit-stride or broadcast
537    /// (stride 0) input streams along with it. An input read at any other
538    /// stride along the fast axis (a transposed operand) is swept across the
539    /// whole fast axis once per row, touching a new cache line per element and
540    /// again on the next row; the blocked kernel tiles that access. The one
541    /// exception is the transposed-scalar tile, which blocks itself.
542    fn walks_inputs_without_blocking(&self) -> bool {
543        let streams = |stride: isize| stride == 0 || stride == 1;
544        if streams(self.a_fast_stride) && streams(self.b_fast_stride) {
545            return true;
546        }
547        if transposed_scalar_tile_kind(self).is_some() {
548            return true;
549        }
550        false
551    }
552}
553
554#[derive(Clone, Copy, Debug, Eq, PartialEq)]
555enum TransposedScalarTileKind {
556    RhsScalar,
557    LhsScalar,
558}
559
560fn transposed_scalar_tile_kind(plan: &ContiguousMulRangePlan) -> Option<TransposedScalarTileKind> {
561    if plan.inner_len > TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT.saturating_mul(plan.row_len) {
562        return None;
563    }
564    let row_len = isize::try_from(plan.row_len).ok()?;
565    if plan.a_fast_stride == row_len
566        && plan.a_row_stride == 1
567        && plan.b_fast_stride == 0
568        && plan.b_row_stride == 0
569    {
570        return Some(TransposedScalarTileKind::RhsScalar);
571    }
572
573    if plan.b_fast_stride == row_len
574        && plan.b_row_stride == 1
575        && plan.a_fast_stride == 0
576        && plan.a_row_stride == 0
577    {
578        return Some(TransposedScalarTileKind::LhsScalar);
579    }
580
581    None
582}
583
584fn compact_axis_order(dims: &[usize], strides: &[isize]) -> Option<AxisVec<usize>> {
585    if dims.len() != strides.len() {
586        return None;
587    }
588
589    let mut active = AxisVec::<usize>::new();
590    let mut inactive = AxisVec::<usize>::new();
591    for (axis, (&dim, &stride)) in dims.iter().zip(strides.iter()).enumerate() {
592        if stride < 0 {
593            return None;
594        }
595        if dim > 1 {
596            active.push(axis);
597        } else {
598            inactive.push(axis);
599        }
600    }
601
602    active.sort_by(|&lhs, &rhs| strides[lhs].cmp(&strides[rhs]).then_with(|| lhs.cmp(&rhs)));
603
604    let mut expected = 1isize;
605    for &axis in &active {
606        if strides[axis] != expected {
607            return None;
608        }
609        expected = expected.saturating_mul(dims[axis] as isize);
610    }
611
612    active.extend(inactive);
613    Some(active)
614}
615
616fn can_fuse_contiguous_range_axis(dim: usize, prev_stride: isize, next_stride: isize) -> bool {
617    dim <= 1 || (prev_stride == 0 && next_stride == 0) || next_stride == prev_stride * dim as isize
618}
619
620fn contiguous_mul_range_plan(
621    dims: &[usize],
622    dst_strides: &[isize],
623    a_strides: &[isize],
624    b_strides: &[isize],
625) -> Option<ContiguousMulRangePlan> {
626    let axis_order = compact_axis_order(dims, dst_strides)?;
627    if dims.is_empty() {
628        return Some(ContiguousMulRangePlan {
629            axis_order,
630            inner_len: 1,
631            inner_axis_count: 0,
632            row_len: 1,
633            outer_axis_start: 0,
634            fast_axis: 0,
635            a_fast_stride: 0,
636            b_fast_stride: 0,
637            a_row_stride: 0,
638            b_row_stride: 0,
639        });
640    }
641
642    let first_pos = axis_order
643        .iter()
644        .position(|&axis| dims[axis] > 1)
645        .unwrap_or(0);
646    let first_axis = axis_order[first_pos];
647    let mut inner_len = dims[first_axis].max(1);
648    let mut inner_axis_count = first_pos + 1;
649    let mut prev_axis = first_axis;
650
651    for &axis in axis_order.iter().skip(first_pos + 1) {
652        if can_fuse_contiguous_range_axis(
653            dims[prev_axis],
654            dst_strides[prev_axis],
655            dst_strides[axis],
656        ) && can_fuse_contiguous_range_axis(
657            dims[prev_axis],
658            a_strides[prev_axis],
659            a_strides[axis],
660        ) && can_fuse_contiguous_range_axis(
661            dims[prev_axis],
662            b_strides[prev_axis],
663            b_strides[axis],
664        ) {
665            inner_len = inner_len.checked_mul(dims[axis].max(1))?;
666            inner_axis_count += 1;
667            prev_axis = axis;
668        } else {
669            break;
670        }
671    }
672
673    let row = axis_order
674        .iter()
675        .enumerate()
676        .skip(inner_axis_count)
677        .find(|&(_, &axis)| dims[axis] > 1);
678    let (row_len, outer_axis_start, a_row_stride, b_row_stride) =
679        if let Some((row_pos, &row_axis)) = row {
680            (
681                dims[row_axis],
682                row_pos + 1,
683                a_strides[row_axis],
684                b_strides[row_axis],
685            )
686        } else {
687            (1, axis_order.len(), 0, 0)
688        };
689
690    Some(ContiguousMulRangePlan {
691        axis_order,
692        inner_len,
693        inner_axis_count,
694        row_len,
695        outer_axis_start,
696        fast_axis: first_axis,
697        a_fast_stride: a_strides[first_axis],
698        b_fast_stride: b_strides[first_axis],
699        a_row_stride,
700        b_row_stride,
701    })
702}
703
704struct ContiguousMulOuterCursor<'a> {
705    dims: &'a [usize],
706    a_strides: &'a [isize],
707    b_strides: &'a [isize],
708    axes: AxisVec<usize>,
709    coords: AxisVec<usize>,
710    a_offset: isize,
711    b_offset: isize,
712}
713
714impl<'a> ContiguousMulOuterCursor<'a> {
715    fn new(
716        dims: &'a [usize],
717        a_strides: &'a [isize],
718        b_strides: &'a [isize],
719        plan: &ContiguousMulRangePlan,
720        outer_group: usize,
721    ) -> Self {
722        let axes: AxisVec<usize> = plan
723            .axis_order
724            .iter()
725            .skip(plan.outer_axis_start)
726            .copied()
727            .collect();
728        let mut coords = AxisVec::<usize>::with_capacity(axes.len());
729        let mut rem = outer_group;
730        let mut a_offset = 0isize;
731        let mut b_offset = 0isize;
732
733        for &axis in &axes {
734            let dim = dims[axis].max(1);
735            let coord = rem % dim;
736            rem /= dim;
737            coords.push(coord);
738            a_offset += coord as isize * a_strides[axis];
739            b_offset += coord as isize * b_strides[axis];
740        }
741
742        Self {
743            dims,
744            a_strides,
745            b_strides,
746            axes,
747            coords,
748            a_offset,
749            b_offset,
750        }
751    }
752
753    fn advance(&mut self) {
754        for (i, &axis) in self.axes.iter().enumerate() {
755            let dim = self.dims[axis].max(1);
756            if dim <= 1 {
757                continue;
758            }
759
760            // Advance only when another position along this axis follows,
761            // and rewind from the last one, so offsets stay reachable.
762            if self.coords[i] + 1 < dim {
763                self.coords[i] += 1;
764                self.a_offset += self.a_strides[axis];
765                self.b_offset += self.b_strides[axis];
766                break;
767            }
768
769            let last = (dim - 1) as isize;
770            self.coords[i] = 0;
771            self.a_offset -= last * self.a_strides[axis];
772            self.b_offset -= last * self.b_strides[axis];
773        }
774    }
775}
776
777// Row block: both operand bases with their strides, the plan, the start index and
778// the element count.
779#[allow(clippy::too_many_arguments)]
780#[inline(always)]
781unsafe fn run_contiguous_mul_row_block<
782    O: MulOutput<D>,
783    D: Copy + 'static,
784    A: Copy + Mul<B, Output = D> + 'static,
785    B: Copy + 'static,
786>(
787    dst_ptr: *mut O::Slot,
788    a_ptr: *const A,
789    b_ptr: *const B,
790    plan: &ContiguousMulRangePlan,
791    base_index: usize,
792    total: usize,
793    base_a_offset: isize,
794    base_b_offset: isize,
795) {
796    let inner_len = plan.inner_len.max(1);
797    let row_len = plan.row_len.max(1);
798    let block_len = inner_len.saturating_mul(row_len);
799
800    if total.saturating_sub(base_index) >= block_len {
801        let tiled = match transposed_scalar_tile_kind(plan) {
802            Some(TransposedScalarTileKind::RhsScalar) => {
803                simd::try_mul_transposed_scalar_rhs_2d::<D, A, B>(
804                    dst_ptr.add(base_index).cast::<D>(),
805                    a_ptr.offset(base_a_offset),
806                    b_ptr.offset(base_b_offset),
807                    inner_len,
808                    row_len,
809                    plan.a_fast_stride,
810                    plan.a_row_stride,
811                )
812            }
813            Some(TransposedScalarTileKind::LhsScalar) => {
814                simd::try_mul_transposed_scalar_lhs_2d::<D, A, B>(
815                    dst_ptr.add(base_index).cast::<D>(),
816                    a_ptr.offset(base_a_offset),
817                    b_ptr.offset(base_b_offset),
818                    inner_len,
819                    row_len,
820                    plan.b_fast_stride,
821                    plan.b_row_stride,
822                )
823            }
824            None => false,
825        };
826        if tiled {
827            return;
828        }
829    }
830
831    let mut index = base_index;
832    let mut a_offset = base_a_offset;
833    let mut b_offset = base_b_offset;
834
835    for _ in 0..row_len {
836        if index >= total {
837            break;
838        }
839        let len = inner_len.min(total - index);
840        inner_loop_mul2::<O, D, A, B>(
841            dst_ptr.add(index),
842            1,
843            a_ptr.offset(a_offset),
844            plan.a_fast_stride,
845            b_ptr.offset(b_offset),
846            plan.b_fast_stride,
847            len,
848        );
849        index += inner_len;
850        a_offset += plan.a_row_stride;
851        b_offset += plan.b_row_stride;
852    }
853}
854
855fn strided_offset_for_contiguous_linear_index(
856    dims: &[usize],
857    strides: &[isize],
858    axis_order: &[usize],
859    mut index: usize,
860) -> isize {
861    let mut offset = 0isize;
862    for &axis in axis_order {
863        let dim = dims[axis];
864        if dim == 0 {
865            return 0;
866        }
867        let coord = index % dim;
868        index /= dim;
869        offset += coord as isize * strides[axis];
870    }
871    offset
872}
873
874// Raw parts entry for the whole contiguous product: the three base pointers, the
875// dims and strides of both operands and the executor. The argument list is wide
876// because this function splits the work itself (`parallel_for_each` below) from
877// exactly these values; nothing calls it once per worker range.
878#[allow(clippy::too_many_arguments)]
879fn try_contiguous_range_mul<
880    O: MulOutput<D>,
881    D: Copy + Send + Sync + 'static,
882    A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
883    B: Copy + Send + Sync + 'static,
884>(
885    dst_ptr: *mut O::Slot,
886    dims: &[usize],
887    dst_strides: &[isize],
888    a_ptr: *const A,
889    a_strides: &[isize],
890    b_ptr: *const B,
891    b_strides: &[isize],
892    exec: Exec<'_>,
893) -> bool {
894    // An overflowing element count falls back to the general path, which
895    // reports it as `OffsetOverflow`.
896    let Ok(total) = total_len(dims) else {
897        return false;
898    };
899    if total == 0 {
900        return true;
901    }
902    // One flat section over the chunks: the width is the token's decision.
903    let exec = for_len(exec, KernelClass::Map, total);
904    if exec.is_serial() {
905        return false;
906    }
907
908    let Some(plan) = contiguous_mul_range_plan(dims, dst_strides, a_strides, b_strides) else {
909        return false;
910    };
911    if !plan.walks_inputs_without_blocking() {
912        return false;
913    }
914
915    let inner_len = plan.inner_len.max(1);
916    let row_len = plan.row_len.max(1);
917    let block_len = inner_len.saturating_mul(row_len).max(1);
918    let outer_groups = total.div_ceil(block_len);
919
920    let nthreads = exec.budget().get();
921    if nthreads > 1 {
922        use crate::threading::{parallel_for_each, SendPtr};
923
924        let dst = SendPtr(dst_ptr);
925        let a = SendPtr(a_ptr as *mut A);
926        let b = SendPtr(b_ptr as *mut B);
927
928        if outer_groups < nthreads {
929            let chunk_len = total.div_ceil(nthreads);
930            let nchunks = total.div_ceil(chunk_len);
931
932            parallel_for_each(0..nchunks, exec, &|chunks| {
933                for chunk in chunks {
934                    let start = chunk * chunk_len;
935                    let end = (start + chunk_len).min(total);
936                    let mut index = start;
937
938                    while index < end {
939                        let in_inner = index % inner_len;
940                        let len = (inner_len - in_inner).min(end - index);
941                        let a_offset = strided_offset_for_contiguous_linear_index(
942                            dims,
943                            a_strides,
944                            &plan.axis_order,
945                            index,
946                        );
947                        let b_offset = strided_offset_for_contiguous_linear_index(
948                            dims,
949                            b_strides,
950                            &plan.axis_order,
951                            index,
952                        );
953
954                        unsafe {
955                            inner_loop_mul2::<O, D, A, B>(
956                                dst.as_ptr().add(index),
957                                1,
958                                a.as_const().offset(a_offset),
959                                plan.a_fast_stride,
960                                b.as_const().offset(b_offset),
961                                plan.b_fast_stride,
962                                len,
963                            );
964                        }
965                        index += len;
966                    }
967                }
968            });
969
970            return true;
971        }
972
973        let groups_per_chunk = outer_groups.div_ceil(nthreads);
974        let nchunks = outer_groups.div_ceil(groups_per_chunk);
975
976        parallel_for_each(0..nchunks, exec, &|chunks| {
977            for chunk in chunks {
978                let group_start = chunk * groups_per_chunk;
979                let group_end = (group_start + groups_per_chunk).min(outer_groups);
980                let mut cursor =
981                    ContiguousMulOuterCursor::new(dims, a_strides, b_strides, &plan, group_start);
982
983                for group in group_start..group_end {
984                    let index = group * block_len;
985                    unsafe {
986                        run_contiguous_mul_row_block::<O, D, A, B>(
987                            dst.as_ptr(),
988                            a.as_const(),
989                            b.as_const(),
990                            &plan,
991                            index,
992                            total,
993                            cursor.a_offset,
994                            cursor.b_offset,
995                        );
996                    }
997                    cursor.advance();
998                }
999            }
1000        });
1001
1002        true
1003    } else {
1004        run_contiguous_range_mul_single_thread::<O, D, A, B>(
1005            dst_ptr,
1006            dims,
1007            a_ptr,
1008            a_strides,
1009            b_ptr,
1010            b_strides,
1011            &plan,
1012            total,
1013            block_len,
1014            outer_groups,
1015        )
1016    }
1017}
1018
1019// Worker-range leaf of the contiguous multiply: both operand bases and stride
1020// tables, the plan, the extents and the range.
1021#[allow(clippy::too_many_arguments)]
1022fn run_contiguous_range_mul_single_thread<
1023    O: MulOutput<D>,
1024    D: Copy + Send + Sync + 'static,
1025    A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
1026    B: Copy + Send + Sync + 'static,
1027>(
1028    dst_ptr: *mut O::Slot,
1029    dims: &[usize],
1030    a_ptr: *const A,
1031    a_strides: &[isize],
1032    b_ptr: *const B,
1033    b_strides: &[isize],
1034    plan: &ContiguousMulRangePlan,
1035    total: usize,
1036    block_len: usize,
1037    outer_groups: usize,
1038) -> bool {
1039    let mut cursor = ContiguousMulOuterCursor::new(dims, a_strides, b_strides, plan, 0);
1040    for group in 0..outer_groups {
1041        let index = group * block_len;
1042        unsafe {
1043            run_contiguous_mul_row_block::<O, D, A, B>(
1044                dst_ptr,
1045                a_ptr,
1046                b_ptr,
1047                plan,
1048                index,
1049                total,
1050                cursor.a_offset,
1051                cursor.b_offset,
1052            );
1053        }
1054        cursor.advance();
1055    }
1056    true
1057}
1058
1059/// Ternary inner loop: `dest[i] = f(a[i], b[i], c[i])`.
1060// Three operand pointers with their strides, the destination, the length and the
1061// element op: the call's inputs, not a block's.
1062#[allow(clippy::too_many_arguments)]
1063#[inline(always)]
1064unsafe fn inner_loop_map3<
1065    D: Copy,
1066    A: Copy,
1067    B: Copy,
1068    C: Copy,
1069    OpA: ElementOp<A>,
1070    OpB: ElementOp<B>,
1071    OpC: ElementOp<C>,
1072>(
1073    dp: *mut D,
1074    ds: isize,
1075    ap: *const A,
1076    a_s: isize,
1077    bp: *const B,
1078    b_s: isize,
1079    cp: *const C,
1080    c_s: isize,
1081    len: usize,
1082    f: &impl Fn(A, B, C) -> D,
1083) {
1084    if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 {
1085        let src_a = std::slice::from_raw_parts(ap, len);
1086        let src_b = std::slice::from_raw_parts(bp, len);
1087        let src_c = std::slice::from_raw_parts(cp, len);
1088        let dst = std::slice::from_raw_parts_mut(dp, len);
1089        simd::dispatch_if_large(len, || {
1090            for (((d, &a), &b), &c) in dst.iter_mut().zip(src_a).zip(src_b).zip(src_c) {
1091                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
1092            }
1093        });
1094    } else {
1095        let mut dp = dp;
1096        let mut ap = ap;
1097        let mut bp = bp;
1098        let mut cp = cp;
1099        simd::dispatch_if_large(len, || {
1100            for _ in 0..len {
1101                *dp = f(OpA::apply(*ap), OpB::apply(*bp), OpC::apply(*cp));
1102                dp = dp.wrapping_offset(ds);
1103                ap = ap.wrapping_offset(a_s);
1104                bp = bp.wrapping_offset(b_s);
1105                cp = cp.wrapping_offset(c_s);
1106            }
1107        });
1108    }
1109}
1110
1111/// Quaternary inner loop: `dest[i] = f(a[i], b[i], c[i], e[i])`.
1112// Four operand pointers with their strides, the destination, the length and the
1113// element op: the call's inputs, not a block's.
1114#[allow(clippy::too_many_arguments)]
1115#[inline(always)]
1116unsafe fn inner_loop_map4<
1117    D: Copy,
1118    A: Copy,
1119    B: Copy,
1120    C: Copy,
1121    E: Copy,
1122    OpA: ElementOp<A>,
1123    OpB: ElementOp<B>,
1124    OpC: ElementOp<C>,
1125    OpE: ElementOp<E>,
1126>(
1127    dp: *mut D,
1128    ds: isize,
1129    ap: *const A,
1130    a_s: isize,
1131    bp: *const B,
1132    b_s: isize,
1133    cp: *const C,
1134    c_s: isize,
1135    ep: *const E,
1136    e_s: isize,
1137    len: usize,
1138    f: &impl Fn(A, B, C, E) -> D,
1139) {
1140    if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 && e_s == 1 {
1141        let src_a = std::slice::from_raw_parts(ap, len);
1142        let src_b = std::slice::from_raw_parts(bp, len);
1143        let src_c = std::slice::from_raw_parts(cp, len);
1144        let src_e = std::slice::from_raw_parts(ep, len);
1145        let dst = std::slice::from_raw_parts_mut(dp, len);
1146        // Zipped iterators keep the loop free of bounds checks, which keeps the
1147        // dispatched closure small enough for pulp's target-feature function to
1148        // inline (tensor4all/strided-rs#340).
1149        simd::dispatch_if_large(len, || {
1150            for ((((d, &a), &b), &c), &e) in
1151                dst.iter_mut().zip(src_a).zip(src_b).zip(src_c).zip(src_e)
1152            {
1153                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
1154            }
1155        });
1156    } else {
1157        let mut dp = dp;
1158        let mut ap = ap;
1159        let mut bp = bp;
1160        let mut cp = cp;
1161        let mut ep = ep;
1162        simd::dispatch_if_large(len, || {
1163            for _ in 0..len {
1164                *dp = f(
1165                    OpA::apply(*ap),
1166                    OpB::apply(*bp),
1167                    OpC::apply(*cp),
1168                    OpE::apply(*ep),
1169                );
1170                dp = dp.wrapping_offset(ds);
1171                ap = ap.wrapping_offset(a_s);
1172                bp = bp.wrapping_offset(b_s);
1173                cp = cp.wrapping_offset(c_s);
1174                ep = ep.wrapping_offset(e_s);
1175            }
1176        });
1177    }
1178}
1179
1180/// An element function that carries its [`KernelClass`].
1181///
1182/// [`map_into`] and its siblings gate their parallel fork on the serial time of the
1183/// region, which is the element count times the throughput of the function's class.
1184/// A plain closure or function item is a [`KernelClass::Map`] (a few arithmetic
1185/// operations per element) through the blanket implementation, so existing callers
1186/// are unchanged. A caller whose function is expensive declares it with
1187/// [`Classed`], usually through [`KernelClass::with`], so a region too short to pay
1188/// for a fork as arithmetic is still forked when each element costs a libm call.
1189///
1190/// The trait is implemented for every `Fn(A) -> D + Sync` and for [`Classed`]; it is
1191/// not meant to be implemented by hand.
1192///
1193/// # Examples
1194/// ```
1195/// use strided_basic::execution::KernelClass;
1196/// use strided_basic::{map_into, Exec, StridedView, StridedViewMut};
1197///
1198/// let src = vec![0.0f64, 1.0, 2.0];
1199/// let mut out = vec![0.0f64; 3];
1200/// let mut dest = StridedViewMut::new(&mut out, &[3], &[1], 0).unwrap();
1201/// let source = StridedView::<f64>::new(&src, &[3], &[1], 0).unwrap();
1202/// // `exp` costs about as much as ten arithmetic operations: say so.
1203/// let exp = KernelClass::Transcendental.with(|x: f64| x.exp());
1204/// map_into(&mut dest, &source, exp, Exec::serial()).unwrap();
1205/// assert_eq!(out[0], 1.0);
1206/// ```
1207pub trait MapFn<A, D>: Sync {
1208    /// Apply the function to one element.
1209    fn call(&self, a: A) -> D;
1210
1211    /// Serial cost class the parallel gate charges per element; `Map` by default.
1212    fn class(&self) -> KernelClass {
1213        KernelClass::Map
1214    }
1215}
1216
1217impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for F {
1218    #[inline(always)]
1219    fn call(&self, a: A) -> D {
1220        self(a)
1221    }
1222}
1223
1224/// An element function together with the [`KernelClass`] it is charged as.
1225///
1226/// Build one with [`Classed::new`] or [`KernelClass::with`]; see [`MapFn`].
1227#[derive(Debug, Clone, Copy)]
1228pub struct Classed<F> {
1229    /// The element function.
1230    pub f: F,
1231    /// The class the parallel gate charges per element.
1232    pub class: KernelClass,
1233}
1234
1235impl<F> Classed<F> {
1236    /// Declare `f` as a function of cost class `class`.
1237    pub const fn new(class: KernelClass, f: F) -> Self {
1238        Self { f, class }
1239    }
1240}
1241
1242impl KernelClass {
1243    /// Declare an element function of this class, for [`map_into`] and its siblings.
1244    pub const fn with<F>(self, f: F) -> Classed<F> {
1245        Classed::new(self, f)
1246    }
1247}
1248
1249impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for Classed<F> {
1250    #[inline(always)]
1251    fn call(&self, a: A) -> D {
1252        (self.f)(a)
1253    }
1254
1255    fn class(&self) -> KernelClass {
1256        self.class
1257    }
1258}
1259
1260/// Apply a function element-wise from source to destination.
1261///
1262/// The element operation `Op` is applied lazily when reading from `src`.
1263/// Source and destination may have different element types.
1264///
1265/// `f` is any `Fn(A) -> D + Sync`, gated as [`KernelClass::Map`], or a function declared
1266/// with a class of its own (see [`MapFn`]), so an expensive element function forks sooner.
1267///
1268/// # Errors
1269///
1270/// Returns [`StridedError::RankMismatch`] or
1271/// [`StridedError::ShapeMismatch`] when the views have different ranks or
1272/// extents, and [`StridedError::NonInjectiveOutputLayout`] when the
1273/// destination layout cannot be proven representable and injective: two
1274/// logical indices name one offset, or the element count or address span is
1275/// too large to represent. Nothing is written unless validation succeeds.
1276pub fn map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1277    dest: &mut StridedViewMut<D>,
1278    src: &StridedView<A, Op>,
1279    f: impl MapFn<A, D>,
1280    exec: Exec<'_>,
1281) -> Result<()> {
1282    let class = f.class();
1283    map_parts_into::<D, A, Op, NoKernel>(
1284        dest.as_mut_ptr(),
1285        dest.dims(),
1286        dest.strides(),
1287        src.ptr(),
1288        src.dims(),
1289        src.strides(),
1290        |a| f.call(a),
1291        &NoKernel,
1292        class,
1293        exec,
1294    )
1295}
1296
1297/// `map_into` for a pure copy: the same traversal, gated as [`KernelClass::Copy`].
1298///
1299/// A copy has no arithmetic per element, so a fork over it pays later than over
1300/// a general element map; asking as `Map` forked it too early.
1301pub(crate) fn copy_map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1302    dest: &mut StridedViewMut<D>,
1303    src: &StridedView<A, Op>,
1304    f: impl Fn(A) -> D + Sync,
1305    exec: Exec<'_>,
1306) -> Result<()> {
1307    map_parts_into::<D, A, Op, NoKernel>(
1308        dest.as_mut_ptr(),
1309        dest.dims(),
1310        dest.strides(),
1311        src.ptr(),
1312        src.dims(),
1313        src.strides(),
1314        f,
1315        &NoKernel,
1316        KernelClass::Copy,
1317        exec,
1318    )
1319}
1320
1321pub(crate) fn map_into_validated<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1322    dest: &mut StridedViewMut<D>,
1323    src: &StridedView<A, Op>,
1324    f: impl MapFn<A, D>,
1325    validated: ValidatedDestinationLayout,
1326    exec: Exec<'_>,
1327) -> Result<()> {
1328    ensure_same_shape(dest.dims(), src.dims())?;
1329    let class = f.class();
1330    map_parts_into_validated::<D, A, Op, NoKernel>(
1331        dest.as_mut_ptr(),
1332        dest.dims(),
1333        dest.strides(),
1334        src.ptr(),
1335        src.strides(),
1336        |a| f.call(a),
1337        &NoKernel,
1338        validated,
1339        class,
1340        exec,
1341    )
1342}
1343
1344/// Apply an element-wise function into a potentially uninitialized destination, handing a
1345/// vectorized kernel the flat runs it can take.
1346///
1347/// `kernel` is offered a flat, unit-stride run whenever this crate has one to give: the whole
1348/// buffer when the two views share a contiguous layout and the operation runs sequentially, and
1349/// one block at a time on the parallel path. It must write every element of the run or write
1350/// nothing — returning `false` makes that run fall back to `scalar`, element by element. Runs whose
1351/// strides are not both 1 never reach it. It is called from whichever tasks the budget of
1352/// `exec` selected, so it must be `Sync`.
1353///
1354/// The point of the entry point is that a caller which owns vector kernels — a SIMD math library,
1355/// say — reuses this crate's layout logic and this crate's threading decision instead of
1356/// rebuilding either. `scalar` and `kernel` must agree up to the caller's own error bound; this
1357/// crate only guarantees that every destination element is written exactly once.
1358///
1359/// `scalar` may carry a [`KernelClass`] (see [`MapFn`]); the gate charges the region as that
1360/// class, so a transcendental `scalar` forks sooner.
1361///
1362/// # Examples
1363/// ```
1364/// use std::mem::MaybeUninit;
1365/// use strided_basic::{map_into_uninit_with_kernel, Exec, StridedView, StridedViewMut};
1366///
1367/// let src = vec![1.0f64, 2.0, 3.0];
1368/// let mut out = vec![MaybeUninit::<f64>::uninit(); 3];
1369/// let mut dest = StridedViewMut::new(&mut out, &[3], &[1], 0).unwrap();
1370/// let source = StridedView::new(&src, &[3], &[1], 0).unwrap();
1371/// let kernel = |src: &[f64], dst: &mut [MaybeUninit<f64>]| {
1372///     for (slot, value) in dst.iter_mut().zip(src) {
1373///         slot.write(value * 2.0);
1374///     }
1375///     true
1376/// };
1377/// map_into_uninit_with_kernel(&mut dest, &source, kernel, |x| x * 2.0, Exec::serial()).unwrap();
1378/// // SAFETY: the kernel wrote every element.
1379/// let values: Vec<f64> = out.into_iter().map(|x| unsafe { x.assume_init() }).collect();
1380/// assert_eq!(values, [2.0, 4.0, 6.0]);
1381/// ```
1382///
1383/// # Errors
1384/// Returns [`StridedError::ShapeMismatch`] when the shapes differ and
1385/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout cannot be proven
1386/// injective.
1387pub fn map_into_uninit_with_kernel<T: Copy + Send + Sync>(
1388    dest: &mut StridedViewMut<MaybeUninit<T>>,
1389    src: &StridedView<T>,
1390    kernel: impl Fn(&[T], &mut [MaybeUninit<T>]) -> bool + Sync,
1391    scalar: impl MapFn<T, T>,
1392    exec: Exec<'_>,
1393) -> Result<()> {
1394    let class = scalar.class();
1395    map_parts_into::<MaybeUninit<T>, T, Identity, _>(
1396        dest.as_mut_ptr(),
1397        dest.dims(),
1398        dest.strides(),
1399        src.ptr(),
1400        src.dims(),
1401        src.strides(),
1402        |value| MaybeUninit::new(scalar.call(value)),
1403        &SliceKernel(kernel),
1404        class,
1405        exec,
1406    )
1407}
1408
1409/// Map a raw destination from a raw source without preparing a plan.
1410///
1411/// Only the raw-replay unit tests need this entry now; the crate's production
1412/// callers either go through the fused traversal (raw ops, `CopyPlan`) or
1413/// [`map_raw_into_validated`].
1414#[cfg(test)]
1415pub(crate) fn map_raw_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1416    dest: &mut crate::RawStridedMut<'_, D>,
1417    src: &crate::RawStridedRef<'_, A>,
1418    f: impl Fn(A) -> D + Sync,
1419    exec: Exec<'_>,
1420) -> Result<()> {
1421    map_parts_into::<D, A, Op, NoKernel>(
1422        dest.as_mut_ptr(),
1423        dest.dims(),
1424        dest.strides(),
1425        src.ptr(),
1426        src.dims(),
1427        src.strides(),
1428        f,
1429        &NoKernel,
1430        KernelClass::Map,
1431        exec,
1432    )
1433}
1434
1435pub(crate) fn map_raw_into_validated<
1436    D: Copy + Send + Sync,
1437    A: Copy + Send + Sync,
1438    Op: ElementOp<A>,
1439>(
1440    dest: &mut crate::RawStridedMut<'_, D>,
1441    src: &crate::RawStridedRef<'_, A>,
1442    f: impl Fn(A) -> D + Sync,
1443    validated: ValidatedDestinationLayout,
1444    exec: Exec<'_>,
1445) -> Result<()> {
1446    ensure_same_shape(dest.dims(), src.dims())?;
1447    map_parts_into_validated::<D, A, Op, NoKernel>(
1448        dest.as_mut_ptr(),
1449        dest.dims(),
1450        dest.strides(),
1451        src.ptr(),
1452        src.strides(),
1453        f,
1454        &NoKernel,
1455        validated,
1456        KernelClass::Map,
1457        exec,
1458    )
1459}
1460
1461// Raw parts for the whole operation, not one block: this entry point checks the shapes and
1462// the destination layout, then forwards the same parts to `map_parts_into_validated`.
1463#[allow(clippy::too_many_arguments)]
1464fn map_parts_into<
1465    D: Copy + Send + Sync,
1466    A: Copy + Send + Sync,
1467    Op: ElementOp<A>,
1468    K: MapKernel<A, D> + Sync,
1469>(
1470    dst_ptr: *mut D,
1471    dst_dims: &[usize],
1472    dst_strides: &[isize],
1473    src_ptr: *const A,
1474    src_dims: &[usize],
1475    src_strides: &[isize],
1476    f: impl Fn(A) -> D + Sync,
1477    kernel: &K,
1478    class: KernelClass,
1479    exec: Exec<'_>,
1480) -> Result<()> {
1481    ensure_same_shape(dst_dims, src_dims)?;
1482    let validated = validate_destination_layout(dst_dims, dst_strides)?;
1483    map_parts_into_validated::<D, A, Op, K>(
1484        dst_ptr,
1485        dst_dims,
1486        dst_strides,
1487        src_ptr,
1488        src_strides,
1489        f,
1490        kernel,
1491        validated,
1492        class,
1493        exec,
1494    )
1495}
1496
1497#[allow(clippy::too_many_arguments)]
1498/// Apply `f` (with the element operation) over a fully contiguous pair.
1499///
1500/// Kept out of line, and out of the caller's dispatch frame, so that it cannot
1501/// change the code shape of the blocked kernels below; the operation is moved in
1502/// by value because a dispatched closure that only borrows it stays scalar
1503/// (tensor4all/strided-rs#301).
1504#[inline(never)]
1505fn map_contiguous<D: Copy, A: Copy, Op: ElementOp<A>>(
1506    dst: &mut [D],
1507    src: &[A],
1508    f: impl Fn(A) -> D,
1509) {
1510    simd::dispatch_if_large(dst.len(), move || {
1511        for i in 0..dst.len() {
1512            dst[i] = f(Op::apply(src[i]));
1513        }
1514    });
1515}
1516
1517// Raw parts, the element op, the kernel, the class and the token: the operands are the
1518// operation's, not a block's.
1519#[allow(clippy::too_many_arguments)]
1520fn map_parts_into_validated<
1521    D: Copy + Send + Sync,
1522    A: Copy + Send + Sync,
1523    Op: ElementOp<A>,
1524    K: MapKernel<A, D> + Sync,
1525>(
1526    dst_ptr: *mut D,
1527    dst_dims: &[usize],
1528    dst_strides: &[isize],
1529    src_ptr: *const A,
1530    src_strides: &[isize],
1531    f: impl Fn(A) -> D + Sync,
1532    kernel: &K,
1533    _validated: ValidatedDestinationLayout,
1534    class: KernelClass,
1535    exec: Exec<'_>,
1536) -> Result<()> {
1537    if sequential_contiguous_layout(class, dst_dims, &[dst_strides, src_strides], &exec)?.is_some()
1538    {
1539        let len = total_len(dst_dims)?;
1540        // SAFETY: the contiguous-layout check proves that `dst` and `src` address `len` elements in
1541        // the same order starting at their base pointer, so each is exactly a flat run of `len`
1542        // elements.
1543        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1544        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
1545        if kernel.apply(src, dst) {
1546            return Ok(());
1547        }
1548        map_contiguous::<D, A, Op>(dst, src, f);
1549        return Ok(());
1550    }
1551
1552    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
1553    let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<A>());
1554    let total = total_len(dst_dims)?;
1555
1556    // Small tensor fast path: skip compute_order and compute_block_sizes
1557    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
1558        build_plan_fused_small(dst_dims, &strides_list)
1559    } else {
1560        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
1561    };
1562
1563    let total = total_len(&fused_dims)?;
1564    if is_parallel(class, total, &exec) {
1565        use crate::threading::SendPtr;
1566        let dst_send = SendPtr(dst_ptr);
1567        let src_send = SendPtr(src_ptr as *mut A);
1568
1569        let costs = compute_costs(&ordered_strides);
1570        let initial_offsets: crate::inline::OperandVec<isize> =
1571            strides_list.iter().map(|_| 0isize).collect();
1572        return mapreduce_threaded(
1573            &fused_dims,
1574            &plan.block,
1575            &ordered_strides,
1576            &initial_offsets,
1577            &costs,
1578            class,
1579            exec,
1580            &|dims, blocks, strides_list, offsets| {
1581                for_each_inner_block_with_offsets(
1582                    dims,
1583                    blocks,
1584                    strides_list,
1585                    offsets,
1586                    |offsets, len, strides| {
1587                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1588                        let sp = unsafe { src_send.as_const().offset(offsets[1]) };
1589                        unsafe {
1590                            inner_loop_map1::<D, A, Op, K>(
1591                                dp, strides[0], sp, strides[1], len, &f, kernel,
1592                            )
1593                        };
1594                        Ok(())
1595                    },
1596                )
1597            },
1598        );
1599    }
1600
1601    let initial_offsets: crate::inline::OperandVec<isize> =
1602        ordered_strides.iter().map(|_| 0isize).collect();
1603    for_each_inner_block_preordered(
1604        &fused_dims,
1605        &plan.block,
1606        &ordered_strides,
1607        &initial_offsets,
1608        |offsets, len, strides| {
1609            let dp = unsafe { dst_ptr.offset(offsets[0]) };
1610            let sp = unsafe { src_ptr.offset(offsets[1]) };
1611            unsafe {
1612                inner_loop_map1::<D, A, Op, K>(dp, strides[0], sp, strides[1], len, &f, kernel)
1613            };
1614            Ok(())
1615        },
1616    )
1617}
1618
1619/// Binary element-wise operation: `dest[i] = f(a[i], b[i])`.
1620///
1621/// Source operands `a` and `b` may have different element types from each other
1622/// and from `dest`. The closure `f` handles per-element type conversion.
1623///
1624/// # Errors
1625///
1626/// Returns [`StridedError::RankMismatch`] or
1627/// [`StridedError::ShapeMismatch`] when `dest` and either source view have
1628/// different ranks or extents, and
1629/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1630/// cannot be proven representable and injective: two logical indices name
1631/// one offset, or the element count or address span is too large to
1632/// represent. Nothing is written unless validation succeeds.
1633pub fn zip_map2_into<
1634    D: Copy + Send + Sync,
1635    A: Copy + Send + Sync,
1636    B: Copy + Send + Sync,
1637    OpA: ElementOp<A>,
1638    OpB: ElementOp<B>,
1639>(
1640    dest: &mut StridedViewMut<D>,
1641    a: &StridedView<A, OpA>,
1642    b: &StridedView<B, OpB>,
1643    f: impl Fn(A, B) -> D + Sync,
1644    exec: Exec<'_>,
1645) -> Result<()> {
1646    zip_map2_parts_into::<D, A, B, OpA, OpB>(
1647        dest.as_mut_ptr(),
1648        dest.dims(),
1649        dest.strides(),
1650        a.ptr(),
1651        a.dims(),
1652        a.strides(),
1653        b.ptr(),
1654        b.dims(),
1655        b.strides(),
1656        f,
1657        exec,
1658    )
1659}
1660
1661pub(crate) fn zip_map2_into_validated<
1662    D: Copy + Send + Sync,
1663    A: Copy + Send + Sync,
1664    B: Copy + Send + Sync,
1665    OpA: ElementOp<A>,
1666    OpB: ElementOp<B>,
1667>(
1668    dest: &mut StridedViewMut<D>,
1669    a: &StridedView<A, OpA>,
1670    b: &StridedView<B, OpB>,
1671    f: impl Fn(A, B) -> D + Sync,
1672    validated: ValidatedDestinationLayout,
1673    exec: Exec<'_>,
1674) -> Result<()> {
1675    zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1676        dest.as_mut_ptr(),
1677        dest.dims(),
1678        dest.strides(),
1679        a.ptr(),
1680        a.strides(),
1681        b.ptr(),
1682        b.strides(),
1683        f,
1684        validated,
1685        exec,
1686    )
1687}
1688
1689/// Runtime comparison selected once before entering the element loop.
1690#[non_exhaustive]
1691#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1692pub enum CompareOp {
1693    Eq,
1694    Lt,
1695    Le,
1696    Gt,
1697    Ge,
1698}
1699
1700/// Compare two ordered views elementwise into a Boolean destination.
1701///
1702/// Unlike embedding a runtime operation match inside a [`zip_map2_into`]
1703/// closure, this entry point selects a fixed comparison before traversal.
1704///
1705/// # Errors
1706///
1707/// Returns [`StridedError::ShapeMismatch`] when the source and destination
1708/// shapes differ, or [`StridedError::NonInjectiveOutputLayout`] when distinct
1709/// logical destination elements may overlap.
1710pub fn compare_into<T, OpA, OpB>(
1711    dest: &mut StridedViewMut<bool>,
1712    a: &StridedView<T, OpA>,
1713    b: &StridedView<T, OpB>,
1714    op: CompareOp,
1715    exec: Exec<'_>,
1716) -> Result<()>
1717where
1718    T: Copy + Send + Sync + PartialOrd,
1719    OpA: ElementOp<T>,
1720    OpB: ElementOp<T>,
1721{
1722    match op {
1723        CompareOp::Eq => zip_map2_into(dest, a, b, |lhs, rhs| lhs == rhs, exec),
1724        CompareOp::Lt => zip_map2_into(dest, a, b, |lhs, rhs| lhs < rhs, exec),
1725        CompareOp::Le => zip_map2_into(dest, a, b, |lhs, rhs| lhs <= rhs, exec),
1726        CompareOp::Gt => zip_map2_into(dest, a, b, |lhs, rhs| lhs > rhs, exec),
1727        CompareOp::Ge => zip_map2_into(dest, a, b, |lhs, rhs| lhs >= rhs, exec),
1728    }
1729}
1730
1731/// Compare two views into a fully overwritten uninitialized Boolean output.
1732///
1733/// Dtype-independent shape, destination-injectivity, and reachable-byte
1734/// overlap validation completes before the first write. Safe Rust borrows
1735/// already prevent input/output aliasing; the explicit overlap check preserves
1736/// the contract for views produced through unsafe constructors.
1737///
1738/// `Ok(())` means every logical destination element is initialized. An error
1739/// occurs before writes. A panic during replay may leave a partially initialized
1740/// destination, which remains safe to drop as `MaybeUninit<bool>`.
1741///
1742/// # Errors
1743///
1744/// Returns [`StridedError::ShapeMismatch`] for unequal shapes,
1745/// [`StridedError::NonInjectiveOutputLayout`] for an overlapping output layout,
1746/// [`StridedError::OverlappingInputOutput`] for aliased storage, or
1747/// [`StridedError::OffsetOverflow`] when a reachable byte range is not
1748/// representable.
1749pub fn compare_into_uninit<T, OpA, OpB>(
1750    dest: &mut StridedViewMut<MaybeUninit<bool>>,
1751    a: &StridedView<T, OpA>,
1752    b: &StridedView<T, OpB>,
1753    op: CompareOp,
1754    exec: Exec<'_>,
1755) -> Result<()>
1756where
1757    T: Copy + Send + Sync + PartialOrd,
1758    OpA: ElementOp<T>,
1759    OpB: ElementOp<T>,
1760{
1761    ensure_same_shape(dest.dims(), a.dims())?;
1762    ensure_same_shape(dest.dims(), b.dims())?;
1763    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
1764    validate_typed_no_overlap(dest, a, 0)?;
1765    validate_typed_no_overlap(dest, b, 1)?;
1766    match op {
1767        CompareOp::Eq => zip_map2_into_validated(
1768            dest,
1769            a,
1770            b,
1771            |lhs, rhs| MaybeUninit::new(lhs == rhs),
1772            validated,
1773            exec,
1774        ),
1775        CompareOp::Lt => zip_map2_into_validated(
1776            dest,
1777            a,
1778            b,
1779            |lhs, rhs| MaybeUninit::new(lhs < rhs),
1780            validated,
1781            exec,
1782        ),
1783        CompareOp::Le => zip_map2_into_validated(
1784            dest,
1785            a,
1786            b,
1787            |lhs, rhs| MaybeUninit::new(lhs <= rhs),
1788            validated,
1789            exec,
1790        ),
1791        CompareOp::Gt => zip_map2_into_validated(
1792            dest,
1793            a,
1794            b,
1795            |lhs, rhs| MaybeUninit::new(lhs > rhs),
1796            validated,
1797            exec,
1798        ),
1799        CompareOp::Ge => zip_map2_into_validated(
1800            dest,
1801            a,
1802            b,
1803            |lhs, rhs| MaybeUninit::new(lhs >= rhs),
1804            validated,
1805            exec,
1806        ),
1807    }
1808}
1809
1810pub(crate) fn zip_map2_raw_into_validated<
1811    D: Copy + Send + Sync,
1812    A: Copy + Send + Sync,
1813    B: Copy + Send + Sync,
1814    OpA: ElementOp<A>,
1815    OpB: ElementOp<B>,
1816>(
1817    dest: &mut crate::RawStridedMut<'_, D>,
1818    a: &crate::RawStridedRef<'_, A>,
1819    b: &crate::RawStridedRef<'_, B>,
1820    f: impl Fn(A, B) -> D + Sync,
1821    validated: ValidatedDestinationLayout,
1822    exec: Exec<'_>,
1823) -> Result<()> {
1824    ensure_same_shape(dest.dims(), a.dims())?;
1825    ensure_same_shape(dest.dims(), b.dims())?;
1826    zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1827        dest.as_mut_ptr(),
1828        dest.dims(),
1829        dest.strides(),
1830        a.ptr(),
1831        a.strides(),
1832        b.ptr(),
1833        b.strides(),
1834        f,
1835        validated,
1836        exec,
1837    )
1838}
1839
1840// Raw parts for the whole binary operation: the shape and destination-layout checks live
1841// here, and `zip_map2_parts_into_validated` below does the planning and dispatch.
1842#[allow(clippy::too_many_arguments)]
1843fn zip_map2_parts_into<
1844    D: Copy + Send + Sync,
1845    A: Copy + Send + Sync,
1846    B: Copy + Send + Sync,
1847    OpA: ElementOp<A>,
1848    OpB: ElementOp<B>,
1849>(
1850    dst_ptr: *mut D,
1851    dst_dims: &[usize],
1852    dst_strides: &[isize],
1853    a_ptr: *const A,
1854    a_dims: &[usize],
1855    a_strides: &[isize],
1856    b_ptr: *const B,
1857    b_dims: &[usize],
1858    b_strides: &[isize],
1859    f: impl Fn(A, B) -> D + Sync,
1860    exec: Exec<'_>,
1861) -> Result<()> {
1862    ensure_same_shape(dst_dims, a_dims)?;
1863    ensure_same_shape(dst_dims, b_dims)?;
1864    let validated = validate_destination_layout(dst_dims, dst_strides)?;
1865    zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1866        dst_ptr,
1867        dst_dims,
1868        dst_strides,
1869        a_ptr,
1870        a_strides,
1871        b_ptr,
1872        b_strides,
1873        f,
1874        validated,
1875        exec,
1876    )
1877}
1878
1879// The same three raw-parts operands plus the closure, the `validated` marker and the
1880// token: the planned traversal for the whole operation, entered once, not per block.
1881#[allow(clippy::too_many_arguments)]
1882fn zip_map2_parts_into_validated<
1883    D: Copy + Send + Sync,
1884    A: Copy + Send + Sync,
1885    B: Copy + Send + Sync,
1886    OpA: ElementOp<A>,
1887    OpB: ElementOp<B>,
1888>(
1889    dst_ptr: *mut D,
1890    dst_dims: &[usize],
1891    dst_strides: &[isize],
1892    a_ptr: *const A,
1893    a_strides: &[isize],
1894    b_ptr: *const B,
1895    b_strides: &[isize],
1896    f: impl Fn(A, B) -> D + Sync,
1897    _validated: ValidatedDestinationLayout,
1898    exec: Exec<'_>,
1899) -> Result<()> {
1900    if sequential_contiguous_layout(
1901        KernelClass::Map,
1902        dst_dims,
1903        &[dst_strides, a_strides, b_strides],
1904        &exec,
1905    )?
1906    .is_some()
1907    {
1908        let len = total_len(dst_dims)?;
1909        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1910        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
1911        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
1912        simd::dispatch_if_large(len, || {
1913            for i in 0..len {
1914                dst[i] = f(OpA::apply(sa[i]), OpB::apply(sb[i]));
1915            }
1916        });
1917        return Ok(());
1918    }
1919
1920    let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
1921    let elem_size = std::mem::size_of::<D>()
1922        .max(std::mem::size_of::<A>())
1923        .max(std::mem::size_of::<B>());
1924    let total = total_len(dst_dims)?;
1925
1926    // Small tensor fast path: skip compute_order and compute_block_sizes
1927    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
1928        build_plan_fused_small(dst_dims, &strides_list)
1929    } else {
1930        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
1931    };
1932
1933    let total = total_len(&fused_dims)?;
1934    if is_parallel(KernelClass::Map, total, &exec) {
1935        use crate::threading::SendPtr;
1936        let dst_send = SendPtr(dst_ptr);
1937        let a_send = SendPtr(a_ptr as *mut A);
1938        let b_send = SendPtr(b_ptr as *mut B);
1939
1940        let costs = compute_costs(&ordered_strides);
1941        let initial_offsets: crate::inline::OperandVec<isize> =
1942            strides_list.iter().map(|_| 0isize).collect();
1943        return mapreduce_threaded(
1944            &fused_dims,
1945            &plan.block,
1946            &ordered_strides,
1947            &initial_offsets,
1948            &costs,
1949            KernelClass::Map,
1950            exec,
1951            &|dims, blocks, strides_list, offsets| {
1952                for_each_inner_block_with_offsets(
1953                    dims,
1954                    blocks,
1955                    strides_list,
1956                    offsets,
1957                    |offsets, len, strides| {
1958                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1959                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
1960                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
1961                        unsafe {
1962                            inner_loop_map2::<D, A, B, OpA, OpB>(
1963                                dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1964                            )
1965                        };
1966                        Ok(())
1967                    },
1968                )
1969            },
1970        );
1971    }
1972
1973    let initial_offsets: crate::inline::OperandVec<isize> =
1974        ordered_strides.iter().map(|_| 0isize).collect();
1975    for_each_inner_block_preordered(
1976        &fused_dims,
1977        &plan.block,
1978        &ordered_strides,
1979        &initial_offsets,
1980        |offsets, len, strides| {
1981            let dp = unsafe { dst_ptr.offset(offsets[0]) };
1982            let ap = unsafe { a_ptr.offset(offsets[1]) };
1983            let bp = unsafe { b_ptr.offset(offsets[2]) };
1984            unsafe {
1985                inner_loop_map2::<D, A, B, OpA, OpB>(
1986                    dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1987                )
1988            };
1989            Ok(())
1990        },
1991    )
1992}
1993
1994// Raw parts for the whole multiplication: it takes the contiguous fast path when the three
1995// layouts agree, and otherwise walks the strided one. The operands here are the
1996// operation's, not a block's.
1997#[allow(clippy::too_many_arguments)]
1998fn mul_identity_into_raw<
1999    O: MulOutput<D>,
2000    D: Copy + Send + Sync + 'static,
2001    A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
2002    B: Copy + Send + Sync + 'static,
2003>(
2004    dst_ptr: *mut O::Slot,
2005    dst_dims: &[usize],
2006    dst_strides: &[isize],
2007    a_ptr: *const A,
2008    a_strides: &[isize],
2009    b_ptr: *const B,
2010    b_strides: &[isize],
2011    _validated: ValidatedDestinationLayout,
2012    mut exec: Exec<'_>,
2013) -> Result<()> {
2014    debug_assert_eq!(dst_dims.len(), a_strides.len());
2015    debug_assert_eq!(dst_dims.len(), b_strides.len());
2016
2017    if sequential_contiguous_layout(
2018        KernelClass::Map,
2019        dst_dims,
2020        &[dst_strides, a_strides, b_strides],
2021        &exec,
2022    )?
2023    .is_some()
2024    {
2025        let len = total_len(dst_dims)?;
2026        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2027        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2028        if unsafe { O::try_contiguous(dst_ptr, len, sa, sb) } {
2029            return Ok(());
2030        }
2031        for i in 0..len {
2032            unsafe { O::write(dst_ptr.add(i), multiply_value(sa[i], sb[i])) };
2033        }
2034        return Ok(());
2035    }
2036
2037    let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
2038    let elem_size = std::mem::size_of::<D>()
2039        .max(std::mem::size_of::<A>())
2040        .max(std::mem::size_of::<B>());
2041    let total = total_len(dst_dims)?;
2042
2043    if try_contiguous_range_mul::<O, D, A, B>(
2044        dst_ptr,
2045        dst_dims,
2046        dst_strides,
2047        a_ptr,
2048        a_strides,
2049        b_ptr,
2050        b_strides,
2051        exec.reborrow(),
2052    ) {
2053        return Ok(());
2054    }
2055
2056    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2057        build_plan_fused_small(dst_dims, &strides_list)
2058    } else {
2059        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2060    };
2061
2062    let total = total_len(&fused_dims)?;
2063    if is_parallel(KernelClass::Map, total, &exec) {
2064        use crate::threading::SendPtr;
2065        let dst_send = SendPtr(dst_ptr);
2066        let a_send = SendPtr(a_ptr as *mut A);
2067        let b_send = SendPtr(b_ptr as *mut B);
2068
2069        let costs = compute_costs(&ordered_strides);
2070        let initial_offsets: crate::inline::OperandVec<isize> =
2071            strides_list.iter().map(|_| 0isize).collect();
2072        return mapreduce_threaded(
2073            &fused_dims,
2074            &plan.block,
2075            &ordered_strides,
2076            &initial_offsets,
2077            &costs,
2078            KernelClass::Map,
2079            exec,
2080            &|dims, blocks, strides_list, offsets| {
2081                for_each_inner_block_with_offsets(
2082                    dims,
2083                    blocks,
2084                    strides_list,
2085                    offsets,
2086                    |offsets, len, strides| {
2087                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2088                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2089                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2090                        unsafe {
2091                            inner_loop_mul2::<O, D, A, B>(
2092                                dp, strides[0], ap, strides[1], bp, strides[2], len,
2093                            )
2094                        };
2095                        Ok(())
2096                    },
2097                )
2098            },
2099        );
2100    }
2101
2102    let initial_offsets: crate::inline::OperandVec<isize> =
2103        ordered_strides.iter().map(|_| 0isize).collect();
2104    for_each_inner_block_preordered(
2105        &fused_dims,
2106        &plan.block,
2107        &ordered_strides,
2108        &initial_offsets,
2109        |offsets, len, strides| {
2110            let dp = unsafe { dst_ptr.offset(offsets[0]) };
2111            let ap = unsafe { a_ptr.offset(offsets[1]) };
2112            let bp = unsafe { b_ptr.offset(offsets[2]) };
2113            unsafe {
2114                inner_loop_mul2::<O, D, A, B>(dp, strides[0], ap, strides[1], bp, strides[2], len)
2115            };
2116            Ok(())
2117        },
2118    )
2119}
2120
2121/// Element-wise multiplication: `dest[i] = a[i] * b[i]`.
2122///
2123/// All views must have the same shape. Broadcast operands should be represented
2124/// as stride-0 views before calling this function.
2125///
2126/// # Errors
2127///
2128/// Returns [`StridedError::RankMismatch`] or
2129/// [`StridedError::ShapeMismatch`] when `dest` and either operand have
2130/// different ranks or extents, and
2131/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2132/// cannot be proven representable and injective: two logical indices name
2133/// one offset, or the element count or address span is too large to
2134/// represent. Nothing is written unless validation succeeds.
2135pub fn mul_into<
2136    D: Copy + Send + Sync + 'static,
2137    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2138    B: Copy + Send + Sync + 'static,
2139    OpA: ElementOp<A>,
2140    OpB: ElementOp<B>,
2141>(
2142    dest: &mut StridedViewMut<D>,
2143    a: &StridedView<A, OpA>,
2144    b: &StridedView<B, OpB>,
2145    exec: Exec<'_>,
2146) -> Result<()> {
2147    ensure_same_shape(dest.dims(), a.dims())?;
2148    ensure_same_shape(dest.dims(), b.dims())?;
2149    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2150
2151    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2152        return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2153            dest.as_mut_ptr(),
2154            dest.dims(),
2155            dest.strides(),
2156            a.ptr(),
2157            a.strides(),
2158            b.ptr(),
2159            b.strides(),
2160            validated,
2161            exec,
2162        );
2163    }
2164
2165    zip_map2_into_validated(dest, a, b, multiply_value, validated, exec)
2166}
2167
2168/// Multiply two views into a fully overwritten uninitialized output.
2169///
2170/// Shape, destination-injectivity, and reachable-byte overlap validation
2171/// completes before the first write. Safe Rust borrows already prevent
2172/// input/output aliasing; the explicit overlap check preserves the contract for
2173/// views produced through unsafe constructors.
2174///
2175/// `Ok(())` means every logical destination element is initialized. An error
2176/// occurs before writes. A panic during replay may leave a partially initialized
2177/// destination, which remains safe to drop as `MaybeUninit<D>`.
2178///
2179/// # Errors
2180///
2181/// Returns [`StridedError::ShapeMismatch`] for unequal shapes,
2182/// [`StridedError::NonInjectiveOutputLayout`] for an overlapping output layout,
2183/// [`StridedError::OverlappingInputOutput`] for aliased storage, or
2184/// [`StridedError::OffsetOverflow`] when a reachable byte range is not
2185/// representable.
2186pub fn mul_into_uninit<
2187    D: Copy + Send + Sync + 'static,
2188    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2189    B: Copy + Send + Sync + 'static,
2190    OpA: ElementOp<A>,
2191    OpB: ElementOp<B>,
2192>(
2193    dest: &mut StridedViewMut<MaybeUninit<D>>,
2194    a: &StridedView<A, OpA>,
2195    b: &StridedView<B, OpB>,
2196    exec: Exec<'_>,
2197) -> Result<()> {
2198    ensure_same_shape(dest.dims(), a.dims())?;
2199    ensure_same_shape(dest.dims(), b.dims())?;
2200    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2201    validate_typed_no_overlap(dest, a, 0)?;
2202    validate_typed_no_overlap(dest, b, 1)?;
2203
2204    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2205        return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2206            dest.as_mut_ptr(),
2207            dest.dims(),
2208            dest.strides(),
2209            a.ptr(),
2210            a.strides(),
2211            b.ptr(),
2212            b.strides(),
2213            validated,
2214            exec,
2215        );
2216    }
2217
2218    zip_map2_into_validated(
2219        dest,
2220        a,
2221        b,
2222        |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2223        validated,
2224        exec,
2225    )
2226}
2227
2228fn broadcast_strides_for_axes(
2229    source_dims: &[usize],
2230    source_strides: &[isize],
2231    target_dims: &[usize],
2232    axes: &[usize],
2233) -> Result<AxisVec<isize>> {
2234    if source_dims.len() != axes.len() {
2235        return Err(StridedError::RankMismatch(source_dims.len(), axes.len()));
2236    }
2237    debug_assert_eq!(source_dims.len(), source_strides.len());
2238
2239    let mut seen = AxisVec::<bool>::new();
2240    seen.resize(target_dims.len(), false);
2241    let mut strides = AxisVec::<isize>::new();
2242    strides.resize(target_dims.len(), 0);
2243    for (src_axis, &dst_axis) in axes.iter().enumerate() {
2244        if dst_axis >= target_dims.len() {
2245            return Err(StridedError::InvalidAxis {
2246                axis: dst_axis,
2247                rank: target_dims.len(),
2248            });
2249        }
2250        if seen[dst_axis] {
2251            return Err(StridedError::InvalidAxis {
2252                axis: dst_axis,
2253                rank: target_dims.len(),
2254            });
2255        }
2256        seen[dst_axis] = true;
2257
2258        let source_dim = source_dims[src_axis];
2259        let target_dim = target_dims[dst_axis];
2260        if source_dim != target_dim && source_dim != 1 {
2261            return Err(StridedError::ShapeMismatch(
2262                source_dims.to_vec(),
2263                target_dims.to_vec(),
2264            ));
2265        }
2266        if source_dim == target_dim {
2267            strides[dst_axis] = source_strides[src_axis];
2268        }
2269    }
2270
2271    Ok(strides)
2272}
2273
2274/// A view of `view`'s data with stride-0 broadcast axes.
2275///
2276/// The metadata is computed for this call only; [`StridedView::new_unchecked`]
2277/// copies it into owned metadata (inline up to [`INLINE_RANK`](crate::INLINE_RANK)
2278/// axes, boxed above, so a rank-9 view allocates), and the view still borrows
2279/// only the element data.
2280fn broadcast_view_with_strides<'a, T, Op: ElementOp<T>>(
2281    view: &StridedView<'a, T, Op>,
2282    target_dims: &[usize],
2283    strides: &[isize],
2284) -> StridedView<'a, T, Op> {
2285    // SAFETY: `broadcast_strides_for_axes` (:2192) keeps a source stride only on
2286    // axes where `source_dims[src_axis] == target_dims[dst_axis]`, sets every other
2287    // target axis to stride 0, rejects repeated and out-of-range target axes, and
2288    // requires `source_dims.len() == axes.len()`, so this view reaches a subset of
2289    // the offsets `view` reaches; `StridedView` construction (`new`,
2290    // strided-view/src/view.rs:508 via `validate_bounds`, :186) already proved those
2291    // in bounds, and `target_dims`/`strides` come from the caller's `dest.dims()`
2292    // and that validation. `new_unchecked` copies the metadata into owned
2293    // storage, not inline: `ViewMeta::from_slices` (strided-view/src/view.rs:128)
2294    // keeps it inline up to `INLINE_RANK` = 8 and boxes it above (strided-view/src/view.rs:140), which
2295    // is why a rank-9 call allocates here.
2296    unsafe { StridedView::new_unchecked(view.data(), target_dims, strides, view.offset()) }
2297}
2298
2299/// Broadcasted element-wise multiplication: `dest[i] = a[i] * b[i]`.
2300///
2301/// `a_axes` and `b_axes` map each source axis to an axis of `dest`. Output axes
2302/// not referenced by a source operand are treated as stride-0 broadcast axes.
2303///
2304/// # Errors
2305///
2306/// Returns [`StridedError::RankMismatch`] when an axis list does not have one
2307/// entry per source axis, [`StridedError::InvalidAxis`] when an axis lies
2308/// outside `dest` or is mapped twice, [`StridedError::ShapeMismatch`] when a
2309/// source extent is neither the destination extent nor 1, and
2310/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2311/// cannot be proven representable and injective: two logical indices name
2312/// one offset, or the element count or address span is too large to
2313/// represent. Nothing is written unless validation succeeds.
2314pub fn broadcast_mul_into<
2315    D: Copy + Send + Sync + 'static,
2316    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2317    B: Copy + Send + Sync + 'static,
2318    OpA: ElementOp<A>,
2319    OpB: ElementOp<B>,
2320>(
2321    dest: &mut StridedViewMut<D>,
2322    a: &StridedView<A, OpA>,
2323    a_axes: &[usize],
2324    b: &StridedView<B, OpB>,
2325    b_axes: &[usize],
2326    exec: Exec<'_>,
2327) -> Result<()> {
2328    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2329    let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2330    let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2331
2332    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2333        return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2334            dest.as_mut_ptr(),
2335            dest.dims(),
2336            dest.strides(),
2337            a.ptr(),
2338            &a_strides,
2339            b.ptr(),
2340            &b_strides,
2341            validated,
2342            exec,
2343        );
2344    }
2345
2346    let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2347    let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2348    zip_map2_into_validated(dest, &a, &b, multiply_value, validated, exec)
2349}
2350
2351/// Broadcast and multiply into a fully overwritten uninitialized output.
2352///
2353/// Axis mappings, shapes, destination injectivity, and reachable-byte overlap
2354/// are validated before the first write. Safe Rust borrows already prevent
2355/// input/output aliasing; the explicit overlap check preserves the contract for
2356/// views produced through unsafe constructors.
2357///
2358/// `Ok(())` means every logical destination element is initialized. An error
2359/// occurs before writes. A panic during replay may leave a partially initialized
2360/// destination, which remains safe to drop as `MaybeUninit<D>`.
2361///
2362/// # Errors
2363///
2364/// Returns a typed rank, axis, or shape error for an invalid broadcast mapping,
2365/// [`StridedError::NonInjectiveOutputLayout`] for an overlapping output layout,
2366/// [`StridedError::OverlappingInputOutput`] for aliased storage, or
2367/// [`StridedError::OffsetOverflow`] when a reachable byte range is not
2368/// representable.
2369pub fn broadcast_mul_into_uninit<
2370    D: Copy + Send + Sync + 'static,
2371    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2372    B: Copy + Send + Sync + 'static,
2373    OpA: ElementOp<A>,
2374    OpB: ElementOp<B>,
2375>(
2376    dest: &mut StridedViewMut<MaybeUninit<D>>,
2377    a: &StridedView<A, OpA>,
2378    a_axes: &[usize],
2379    b: &StridedView<B, OpB>,
2380    b_axes: &[usize],
2381    exec: Exec<'_>,
2382) -> Result<()> {
2383    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2384    let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2385    let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2386    validate_typed_no_overlap(dest, a, 0)?;
2387    validate_typed_no_overlap(dest, b, 1)?;
2388
2389    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2390        return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2391            dest.as_mut_ptr(),
2392            dest.dims(),
2393            dest.strides(),
2394            a.ptr(),
2395            &a_strides,
2396            b.ptr(),
2397            &b_strides,
2398            validated,
2399            exec,
2400        );
2401    }
2402
2403    let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2404    let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2405    zip_map2_into_validated(
2406        dest,
2407        &a,
2408        &b,
2409        |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2410        validated,
2411        exec,
2412    )
2413}
2414
2415/// Ternary element-wise operation: `dest[i] = f(a[i], b[i], c[i])`.
2416///
2417/// # Errors
2418///
2419/// Returns [`StridedError::RankMismatch`] or
2420/// [`StridedError::ShapeMismatch`] when `dest` and any of the three source
2421/// views have different ranks or extents, and
2422/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2423/// cannot be proven representable and injective: two logical indices name
2424/// one offset, or the element count or address span is too large to
2425/// represent. Nothing is written unless validation succeeds.
2426pub fn zip_map3_into<
2427    D: Copy + Send + Sync,
2428    A: Copy + Send + Sync,
2429    B: Copy + Send + Sync,
2430    C: Copy + Send + Sync,
2431    OpA: ElementOp<A>,
2432    OpB: ElementOp<B>,
2433    OpC: ElementOp<C>,
2434>(
2435    dest: &mut StridedViewMut<D>,
2436    a: &StridedView<A, OpA>,
2437    b: &StridedView<B, OpB>,
2438    c: &StridedView<C, OpC>,
2439    f: impl Fn(A, B, C) -> D + Sync,
2440    exec: Exec<'_>,
2441) -> Result<()> {
2442    ensure_same_shape(dest.dims(), a.dims())?;
2443    ensure_same_shape(dest.dims(), b.dims())?;
2444    ensure_same_shape(dest.dims(), c.dims())?;
2445    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2446    zip_map3_into_validated(dest, a, b, c, f, validated, exec)
2447}
2448
2449pub(crate) fn zip_map3_into_validated<
2450    D: Copy + Send + Sync,
2451    A: Copy + Send + Sync,
2452    B: Copy + Send + Sync,
2453    C: Copy + Send + Sync,
2454    OpA: ElementOp<A>,
2455    OpB: ElementOp<B>,
2456    OpC: ElementOp<C>,
2457>(
2458    dest: &mut StridedViewMut<D>,
2459    a: &StridedView<A, OpA>,
2460    b: &StridedView<B, OpB>,
2461    c: &StridedView<C, OpC>,
2462    f: impl Fn(A, B, C) -> D + Sync,
2463    _validated: ValidatedDestinationLayout,
2464    exec: Exec<'_>,
2465) -> Result<()> {
2466    ensure_same_shape(dest.dims(), a.dims())?;
2467    ensure_same_shape(dest.dims(), b.dims())?;
2468    ensure_same_shape(dest.dims(), c.dims())?;
2469    let dst_ptr = dest.as_mut_ptr();
2470    let a_ptr = a.ptr();
2471    let b_ptr = b.ptr();
2472    let c_ptr = c.ptr();
2473
2474    let dst_dims = dest.dims();
2475    let dst_strides = dest.strides();
2476
2477    if sequential_contiguous_layout(
2478        KernelClass::Map,
2479        dst_dims,
2480        &[dst_strides, a.strides(), b.strides(), c.strides()],
2481        &exec,
2482    )?
2483    .is_some()
2484    {
2485        let len = total_len(dst_dims)?;
2486        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2487        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2488        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2489        let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2490        simd::dispatch_if_large(len, || {
2491            for (((d, &a), &b), &c) in dst.iter_mut().zip(sa).zip(sb).zip(sc) {
2492                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
2493            }
2494        });
2495        return Ok(());
2496    }
2497
2498    let strides_list: [&[isize]; 4] = [dst_strides, a.strides(), b.strides(), c.strides()];
2499    let elem_size = std::mem::size_of::<D>()
2500        .max(std::mem::size_of::<A>())
2501        .max(std::mem::size_of::<B>())
2502        .max(std::mem::size_of::<C>());
2503    let total = total_len(dst_dims)?;
2504
2505    // Small tensor fast path: skip compute_order and compute_block_sizes
2506    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2507        build_plan_fused_small(dst_dims, &strides_list)
2508    } else {
2509        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2510    };
2511
2512    let total = total_len(&fused_dims)?;
2513    if is_parallel(KernelClass::Map, total, &exec) {
2514        use crate::threading::SendPtr;
2515        let dst_send = SendPtr(dst_ptr);
2516        let a_send = SendPtr(a_ptr as *mut A);
2517        let b_send = SendPtr(b_ptr as *mut B);
2518        let c_send = SendPtr(c_ptr as *mut C);
2519
2520        let costs = compute_costs(&ordered_strides);
2521        let initial_offsets: crate::inline::OperandVec<isize> =
2522            strides_list.iter().map(|_| 0isize).collect();
2523        return mapreduce_threaded(
2524            &fused_dims,
2525            &plan.block,
2526            &ordered_strides,
2527            &initial_offsets,
2528            &costs,
2529            KernelClass::Map,
2530            exec,
2531            &|dims, blocks, strides_list, offsets| {
2532                for_each_inner_block_with_offsets(
2533                    dims,
2534                    blocks,
2535                    strides_list,
2536                    offsets,
2537                    |offsets, len, strides| {
2538                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2539                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2540                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2541                        let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2542                        unsafe {
2543                            inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2544                                dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3],
2545                                len, &f,
2546                            )
2547                        };
2548                        Ok(())
2549                    },
2550                )
2551            },
2552        );
2553    }
2554
2555    let initial_offsets: crate::inline::OperandVec<isize> =
2556        ordered_strides.iter().map(|_| 0isize).collect();
2557    for_each_inner_block_preordered(
2558        &fused_dims,
2559        &plan.block,
2560        &ordered_strides,
2561        &initial_offsets,
2562        |offsets, len, strides| {
2563            let dp = unsafe { dst_ptr.offset(offsets[0]) };
2564            let ap = unsafe { a_ptr.offset(offsets[1]) };
2565            let bp = unsafe { b_ptr.offset(offsets[2]) };
2566            let cp = unsafe { c_ptr.offset(offsets[3]) };
2567            unsafe {
2568                inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2569                    dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], len, &f,
2570                )
2571            };
2572            Ok(())
2573        },
2574    )
2575}
2576
2577/// Quaternary element-wise operation: `dest[i] = f(a[i], b[i], c[i], e[i])`.
2578///
2579/// # Errors
2580///
2581/// Returns [`StridedError::RankMismatch`] or
2582/// [`StridedError::ShapeMismatch`] when `dest` and any of the four source
2583/// views have different ranks or extents, and
2584/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2585/// cannot be proven representable and injective: two logical indices name
2586/// one offset, or the element count or address span is too large to
2587/// represent. Nothing is written unless validation succeeds.
2588pub fn zip_map4_into<
2589    D: Copy + Send + Sync,
2590    A: Copy + Send + Sync,
2591    B: Copy + Send + Sync,
2592    C: Copy + Send + Sync,
2593    E: Copy + Send + Sync,
2594    OpA: ElementOp<A>,
2595    OpB: ElementOp<B>,
2596    OpC: ElementOp<C>,
2597    OpE: ElementOp<E>,
2598>(
2599    dest: &mut StridedViewMut<D>,
2600    a: &StridedView<A, OpA>,
2601    b: &StridedView<B, OpB>,
2602    c: &StridedView<C, OpC>,
2603    e: &StridedView<E, OpE>,
2604    f: impl Fn(A, B, C, E) -> D + Sync,
2605    exec: Exec<'_>,
2606) -> Result<()> {
2607    ensure_same_shape(dest.dims(), a.dims())?;
2608    ensure_same_shape(dest.dims(), b.dims())?;
2609    ensure_same_shape(dest.dims(), c.dims())?;
2610    ensure_same_shape(dest.dims(), e.dims())?;
2611    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2612    zip_map4_into_validated(dest, a, b, c, e, f, validated, exec)
2613}
2614
2615// Four operand views plus the destination, the closure, the `validated` marker and
2616// the token: the entry's shape.
2617#[allow(clippy::too_many_arguments)]
2618pub(crate) fn zip_map4_into_validated<
2619    D: Copy + Send + Sync,
2620    A: Copy + Send + Sync,
2621    B: Copy + Send + Sync,
2622    C: Copy + Send + Sync,
2623    E: Copy + Send + Sync,
2624    OpA: ElementOp<A>,
2625    OpB: ElementOp<B>,
2626    OpC: ElementOp<C>,
2627    OpE: ElementOp<E>,
2628>(
2629    dest: &mut StridedViewMut<D>,
2630    a: &StridedView<A, OpA>,
2631    b: &StridedView<B, OpB>,
2632    c: &StridedView<C, OpC>,
2633    e: &StridedView<E, OpE>,
2634    f: impl Fn(A, B, C, E) -> D + Sync,
2635    _validated: ValidatedDestinationLayout,
2636    exec: Exec<'_>,
2637) -> Result<()> {
2638    ensure_same_shape(dest.dims(), a.dims())?;
2639    ensure_same_shape(dest.dims(), b.dims())?;
2640    ensure_same_shape(dest.dims(), c.dims())?;
2641    ensure_same_shape(dest.dims(), e.dims())?;
2642    let dst_ptr = dest.as_mut_ptr();
2643    let a_ptr = a.ptr();
2644    let b_ptr = b.ptr();
2645    let c_ptr = c.ptr();
2646    let e_ptr = e.ptr();
2647
2648    let dst_dims = dest.dims();
2649    let dst_strides = dest.strides();
2650
2651    if sequential_contiguous_layout(
2652        KernelClass::Map,
2653        dst_dims,
2654        &[
2655            dst_strides,
2656            a.strides(),
2657            b.strides(),
2658            c.strides(),
2659            e.strides(),
2660        ],
2661        &exec,
2662    )?
2663    .is_some()
2664    {
2665        let len = total_len(dst_dims)?;
2666        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2667        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2668        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2669        let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2670        let se = unsafe { std::slice::from_raw_parts(e_ptr, len) };
2671        // As in `inner_loop_map4`: zipped iterators keep the dispatched closure
2672        // small enough to inline (tensor4all/strided-rs#340).
2673        simd::dispatch_if_large(len, || {
2674            for ((((d, &a), &b), &c), &e) in dst.iter_mut().zip(sa).zip(sb).zip(sc).zip(se) {
2675                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
2676            }
2677        });
2678        return Ok(());
2679    }
2680
2681    let strides_list: [&[isize]; 5] = [
2682        dst_strides,
2683        a.strides(),
2684        b.strides(),
2685        c.strides(),
2686        e.strides(),
2687    ];
2688    let elem_size = std::mem::size_of::<D>()
2689        .max(std::mem::size_of::<A>())
2690        .max(std::mem::size_of::<B>())
2691        .max(std::mem::size_of::<C>())
2692        .max(std::mem::size_of::<E>());
2693    let total = total_len(dst_dims)?;
2694
2695    // Small tensor fast path: skip compute_order and compute_block_sizes
2696    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
2697        build_plan_fused_small(dst_dims, &strides_list)
2698    } else {
2699        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
2700    };
2701
2702    let total = total_len(&fused_dims)?;
2703    if is_parallel(KernelClass::Map, total, &exec) {
2704        use crate::threading::SendPtr;
2705        let dst_send = SendPtr(dst_ptr);
2706        let a_send = SendPtr(a_ptr as *mut A);
2707        let b_send = SendPtr(b_ptr as *mut B);
2708        let c_send = SendPtr(c_ptr as *mut C);
2709        let e_send = SendPtr(e_ptr as *mut E);
2710
2711        let costs = compute_costs(&ordered_strides);
2712        let initial_offsets: crate::inline::OperandVec<isize> =
2713            strides_list.iter().map(|_| 0isize).collect();
2714        return mapreduce_threaded(
2715            &fused_dims,
2716            &plan.block,
2717            &ordered_strides,
2718            &initial_offsets,
2719            &costs,
2720            KernelClass::Map,
2721            exec,
2722            &|dims, blocks, strides_list, offsets| {
2723                for_each_inner_block_with_offsets(
2724                    dims,
2725                    blocks,
2726                    strides_list,
2727                    offsets,
2728                    |offsets, len, strides| {
2729                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2730                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2731                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2732                        let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2733                        let ep = unsafe { e_send.as_const().offset(offsets[4]) };
2734                        unsafe {
2735                            inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2736                                dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep,
2737                                strides[4], len, &f,
2738                            )
2739                        };
2740                        Ok(())
2741                    },
2742                )
2743            },
2744        );
2745    }
2746
2747    let initial_offsets: crate::inline::OperandVec<isize> =
2748        ordered_strides.iter().map(|_| 0isize).collect();
2749    for_each_inner_block_preordered(
2750        &fused_dims,
2751        &plan.block,
2752        &ordered_strides,
2753        &initial_offsets,
2754        |offsets, len, strides| {
2755            let dp = unsafe { dst_ptr.offset(offsets[0]) };
2756            let ap = unsafe { a_ptr.offset(offsets[1]) };
2757            let bp = unsafe { b_ptr.offset(offsets[2]) };
2758            let cp = unsafe { c_ptr.offset(offsets[3]) };
2759            let ep = unsafe { e_ptr.offset(offsets[4]) };
2760            unsafe {
2761                inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2762                    dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep, strides[4],
2763                    len, &f,
2764                )
2765            };
2766            Ok(())
2767        },
2768    )
2769}
2770
2771#[cfg(test)]
2772#[path = "map_view/tests/scalar_branch_tests.rs"]
2773mod scalar_branch_tests;
2774
2775#[cfg(test)]
2776#[path = "map_view/tests/tests.rs"]
2777mod tests;